Skip to content
Snippets Groups Projects
Commit 20a921c2 authored by David Hoese's avatar David Hoese
Browse files

Remove separate satpy notebook

parent 932df900
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
Showing
with 0 additions and 274 deletions
g16_abi_20190117.ipynb deleted 100644 → 0
+ 0
− 274
View file @ 932df900
%% Cell type:code id: tags:
``` python
import os
os.environ['HDF5_USE_FILE_LOCKING'] = "FALSE"
os.environ['PYTROLL_CHUNK_SIZE'] = "2048"
from glob import glob
from satpy import MultiScene
import dask
from multiprocessing.pool import ThreadPool
from dask.diagnostics import ProgressBar
```
%% Cell type:code id: tags:
``` python
input_files = glob('/arcdata/goes/grb/goes16/2019/2019_01_17_017/abi/L1b/RadC/*C01*s201901712*.nc')
# input_files = glob('/arcdata/goes/grb/goes16/2019/2019_01_17_017/abi/L1b/RadC/*s2019017[12]*.nc')
#input_files = glob('/arcdata/goes/grb/goes16/2019/2019_01_18_018/abi/L1b/RadC/*s2019018[12]*.nc')
#input_files += glob('/arcdata/goes/grb/goes16/2019/2019_01_19_019/abi/L1b/RadC/*s20190190*.nc')
print(len(input_files))
```
%% Output
12
%% Cell type:code id: tags:
``` python
mscn = MultiScene.from_files(input_files, reader='abi_l1b')
mscn.load(['C01'])
# mscn.load(['true_color_night'])
# res_mscn = mscn.resample(resampler='native')
```
%% Cell type:code id: tags:
``` python
import sys
from dask_jobqueue import SLURMCluster
cluster = SLURMCluster(partition='all',
walltime='02:00:00',
name='davidh_dask',
cores=4,
processes=2,
memory='20GB',
python=sys.executable, # '/home/davidh/miniconda3/envs/pangeo/bin/python',
local_directory='/scratch',
diagnostics_port=int(os.getenv('DASK_PORT', 8787)),
)
cluster.scale(5)
cluster
```
%% Output
/home/davidh/miniconda3/envs/pangeo/lib/python3.6/site-packages/distributed/bokeh/core.py:56: UserWarning:
Port 33121 is already in use.
Perhaps you already have a cluster running?
Hosting the diagnostics dashboard on a random port instead.
warnings.warn('\n' + msg)
%% Cell type:code id: tags:
``` python
from dask.distributed import Client
client = Client(cluster)
client
```
%% Output
<Client: scheduler='tcp://10.23.255.247:33990' processes=0 cores=0>
%% Cell type:code id: tags:
``` python
import numpy as np
c01_avg = np.mean([scn['C01'] for scn in mscn], axis=0)
```
%% Cell type:code id: tags:
``` python
c01_avg
```
%% Cell type:code id: tags:
``` python
%%time
all_jobs = []
for idx, scn in enumerate(res_mscn):
print("Scene {:d}".format(idx))
res = scn.save_datasets(writer='scmi', base_dir='/odyssey/isis/tmp/davidh', sector_id="GOES_EAST", source_name="SSEC", compute=False)
all_jobs.append(res)
```
%% Output
Scene 0
Scene 1
Scene 2
Scene 3
Scene 4
Scene 5
Scene 6
Scene 7
Scene 8
Scene 9
Scene 10
CPU times: user 39.1 s, sys: 6.36 s, total: 45.4 s
Wall time: 1min 7s
%% Cell type:code id: tags:
``` python
all_jobs = [x for y in all_jobs for x in y]
print(len(all_jobs))
all_jobs
```
%% Cell type:code id: tags:
``` python
dask.compute(all_jobs)
```
%% Cell type:code id: tags:
``` python
%%time
mscn.save_animation('/odyssey/isis/tmp/davidh/g16_abi_{name}_{start_time:%Y%m%d_%H%M%S}/g16_abi_{name}_{start_time:%Y%m%d_%H%M%S}.mp4', fps=24, batch_size=16)
```
%% Cell type:code id: tags:
``` python
!png2mp4.sh /odyssey/isis/tmp/davidh/g16_abi_true_color_night.mp4 /odyssey/isis/tmp/davidh/g16_abi_true_color_night_2*/*.png
```
%% Cell type:code id: tags:
``` python
del client
del cluster
```
%% Cell type:code id: tags:
``` python
```
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment