... | ... | @@ -37,11 +37,11 @@ Verbosity is 2 |
|
|
(INFO): Submitting interval 2018-04-10 00:00:00 -> 2018-04-10 23:59:59
|
|
|
(INFO): Opening log file fusion_matlab_snpp_s201804100000_e201804102359_c20201124170636.log
|
|
|
(INFO): There are 240 contexts in this interval
|
|
|
(INFO): First context: {'satellite': 'snpp', 'version': '1.1.0dev4', 'granule': datetime.datetime(2018, 4, 10, 0, 0)}
|
|
|
(INFO): Last context: {'satellite': 'snpp', 'version': '1.1.0dev4', 'granule': datetime.datetime(2018, 4, 10, 23, 54)}
|
|
|
(INFO): First context: {'satellite': 'snpp', 'version': '1.2.0.1dev0', 'granule': datetime.datetime(2018, 4, 10, 0, 0)}
|
|
|
(INFO): Last context: {'satellite': 'snpp', 'version': '1.2.0.1dev0', 'granule': datetime.datetime(2018, 4, 10, 23, 54)}
|
|
|
(INFO): Attempted 240 submits; results: 239/1/0 (submitted/not needed/not ready) [jobs 187647274 to 187647512]
|
|
|
(INFO): Job nums: range(187647274, 187647513)
|
|
|
(INFO): contexts: [{'satellite': 'snpp', 'version': '1.1.0dev4', 'granule': datetime.datetime(2018, 4, 10, 0, 0)}, {'satellite': 'snpp', 'version': '1.1.0dev4', 'granule': datetime.datetime(2018, 4, 10, 23, 54)}]; job numbers: [187647274,187647512]
|
|
|
(INFO): contexts: [{'satellite': 'snpp', 'version': '1.2.0.1dev0', 'granule': datetime.datetime(2018, 4, 10, 0, 0)}, {'satellite': 'snpp', 'version': '1.2.0.1dev0', 'granule': datetime.datetime(2018, 4, 10, 23, 54)}]; job numbers: [187647274,187647512]
|
|
|
(INFO): job numbers: [187647274..187647512]
|
|
|
|
|
|
(INFO): Closing log file fusion_matlab_snpp_s201804100000_e201804102359_c20201124170636.log
|
... | ... | @@ -77,51 +77,49 @@ flo_user='-h ratchet -U ro flo3' |
|
|
> psql $flo_user -c "SELECT job,size,output,context,file_name from stored_products where computation='flo.sw.fusion_matlab:FUSION_MATLAB' and output='fused_l1b' order by file_name;"
|
|
|
job | size | output | context | file_name
|
|
|
----------+-----------+-----------+-------------------------------------------------------------------------------------------------------+-------------------------------------------------
|
|
|
91073252 | 366596456 | fused_l1b | "granule"=>"datetime.datetime(2015, 4, 18, 6, 6)", "version"=>"'1.0dev0'", "satellite"=>"'snpp'" | VNP02FSN.A2015108.0606.001.2018025180544.nc
|
|
|
91073252 | 366596456 | fused_l1b | "granule"=>"datetime.datetime(2015, 4, 18, 6, 6)", "version"=>"'2.0.1dev0'", "satellite"=>"'snpp'" | VNP02FSN.A2015108.0606.001.2018025180544.nc
|
|
|
(1 row)
|
|
|
```
|
|
|
|
|
|
To group granules by day/month etc...
|
|
|
```sql
|
|
|
psql $flo_user -c "SELECT date_trunc('months',pydt(context->'granule')) as m,count(*) from stored_products where computation='flo.sw.fusion_matlab:FUSION_MATLAB' and context->'satellite'='''snpp''' and context->'version'='''1.0dev0''' group by m order by m" | less
|
|
|
psql $flo_user -c "SELECT date_trunc('months',pydt(context->'granule')) as m,count(*) from stored_products where computation='flo.sw.fusion_matlab:FUSION_MATLAB' and context->>'satellite'='snpp' and context->>'version'='2.0.1dev0' group by m order by m" | less
|
|
|
```
|
|
|
To select granules which match or are between certain dates:
|
|
|
```sql
|
|
|
psql $flo_user -c "SELECT job,size,context,file_name from stored_products where computation='flo.sw.fusion_matlab:FUSION_MATLAB' and context->'satellite'='''snpp''' and context->'version'='''1.0dev0''' and date_trunc('days',pydt(context->'granule'))='2015-01-01' order by file_name;" | less
|
|
|
|
|
|
psql $flo_user -c "SELECT job,size,context,file_name from stored_products where computation='flo.sw.fusion_matlab:FUSION_MATLAB' and context->'satellite'='''snpp''' and context->'version'='''1.0dev0''' and date_trunc('days',pydt(context->'granule'))>'2015-01-01' and date_trunc('days',pydt(context->'granule'))<'2015-01-03' order by file_name;" | less
|
|
|
psql $flo_user -c "SELECT job,size,context,file_name from stored_products where computation='flo.sw.fusion_matlab:FUSION_MATLAB' and context->>'satellite'='snpp' and context->>'version'='2.0.1dev0' and (context#>>'{granule,value}')::timestamp between '2019-01-01' and '2019-01-02' order by file_name;" | less
|
|
|
```
|
|
|
To remove old files:
|
|
|
```sql
|
|
|
sudo su - flo
|
|
|
flo_user_rw='-h ratchet flo3'
|
|
|
psql $flo_user_rw -c "SELECT job, size, context, file_name FROM stored_products WHERE computation='flo.sw.fusion_matlab:FUSION_MATLAB' and context->'satellite'='''snpp''' and context->'version'='''1.0dev0''' order by file_name" | less
|
|
|
psql $flo_user_rw -c "DELETE FROM stored_products WHERE computation='flo.sw.fusion_matlab:FUSION_MATLAB' and context->'satellite'='''snpp''' and context->'version'='''1.0dev0'''"
|
|
|
psql $flo_user_rw -c "SELECT job, size, context, file_name FROM stored_products WHERE computation='flo.sw.fusion_matlab:FUSION_MATLAB' and context->>'satellite'='snpp' and context->>'version'='2.0.1dev0' order by file_name" | less
|
|
|
psql $flo_user_rw -c "DELETE FROM stored_products WHERE computation='flo.sw.fusion_matlab:FUSION_MATLAB' and context->>'satellite'='snpp' and context->>'version'='2.0.1dev0'"
|
|
|
```
|
|
|
## Other Database Querys
|
|
|
|
|
|
```sql
|
|
|
psql $flo_user -c "SELECT pydt(context->'granule') as d,count(*) FROM stored_products WHERE computation='flo.sw.fusion_matlab:FUSION_MATLAB' and context->'satellite'='''snpp''' and context->'version'='''1.0dev0''' group by d order by d order by file_name;" | less
|
|
|
psql $flo_user -c "SELECT pydt(context->'granule') as d,count(*) FROM stored_products WHERE computation='flo.sw.fusion_matlab:FUSION_MATLAB' and context->>'satellite'='snpp' and context->>'version'='2.0.1dev0' group by d order by d order by file_name;" | less
|
|
|
|
|
|
psql $flo_user -c "SELECT pydt(context->'granule') as d,count(*) FROM stored_products WHERE computation='flo.sw.fusion_matlab:FUSION_MATLAB' and context->'satellite'='''snpp''' and context->'version'='''1.0dev0''' and date_trunc('days',pydt(context->'granule'))='2014-01-01' group by d order by d;" | less
|
|
|
psql $flo_user -c "SELECT pydt(context->'granule') as d,count(*) FROM stored_products WHERE computation='flo.sw.fusion_matlab:FUSION_MATLAB' and context->>'satellite'='snpp' and context->>'version'='2.0.1dev0' and date_trunc('days',pydt(context->'granule'))='2014-01-01' group by d order by d;" | less
|
|
|
|
|
|
psql $flo_user -c "SELECT job,size,context,file_name from stored_products where computation='flo.sw.fusion_matlab:FUSION_MATLAB' and context->'satellite'='''snpp''' and context->'version'='''1.0dev0''' order by file_name;" | less
|
|
|
psql $flo_user -c "SELECT job,size,context,file_name from stored_products where computation='flo.sw.fusion_matlab:FUSION_MATLAB' and context->>'satellite'='snpp' and context->>'version'='2.0.1dev0' order by file_name;" | less
|
|
|
|
|
|
psql $flo_user -c "SELECT job,size,context,file_name from stored_products where computation='flo.sw.fusion_matlab:FUSION_MATLAB' and context->'satellite'='''snpp''' and context->'version'='''1.0dev0''' and date_trunc('days',pydt(context->'granule'))='2014-01-01' order by file_name;" | less
|
|
|
psql $flo_user -c "SELECT job,size,context,file_name from stored_products where computation='flo.sw.fusion_matlab:FUSION_MATLAB' and context->>'satellite'='snpp' and context->>'version'='2.0.1dev0' and date_trunc('days',pydt(context->'granule'))='2014-01-01' order by file_name;" | less
|
|
|
|
|
|
flo3=> select x FROM generate_series('2015-04-01'::timestamp, '2015-04-30 23:59', '6 minutes') as x where not exists (select null from stored_products where computation='flo.sw.fusion_matlab:FUSION_MATLAB' and context->'satellite'='''snpp''' and output='fused_l1b' and x=pydt(context->'granule'));
|
|
|
flo3=> select x FROM generate_series('2015-04-01'::timestamp, '2015-04-30 23:59', '6 minutes') as x where not exists (select null from stored_products where computation='flo.sw.fusion_matlab:FUSION_MATLAB' and context->>'satellite'='snpp' and output='fused_l1b' and x=pydt(context->'granule'));
|
|
|
|
|
|
# List files keys
|
|
|
psql $flo_user -tA -c "SELECT format ('flo3/%s/%s',job,file_name) FROM stored_products WHERE computation='flo.sw.fusion_matlab:FUSION_MATLAB' and context->'satellite'='''snpp''' and context->'version'='''1.0dev0''' order by file_name limit 5;"
|
|
|
psql $flo_user -tA -c "SELECT format ('flo3/%s/%s',job,file_name) FROM stored_products WHERE computation='flo.sw.fusion_matlab:FUSION_MATLAB' and context->>'satellite'='snpp' and context->>'version'='2.0.1dev0' order by file_name limit 5;"
|
|
|
|
|
|
# List file keys and status
|
|
|
psql $flo_user -tA -c "SELECT format ('flo3/%s/%s',job,file_name) FROM stored_products WHERE computation='flo.sw.fusion_matlab:FUSION_MATLAB' and context->'satellite'='''snpp''' and context->'version'='''1.0dev0''' order by file_name limit 5;" | xargs -n1 -IXX rados -p dev --id flo stat XX
|
|
|
psql $flo_user -tA -c "SELECT format ('flo3/%s/%s',job,file_name) FROM stored_products WHERE computation='flo.sw.fusion_matlab:FUSION_MATLAB' and context->>'satellite'='snpp' and context->>'version'='2.0.1dev0' order by file_name limit 5;" | xargs -n1 -IXX rados -p dev --id flo stat XX
|
|
|
|
|
|
# List file key basenames
|
|
|
psql $flo_user -tA -c "SELECT format ('flo3/%s/%s',job,file_name) FROM stored_products WHERE computation='flo.sw.fusion_matlab:FUSION_MATLAB' and context->'satellite'='''snpp''' and context->'version'='''1.0dev0''' order by file_name limit 5;" | xargs -n1 -IXX basename XX
|
|
|
psql $flo_user -tA -c "SELECT format ('flo3/%s/%s',job,file_name) FROM stored_products WHERE computation='flo.sw.fusion_matlab:FUSION_MATLAB' and context->>'satellite'='snpp' and context->>'version'='2.0.1dev0' order by file_name limit 5;" | xargs -n1 -IXX basename XX
|
|
|
|
|
|
# List the rados commands to download files using the database file keys.
|
|
|
psql $flo_user -tA -c "SELECT format ('flo3/%s/%s',job,file_name) FROM stored_products WHERE computation='flo.sw.fusion_matlab:FUSION_MATLAB' and context->'satellite'='''snpp''' and context->'version'='''1.0dev0''' order by file_name limit 5;" | xargs -n1 -IXX echo rados -p dev --id flo get XX "~/fusion_matlab/work/links/"$(basename XX)
|
|
|
psql $flo_user -tA -c "SELECT format ('flo3/%s/%s',job,file_name) FROM stored_products WHERE computation='flo.sw.fusion_matlab:FUSION_MATLAB' and context->>'satellite'='snpp' and context->>'version'='2.0.1dev0' order by file_name limit 5;" | xargs -n1 -IXX echo rados -p dev --id flo get XX "~/fusion_matlab/work/links/"$(basename XX)
|
|
|
|
|
|
# rados commands
|
|
|
rados -p dev --id flo get flo3/91069111/VNP02FSN.A2015091.0000.001.2018025170339.nc VNP02FSN.A2015091.0000.001.2018025170339.nc
|
... | ... | @@ -155,7 +153,7 @@ VALUES ( |
|
|
'FusionMatlab',
|
|
|
'-4 Days'::interval,
|
|
|
'00:00:00'::interval,
|
|
|
'version=>"''1.0dev4''", satellite=>"''snpp''"'::hstore,
|
|
|
'version=>"''2.0.1dev0''", satellite=>"''snpp''"'::hstore,
|
|
|
'flo.sw.fusion_matlab:FUSION_MATLAB',
|
|
|
'{flo.sw.fusion_matlab:FUSION_MATLAB;fused_l1b}'::text[],
|
|
|
'requests=>"[''Memory=8000'']"'::hstore
|
... | ... | @@ -209,16 +207,16 @@ flo3=> select * from forward_streams where name = 'FusionMatlab'; |
|
|
```
|
|
|
we can preview the change by running
|
|
|
```sql
|
|
|
select find_contexts_arguments || 'version=>"''1.0dev6''"' from forward_streams where name = 'FusionMatlab';
|
|
|
select find_contexts_arguments || 'version=>"''2.0.1dev0''"' from forward_streams where name = 'FusionMatlab';
|
|
|
```
|
|
|
When we are happy with the proposed changes we can modify the existing forward stream
|
|
|
```sql
|
|
|
update forward_streams set find_contexts_arguments = find_contexts_arguments || 'version=>"''1.0dev6''"' where name = 'FusionMatlab';
|
|
|
update forward_streams set find_contexts_arguments = find_contexts_arguments || 'version=>"''2.0.1dev0''"' where name = 'FusionMatlab';
|
|
|
```
|
|
|
For the Fusion Quicklooks we similarly have
|
|
|
```sql
|
|
|
select find_contexts_arguments || 'version=>"''1.0dev4''"' from forward_streams where name = 'FusionMatlabDailyQL';
|
|
|
update forward_streams set find_contexts_arguments = find_contexts_arguments || 'version=>"''1.0dev4''"' where name = 'FusionMatlabDailyQL';
|
|
|
select find_contexts_arguments || 'version=>"''2.0.1dev0''"' from forward_streams where name = 'FusionMatlabDailyQL';
|
|
|
update forward_streams set find_contexts_arguments = find_contexts_arguments || 'version=>"''2.0.1dev0''"' where name = 'FusionMatlabDailyQL';
|
|
|
```
|
|
|
## Deleting a Forward Stream Instance
|
|
|
|
... | ... | @@ -229,9 +227,9 @@ psql $flo_user -c "select name,find_contexts_arguments,workflow_head from forwar |
|
|
```
|
|
|
resulting in
|
|
|
```
|
|
|
FusionMatlab | "version"=>"'1.0dev10'", "satellite"=>"'snpp'" | flo.sw.fusion_matlab:FUSION_MATLAB
|
|
|
FusionMatlab | "version"=>"'2.0.1dev0'", "satellite"=>"'snpp'" | flo.sw.fusion_matlab:FUSION_MATLAB
|
|
|
FusionMatlabDailyQL | "version"=>"'1.0dev7'", "satellite"=>"'snpp'" | flo.sw.fusion_matlab:FUSION_MATLAB_QL
|
|
|
FusionMatlab_jpss1 | "version"=>"'1.0dev10'", "satellite"=>"'noaa20'" | flo.sw.fusion_matlab:FUSION_MATLAB
|
|
|
FusionMatlab_jpss1 | "version"=>"'2.0.1dev0'", "satellite"=>"'noaa20'" | flo.sw.fusion_matlab:FUSION_MATLAB
|
|
|
FusionMatlabDailyQL_jpss1 | "version"=>"'1.0dev7'", "satellite"=>"'noaa20'" | flo.sw.fusion_matlab:FUSION_MATLAB_QL
|
|
|
```
|
|
|
To delete each of these, we enter the `psql` shell and execute a series of `begin-delete-commit` transactions...
|
... | ... | @@ -268,19 +266,19 @@ flo3=> |
|
|
The details o failed jobs can be found from
|
|
|
|
|
|
```sql
|
|
|
psql $flo_user -c "select * from failed_jobs where head_computation = 'flo.sw.fusion_matlab:FUSION_MATLAB' and context->'version'='''1.0dev10''' and timestamp > '2018-01-30';"
|
|
|
psql $flo_user -c "select * from failed_jobs where head_computation = 'flo.sw.fusion_matlab:FUSION_MATLAB' and context->>'version'='2.0.1dev0' and timestamp > '2018-01-30';"
|
|
|
```
|
|
|
|
|
|
Generate a list of jobnumbers for failed jobs:
|
|
|
|
|
|
```sql
|
|
|
psql $flo_user -c "SELECT job, context FROM failed_jobs WHERE head_computation='flo.sw.fusion_matlab:FUSION_MATLAB' and context->'version'='''1.0dev10''' and timestamp > '2018-01-30' order by context;" | grep granule | gawk '{print $1}' > fusion_matlab_v1.0dev10_failed_granules.txt
|
|
|
psql $flo_user -c "SELECT job, context FROM failed_jobs WHERE head_computation='flo.sw.fusion_matlab:FUSION_MATLAB' and context->>'version'='2.0.1dev0' and timestamp > '2018-01-30' order by context;" | grep granule | gawk '{print $1}' > fusion_matlab_v2.0.1dev0_failed_granules.txt
|
|
|
```
|
|
|
|
|
|
Read a file containing the job numbers of failed jobs, and do something with them...
|
|
|
|
|
|
```python
|
|
|
file_obj = open('fusion_matlab_v1.0dev10_failed_granules.txt','r')
|
|
|
file_obj = open('fusion_matlab_v2.0.1dev0_failed_granules.txt','r')
|
|
|
jobnums = file_obj.readlines()
|
|
|
file_obj.close()
|
|
|
jobnums = [int(x) for x in jobnums]
|
... | ... | |