... | ... | @@ -83,43 +83,40 @@ flo_user='-h ratchet -U ro flo3' |
|
|
|
|
|
To group granules by day/month etc...
|
|
|
```sql
|
|
|
psql $flo_user -c "SELECT date_trunc('months',pydt(context->'granule')) as m,count(*) from stored_products where computation='flo.sw.fusion_matlab:FUSION_MATLAB' and context->>'satellite'='snpp' and context->>'version'='2.0.1dev0' group by m order by m" | less
|
|
|
satellite='snpp'; psql $flo_user -c "select date_trunc('months', jts(context->'granule')) as m,count(*) from stored_products where computation='flo.sw.fusion_matlab:FUSION_MATLAB' and context->>'satellite'='$satellite' and context->>'version'='2.0.0' group by m order by m;"
|
|
|
```
|
|
|
To select granules which match or are between certain dates:
|
|
|
```sql
|
|
|
psql $flo_user -c "SELECT job,size,context,file_name from stored_products where computation='flo.sw.fusion_matlab:FUSION_MATLAB' and context->>'satellite'='snpp' and context->>'version'='2.0.1dev0' and (context#>>'{granule,value}')::timestamp between '2019-01-01' and '2019-01-02' order by file_name;" | less
|
|
|
satellite='snpp'; psql $flo_user -c "SELECT job,size,context,file_name from stored_products where computation='flo.sw.fusion_matlab:FUSION_MATLAB' and context->>'satellite'='$satellite' and context->>'version'='2.0.1dev0' and (context#>>'{granule,value}')::timestamp between '2019-01-01' and '2019-01-02' order by file_name;" | less
|
|
|
```
|
|
|
To remove old files:
|
|
|
```sql
|
|
|
sudo su - flo
|
|
|
flo_user_rw='-h ratchet flo3'
|
|
|
psql $flo_user_rw -c "SELECT job, size, context, file_name FROM stored_products WHERE computation='flo.sw.fusion_matlab:FUSION_MATLAB' and context->>'satellite'='snpp' and context->>'version'='2.0.1dev0' order by file_name" | less
|
|
|
psql $flo_user_rw -c "DELETE FROM stored_products WHERE computation='flo.sw.fusion_matlab:FUSION_MATLAB' and context->>'satellite'='snpp' and context->>'version'='2.0.1dev0'"
|
|
|
satellite='snpp'; psql $flo_user_rw -c "SELECT job, size, context, file_name FROM stored_products WHERE computation='flo.sw.fusion_matlab:FUSION_MATLAB' and context->>'satellite'='$satellite' and context->>'version'='2.0.1dev0' order by file_name" | less
|
|
|
satellite='snpp'; psql $flo_user_rw -c "DELETE FROM stored_products WHERE computation='flo.sw.fusion_matlab:FUSION_MATLAB' and context->>'satellite'='$satellite' and context->>'version'='2.0.1dev0'"
|
|
|
```
|
|
|
## Other Database Querys
|
|
|
|
|
|
```sql
|
|
|
psql $flo_user -c "SELECT pydt(context->'granule') as d,count(*) FROM stored_products WHERE computation='flo.sw.fusion_matlab:FUSION_MATLAB' and context->>'satellite'='snpp' and context->>'version'='2.0.1dev0' group by d order by d order by file_name;" | less
|
|
|
satellite='snpp'; $flo_user -c "SELECT jts(context->'granule') as d,count(*) FROM stored_products WHERE computation='flo.sw.fusion_matlab:FUSION_MATLAB' and context->>'satellite'='$satellite' and context->>'version'='2.0.1dev0' group by d order by d order by file_name;" | less
|
|
|
satellite='snpp'; $flo_user -c "SELECT jts(context->'granule') as d,count(*) FROM stored_products WHERE computation='flo.sw.fusion_matlab:FUSION_MATLAB' and context->>'satellite'='$satellite' and context->>'version'='2.0.1dev0' and date_trunc('days',jts(context->'granule'))='2014-01-01' group by d order by d;" | less
|
|
|
satellite='snpp'; $flo_user -c "SELECT job,size,context,file_name from stored_products where computation='flo.sw.fusion_matlab:FUSION_MATLAB' and context->>'satellite'='$satellite' and context->>'version'='2.0.1dev0' order by file_name;" | less
|
|
|
satellite='snpp'; $flo_user -c "SELECT job,size,context,file_name from stored_products where computation='flo.sw.fusion_matlab:FUSION_MATLAB' and context->>'satellite'='$satellite' and context->>'version'='2.0.1dev0' and date_trunc('days',jts(context->'granule'))='2014-01-01' order by file_name;" | less
|
|
|
|
|
|
psql $flo_user -c "SELECT pydt(context->'granule') as d,count(*) FROM stored_products WHERE computation='flo.sw.fusion_matlab:FUSION_MATLAB' and context->>'satellite'='snpp' and context->>'version'='2.0.1dev0' and date_trunc('days',pydt(context->'granule'))='2014-01-01' group by d order by d;" | less
|
|
|
|
|
|
psql $flo_user -c "SELECT job,size,context,file_name from stored_products where computation='flo.sw.fusion_matlab:FUSION_MATLAB' and context->>'satellite'='snpp' and context->>'version'='2.0.1dev0' order by file_name;" | less
|
|
|
|
|
|
psql $flo_user -c "SELECT job,size,context,file_name from stored_products where computation='flo.sw.fusion_matlab:FUSION_MATLAB' and context->>'satellite'='snpp' and context->>'version'='2.0.1dev0' and date_trunc('days',pydt(context->'granule'))='2014-01-01' order by file_name;" | less
|
|
|
|
|
|
flo3=> select x FROM generate_series('2015-04-01'::timestamp, '2015-04-30 23:59', '6 minutes') as x where not exists (select null from stored_products where computation='flo.sw.fusion_matlab:FUSION_MATLAB' and context->>'satellite'='snpp' and output='fused_l1b' and x=pydt(context->'granule'));
|
|
|
flo3=> select x FROM generate_series('2015-04-01'::timestamp, '2015-04-30 23:59', '6 minutes') as x where not exists (select null from stored_products where computation='flo.sw.fusion_matlab:FUSION_MATLAB' and context->>'satellite'='$satellite' and output='fused_l1b' and x=jts(context->'granule'));
|
|
|
|
|
|
# List files keys
|
|
|
psql $flo_user -tA -c "SELECT format ('flo3/%s/%s',job,file_name) FROM stored_products WHERE computation='flo.sw.fusion_matlab:FUSION_MATLAB' and context->>'satellite'='snpp' and context->>'version'='2.0.1dev0' order by file_name limit 5;"
|
|
|
satellite='snpp'; $flo_user -tA -c "SELECT format ('flo3/%s/%s',job,file_name) FROM stored_products WHERE computation='flo.sw.fusion_matlab:FUSION_MATLAB' and context->>'satellite'='$satellite' and context->>'version'='2.0.1dev0' order by file_name limit 5;"
|
|
|
|
|
|
# List file keys and status
|
|
|
psql $flo_user -tA -c "SELECT format ('flo3/%s/%s',job,file_name) FROM stored_products WHERE computation='flo.sw.fusion_matlab:FUSION_MATLAB' and context->>'satellite'='snpp' and context->>'version'='2.0.1dev0' order by file_name limit 5;" | xargs -n1 -IXX rados -p dev --id flo stat XX
|
|
|
satellite='snpp'; $flo_user -tA -c "SELECT format ('flo3/%s/%s',job,file_name) FROM stored_products WHERE computation='flo.sw.fusion_matlab:FUSION_MATLAB' and context->>'satellite'='$satellite' and context->>'version'='2.0.1dev0' order by file_name limit 5;" | xargs -n1 -IXX rados -p dev --id flo stat XX
|
|
|
|
|
|
# List file key basenames
|
|
|
psql $flo_user -tA -c "SELECT format ('flo3/%s/%s',job,file_name) FROM stored_products WHERE computation='flo.sw.fusion_matlab:FUSION_MATLAB' and context->>'satellite'='snpp' and context->>'version'='2.0.1dev0' order by file_name limit 5;" | xargs -n1 -IXX basename XX
|
|
|
satellite='snpp'; $flo_user -tA -c "SELECT format ('flo3/%s/%s',job,file_name) FROM stored_products WHERE computation='flo.sw.fusion_matlab:FUSION_MATLAB' and context->>'satellite'='$satellite' and context->>'version'='2.0.1dev0' order by file_name limit 5;" | xargs -n1 -IXX basename XX
|
|
|
|
|
|
# List the rados commands to download files using the database file keys.
|
|
|
psql $flo_user -tA -c "SELECT format ('flo3/%s/%s',job,file_name) FROM stored_products WHERE computation='flo.sw.fusion_matlab:FUSION_MATLAB' and context->>'satellite'='snpp' and context->>'version'='2.0.1dev0' order by file_name limit 5;" | xargs -n1 -IXX echo rados -p dev --id flo get XX "~/fusion_matlab/work/links/"$(basename XX)
|
|
|
satellite='snpp'; $flo_user -tA -c "SELECT format ('flo3/%s/%s',job,file_name) FROM stored_products WHERE computation='flo.sw.fusion_matlab:FUSION_MATLAB' and context->>'satellite'='$satellite' and context->>'version'='2.0.1dev0' order by file_name limit 5;" | xargs -n1 -IXX echo rados -p dev --id flo get XX "~/fusion_matlab/work/links/"$(basename XX)
|
|
|
|
|
|
# rados commands
|
|
|
rados -p dev --id flo get flo3/91069111/VNP02FSN.A2015091.0000.001.2018025170339.nc VNP02FSN.A2015091.0000.001.2018025170339.nc
|
... | ... | @@ -273,81 +270,11 @@ Generate a list of jobnumbers for failed jobs: |
|
|
|
|
|
```sql
|
|
|
psql $flo_user -c "SELECT job, context FROM failed_jobs WHERE head_computation='flo.sw.fusion_matlab:FUSION_MATLAB' and context->>'version'='2.0.1dev0' and timestamp > '2018-01-30' order by context;" | grep granule | gawk '{print $1}' > fusion_matlab_v2.0.1dev0_failed_granules.txt
|
|
|
satellite='snpp'; psql $flo_user -c "select job,context,timestamp,exit_code from failed_jobs where head_computation='flo.sw.fusion_matlab:FUSION_MATLAB' and context->>'satellite'='$satellite' and context->>'version'='2.0.1dev0' and timestamp > '2023-04-22' and exit_code != 6000 order by timestamp;"
|
|
|
```
|
|
|
Generate a list of jobnumbers from the 'dev' and 'ops' cluster, and examine the last two lines of the stdout files associated with the jobs...
|
|
|
|
|
|
Read a file containing the job numbers of failed jobs, and do something with them...
|
|
|
|
|
|
```python
|
|
|
file_obj = open('fusion_matlab_v2.0.1dev0_failed_granules.txt','r')
|
|
|
jobnums = file_obj.readlines()
|
|
|
file_obj.close()
|
|
|
jobnums = [int(x) for x in jobnums]
|
|
|
|
|
|
run -e /mnt/sdata/geoffc/git/sips_utils/snippets.py
|
|
|
|
|
|
_ = [os.system('cat {} >> test.log; echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>" >> test.log'.format(job_number_to_dir('/scratch/flo/jobs', job_num,suffix='-stdout'))) for job_num in jobnums]
|
|
|
|
|
|
job_file_branches = [job_number_to_dir('/scratch/flo/jobs',job) for job in jobnums]
|
|
|
job_stdout_files = list(np.squeeze([glob(dir+'-stdout') for dir in job_file_branches]))
|
|
|
job_stderr_files = list(np.squeeze([glob(dir+'-stderr') for dir in job_file_branches]))
|
|
|
|
|
|
for files in job_stdout_files:
|
|
|
result = search_logfile_for_string(files, 'input sounder_0')
|
|
|
if result != []:
|
|
|
result = search_logfile_for_string(files, 'Dateline granule')
|
|
|
if result != []:
|
|
|
print(result[0].replace('\n',''))
|
|
|
else:
|
|
|
print(files)
|
|
|
else:
|
|
|
pass
|
|
|
```
|
|
|
|
|
|
```python
|
|
|
for stdout_file, stderr_file in zip(job_stdout_files,job_stderr_files):
|
|
|
try:
|
|
|
if os.path.isfile(stderr_file) and (os.stat(stderr_file).st_size > 0):
|
|
|
print('\n>>> stderr_file = {}'.format(stderr_file))
|
|
|
file_obj = open(stdout_file,'r')
|
|
|
for line in file_obj.readlines():
|
|
|
searchObj = re.search( r'Dateline granule', line, re.M)
|
|
|
if searchObj:
|
|
|
line = line.replace('\n','')
|
|
|
print('Checking {}: {}'.format(stdout_file, line))
|
|
|
else:
|
|
|
print('Checking {}:'.format(stdout_file))
|
|
|
|
|
|
file_obj.seek(3)
|
|
|
line = file_obj.readline()
|
|
|
line = os.pathbasename(line.replace('\n','').split(' ')[-1])
|
|
|
print(line)
|
|
|
|
|
|
file_obj.close()
|
|
|
else:
|
|
|
pass
|
|
|
#print('stderr_file {} does not exist or has zero size.'.format(stderr_file))
|
|
|
except Exception:
|
|
|
file_obj.close()
|
|
|
print('There was a problem with stderr_file {}'.format(stderr_file))
|
|
|
print(traceback.format_exc())
|
|
|
|
|
|
print('stdout_file = {}'.format(stdout_file))
|
|
|
```
|
|
|
```python
|
|
|
for stderr_file in [glob(dir+'-stdout') for dir in [job_number_to_dir('/scratch/flo/jobs',job) for job in range(77666696, 77667532)]]: check_call('tail -n 1 {}'.format(stderr_file[0]).split(' '))
|
|
|
|
|
|
logfile_obj = open(logpath,'w')
|
|
|
|
|
|
# Write the geocat output to a log file, and parse it to determine the output
|
|
|
# HDF4 files.
|
|
|
hdf_files = []
|
|
|
for line in exe_out.splitlines():
|
|
|
logfile_obj.write(line+"\n")
|
|
|
searchObj = re.search( r'geocat[LR].*\.hdf', line, re.M)
|
|
|
if searchObj:
|
|
|
hdf_files.append(string.split(line," ")[-1])
|
|
|
else:
|
|
|
pass
|
|
|
|
|
|
logfile_obj.close()
|
|
|
``` |
|
|
```sql
|
|
|
satellite='snpp'; psql $flo_user -tA -c "select job from failed_jobs where head_computation='flo.sw.fusion_matlab:FUSION_MATLAB' and context->>'satellite'='$satellite' and context->>'version'='2.0.0' and timestamp > '2023-04-01' and exit_code is null order by timestamp;" | cat | tail | xargs -n 1 -P 10 jobout | xargs -n 1 -P 10 tail -n 2
|
|
|
satellite='snpp'; psql $flo_ops -tA -c "select job from failed_jobs where head_computation='flo.sw.fusion_matlab:FUSION_MATLAB' and context->>'satellite'='$satellite' and context->>'version'='2.0.0' and timestamp > '2023-04-01' and exit_code is null order by timestamp;" | cat | tail | xargs -n 1 -P 10 jobout_ops | xargs -n 1 -P 10 tail -n 2
|
|
|
``` |
|
|
\ No newline at end of file |