Skip to content
Snippets Groups Projects
Commit 5ec81aff authored by R.K.Garcia's avatar R.K.Garcia
Browse files

wrapper script and processing pool

parent 80f291ec
No related branches found
Tags cmi_changer-20170506
No related merge requests found
#!/bin/bash
SOURCE="${BASH_SOURCE[0]}"
while [ -h "$SOURCE" ] ; do SOURCE="$(readlink "$SOURCE")"; done
BASE="$( cd -P "$( dirname "$SOURCE" )/.." && pwd )"
test -n "$PYTHON" || export PYTHON=$BASE/ShellB3/bin/python3
exec $PYTHON -m goesr.cmi_changer "$@"
......@@ -15,6 +15,7 @@ import os, sys
import logging, unittest, argparse
import re
import socket
import multiprocessing as mp
import numpy as np
from datetime import datetime
......@@ -336,7 +337,7 @@ def discard_unwanted_vars(pvda, inclusion_set=ONLY_THESE_VARS):
if (V is None) or (P[-1] in inclusion_set):
yield P,V,D,A
else:
LOG.info('discarding {}'.format('/'.join(P)))
LOG.debug('discarding {}'.format('/'.join(P)))
......@@ -546,7 +547,7 @@ class scmi_l1b_util(object):
yield (path,), None, mt, a
def l1b_makin_the_chimichangas(path, n_tiles=(1, 1), max_pixels=0, region=None):
def l1b_makin_the_chimichangas(path, n_tiles=(1, 1), max_pixels=0, region=None, debug=False):
"""
Let's make CMIchangas!
:param path: source file
......@@ -572,7 +573,8 @@ def l1b_makin_the_chimichangas(path, n_tiles=(1, 1), max_pixels=0, region=None):
# collect navigation information for each tile, then put in additional global attributes before closing the file
seq = scmi.append_global_atts(seq)
# add a debug stage at any point by the way to dump everything to screen as it goes
seq = rf.debug(seq)
if debug:
seq = rf.debug(seq)
# generate all the tiles
rf.nc_write(seq) # run the filter stack and generate output files
......@@ -598,6 +600,13 @@ def _debug(typ, value, tb):
pdb.post_mortem(tb) # more “modern”
class chimicherry(object):
def __init__(self, **kwargs):
self.kwargs=kwargs
def __call__(self, *args):
l1b_makin_the_chimichangas(*args, **self.kwargs)
def main():
parser = argparse.ArgumentParser(
description="slice and convert GOES-R L1B files into SCMI tiles for AWIPS2",
......@@ -614,8 +623,10 @@ def main():
help="number of tiles in Y dimension")
parser.add_argument('-X', '--xtiles', type=int, default=0,
help="number of tiles in X dimension")
parser.add_argument('-P', '--pixels', type=int, default=0,
parser.add_argument('-p', '--pixels', type=int, default=2000,
help="max number of pixels in any direction, for tile counts not specified")
parser.add_argument('-P', '--parallel', type=int, default=0,
help="max number of processors to use, default all-available")
parser.add_argument('-R', '--region', help='region tag to use in filename, e.g. ECONUS')
parser.add_argument('inputs', nargs='*',
help="input files to process")
......@@ -631,8 +642,20 @@ def main():
unittest.main()
return 0
for path in args.inputs:
l1b_makin_the_chimichangas(path, (args.ytiles, args.xtiles), args.pixels, args.region)
cherrychanga = chimicherry(n_tiles=(args.ytiles, args.xtiles), max_pixels=args.pixels, region=args.region, debug=args.debug)
if args.parallel==0:
multi = mp.Pool()
run_all = multi.map
LOG.info('using {} cores'.format(os.cpu_count()))
elif args.parallel==1:
run_all = map
else:
multi = mp.Pool(args.parallel)
run_all = multi.map
LOG.info('using {} cores'.format(args.parallel))
run_all(cherrychanga, args.inputs)
return 0
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment