Commit a036c7be authored by William Roberts's avatar William Roberts
Browse files

Improve dmv_to_ncdf and file_finder

parent b40c6971
......@@ -32,7 +32,7 @@ RUN git clone --depth 1 -b m64_r20200617 git@gitlab.ssec.wisc.edu:aeri/ifg_ret.g
RUN git clone --depth 1 -b m64_r20210721 git@gitlab.ssec.wisc.edu:aeri/ifg_tools.git
RUN git clone --depth 1 -b m64_r20211123 git@gitlab.ssec.wisc.edu:cphillips/aeri_quality_control.git
RUN git clone --depth 1 -b m64_r20200630 git@gitlab.ssec.wisc.edu:cphillips/aeri_tools.git
RUN git clone --depth 1 -b m64_r20211123 git@gitlab.ssec.wisc.edu:aeri/ifg_aeri.git
RUN git clone --depth 1 -b m64_r20211129 git@gitlab.ssec.wisc.edu:aeri/ifg_aeri.git
# The following directories just need to be in the same directory.
RUN mv /ifg_aeri/py/nli /root/nli
......
#!/usr/bin/env python3.8
from os.path import basename, join, dirname, isdir, isfile, realpath
from os import mkdir, getenv
from sys import stderr
import os
import subprocess
import argparse
import shutil
import logging
from utils import file_finder
def process(input_file, output_dir, verbose):
exe = '/usr/bin/dmv-ncdf'
output_file = join(dirname(input_file), f'{basename(input_file)[:-4]}_{str.lower(basename(input_file)[-3:])}.nc')
logfile = join(dirname(input_file), f'{basename(input_file)[:-4]}_{str.lower(basename(input_file)[-3:])}.log')
output_file = os.path.join(os.path.dirname(input_file), f'{os.path.basename(input_file)[:-4]}_{str.lower(os.path.basename(input_file)[-3:])}.nc')
logfile = os.path.join(os.path.dirname(input_file), f'{os.path.basename(input_file)[:-4]}_{str.lower(os.path.basename(input_file)[-3:])}.log')
dmv_input = '%s\n1\n99999\n0\n%s\n\n\n%s\n' % (input_file, output_file, logfile)
if verbose:
if verbose > 1:
p = subprocess.Popen( [exe], stdin=subprocess.PIPE )
else:
p = subprocess.Popen( [exe], stdout=subprocess.PIPE, stdin=subprocess.PIPE )
p.communicate(dmv_input.encode())
if not output_dir:
output_dir = dirname(input_file)
if not isdir(output_dir):
mkdir(output_dir)
output_dir = os.path.dirname(input_file)
if not os.path.isdir(output_dir):
os.mkdir(output_dir)
shutil.move(logfile, join(output_dir,basename(logfile)))
shutil.move(output_file, join(output_dir,basename(output_file)))
shutil.move(logfile, os.path.join(output_dir, os.path.basename(logfile)))
shutil.move(output_file, os.path.join(output_dir, os.path.basename(output_file)))
if __name__=='__main__':
......@@ -44,16 +42,14 @@ if __name__=='__main__':
help='dmv file extensions to convert')
args = parser.parse_args()
logger_levels = [logging.ERROR, logging.WARNING, logging.INFO, logging.DEBUG]
logging_format = '[%(levelname)s: %(asctime)s : %(name)s] %(message)s'
logging.basicConfig(format=logging_format, datefmt='%Y-%m-%d %H:%M:%S', level=logger_levels[min(3, args.verbose)])
extensions = args.extensions
if extensions is None:
extensions = ['.CXS', '.SUM', '.RNC']
if isdir(args.input_file):
ae_dirs = file_finder.getFiles(args.input_file, extensions=extensions)
for d, files in ae_dirs.items():
for f in files:
process(join(d,f), args.output_dir, args.verbose)
elif not isfile(args.input_file):
print("invalid input", file=stderr)
exit(1)
else:
process(realpath(args.input_file), args.output_dir, args.verbose)
for output_dir, files in file_finder.getFiles(args.input_file, extensions=args.extensions).items():
if args.output_dir is not None:
output_dir = args.output_dir
for fp in files:
process(fp, output_dir, args.verbose)
#!/usr/bin/env python3
import os
import logging
from datetime import datetime
logger = logging.getLogger(__name__)
def getDirs(fp):
# helper for list comprehension
def valid_dir(d):
......@@ -25,20 +28,27 @@ def getDirs(fp):
return dirs
def getFiles(fp, extensions=[]):
def getFiles(fp, extensions=None):
# helper for list comprehension
def valid_file(f):
if os.path.isfile(f):
extension = os.path.splitext(f)[1]
if not extensions or extension.upper() in extensions or extension.lower() in extensions:
extension = f.split('.')[-1]
if extensions is None or extension.upper() in extensions or extension.lower() in extensions:
return True
elif not os.path.isdir(f):
logger.error(f'{f} does not exist')
exit(1)
return False
tree = {};
fp = os.path.realpath(fp)
if valid_file(fp):
tree = {os.path.dirname(fp) : (os.path.basename(fp),)}
return {os.path.dirname(fp): (fp,)}
elif os.path.isdir(fp):
tree = {d : tuple(os.path.basename(f) for f in os.listdir(d) if valid_file(os.path.join(d,f))) for d in getDirs(fp)}
return tree
if extensions is None:
extensions = ['CXS', 'SUM', 'RNC']
tree = {d: tuple(os.path.join(d, f) for f in os.listdir(d) if valid_file(os.path.join(d, f))) for d in getDirs(fp)}
if len(tree.keys()) == 0:
logger.warning(f'No valid AEYYMMDD directories found in {fp}')
return tree
logger.warning(f'{fp} does not have the specified extension(s): {" ".join(extensions)}')
return {}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment