From f48c5cc982323242750a475a39f3b145c7f61d79 Mon Sep 17 00:00:00 2001
From: kgao <kenny.gao@ssec.wisc.edu>
Date: Wed, 10 Aug 2016 22:44:47 +0000
Subject: [PATCH] Changed no empty flag Got rid of the --no-empty flag If empty
ASCII file, doesn't create a nc file If all ASCII files are empty, raise IO
ERROR
---
aosstower/level_a0/nc.py | 64 +++++++++++++++++++---------------------
1 file changed, 30 insertions(+), 34 deletions(-)
diff --git a/aosstower/level_a0/nc.py b/aosstower/level_a0/nc.py
index 55f700a..774f29b 100644
--- a/aosstower/level_a0/nc.py
+++ b/aosstower/level_a0/nc.py
@@ -17,7 +17,6 @@ LOG = logging.getLogger(__name__)
# no returns
def writeDimensions(ncFile):
- #ncFile.createDimension('time', len(stamps))
ncFile.createDimension('time', None)
ncFile.createDimension('max_len_station_name', 32)
@@ -119,15 +118,12 @@ def createVariables(ncFile, firstStamp, chunksizes, zlib):
return ncFile
-def getData(inputFiles, no_empty):
+def getData(inputFiles):
dictData = {}
for filename in inputFiles:
getFrames = list(parser.read_frames(filename))
- if(len(getFrames) == 0 and no_empty):
- return [None, False]
-
for frame in getFrames:
if 'stamp' not in frame:
continue
@@ -137,7 +133,7 @@ def getData(inputFiles, no_empty):
dictData[stamp] = frame
- return [pd.DataFrame(dictData).transpose(), True]
+ return pd.DataFrame(dictData).transpose()
def writeVars(ncFile, frame):
stamps = list(frame.index)
@@ -202,7 +198,7 @@ def writeVars(ncFile, frame):
# @param input filenames - list of filenames
# @param output filename - filename of the netcdf file
-def createGiantNetCDF(start, end, inputFiles, outputName, zlib, chunkSize, no_empty):
+def createGiantNetCDF(start, end, inputFiles, outputName, zlib, chunkSize):
default = False
if(chunkSize):
@@ -211,29 +207,12 @@ def createGiantNetCDF(start, end, inputFiles, outputName, zlib, chunkSize, no_em
else:
default = True
- frame = getData(inputFiles, no_empty)
-
- if(not frame[1]):
- raise IOError('An empty ASCII file was found')
-
- if(frame[0].empty):
- ncFile = Dataset(outputName, 'w', format='NETCDF4_CLASSIC')
- #ncfile = writeDimensions(ncFile)
-
- #if(start):
- # ncFile = createVariables(ncFile, start, chunksizes, zlib)
-
- #else:
- # firstEmpty = inputFiles[0].split('/')
- # filename = firstEmpty[len(firstEmpty) - 1]
- # first = dt.strptime(filename, 'rig_tower.%Y-%m-%d.ascii')
- # ncFile = createVariables(ncFile, first, chunksizes, zlib)
+ frame = getData(inputFiles)
- ncFile.close()
-
+ if(frame.empty):
+ return False
+
else:
- frame = frame[0]
-
if(start and end):
frame = frame[start.strftime('%Y-%m-%d %H:%M:%S'): end.strftime('%Y-%m-%d %H:%M:%S')]
@@ -253,14 +232,31 @@ def createGiantNetCDF(start, end, inputFiles, outputName, zlib, chunkSize, no_em
ncFile = writeVars(ncFile, frame)
ncFile.close()
+
+ return True
-def createMultiple(filenames, outputFilenames, zlib, chunkSize, no_empty):
+def createMultiple(filenames, outputFilenames, zlib, chunkSize):
if(outputFilenames and len(filenames) != len(outputFilenames)):
print('USAGE: number of output filenames must equal number of input filenames when start and end times are not specified')
exit(0)
+ results = []
+
for idx, filename in enumerate(filenames):
- createGiantNetCDF(None, None, [filename], outputFilenames[idx], zlib, chunkSize, no_empty)
+ results.append(createGiantNetCDF(None, None, [filename], outputFilenames[idx], zlib, chunkSize))
+
+ print(results)
+
+ allFalse = True
+
+ for result in results:
+ if result == True:
+ print('here')
+ print(result)
+ allFalse = False
+
+ if allFalse == True:
+ raise IOError('All ASCII files were empty')
#The purpose of this method is to take a string in the format
# YYYY-mm-ddTHH:MM:SS and convert that to a datetime object
@@ -288,8 +284,6 @@ def main():
parser.add_argument('-e', '--end-time', type=_dt_convert, help='End time of massive netcdf file')
parser.add_argument('-cs', '--chunk-size', type=int, help='chunk Size for the netCDF file')
parser.add_argument('-z', '--zlib', action='store_true', help='compress netCDF file with zlib')
- parser.add_argument('--no-empty', '--no-empty', action='store_true', help='allow empty nc files or not,' +
- ' if not and there is an empty file, an exception is raised')
parser.add_argument("input_files", nargs="+",
help="aoss_tower level_00 paths")
@@ -304,12 +298,14 @@ def main():
if(args.start_time and args.end_time):
- createGiantNetCDF(args.start_time, args.end_time, args.input_files, args.output[0], args.zlib, args.chunk_size, args.no_empty)
+ result = createGiantNetCDF(args.start_time, args.end_time, args.input_files, args.output[0], args.zlib, args.chunk_size)
+ if(result == False):
+ raise IOError('An empty ASCII file was found')
elif(args.start_time or args.end_time):
print('USAGE: start time and end time must both be specified or not specified')
else:
- createMultiple(args.input_files, args.output, args.zlib, args.chunk_size, args.no_empty)
+ createMultiple(args.input_files, args.output, args.zlib, args.chunk_size)
if __name__ == "__main__":
main()
--
GitLab