diff --git a/aosstower/level_a0/nc.py b/aosstower/level_a0/nc.py index 55f700a1464645039476ee405b3551dfd28f1086..774f29b474e9274973449821d80eb6428c4c195e 100644 --- a/aosstower/level_a0/nc.py +++ b/aosstower/level_a0/nc.py @@ -17,7 +17,6 @@ LOG = logging.getLogger(__name__) # no returns def writeDimensions(ncFile): - #ncFile.createDimension('time', len(stamps)) ncFile.createDimension('time', None) ncFile.createDimension('max_len_station_name', 32) @@ -119,15 +118,12 @@ def createVariables(ncFile, firstStamp, chunksizes, zlib): return ncFile -def getData(inputFiles, no_empty): +def getData(inputFiles): dictData = {} for filename in inputFiles: getFrames = list(parser.read_frames(filename)) - if(len(getFrames) == 0 and no_empty): - return [None, False] - for frame in getFrames: if 'stamp' not in frame: continue @@ -137,7 +133,7 @@ def getData(inputFiles, no_empty): dictData[stamp] = frame - return [pd.DataFrame(dictData).transpose(), True] + return pd.DataFrame(dictData).transpose() def writeVars(ncFile, frame): stamps = list(frame.index) @@ -202,7 +198,7 @@ def writeVars(ncFile, frame): # @param input filenames - list of filenames # @param output filename - filename of the netcdf file -def createGiantNetCDF(start, end, inputFiles, outputName, zlib, chunkSize, no_empty): +def createGiantNetCDF(start, end, inputFiles, outputName, zlib, chunkSize): default = False if(chunkSize): @@ -211,29 +207,12 @@ def createGiantNetCDF(start, end, inputFiles, outputName, zlib, chunkSize, no_em else: default = True - frame = getData(inputFiles, no_empty) - - if(not frame[1]): - raise IOError('An empty ASCII file was found') - - if(frame[0].empty): - ncFile = Dataset(outputName, 'w', format='NETCDF4_CLASSIC') - #ncfile = writeDimensions(ncFile) - - #if(start): - # ncFile = createVariables(ncFile, start, chunksizes, zlib) - - #else: - # firstEmpty = inputFiles[0].split('/') - # filename = firstEmpty[len(firstEmpty) - 1] - # first = dt.strptime(filename, 'rig_tower.%Y-%m-%d.ascii') - # ncFile = createVariables(ncFile, first, chunksizes, zlib) + frame = getData(inputFiles) - ncFile.close() - + if(frame.empty): + return False + else: - frame = frame[0] - if(start and end): frame = frame[start.strftime('%Y-%m-%d %H:%M:%S'): end.strftime('%Y-%m-%d %H:%M:%S')] @@ -253,14 +232,31 @@ def createGiantNetCDF(start, end, inputFiles, outputName, zlib, chunkSize, no_em ncFile = writeVars(ncFile, frame) ncFile.close() + + return True -def createMultiple(filenames, outputFilenames, zlib, chunkSize, no_empty): +def createMultiple(filenames, outputFilenames, zlib, chunkSize): if(outputFilenames and len(filenames) != len(outputFilenames)): print('USAGE: number of output filenames must equal number of input filenames when start and end times are not specified') exit(0) + results = [] + for idx, filename in enumerate(filenames): - createGiantNetCDF(None, None, [filename], outputFilenames[idx], zlib, chunkSize, no_empty) + results.append(createGiantNetCDF(None, None, [filename], outputFilenames[idx], zlib, chunkSize)) + + print(results) + + allFalse = True + + for result in results: + if result == True: + print('here') + print(result) + allFalse = False + + if allFalse == True: + raise IOError('All ASCII files were empty') #The purpose of this method is to take a string in the format # YYYY-mm-ddTHH:MM:SS and convert that to a datetime object @@ -288,8 +284,6 @@ def main(): parser.add_argument('-e', '--end-time', type=_dt_convert, help='End time of massive netcdf file') parser.add_argument('-cs', '--chunk-size', type=int, help='chunk Size for the netCDF file') parser.add_argument('-z', '--zlib', action='store_true', help='compress netCDF file with zlib') - parser.add_argument('--no-empty', '--no-empty', action='store_true', help='allow empty nc files or not,' + - ' if not and there is an empty file, an exception is raised') parser.add_argument("input_files", nargs="+", help="aoss_tower level_00 paths") @@ -304,12 +298,14 @@ def main(): if(args.start_time and args.end_time): - createGiantNetCDF(args.start_time, args.end_time, args.input_files, args.output[0], args.zlib, args.chunk_size, args.no_empty) + result = createGiantNetCDF(args.start_time, args.end_time, args.input_files, args.output[0], args.zlib, args.chunk_size) + if(result == False): + raise IOError('An empty ASCII file was found') elif(args.start_time or args.end_time): print('USAGE: start time and end time must both be specified or not specified') else: - createMultiple(args.input_files, args.output, args.zlib, args.chunk_size, args.no_empty) + createMultiple(args.input_files, args.output, args.zlib, args.chunk_size) if __name__ == "__main__": main()