Skip to content
Snippets Groups Projects
Commit f48c5cc9 authored by kgao's avatar kgao
Browse files

Changed no empty flag

Got rid of the --no-empty flag
If empty ASCII file, doesn't create a nc file
If all ASCII files are empty, raise IO ERROR
parent 0e1747de
No related branches found
No related tags found
No related merge requests found
......@@ -17,7 +17,6 @@ LOG = logging.getLogger(__name__)
# no returns
def writeDimensions(ncFile):
#ncFile.createDimension('time', len(stamps))
ncFile.createDimension('time', None)
ncFile.createDimension('max_len_station_name', 32)
......@@ -119,15 +118,12 @@ def createVariables(ncFile, firstStamp, chunksizes, zlib):
return ncFile
def getData(inputFiles, no_empty):
def getData(inputFiles):
dictData = {}
for filename in inputFiles:
getFrames = list(parser.read_frames(filename))
if(len(getFrames) == 0 and no_empty):
return [None, False]
for frame in getFrames:
if 'stamp' not in frame:
continue
......@@ -137,7 +133,7 @@ def getData(inputFiles, no_empty):
dictData[stamp] = frame
return [pd.DataFrame(dictData).transpose(), True]
return pd.DataFrame(dictData).transpose()
def writeVars(ncFile, frame):
stamps = list(frame.index)
......@@ -202,7 +198,7 @@ def writeVars(ncFile, frame):
# @param input filenames - list of filenames
# @param output filename - filename of the netcdf file
def createGiantNetCDF(start, end, inputFiles, outputName, zlib, chunkSize, no_empty):
def createGiantNetCDF(start, end, inputFiles, outputName, zlib, chunkSize):
default = False
if(chunkSize):
......@@ -211,29 +207,12 @@ def createGiantNetCDF(start, end, inputFiles, outputName, zlib, chunkSize, no_em
else:
default = True
frame = getData(inputFiles, no_empty)
if(not frame[1]):
raise IOError('An empty ASCII file was found')
if(frame[0].empty):
ncFile = Dataset(outputName, 'w', format='NETCDF4_CLASSIC')
#ncfile = writeDimensions(ncFile)
#if(start):
# ncFile = createVariables(ncFile, start, chunksizes, zlib)
#else:
# firstEmpty = inputFiles[0].split('/')
# filename = firstEmpty[len(firstEmpty) - 1]
# first = dt.strptime(filename, 'rig_tower.%Y-%m-%d.ascii')
# ncFile = createVariables(ncFile, first, chunksizes, zlib)
frame = getData(inputFiles)
ncFile.close()
if(frame.empty):
return False
else:
frame = frame[0]
if(start and end):
frame = frame[start.strftime('%Y-%m-%d %H:%M:%S'): end.strftime('%Y-%m-%d %H:%M:%S')]
......@@ -253,14 +232,31 @@ def createGiantNetCDF(start, end, inputFiles, outputName, zlib, chunkSize, no_em
ncFile = writeVars(ncFile, frame)
ncFile.close()
return True
def createMultiple(filenames, outputFilenames, zlib, chunkSize, no_empty):
def createMultiple(filenames, outputFilenames, zlib, chunkSize):
if(outputFilenames and len(filenames) != len(outputFilenames)):
print('USAGE: number of output filenames must equal number of input filenames when start and end times are not specified')
exit(0)
results = []
for idx, filename in enumerate(filenames):
createGiantNetCDF(None, None, [filename], outputFilenames[idx], zlib, chunkSize, no_empty)
results.append(createGiantNetCDF(None, None, [filename], outputFilenames[idx], zlib, chunkSize))
print(results)
allFalse = True
for result in results:
if result == True:
print('here')
print(result)
allFalse = False
if allFalse == True:
raise IOError('All ASCII files were empty')
#The purpose of this method is to take a string in the format
# YYYY-mm-ddTHH:MM:SS and convert that to a datetime object
......@@ -288,8 +284,6 @@ def main():
parser.add_argument('-e', '--end-time', type=_dt_convert, help='End time of massive netcdf file')
parser.add_argument('-cs', '--chunk-size', type=int, help='chunk Size for the netCDF file')
parser.add_argument('-z', '--zlib', action='store_true', help='compress netCDF file with zlib')
parser.add_argument('--no-empty', '--no-empty', action='store_true', help='allow empty nc files or not,' +
' if not and there is an empty file, an exception is raised')
parser.add_argument("input_files", nargs="+",
help="aoss_tower level_00 paths")
......@@ -304,12 +298,14 @@ def main():
if(args.start_time and args.end_time):
createGiantNetCDF(args.start_time, args.end_time, args.input_files, args.output[0], args.zlib, args.chunk_size, args.no_empty)
result = createGiantNetCDF(args.start_time, args.end_time, args.input_files, args.output[0], args.zlib, args.chunk_size)
if(result == False):
raise IOError('An empty ASCII file was found')
elif(args.start_time or args.end_time):
print('USAGE: start time and end time must both be specified or not specified')
else:
createMultiple(args.input_files, args.output, args.zlib, args.chunk_size, args.no_empty)
createMultiple(args.input_files, args.output, args.zlib, args.chunk_size)
if __name__ == "__main__":
main()
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment