Skip to content
Snippets Groups Projects
Commit 7f879abc authored by Alex Diebold's avatar Alex Diebold
Browse files

completed _sync() functionality with option [exp/] directory level. optional... 

completed _sync() functionality with option [exp/] directory level. optional [exp/] now included in relative_path when needed in _add(). improved documentation for all methods
parent 9aaa9e83
Branches
No related tags found
No related merge requests found
Show whitespace changes
Inline Side-by-side
metobscommon/archive/db.py
+ 76
− 98
View file @ 7f879abc
......@@ -507,9 +507,9 @@ def _hardcoded_create(args):
else:
raise FileExistsError('database \'{}\' already exists'.format(args.filename))
'''creates a new database
'''from config.yaml, creates a new database or updates a pre-existing one
will output an error message if the database name given already exists
if database creation fails partway through, partial database is NOT made
'''
def _create(args):
......@@ -620,8 +620,7 @@ def _create(args):
print(traceback.format_exc())
print('Error in creating database -- process aborted.\n')
'''cycles through directory, looking at instrument, site, and filetype to add
files found in directory to database
'''add file to database from directory -- /data/site/[exp/]inst/
'''
def _sync(args):
......@@ -631,119 +630,64 @@ def _sync(args):
Session = sessionmaker(bind=engine)
session = Session()
''' *meant for different function*
#read in the config.yaml file
stream = open('config.yaml')
docs = yaml.load(stream)
'''
#get all instruments and sites
all_instruments = session.query(Instrument).all()
insts = [i.name for i in all_instruments]
all_sites = session.query(Site).all()
sites = [s.name for s in all_sites]
all_experiments = session.query(Experiment).all()
exps = [e.name for e in all_experiments]
#make a temp exp
curr_exp = Experiment(name='sad')
print('\nFILE sync output')
print(OUTPUT.FILE.value)
'''
for step in os.walk('data'):
path = step[0]
dirname = os.path.dirname(path).split('/')[len(os.path.dirname(path).split('/'))-1]
basename = os.path.basename(path)
files = step[2]
if dirname in sites:
if basename in insts:
curr_site = session.query(Site).filter(Site.name == dirname).all()[0]
#curr_exp = session.query(Experiment).filter(Experiment.name == 'sad').all()[0]
curr_inst = session.query(Instrument).filter(Instrument.name == basename)
curr_inst = curr_inst.filter(Instrument.site.has(name = dirname)).all()[0]
for f in files:
curr_path = '{}/{}/{}'.format(dirname, basename, f)
for curr_filetype in curr_inst.filetype:
if glob.fnmatch.fnmatch(f, curr_filetype.glob_pattern):
curr_file = File(instrument=curr_inst, experiment=curr_exp, filetype=curr_filetype, relative_path=curr_path)
print(curr_file)
session.add(curr_file)
break
'''
curr_num = 1
#goes through data directory
for step in os.walk('data'):
#step[2] only true in deepest part of directory -- when there are files
if step[2]:
# /data/site/[exp/]/inst/
path = step[0]
site_name = path.split('/')[1]
inst_name = os.path.basename(path)
#checks if there is the optional exp part
if len(path.split('/')) == 4:
exp_name = path.split('/')[2]
#no experiment, set to default name
else:
exp_name = 'n/a'
files = step[2]
site = session.query(Site).filter(Site.name == site_name).all()
inst = session.query(Instrument).filter(Instrument.anem == inst_name)
inst = session.query(Instrument).filter(Instrument.name == inst_name)
inst = inst.filter(Instrument.site.has(name = site_name)).all()
exp = session.query(Experiment).filter(Experiment.name == exp_name).all()
#make sure the site, instrument, and experiment being used exist in database
if site and inst and exp:
site = site[0]
inst = inst[0]
exp = exp[0]
#go through all files in that particular directory
for f in files:
curr_path = '{}/{}'.format(path, f)
#check files glob_pattern against glob_pattern of instrument to determine filetype
for curr_filetype in inst.filetype:
if glob.fnmatch.fnmatch(f, curr_filetype.glob_pattern):
new_file = File(instrument=inst, experiment=exp, filetype=curr_filetype, relative_path=curr_path)
print('{} - {}'.format(curr_num, new_file))
curr_num += 1
session.add(new_file)
break
# site, instrument, or experiment doesn't exist in database
else:
if not site:
raise(ValueError('Error -- site not in database'))
if not inst:
raise(ValueError('Error -- instrument not in database'))
if not exp:
raise(ValueError('Error -- experiment not in database'))
session.commit()
'''
for step in os.walk('data'):
...: if step[2]:
...: path = step[0]
...: #dirname = os.path.dirname(path).split('/')[len(os.path.dirname(path).split('/'))-1]
...: #basename = os.path.basename(path)
...: site_name = path.split('/')[2]
...: inst_name = os.path.basename(path)
...: if len(path.split('/')) == 4:
...: exp_name = path.split('/')[3]
...: else:
...: exp_name = 'n/a'
...: files = step[2]
...: #print('site = {} --- inst = {} --- exp = {} --- files = {}'.format(site_name, inst_name, exp_name, files))
...: #print('\t\t{}'.format(step))
...: site = session.query(Site).filter(Site.name == site_name).all()
...: inst = session.query(Instrument).filter(Instrument.name == inst_name)
...: inst = inst.filter(Instrument.site.has(name = site_name)).all()
...: exp = session.query(Experiment).filter(Experiment.name == exp_name).all()
...: if site and inst and exp:
...: site = site[0]
...: inst = inst[0]
...: exp = exp[0]
...: print('{} - {} - {}'.format(site.name, inst.name, exp.name))
...: #if dirname in sites:
...: #print('0 - dirname = {} -- basename = {} -- files = {}'.format(dirname, basename, files))
...: #if basename in insts:
...: #print('1 - dirname = {} -- basename = {} -- files = {}'.format(dirname, basename, files))
...: #for f in files:
...: #curr_path = 'data/{}/{}/{}'.format(dirname, basename, f)
...: #print(curr_path)
...: #elif basename in exps:
...: #print('2 - dirname = {} -- basename = {} -- files = {}'.format(dirname, basename, files))
...: #sitename = os.path.dirname(path).split('/')[len(os.path.dirname(path).split('/'))-1]
...: #print('site = {} -- files = {}'.format(sitename, files))
'''
'''query DataBase --- can choose multiple columns to query
optional arguments:
-h, --help show this help message and exit
-i INSTRUMENT, --instrument INSTRUMENT
query Instruments
-s SITE, --site SITE query Sites
-e EXPERIMENT, --experiment EXPERIMENT
query Experiments
-t FILETYPE, --filetype FILETYPE
query FileTypes
-f FILE, --file FILE query Files
-i, --instrument query Instruments
-s, --site query Sites
-e, --experiment query Experiments
-t, --filetype query FileTypes
-f, --file query Files
'''
def _query(args):
......@@ -1108,7 +1052,7 @@ optional arguments:
format as '-t pattern.interval'
-f FILE, --file FILE format as '-f instrument.experiment.filetype.relative
_path.start_time.end_time'
**currently not in use**
'''
......@@ -1240,6 +1184,24 @@ def _original_add(args):
else:
raise FileNotFoundError('database \'{}\' does not exist'.format(args.filename))
'''add Files to database
optional arguments:
-h, --help show this help message and exit
-i INSTRUMENT, --instrument INSTRUMENT
specify instrument
-s SITE, --site SITE specify site
-S START_TIME, --start_time START_TIME
format as YYYY-mm-dd or YYYY-mm-ddTHH:MM:SS
-E END_TIME, --end_time END_TIME
defaults to start_time + period
-e EXPERIMENT, --experiment EXPERIMENT
specify experiment -- optional -- defaults to n/a
-f FILE [FILE ...], --file FILE [FILE ...]
specify files
'''
def _add(args):
#check to make sure database given exists
......@@ -1249,34 +1211,46 @@ def _add(args):
Session = sessionmaker(bind=engine)
session = Session()
#determine specified instrument
inst = session.query(Instrument).filter(Instrument.name == args.instrument)
inst = inst.filter(Instrument.site.has(name = args.site)).all()
if len(inst) == 0:
raise ValueError('INSTRUMENT does not exist in database')
else:
inst = inst[0]
#if experiment specified, find experiment
if args.experiment:
exp = session.query(Experiment).filter(Experiment.name == args.experiment).all()
if len(exp) == 0:
raise ValueError('EXPERIMENT does not exist in database')
else:
exp = exp[0]
#experiment not specified, use default experiment
else:
exp = session.query(Experiment).filter(Experiment.name == 'n/a').all()[0]
print(OUTPUT.FILE.value)
i = 0
#loop through filenames given
for f in args.file:
i = i + 1
#loop through filetypes for specified instrument
for ft in inst.filetype:
#check glob_pattern of file against glob_pattern of filetypes for a match
if glob.fnmatch.fnmatch(f, ft.glob_pattern):
#endtime not given, defaults to start_time + period
if not args.endtime:
if ft.period == 'Daily':
end_time = args.start_time + timedelta(hours=24)
elif ft.period == 'Half-Daily':
end_time = args.start_time + timedelta(hours=12)
#end time is given
else:
end_time = args.end_time
path = 'data/' + inst.site.name + '/' + inst.name + '/' + f
#create relative path -- /data/site/[exp/]inst/
path = 'data/' + inst.site.name + '/'
if exp.name != 'n/a':
path = path + exp.name + '/'
path = path + inst.name + '/' + f
new_file = File(instrument=inst, experiment=exp, filetype=ft, relative_path=path, start_time=args.start_time, end_time=end_time)
print('{} - {}'.format(i, new_file))
session.add(new_file)
......@@ -1417,11 +1391,11 @@ def main():
subparsers = parser.add_subparsers()
#create subparser for 'create' option
parser_create = subparsers.add_parser('create', help='creates a new database')
parser_create = subparsers.add_parser('create', help='from config.yaml, creates a new database or updates a pre-existing one')
parser_create.set_defaults(func=_create)
#create subparser for 'sync' option
parser_sync = subparsers.add_parser('sync', help='sync files from directory -- /data/site/inst/')
parser_sync = subparsers.add_parser('sync', help='add file to database from directory -- /data/site/[exp/]inst/')
parser_sync.set_defaults(func=_sync)
#create subparser for 'query' option
......@@ -1433,6 +1407,7 @@ def main():
parser_query.add_argument('-t', '--filetype', action='store_true', help='query FileTypes')
parser_query.add_argument('-f', '--file', action='store_true', help='query Files')
'''
#create subparser for 'add' option -- original option
parser_o_add = subparsers.add_parser('original_add')
parser_o_add.set_defaults(func=_original_add)
......@@ -1441,17 +1416,19 @@ def main():
parser_o_add.add_argument('-e', '--experiment', help='format as \'-e name.start_time.end_time\'')
parser_o_add.add_argument('-t', '--filetype', help='format as \'-t pattern.interval\'')
parser_o_add.add_argument('-f', '--file', help='format as \'-f instrument.experiment.filetype.relative_path.start_time.end_time\'')
'''
#create subparser for 'add' option -- current, only-file option
parser_add = subparsers.add_parser('add')
parser_add = subparsers.add_parser('add', help='add Files to database')
parser_add.set_defaults(func=_add)
parser_add.add_argument('-i', '--instrument', required=True, help='specify instrument')
parser_add.add_argument('-s', '--site', required=True, help='specify site')
parser_add.add_argument('-S', '--start_time', required=True, type=_parse_datetime, help='format as YYYY-mm-dd or YYYY-mm-ddTHH:MM:SS')
parser_add.add_argument('-E', '--end_time', type=_parse_datetime, help='defaults to start_time + period')
parser_add.add_argument('-e', '--experiment', help='specify experiment -- optional')
parser_add.add_argument('-e', '--experiment', help='specify experiment -- optional -- defaults to n/a')
parser_add.add_argument('-f', '--file', nargs='+', required=True, help='specify files')
'''
#method for testing stuff
parser_test = subparsers.add_parser('test')
parser_test.set_defaults(func=_test)
......@@ -1465,6 +1442,7 @@ def main():
parser_test.add_argument('-f', '--file', nargs='?', type=bool, const=True, help='query files')
parser_test.add_argument('-v', '--testing_stuff', nargs='?', type=bool, const=True, help='testing stuff out')
parser_test.add_argument('-c', '--testing_create', help='testing _create() functionality with yaml')
'''
args = parser.parse_args()
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment