updating _create(). now allows for _create() to be run when the database...

updating _create(). now allows for _create() to be run when the database already exists. working to update from yaml. currently deletes then re-adds if something is already exists, but this causes errors with Files in the database

metobscommon/archive/db.py

@@ -405,49 +405,59 @@ will output an error message if the database name given already exists
 '''
 def _create(args):
 
-    if not os.path.exists(args.filename):
+    try:
 
-        try:
+        engine = create_engine(('sqlite:///' + args.filename))
 
-            engine = create_engine(('sqlite:///' + args.filename))
-            #create empty table
+        #create new table if one doesn't exists
+        if not os.path.exists(args.filename):
             Base.metadata.create_all(engine)
 
-            DBsession = sessionmaker(bind=engine)
-            session = DBsession()
+        Session = sessionmaker(bind=engine)
+        session = Session()
 
-            #read from yaml
-            stream = open('config.yaml')
-            docs = yaml.load(stream)
+        #read from yaml
+        stream = open('config.yaml')
+        docs = yaml.load(stream)
 
-            for s in docs['instruments'].keys():
-                curr_site = Site(name = s)
-                session.add(curr_site)
+        for s in docs['instruments'].keys():
+            curr_site = Site(name = s)
+            site_q = session.query(Site).filter(Site.name == curr_site.name).all()
+            if len(site_q) > 0:
+                session.delete(site_q[0])
                 session.commit()
-                for i in docs['instruments'][s].keys():
-                    if i != 'display_name':
-                        curr_inst = Instrument(name=i, site=curr_site)
-                        session.add(curr_inst)
+            session.add(curr_site)
+            session.commit()
+            for i in docs['instruments'][s].keys():
+                if i != 'display_name':
+                    curr_inst = Instrument(name=i, site=curr_site)
+                    inst_q = session.query(Instrument).filter(Instrument.name == curr_inst.name)
+                    inst_q = inst_q.filter(Instrument.site.has(name = curr_inst.site.name)).all()
+                    if len(inst_q) > 0:
+                        session.delete(inst_q[0])
                         session.commit()
-                        for f in docs['instruments'][s][i]['filetypes']:
-                            if f != 'display_name':
-                                #current dictionary of filetype values
-                                d = docs['filetypes'][f]
-                                if 'level' in d.keys():
-                                    level = d['level']
-                                else:
-                                    level = None
-                                curr_filetype = FileType(name=f, glob_pattern=d['glob_pattern'], format_pattern=d['format_pattern'], level=level, period=d['period'])
-                                curr_filetype.add_instrument(curr_inst)
-
-        #remove database if process failed
-        except:
-            os.remove(args.filename)
-            print(traceback.format_exc())
-            print('Error in creating database -- process aborted.\n')
-
-    else:
-        raise FileExistsError('database \'{}\' already exists'.format(args.filename))
+                    session.add(curr_inst)
+                    session.commit()
+                    for f in docs['instruments'][s][i]['filetypes']:
+                        if f != 'display_name':
+                            #current dictionary of filetype values
+                            d = docs['filetypes'][f]
+                            if 'level' in d.keys():
+                                level = d['level']
+                            else:
+                                level = None
+                            curr_filetype = FileType(name=f, glob_pattern=d['glob_pattern'], format_pattern=d['format_pattern'], level=level, period=d['period'])
+                            filetype_q = session.query(FileType).filter(FileType.name == curr_filetype.name).all()
+                            if len(filetype_q) > 0:
+                                session.delete(filetype_q[0])
+                                session.commit()
+                            curr_filetype.add_instrument(curr_inst)
+
+    #remove database if process failed
+    except:
+        os.remove(args.filename)
+        print(traceback.format_exc())
+        print('Error in creating database -- process aborted.\n')
 
 '''cycles through directory, looking at instrument, site, and filetype to add
     files found in directory to database
@@ -466,16 +476,15 @@ def _sync(args):
     docs = yaml.load(stream)
     '''
 
-    #get all filetypes
-    all_filetypes = session.query(FileType).all()
-    patterns = [f.pattern for f in all_filetypes]
-
     #get all instruments and sites
     all_instruments = session.query(Instrument).all()
     insts = [i.name for i in all_instruments]
     all_sites = session.query(Site).all()
     sites = [s.name for s in all_sites]
 
+    #make a temp exp
+    curr_exp = Experiment(name='sad')
+
     print('\nFILE sync output')
     print(OUTPUT.FILE.value)
     for step in os.walk('data'):
@@ -486,16 +495,16 @@ def _sync(args):
         if dirname in sites:
             if basename in insts:
                 curr_site = session.query(Site).filter(Site.name == dirname).all()[0]
-                curr_exp = session.query(Experiment).filter(Experiment.name == 'sad').all()[0]
+                #curr_exp = session.query(Experiment).filter(Experiment.name == 'sad').all()[0]
                 curr_inst = session.query(Instrument).filter(Instrument.name == basename).all()[0]
                 for f in files:
                     curr_path = '{}/{}/{}'.format(dirname, basename, f)
-                    curr_filetype = session.query(FileType).filter(FileType.pattern == f.split('.')[1]).all()
-                    if curr_filetype:
-                        curr_filetype = curr_filetype[0]
-                        curr_file = File(instrument=curr_inst, experiment=curr_exp, filetype=curr_filetype, relative_path=curr_path)
-                        print(curr_file)
-                        session.add(curr_file)
+                    for curr_filetype in curr_inst.filetype:
+                        if glob.fnmatch.fnmatch(f, curr_filetype.glob_pattern):
+                            curr_file = File(instrument=curr_inst, experiment=curr_exp, filetype=curr_filetype, relative_path=curr_path)
+                            print(curr_file)
+                            session.add(curr_file)
+                            break
     session.commit()
 
 '''query DataBase --- can choose multiple columns to query