fixed _create() to correctly add experiments. worked on config.yaml to have...

fixed _create() to correctly add experiments. worked on config.yaml to have proper test inst/site/exp/filetype

metobscommon/archive/config.yaml

@@ -53,15 +53,23 @@ instruments:
         - mendota.buoy.png
   test_site_1:
     display_name: 'test_site_1'
+    test_inst_1:
+      display_name: 'test_inst_1'
+      filetypes:
+        - test_site_1.test_inst_1.qc
+        - test_site_1.test_inst_1.ascii
   test_site_2:
     display_name: 'test_site_2'
-    test_inst:
+    test_inst_2:
       display_name: 'test_inst'
       filetypes:
-        - test_site_2.test_inst.ascii
-        - test_site_2.test_inst.nc
+        - test_site_2.test_inst_2.nc
   test_site_3:
     display_name: 'test_site_3'
+    test_inst_3:
+      display_name: 'test_inst_3'
+      filetypes:
+        - test_site_3.test_inst_3.png
 
 filetypes:
   aoss.aeri.par.l00.v00:
@@ -222,15 +230,19 @@ filetypes:
     period: 'Daily'
     format_pattern: '{site}_{inst}.{measurement}.{start_time:%y-%m-%d_%h%m%s}_{end_time:%h%m%s}.png'
     glob_pattern: '*_*.*.????-??-??_??????_??????*.png'
-  # test_site_1.test_inst_1.png:
-  #   period: 'Daily'
-  #   format_pattern: '{start_time:%y%m%d}.png'
-  #   glob_pattern: '??????.png'
-  test_site_2.test_inst.ascii:
+  test_site_1.test_inst_1.qc:
     period: 'Daily'
-    format_pattern: '{start_time:%y%m%d}.png'
-    glob_pattern: '??????.png'
-  test_site_2.test_inst.nc:
+    format_pattern: '{start_time:%y%m%d}.qc'
+    glob_pattern: '??????.qc'
+  test_site_1.test_inst_1.ascii:
+    period: 'Daily'
+    format_pattern: '{start_time:%y%m%d}.ascii'
+    glob_pattern: '??????.ascii'
+  test_site_2.test_inst_2.nc:
+    period: 'Daily'
+    format_pattern: '{start_time:%y%m%d}.nc'
+    glob_pattern: '??????.nc'
+  test_site_3.test_inst_3.png:
     period: 'Daily'
     format_pattern: '{start_time:%y%m%d}.png'
     glob_pattern: '??????.png'

metobscommon/archive/db.py

@@ -446,25 +446,36 @@ def _create(args):
         stream = open('config.yaml')
         docs = yaml.load(stream)
 
+        #cycle through sites of yaml
         for s in docs['instruments'].keys():
+            #check if site exists in database
             site_q = session.query(Site).filter(Site.name == s).all()
+            #site does exist
             if len(site_q) > 0:
                 curr_site = site_q[0]
+            #site doesn't exist; new site is made
             else:
                 curr_site = Site(name = s)
             session.add(curr_site)
             session.commit()
+            #cycle through instruments of yaml
             for i in docs['instruments'][s].keys():
+                #skip display name
                 if i != 'display_name':
+                    #check if instrument exists with specific site
                     inst_q = session.query(Instrument).filter(Instrument.name == i)
                     inst_q = inst_q.filter(Instrument.site.has(name = curr_site.name)).all()
+                    #isntrument exists
                     if len(inst_q) > 0:
                         curr_inst = inst_q[0]
+                    #instrument doesn't exist; new instrument is made
                     else:
                         curr_inst = Instrument(name=i, site=curr_site)
                     session.add(curr_inst)
                     session.commit()
+                    #cycle through filetypes in yaml
                     for f in docs['instruments'][s][i]['filetypes']:
+                        #skip display name
                         if f != 'display_name':
                             #current dictionary of filetype values
                             d = docs['filetypes'][f]
@@ -473,6 +484,7 @@ def _create(args):
                             else:
                                 level = None
                             filetype_q = session.query(FileType).filter(FileType.name == f).all()
+                            #filetype exists; update from yaml
                             if len(filetype_q) > 0:
                                 curr_filetype = filetype_q[0]
                                 curr_filetype.glob_pattern = d['glob_pattern']
@@ -481,26 +493,37 @@ def _create(args):
                                 curr_filetype.period = d['period']
                                 session.add(curr_filetype)
                                 session.commit()
+                            #filetype doensn't exist; create new
                             else:
                                 curr_filetype = FileType(name=f, glob_pattern=d['glob_pattern'], format_pattern=d['format_pattern'], level=level, period=d['period'])
                                 curr_filetype.add_instrument(curr_inst)
+        #cycle through experiments in yaml
         for e in docs['experiments'].keys():
+            #check if experiment exists
             exp_q = session.query(Experiment).filter(Experiment.name == e)
+            #experiment exists
             if len(exp_q.all()) > 0:
                 curr_exp = exp_q.all()[0]
+            #experiment doesn't exist; new is created
             else:
                 curr_exp = Experiment(name = e)
+            #update start_time and end_time from yaml
             curr_exp.start_time = _parse_datetime(docs['experiments'][e]['start_time'])
             curr_exp.end_time = _parse_datetime(docs['experiments'][e]['end_time'])
+            #remove current site relationships
+            curr_exp.site_experiments.clear()
             session.add(curr_exp)
             session.commit()
+            #cycle through sites for the experiments and add
             for s in docs['experiments'][e]['site'].replace(' ', '').split(','):
-                if len(exp_q.filter(Experiment.site.any(name = s)).all()) == 0:
-                    curr_site = session.query(Site).filter(Site.name == s).all()
-                    if len(curr_site) > 0:
-                        curr_exp.add_site(curr_site[0])
-                    else:
-                        raise ValueError('Error in YAML: {} site does not exist'.format(s))
+                # if len(exp_q.filter(Experiment.site.any(name = s)).all()) == 0:
+                #find the site to link to
+                curr_site = session.query(Site).filter(Site.name == s).all()
+                if len(curr_site) > 0:
+                    curr_exp.add_site(curr_site[0])
+                else:
+                    raise ValueError('Error in YAML: {} site does not exist'.format(s))
+                session.commit()
 
     #remove database if process failed
     except: