Updated _query() to filter using start_time and end_time. made a test experiment in config.yaml

metobscommon/archive/config.yaml

@@ -28,7 +28,7 @@ instruments:
         - aoss.ceilo.ascii
         - aoss.ceilo.nc
         - aoss.ceilo.png
-        - aoss.ceilo.tn.png
+        #- aoss.ceilo.tn.png
     tower:
       display_name: 'Tower'
       filetypes:
@@ -211,3 +211,7 @@ filetypes:
     period: 'Daily'
     format_pattern: '{site}_{inst}.{measurement}.{start_time:%y-%m-%d_%h%m%s}_{end_time:%h%m%s}.png'
     glob_pattern: '*_*.*.????-??-??_??????_??????*.png'
+  experiments:
+    test_experiment:
+      start_time: '2000-01-01T01:02:03'
+      end_time: '2000-01-02T01:02:03'

metobscommon/archive/db.py

@@ -249,14 +249,14 @@ def _parse_datetime(date_string):
 
     #check if date follows first format
     try:
-        dt = datetime.strptime(date_string, '%Y-%m-%dT%H:%H:%S')
+        dt = datetime.strptime(date_string, '%Y-%m-%dT%H:%M:%S')
     except ValueError:
         #doesn't follow first format, check second format
         try:
             dt = datetime.strptime(date_string, '%Y-%m-%d')
         #doesn't follow second format, raise error
         except ValueError:
-            raise(ValueError('DATE format incorrect'))
+            raise(ValueError('date format incorrect'))
     #follows one of the two formats, return
     return(dt)
 
@@ -565,8 +565,8 @@ def _query(args):
     query_input_options = {
     C.NAME : 'Name (empty if want all, separate multiple by commas): ',
     C.SITE : 'Site (empty if want all, separate multiple by commas): ',
-    C.START : 'Start Time (YYMMDD -- empty for today): ',
-    C.END : 'End Time (YYMMDD -- empty for Start Time): ',
+    C.START : 'Start Time (YYYY-mm-dd or YYYY-mm-ddTHH:MM:SS -- empty for today): ',
+    C.END : 'End Time (YYYY-mm-dd or YYYY-mm-ddTHH:MM:SS -- empty for Start Time): ',
     C.GLOB : 'Glob Pattern (empty if want all, separate multiple by commas): ',
     C.FORMAT : 'Format Pattern (empty if want all, separate multiple by commas): ',
     C.LEVEL : 'Level (empty if want all, separate multiple by commas): ',
@@ -600,6 +600,9 @@ def _query(args):
         print('\nEXPERIMENT query:')
         for in_option in (C.NAME, C.START, C.END, C.SITE):
             exp_in[in_option] = input(query_input_options[in_option])
+            if (in_option == C.START or in_option == C.END) and exp_in[in_option]:
+                exp_in[in_option] = _parse_datetime(exp_in[in_option])
+        #make sure start_time and end_time are in an accepted format
         query_in[C.EXP] = exp_in
     #get FILETYPE query input
     if args.filetype:
@@ -614,6 +617,8 @@ def _query(args):
         print('\nFILE query:')
         for in_option in (C.INST, C.EXP, C.FILETYPE, C.PATH, C.START, C.END):
             file_in[in_option] = input(query_input_options[in_option])
+            if (in_option == C.START or in_option == C.END) and file_in[in_option]:
+                file_in[in_option] = _parse_datetime(file_in[in_option])
         query_in[C.FILE] = file_in
 
     #separates multiple query specifiers into a list
@@ -709,6 +714,26 @@ def _query(args):
                 queries[C.EXP].append(init_exp_query.filter(Experiment.name == e_name))
         else:
             queries[C.EXP].append(init_exp_query)
+        #check if there is a start_time to use to filter
+        if query_in[C.EXP][C.START]:
+            #make copy of current queries
+            temp = queries[C.EXP].copy()
+            #clear out current queries
+            queries[C.EXP].clear()
+            #loop through each query and filter by experiment name
+            for qry in temp:
+                for e_start in query_in[C.EXP][C.START]:
+                    queries[C.EXP].append(qry.filter(Experiment.start_time >= e_start))
+        #check if there is an end_time to use to filter
+        if query_in[C.EXP][C.END]:
+            #make copy of current queries
+            temp = queries[C.EXP].copy()
+            #clear out current queries
+            queries[C.EXP].clear()
+            #loop through each query and filter by experiment name
+            for qry in temp:
+                for e_end in query_in[C.EXP][C.END]:
+                    queries[C.EXP].append(qry.filter(Experiment.end_time < e_end))
         #check if there are any sites to use to filter
         if query_in[C.EXP][C.SITE][0]:
             #make copy of current queries
@@ -850,6 +875,26 @@ def _query(args):
             for qry in temp:
                 for f_path in query_in[C.FILE][C.PATH]:
                     queries[C.FILE].append(qry.filter(File.relative_path == f_path))
+        #check if there is a start_time to use to filter
+        if query_in[C.FILE][C.START]:
+            #make copy of current queries
+            temp = queries[C.FILE].copy()
+            #clear out current queries
+            queries[C.FILE].clear()
+            #loop through each query and filter by experiment name
+            for qry in temp:
+                for f_start in query_in[C.FILE][C.START]:
+                    queries[C.FILE].append(qry.filter(File.start_time >= f_start))
+        #check if there is an end_time to use to filter
+        if query_in[C.FILE][C.END]:
+            #make copy of current queries
+            temp = queries[C.FILE].copy()
+            #clear out current queries
+            queries[C.FILE].clear()
+            #loop through each query and filter by experiment name
+            for qry in temp:
+                for f_end in query_in[C.FILE][C.END]:
+                    queries[C.FILE].append(qry.filter(File.end_time < f_end))
         print('\nFILE query output')
         print(OUTPUT.FILE.value)
         num = 1