made Site.name and Experiment.name unique. created a many-to-many foreign key...

made Site.name and Experiment.name unique. created a many-to-many foreign key between Instrument and FileType. added 'version' attribute to File. Added 'name', 'level', 'glob_pattern', 'format_pattern', and 'period' attributes to FileType

metobscommon/archive/db.py

@@ -43,7 +43,7 @@ class OUTPUT(Enum):
     SITE = 'num - name - experiment - description'
     EXP = 'num - name - start_time - end_time - site - description'
     FILETYPE = 'num - pattern - interval - description'
-    FILE = 'num - instrument - experiment - file_type - relative_path - start_time - end_time'
+    FILE = 'num - instrument - experiment - filetype - relative_path - start_time - end_time'
 
 # association_table = Table('association', Base.metdadata,
 #     Column('site_id', Integer, ForeignKey('Sites.site_id')),
@@ -59,6 +59,7 @@ class Instrument(Base):
     # Instruments to Sites -- many to one
     site_id = Column('site_id', Integer, ForeignKey('Sites.site_id'))
     site = relationship('Site')
+    filetype = relationship('FileType', secondary='Instrument_FileTypes', viewonly=True)
     description = Column(String())
 
     def __init__(self, name = 'DEFAULT_NAME', site=None, desc = ''):
@@ -66,8 +67,35 @@ class Instrument(Base):
         self.site = site
         self.description = desc
 
+    def add_filetype(self, filetype):
+        self.instrument_filetypes.append(Instrument_FileType(instrument=self, filetype=filetype))
+
+    def __repr__(self):
+        output = '{} - {} - {}'.format(self.name, self.site.name, self.description)
+        if self.instrument_filetypes:
+            for i_f in self.instrument_filetypes:
+                output += '{}~'.format(i_f.filetype.name)
+        output += ' - {}'.format(self.description)
+        return output
+
+class Instrument_FileType(Base):
+
+    _tablename__ = 'Instrument_FileTypes'
+
+    instrument_filetype_id = Column(Integer, primary_key=True, unique=True)
+    instrument_id = Column(Integer, ForeignKey('Instruments.instrument_id'), primary_key=True)
+    filetype_id = Column(Integer, ForeignKey('FileTypes.filetype_id'), primary_key=True)
+
+    instrument = relationship('Instrument', backref=backref('instrument_filetypes', cascade='all, delete-orphan'))
+    filetype = relationship('FileType', backref=backref('instrument_filetypes', cascade='all, delete-orphan'))
+
+    def __init__(self, instrument=None, filetype=None):
+        self.instrument_filetype_id = uuid.uuid4().hex
+        self.instrument = instrument
+        self.filetype = filetype
+
     def __repr__(self):
-        return '{} - {} - {}'.format(self.name, self.site.name, self.description)
+        return 'Instrument -- {} | FileType -- {}'.format(self.instrument.name, self.filetype.name)
 
 class Site_Experiment(Base):
 
@@ -93,7 +121,7 @@ class Site(Base):
     __tablename__ = 'Sites'
 
     site_id = Column(Integer, primary_key=True)
-    name = Column(String(255))
+    name = Column(String(255), unique=True)
     experiment = relationship('Experiment', secondary='Site_Experiments', viewonly=True)
     description = Column(String())
 
@@ -117,7 +145,7 @@ class Experiment(Base):
     __tablename__ = 'Experiments'
 
     experiment_id = Column(Integer, primary_key=True)
-    name = Column(String(255))
+    name = Column(String(255), unique=True)
     start_time = Column(DateTime)
     end_time = Column(DateTime)
     site = relationship('Site', secondary='Site_Experiments', viewonly=True)
@@ -145,10 +173,16 @@ class FileType(Base):
 
     __tablename__ = 'FileTypes'
 
-    file_type_id = Column(Integer, primary_key=True)
-    pattern = Column(String(255))
-    # number of seconds in each record, 1 minute average = 60
+    filetype_id = Column(Integer, primary_key=True)
+    name = Column(String(255))
+    level = Column(String(255))
+    glob_pattern = Column(String(255))
+    format_pattern = Column(String(255))
+    # how many seconds each record represents
     interval = Column(Integer)
+    #size of file
+    period = Column(String(255))
+    instrument = relationship('Instrument', secondary='Instrument_FileTypes', viewonly=True)
     description = Column(String())
 
     def __init__(self, pattern = None, interval = None, desc = ''):
@@ -156,8 +190,16 @@ class FileType(Base):
         self.interval = interval
         self.description = desc
 
+    def add_instrument(self, instrument):
+        self.instrument_filetypes.append(Instrument_FileType(instrument=instrument, filetype=self))
+
     def __repr__(self):
-        return '{} - {} - {}'.format(self.pattern, self.interval, self.description)
+        output = '{} - {} - {} - {} - {} - {} - '.format(self.name, self.level, self.glob_pattern, self.format_pattern, self.interval, self.period)
+        if self.instrument_filetypes:
+            for i_f in self.instrument_filetypes:
+                output += '{}~'.format(i_f.instrument.name)
+        output += ' - {}'.format(self.description)
+        return output
 
 class File(Base):
 
@@ -171,23 +213,24 @@ class File(Base):
     experiment_id = Column('experiment_id', Integer, ForeignKey('Experiments.experiment_id'))
     experiment = relationship('Experiment')
     # Files to FileTypes
-    file_type_id = Column('file_type_id', Integer, ForeignKey('FileTypes.file_type_id'))
-    file_type = relationship('FileType')
+    filetype_id = Column('filetype_id', Integer, ForeignKey('FileTypes.filetype_id'))
+    filetype = relationship('FileType')
     # path relative to /mnt/inst/data/cache
     relative_path = Column(String(255))
     start_time = Column(DateTime)
     end_time = Column(DateTime)
+    version = Column(String(255))
 
-    def __init__(self, instrument=None, experiment=None, file_type=None, relative_path='', start_time=None, end_time=None):
+    def __init__(self, instrument=None, experiment=None, filetype=None, relative_path='', start_time=None, end_time=None, version=None):
         self.instrument = instrument
         self.experiment = experiment
-        self.file_type = file_type
+        self.filetype = filetype
         self.relative_path = relative_path
         self.start_time = start_time
         self.end_time = end_time
 
     def __repr__(self):
-        return '{} - {} - {} - {} - {} - {}'.format(self.instrument.name, self.experiment.name, self.file_type.pattern, self.relative_path, self.start_time, self.end_time)
+        return '{} - {} - {} - {} - {} - {}'.format(self.instrument.name, self.experiment.name, self.filetype.pattern, self.relative_path, self.start_time, self.end_time)
 
 '''creates very basic outline of database
 
@@ -331,9 +374,9 @@ def _create(args):
             # session.add(cxs_filetype)
 
             #add files
-            session.add(File(instrument=aeri_b_inst, experiment=happy_exp, file_type=nc_filetype, relative_path='/mnt/inst-data/cache/bago/aeri/latest_profiles.nc'))
-            session.add(File(instrument=aeri_b_inst, experiment=happy_exp, file_type=png_filetype, relative_path='/mnt/inst-data/cache/bago/aeri/latest_quicklook_at.png'))
-            session.add(File(instrument=aeri_b_inst, experiment=sad_exp, file_type=png_filetype, relative_path='/mnt/inst-data/cache/bago/aeri/latest_quicklook_mr.png'))
+            session.add(File(instrument=aeri_b_inst, experiment=happy_exp, filetype=nc_filetype, relative_path='/mnt/inst-data/cache/bago/aeri/latest_profiles.nc'))
+            session.add(File(instrument=aeri_b_inst, experiment=happy_exp, filetype=png_filetype, relative_path='/mnt/inst-data/cache/bago/aeri/latest_quicklook_at.png'))
+            session.add(File(instrument=aeri_b_inst, experiment=sad_exp, filetype=png_filetype, relative_path='/mnt/inst-data/cache/bago/aeri/latest_quicklook_mr.png'))
 
             #commit changes to database
             session.commit()
@@ -391,7 +434,7 @@ def _sync(args):
                     curr_filetype = session.query(FileType).filter(FileType.pattern == f.split('.')[1]).all()
                     if curr_filetype:
                         curr_filetype = curr_filetype[0]
-                        curr_file = File(instrument=curr_inst, experiment=curr_exp, file_type=curr_filetype, relative_path=curr_path)
+                        curr_file = File(instrument=curr_inst, experiment=curr_exp, filetype=curr_filetype, relative_path=curr_path)
                         print(curr_file)
                         session.add(curr_file)
     session.commit()
@@ -405,7 +448,7 @@ optional arguments:
   -s SITE, --site SITE  query Sites
   -e EXPERIMENT, --experiment EXPERIMENT
                         query Experiments
-  -t FILE_TYPE, --file_type FILE_TYPE
+  -t FILETYPE, --filetype FILETYPE
                         query FileTypes
   -f FILE, --file FILE  query Files
 
@@ -456,7 +499,7 @@ def _query(args):
             exp_in[in_option] = input(query_input_options[in_option])
         query_in[C.EXP] = exp_in
     #get FILETYPE query input
-    if args.file_type:
+    if args.filetype:
         filetype_in = {}
         print('\nFILETYPE query:')
         for in_option in (C.PATTERN, C.INTERVAL):
@@ -614,7 +657,7 @@ def _query(args):
             #loop through each query and filter by filetype
             for qry in temp:
                 for f_ft in query_in[C.FILE][C.FILETYPE]:
-                    queries[C.FILE].append(qry.filter(File.file_type.has(pattern = f_ft)))
+                    queries[C.FILE].append(qry.filter(File.filetype.has(pattern = f_ft)))
         if query_in[C.FILE][C.PATH]:
             #make copy of current queries
             temp = queries[C.FILE].copy()
@@ -643,7 +686,7 @@ optional arguments:
                         format as '-e name.start_time.end_time'
   -t FILETYPE, --filetype FILETYPE
                         format as '-t pattern.interval'
-  -f FILE, --file FILE  format as '-f instrument.experiment.file_type.relative
+  -f FILE, --file FILE  format as '-f instrument.experiment.filetype.relative
                         _path.start_time.end_time'
 
 
@@ -751,7 +794,7 @@ def _add(args):
             if (len(f) >= 4 and len(f) <= 6):
                 add_instrument = f[0]
                 add_experiment = f[1]
-                add_file_type = f[2]
+                add_filetype = f[2]
                 add_rp = f[3]
                 if len(f) > 4:
                     add_st = f[4]
@@ -763,16 +806,16 @@ def _add(args):
                     add_et = None
                 add_i = session.query(Instrument).filter(Instrument.name == add_instrument).all()[0]
                 add_e = session.query(Experiment).filter(Experiment.name == add_experiment).all()[0]
-                add_ft = session.query(FileType).filter(FileType.pattern == add_file_type).all()[0]
-                for add_something, class_name, name in ((add_i, 'INSTRUMENT', add_instrument), (add_e, 'EXPERIMENT', add_experiment), (add_ft, 'FILETYPE', add_file_type)):
+                add_ft = session.query(FileType).filter(FileType.pattern == add_filetype).all()[0]
+                for add_something, class_name, name in ((add_i, 'INSTRUMENT', add_instrument), (add_e, 'EXPERIMENT', add_experiment), (add_ft, 'FILETYPE', add_filetype)):
                     if not add_something:
                         raise ValueError('{} \'{}\' does not exist'.format(class_name, name))
-                new_file = File(instrument=add_i, experiment=add_e, file_type=add_ft, relative_path=add_rp, start_time=add_st, end_time=add_et)
+                new_file = File(instrument=add_i, experiment=add_e, filetype=add_ft, relative_path=add_rp, start_time=add_st, end_time=add_et)
                 print(new_file)
                 session.add(new_file)
                 session.commit()
             else:
-                    raise ValueError('FILE add format is incorrect -- instrument.experiment.file_type.relative_path.start_time.end_time')
+                    raise ValueError('FILE add format is incorrect -- instrument.experiment.filetype.relative_path.start_time.end_time')
 
     else:
         raise FileNotFoundError('database \'{}\' does not exist'.format(args.filename))
@@ -881,7 +924,7 @@ def main():
     parser_create.set_defaults(func=_create)
 
     #create subparser for 'sync' option
-    parser_sync = subparsers.add_parser('sync')
+    parser_sync = subparsers.add_parser('sync', help='sync files from directory -- /data/site/inst/')
     parser_sync.set_defaults(func=_sync)
 
     #create subparser for 'query' option
@@ -890,7 +933,7 @@ def main():
     parser_query.add_argument('-i', '--instrument', nargs='?', type=bool, const=True, help='query Instruments')
     parser_query.add_argument('-s', '--site', nargs='?', type=bool, const=True, help='query Sites')
     parser_query.add_argument('-e', '--experiment', nargs='?', type=bool, const=True, help='query Experiments')
-    parser_query.add_argument('-t', '--file_type', nargs='?', type=bool, const=True, help='query FileTypes')
+    parser_query.add_argument('-t', '--filetype', nargs='?', type=bool, const=True, help='query FileTypes')
     parser_query.add_argument('-f', '--file', nargs='?', type=bool, const=True, help='query Files')
 
     #create subparser for 'add' option
@@ -900,7 +943,7 @@ def main():
     parser_add.add_argument('-s', '--site', help='format as \'-s site\'')
     parser_add.add_argument('-e', '--experiment', help='format as \'-e name.start_time.end_time\'')
     parser_add.add_argument('-t', '--filetype', help='format as \'-t pattern.interval\'')
-    parser_add.add_argument('-f', '--file', help='format as \'-f instrument.experiment.file_type.relative_path.start_time.end_time\'')
+    parser_add.add_argument('-f', '--file', help='format as \'-f instrument.experiment.filetype.relative_path.start_time.end_time\'')
 
     #method for testing stuff
     parser_test = subparsers.add_parser('test')
@@ -911,9 +954,10 @@ def main():
     parser_test.add_argument('-i', '--instrument', nargs='?', type=bool, const=True, help='query instruments')
     parser_test.add_argument('-s', '--site', nargs='?', type=bool, const=True, help='query sites')
     parser_test.add_argument('-e', '--experiment', nargs='?', type=bool, const=True, help='query experiments')
-    parser_test.add_argument('-t', '--file_type', nargs='?', type=bool, const=True, help='query filetypes')
+    parser_test.add_argument('-t', '--filetype', nargs='?', type=bool, const=True, help='query filetypes')
     parser_test.add_argument('-f', '--file', nargs='?', type=bool, const=True, help='query files')
     parser_test.add_argument('-v', '--testing_stuff', nargs='?', type=bool, const=True, help='testing stuff out')
+    parser_test.add_argument('-c', '--testing_create', help='testing _create() functionality with yaml')
 
     args = parser.parse_args()