Skip to content
Snippets Groups Projects
Unverified Commit 9ef1451c authored by David Hoese's avatar David Hoese
Browse files

Add file pattern wildcards in file ordering API

parent a7057b01
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
metobsapi/files.py
+ 11
− 2
View file @ 9ef1451c
...@@ -45,7 +45,7 @@ def handle_begin_end(begin, end): ...@@ -45,7 +45,7 @@ def handle_begin_end(begin, end):
return begin, end return begin, end
def get_data(start, end, streams): def get_data(start, end, streams, frame=True):
from flask import current_app as app from flask import current_app as app
cur_dt = start cur_dt = start
...@@ -58,6 +58,12 @@ def get_data(start, end, streams): ...@@ -58,6 +58,12 @@ def get_data(start, end, streams):
# an error occurred # an error occurred
return stream_id, 'unknown_stream' return stream_id, 'unknown_stream'
if isinstance(stream_info, (list, tuple)):
# special wildcard stream_id
res = get_data(start, end, stream_info, frame=False)
data.extend(res)
continue
relpath = stream_info['relpath'] relpath = stream_info['relpath']
for dt in dts: for dt in dts:
path = dt.strftime(relpath) path = dt.strftime(relpath)
...@@ -70,7 +76,10 @@ def get_data(start, end, streams): ...@@ -70,7 +76,10 @@ def get_data(start, end, streams):
file_info['relpath'] = path file_info['relpath'] = path
data.append(file_info) data.append(file_info)
return pd.DataFrame(data) if frame:
return pd.DataFrame(data)
else:
return data
def handleCSV(frame): def handleCSV(frame):
......
metobsapi/tests/test_files_api.py
+ 25
− 27
View file @ 9ef1451c
import metobsapi import metobsapi
from metobsapi.util import ProductFrequency from metobsapi.util import ProductFrequency, file_responses
import unittest import unittest
import tempfile import tempfile
import shutil import shutil
import json
ARCHIVE_INFO = {
'aoss': {
'tower': {
'level_00': {
'ascii': {
'frequency': ProductFrequency.DAILY_FILE,
'pattern': 'rig_tower.%Y-%m-%d.ascii',
},
},
},
},
'mendota': {
},
}
class TestFilesAPI(unittest.TestCase): class TestFilesAPI(unittest.TestCase):
...@@ -31,7 +15,7 @@ class TestFilesAPI(unittest.TestCase): ...@@ -31,7 +15,7 @@ class TestFilesAPI(unittest.TestCase):
# need now for 'recent' queries # need now for 'recent' queries
now = datetime.utcnow() now = datetime.utcnow()
cls._datetimes = [now, now - timedelta(days=1), now - timedelta(days=2)] cls._datetimes = [now, now - timedelta(days=1), now - timedelta(days=2)]
create_fake_archive(ARCHIVE_INFO, root=cls.archive_dir, datetimes=cls._datetimes) create_fake_archive(file_responses.ARCHIVE_INFO, root=cls.archive_dir, datetimes=cls._datetimes)
# import subprocess # import subprocess
# subprocess.check_call(['/opt/local/bin/tree', cls.archive_dir]) # subprocess.check_call(['/opt/local/bin/tree', cls.archive_dir])
...@@ -47,38 +31,38 @@ class TestFilesAPI(unittest.TestCase): ...@@ -47,38 +31,38 @@ class TestFilesAPI(unittest.TestCase):
def tearDown(self): def tearDown(self):
pass pass
def test_api_doc(self): def test_doc(self):
res = self.app.get('/api/files') res = self.app.get('/api/files')
assert b'File Request Application' in res.data assert b'File Request Application' in res.data
def test_api_tower_daily_ascii_csv(self): def test_tower_daily_ascii_csv(self):
res = self.app.get('/api/files.csv?streams=aoss.tower.ascii.l00.*') res = self.app.get('/api/files.csv?streams=aoss.tower.ascii.l00.*')
fn = bytes(self._datetimes[0].strftime('rig_tower.%Y-%m-%d.ascii'), encoding='utf-8') fn = bytes(self._datetimes[0].strftime('rig_tower.%Y-%m-%d.ascii'), encoding='utf-8')
assert fn in res.data assert fn in res.data
def test_api_tower_daily_ascii_json(self): def test_tower_daily_ascii_json(self):
res = self.app.get('/api/files.json?streams=aoss.tower.ascii.l00.*') res = self.app.get('/api/files.json?streams=aoss.tower.ascii.l00.*')
fn = bytes(self._datetimes[0].strftime('rig_tower.%Y-%m-%d.ascii'), encoding='utf-8') fn = bytes(self._datetimes[0].strftime('rig_tower.%Y-%m-%d.ascii'), encoding='utf-8')
assert fn in res.data assert fn in res.data
def test_api_tower_daily_ascii_sh(self): def test_tower_daily_ascii_sh(self):
res = self.app.get('/api/files.sh?streams=aoss.tower.ascii.l00.*') res = self.app.get('/api/files.sh?streams=aoss.tower.ascii.l00.*')
fn = bytes(self._datetimes[0].strftime('rig_tower.%Y-%m-%d.ascii'), encoding='utf-8') fn = bytes(self._datetimes[0].strftime('rig_tower.%Y-%m-%d.ascii'), encoding='utf-8')
assert fn in res.data assert fn in res.data
def test_api_tower_daily_ascii_bat(self): def test_tower_daily_ascii_bat(self):
res = self.app.get('/api/files.bat?streams=aoss.tower.ascii.l00.*') res = self.app.get('/api/files.bat?streams=aoss.tower.ascii.l00.*')
fn = bytes(self._datetimes[0].strftime('rig_tower.%Y-%m-%d.ascii'), encoding='utf-8') fn = bytes(self._datetimes[0].strftime('rig_tower.%Y-%m-%d.ascii'), encoding='utf-8')
assert fn in res.data assert fn in res.data
def test_api_tower_daily_ascii_dated_json(self): def test_tower_daily_ascii_dated_json(self):
dt = self._datetimes[1] dt = self._datetimes[1]
begin = dt.strftime('%Y-%m-%d') begin = dt.strftime('%Y-%m-%d')
res = self.app.get('/api/files.json?streams=aoss.tower.ascii.l00.*&begin={}'.format(begin)) res = self.app.get('/api/files.json?streams=aoss.tower.ascii.l00.*&begin={}'.format(begin))
fn = bytes(dt.strftime('rig_tower.%Y-%m-%d.ascii'), encoding='utf-8') fn = bytes(dt.strftime('rig_tower.%Y-%m-%d.ascii'), encoding='utf-8')
assert fn in res.data assert fn in res.data
def test_api_tower_daily_ascii_relative_json(self): def test_tower_daily_ascii_relative_json(self):
dt = self._datetimes[1] dt = self._datetimes[1]
begin = dt.strftime('%Y-%m-%d') begin = dt.strftime('%Y-%m-%d')
res = self.app.get('/api/files.json?streams=aoss.tower.ascii.l00.*&begin=-2&end={}'.format(begin)) res = self.app.get('/api/files.json?streams=aoss.tower.ascii.l00.*&begin=-2&end={}'.format(begin))
...@@ -88,7 +72,7 @@ class TestFilesAPI(unittest.TestCase): ...@@ -88,7 +72,7 @@ class TestFilesAPI(unittest.TestCase):
fn = bytes(self._datetimes[0].strftime('rig_tower.%Y-%m-%d.ascii'), encoding='utf-8') fn = bytes(self._datetimes[0].strftime('rig_tower.%Y-%m-%d.ascii'), encoding='utf-8')
assert fn not in res.data assert fn not in res.data
def test_api_tower_daily_ascii_relative_noend_json(self): def test_tower_daily_ascii_relative_noend_json(self):
res = self.app.get('/api/files.json?streams=aoss.tower.ascii.l00.*&begin=-2') res = self.app.get('/api/files.json?streams=aoss.tower.ascii.l00.*&begin=-2')
for dt in self._datetimes: for dt in self._datetimes:
fn = bytes(dt.strftime('rig_tower.%Y-%m-%d.ascii'), encoding='utf-8') fn = bytes(dt.strftime('rig_tower.%Y-%m-%d.ascii'), encoding='utf-8')
...@@ -96,5 +80,19 @@ class TestFilesAPI(unittest.TestCase): ...@@ -96,5 +80,19 @@ class TestFilesAPI(unittest.TestCase):
# fn = bytes(self._datetimes[0].strftime('rig_tower.%Y-%m-%d.ascii'), encoding='utf-8') # fn = bytes(self._datetimes[0].strftime('rig_tower.%Y-%m-%d.ascii'), encoding='utf-8')
# assert fn not in res.data # assert fn not in res.data
def test_tower_all_patterns(self):
res = self.app.get('/api/files.json?streams=aoss.tower.*.l00.*')
res = json.loads(str(res.data, encoding='utf-8'))
fn = self._datetimes[0].strftime('rig_tower.%Y-%m-%d.ascii')
assert res['data'][0]['filename'] == fn
def test_tower_multi_all_patterns(self):
res = self.app.get('/api/files.json?streams=aoss.tower.*.l00.*:aoss.tower.nc-1d-1m.lb1.v00')
res = json.loads(str(res.data, encoding='utf-8'))
fn = self._datetimes[0].strftime('rig_tower.%Y-%m-%d.ascii')
assert res['data'][0]['filename'] == fn
fn = self._datetimes[0].strftime('aoss_tower.%Y-%m-%d.nc')
assert res['data'][1]['filename'] == fn
if __name__ == "__main__": if __name__ == "__main__":
unittest.main() unittest.main()
metobsapi/util/__init__.py
+ 9
− 6
View file @ 9ef1451c
...@@ -21,11 +21,9 @@ class ProductFrequency(Enum): ...@@ -21,11 +21,9 @@ class ProductFrequency(Enum):
MONTHLY_DIR = 'monthly_dir' MONTHLY_DIR = 'monthly_dir'
def create_fake_archive(archive_info, root=FAKE_ARCHIVE_PATH, datetimes=None, versions=None): def create_fake_archive(archive_info, root=FAKE_ARCHIVE_PATH, datetimes=None):
if datetimes is None: if datetimes is None:
datetimes = [datetime.utcnow()] datetimes = [datetime.utcnow()]
if versions is None:
versions = ['00']
os.makedirs(root, exist_ok=True) os.makedirs(root, exist_ok=True)
os.chdir(root) os.chdir(root)
...@@ -38,12 +36,11 @@ def create_fake_archive(archive_info, root=FAKE_ARCHIVE_PATH, datetimes=None, ve ...@@ -38,12 +36,11 @@ def create_fake_archive(archive_info, root=FAKE_ARCHIVE_PATH, datetimes=None, ve
for level_name, level_info in level_info.items(): for level_name, level_info in level_info.items():
os.makedirs(level_name, exist_ok=True) os.makedirs(level_name, exist_ok=True)
os.chdir(level_name) os.chdir(level_name)
for version in versions: for version_name in level_info['version']:
version_name = 'version_' + version
os.makedirs(version_name, exist_ok=True) os.makedirs(version_name, exist_ok=True)
os.chdir(version_name) os.chdir(version_name)
for dt in datetimes: for dt in datetimes:
for pattern_name, pattern_info in level_info.items(): for pattern_name, pattern_info in level_info['patterns'].items():
if pattern_info['frequency'] == ProductFrequency.DAILY_DIR: if pattern_info['frequency'] == ProductFrequency.DAILY_DIR:
dated_dir = dt.strftime('%Y/%m/%d') dated_dir = dt.strftime('%Y/%m/%d')
os.makedirs(dated_dir, exist_ok=True) os.makedirs(dated_dir, exist_ok=True)
...@@ -56,6 +53,12 @@ def create_fake_archive(archive_info, root=FAKE_ARCHIVE_PATH, datetimes=None, ve ...@@ -56,6 +53,12 @@ def create_fake_archive(archive_info, root=FAKE_ARCHIVE_PATH, datetimes=None, ve
os.chdir(dated_dir) os.chdir(dated_dir)
open(dt.strftime(pattern_info['pattern']), 'a') open(dt.strftime(pattern_info['pattern']), 'a')
os.chdir('../../') os.chdir('../../')
elif pattern_info['frequency'] == ProductFrequency.MONTHLY_DIR:
dated_dir = dt.strftime('%Y/%m')
os.makedirs(dated_dir, exist_ok=True)
os.chdir(dated_dir)
open(dt.strftime(pattern_info['pattern']), 'a')
os.chdir('../../')
else: else:
raise RuntimeError("Unknown frequency '%s'", pattern_info['frequency']) raise RuntimeError("Unknown frequency '%s'", pattern_info['frequency'])
os.chdir('..') os.chdir('..')
......
metobsapi/util/file_responses.py
+ 72
− 51
View file @ 9ef1451c
...@@ -24,56 +24,62 @@ ARCHIVE_INFO = { ...@@ -24,56 +24,62 @@ ARCHIVE_INFO = {
'aoss': { 'aoss': {
'tower': { 'tower': {
'level_00': { 'level_00': {
'ascii': { 'version': ('version_00',),
'frequency': ProductFrequency.DAILY_FILE, 'patterns': {
'pattern': 'rig_tower.%Y-%m-%d.ascii', 'ascii': {
'version': ('version_00',), 'frequency': ProductFrequency.DAILY_FILE,
'pattern': 'rig_tower.%Y-%m-%d.ascii',
},
}, },
}, },
'level_b1': { 'level_b1': {
'nc-1mo-1d': { 'version': ('version_00',),
'frequency': ProductFrequency.MONTHLY_DIR, 'patterns': {
'pattern': 'aoss_tower.%Y-%m.nc', 'nc-1mo-1d': {
'version': ('version_00',), 'frequency': ProductFrequency.MONTHLY_DIR,
}, 'pattern': 'aoss_tower.%Y-%m.nc',
'nc-1d-1m': { },
'frequency': ProductFrequency.DAILY_DIR, 'nc-1d-1m': {
'pattern': 'aoss_tower.%Y-%m-%d.nc', 'frequency': ProductFrequency.DAILY_DIR,
'version': ('version_00',), 'pattern': 'aoss_tower.%Y-%m-%d.nc',
},
}, },
}, },
}, },
'aeri': { 'aeri': {
'level_00': { 'level_00': {
'par': { 'version': ('version_00',),
'frequency': ProductFrequency.DAILY_DIR, 'patterns': {
'pattern': '%y%m%d.par', 'par': {
'version': ('version_00',), 'frequency': ProductFrequency.DAILY_DIR,
}, 'pattern': '%y%m%d.par',
'qc': { 'version': ('version_00',),
'frequency': ProductFrequency.DAILY_DIR, },
'pattern': '%y%m%d.qc', 'qc': {
'version': ('version_00',), 'frequency': ProductFrequency.DAILY_DIR,
}, 'pattern': '%y%m%d.qc',
'sum': { 'version': ('version_00',),
'frequency': ProductFrequency.DAILY_DIR, },
'pattern': '%y%m%d.sum', 'sum': {
'version': ('version_00',), 'frequency': ProductFrequency.DAILY_DIR,
}, 'pattern': '%y%m%d.sum',
'scr-aesitter': { 'version': ('version_00',),
'frequency': ProductFrequency.DAILY_DIR, },
'pattern': 'AESITTER.SCR', 'scr-aesitter': {
'version': ('version_00',), 'frequency': ProductFrequency.DAILY_DIR,
}, 'pattern': 'AESITTER.SCR',
'scr-radiance': { 'version': ('version_00',),
'frequency': ProductFrequency.DAILY_DIR, },
'pattern': 'RADIANCE.SCR', 'scr-radiance': {
'version': ('version_00',), 'frequency': ProductFrequency.DAILY_DIR,
}, 'pattern': 'RADIANCE.SCR',
'scr-summary': { 'version': ('version_00',),
'frequency': ProductFrequency.DAILY_DIR, },
'pattern': 'SUMMARY.SCR', 'scr-summary': {
'version': ('version_00',), 'frequency': ProductFrequency.DAILY_DIR,
'pattern': 'SUMMARY.SCR',
'version': ('version_00',),
},
}, },
}, },
}, },
...@@ -99,7 +105,6 @@ for file_suffix in ('B1.CXS', ...@@ -99,7 +105,6 @@ for file_suffix in ('B1.CXS',
nfo = { nfo = {
'frequency': ProductFrequency.DAILY_DIR, 'frequency': ProductFrequency.DAILY_DIR,
'pattern': '%y%m%d{}'.format(stream_pat), 'pattern': '%y%m%d{}'.format(stream_pat),
'version': ('version_00',),
} }
ARCHIVE_INFO['aoss']['aeri']['level_00'][stream_pat] = nfo ARCHIVE_INFO['aoss']['aeri']['level_00'][stream_pat] = nfo
...@@ -109,8 +114,10 @@ stream_id_fmt = "{site}.{inst}.{pattern}.{level}.{version}" ...@@ -109,8 +114,10 @@ stream_id_fmt = "{site}.{inst}.{pattern}.{level}.{version}"
for site, site_info in ARCHIVE_INFO.items(): for site, site_info in ARCHIVE_INFO.items():
for inst, inst_info in site_info.items(): for inst, inst_info in site_info.items():
for level, level_info in inst_info.items(): for level, level_info in inst_info.items():
for stream_pattern, pattern_info in level_info.items(): all_patterns_recent = []
for version in pattern_info['version']: for version in level_info['version']:
all_patterns = []
for stream_pattern, pattern_info in level_info['patterns'].items():
stream_id = stream_id_fmt.format( stream_id = stream_id_fmt.format(
site=site, site=site,
inst=inst, inst=inst,
...@@ -118,6 +125,7 @@ for site, site_info in ARCHIVE_INFO.items(): ...@@ -118,6 +125,7 @@ for site, site_info in ARCHIVE_INFO.items():
pattern=stream_pattern, pattern=stream_pattern,
version=version.replace('version_', 'v'), version=version.replace('version_', 'v'),
) )
all_patterns.append(stream_id)
path = os.path.join(site, path = os.path.join(site,
inst, inst,
...@@ -135,16 +143,29 @@ for site, site_info in ARCHIVE_INFO.items(): ...@@ -135,16 +143,29 @@ for site, site_info in ARCHIVE_INFO.items():
} }
ARCHIVE_STREAMS[stream_id] = stream_info ARCHIVE_STREAMS[stream_id] = stream_info
# Special 'most recent' version stream_id # Special 'most recent' version stream_id
stream_id = stream_id_fmt.format( if version == level_info['version'][0]:
recent_stream_id = stream_id_fmt.format(
site=site,
inst=inst,
level=level.replace('level_', 'l'),
pattern=stream_pattern,
version='*',
)
ARCHIVE_STREAMS[recent_stream_id] = ARCHIVE_STREAMS[stream_id]
all_patterns_id = stream_id_fmt.format(
site=site, site=site,
inst=inst, inst=inst,
level=level.replace('level_', 'l'), level=level.replace('level_', 'l'),
pattern=stream_pattern, pattern='*',
version='*', version=version.replace('version_', 'v'),
) )
newest_stream_id = stream_id.replace('*', pattern_info['version'][0].replace('version_', 'v')) ARCHIVE_STREAMS[all_patterns_id] = all_patterns
ARCHIVE_STREAMS[stream_id] = ARCHIVE_STREAMS[newest_stream_id] if version == level_info['version'][0]:
all_patterns_recent_id = all_patterns_id.replace(
version.replace('version_', 'v'), '*')
ARCHIVE_STREAMS[all_patterns_recent_id] = all_patterns
ERROR_HANDLERS = { ERROR_HANDLERS = {
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment