Commit 3973724b authored by Bruce Flynn's avatar Bruce Flynn

Select parsers based on can_parse

parent cf9dfa39
Pipeline #1006 canceled with stage
in 53 minutes and 33 seconds
......@@ -9,7 +9,7 @@ the file.
import argparse
import logging
from amrc_aws import rdr, db
from amrc_aws import parsers, db, exc
LOG = logging
......@@ -36,14 +36,15 @@ def main():
for filepath in (l.strip() for l in args.filelist):
LOG.info(filepath)
if not rdr.can_parse(filepath):
LOG.warn('Cannot parse using RDR parser: %s', filepath)
parser = parsers.get_parser(filepath)
if not parser:
LOG.warn('Cannot find parser for %s', filepath)
continue
try:
frames = rdr.frames(filepath, error_callback=parse_error)
frames = parser.frames(filepath, error_callback=parse_error)
db.insert_frames(frames)
except (db.IntegrityError, rdr.ParseError) as err:
LOG.error('File insert failed(%s): %s', filepath, err)
except (db.IntegrityError, exc.ParseError) as err:
LOG.error('File insert failed(%s): %s', filepath, err.__class__.__name__)
LOG.debug('%s', filepath, exc_info=1)
......
......@@ -9,14 +9,10 @@ cannot be parsed will be skipped.
import argparse
import logging
from amrc_aws import rdr, q1h, db
from amrc_aws import parsers, db
LOG = logging
parser_modules = [
rdr, q1h
]
def main():
parser = argparse.ArgumentParser(
......@@ -32,11 +28,8 @@ def main():
db.init()
filepath = args.filepath
frame_parser = None
for module in parser_modules:
if hasattr(module, 'can_parse') and module.can_parse(filepath):
frame_parser = module
frame_parser = parsers.get_parser(args.filepath)
if frame_parser is None:
parser.exit(status=1, message='Unable to identify a parser for %s\n' % filepath)
......
......@@ -27,4 +27,5 @@ def main():
if __name__ == '__main__':
main()
\ No newline at end of file
main()
NaN = float('nan')
\ No newline at end of file
from . import rdr, q1h
parsers = [rdr, q1h]
def get_parser(filename):
for parser in parsers:
if parser.can_parse(filename):
return parser
from amrc_aws import parsers, rdr, q1h
def test_get_parser():
filename = '996010416.r'
p = parsers.get_parser(filename)
assert p is rdr
filename = 'wfd201606q1h.txt'
p = parsers.get_parser(filename)
assert p is q1h
filename = 'foo'
assert parsers.get_parser(filename) is None
def test_parser_interface():
for p in parsers.parsers:
assert hasattr(p, 'can_parse')
assert hasattr(p, 'frames')
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment