tidy.py 9.97 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
#!/usr/bin/env python
# encoding: utf-8
"""
tidy.py

Routines to handle ceilometer files on tahiti on a regular basis
No URL or opendap paths should be used, this only does local file manipulation
However, calls to external product creation may not do remote file calls,
such as getting data files from an OpenDap url.

Environment variables:

CEILO_INCOMING_DIR
CEILO_RAW_DIR
CEILO_CACHE_DIR
CEILO_LATEST_DIR

CEILO_DIR_FORMAT=%Y/%m/%d

Created by davidh on 2009-10-08.
Copyright (c) 2009 University of Wisconsin SSEC. All rights reserved.
"""

import os, sys, logging, shutil
from subprocess import check_call as sh
from datetime import datetime, timedelta
from metobs.ceilo import quicklook, nc
from metobs.ceilo import CONFIG as c
29
from metobscommon.util import raw_manager
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
rel = raw_manager.relativedir
import re

RE_DIGITS = re.compile(r'\d+')
LOG = logging.getLogger(__name__)
site = "rig"
description=""


CEILO_INCOMING_DIR = c.CEILO_INCOMING_DIR
CEILO_PRAW_DIR =      c.CEILO_PRAW_DIR
CEILO_CACHE_DIR =    c.CEILO_CACHE_DIR
CEILO_LATEST_DIR =   c.CEILO_LATEST_DIR
CEILO_DIR_FORMAT =   c.CEILO_DIR_FORMAT

def create_quicklook(when=None):
    """run quicklook.py with the proper arguments to create various quicklooks."""
    when = when or datetime.now()
    created_files = quicklook.make_plot(dt=when, site=site, description=description)
    if len(created_files) != 4:
        LOG.error("Expected four quicklooks to be created, latest files were not linked, %d filenames were returned" % len(created_files))
        raise ValueError("Latest quicklooks could not be linked because there were not the correct amount of images")
    ptypes = [ c._handle_plot_type(t) for t in range(1, 2) ]
    for file,ptype,tag in zip(created_files[1::2], ptypes+ptypes, ["","tn"]):
        link_latest_quicklooks(c.get_latest_dir(), file, ptype, tag=tag)

def link_latest_quicklooks(latest_dir, filename, ptype, tag=""):
    _,ext = os.path.splitext(filename)
    tag = tag and "_" + tag or ""
    ptype = ptype and "_" + ptype or ""
    linkname = "latest_quicklook%s%s%s" % (ptype,tag,ext)
    tmpname = "." + linkname
    tmppath = os.path.join( latest_dir, tmpname )
    linkpath = os.path.join( latest_dir, linkname )
    if os.path.islink(tmppath): os.unlink(tmppath)
    LOG.debug('symlink %s -> %s' % (linkpath, filename))
    os.symlink( rel(latest_dir, filename), tmppath )
    os.rename( tmppath, linkpath )

def create_nc(when=None):
    """run external nc_gen module with proper arguments to create a new netCDF
    file.
    """
    when = when or datetime.now()
    # Run for past 3 days
    begin = when - timedelta(days=2)
    # Create temp nc files in cache location
    temp_created_files = nc.make_ceilo_files(begin=begin, end=when, basedir=None, site=site, description=description)
    # Check that some files were created
    if not temp_created_files:
        LOG.warning("No new nc files were created")
        return
    # Rename temp files to original files
    created_files = [ os.path.join(d,f[1:]) for d,f in [ os.path.split(x) for x in temp_created_files ] ]
    for t,f in zip(temp_created_files, created_files):
        if os.path.exists(f): os.remove(f)
        os.rename(t,f)
    # Link latest files
    link_latest_file(c.get_latest_dir(), created_files[0], data_type="nc")

def file_dirs(when = None, data_type='ascii'):
    """ yield camera-name, incoming-dir, raw_dir, cache_dir, latest_dir for a given datetime object (default now)
    """
    when = when or datetime.now()
    if not(os.path.isabs(CEILO_INCOMING_DIR) or os.path.isabs(CEILO_PRAW_DIR) or \
        os.path.isabs(CEILO_CACHE_DIR) or os.path.isabs(CEILO_LATEST_DIR)):
            LOG.warning("Directories should be absolute paths")
    return ( c.get_incoming_dir(),
             c.get_praw_dir(when),
             c.get_cache_dir(data_type, when),
             c.get_latest_dir()
             )

def _keyed_filename(f):
    return tuple(int(x) for x in RE_DIGITS.findall(f)), f

def pending_incoming_files(incoming_dir):
    "return list of ascii files remaining in a given incoming directory, sorted latest-first"
    files = [ _keyed_filename(f)
                for f in os.listdir(incoming_dir)
                if "ascii" in f
                ]
    return [ filename for _,filename in reversed(sorted(files)) ]

def link_latest_file(latest_dir, filename, data_type='ascii'):
    """link latest data_type files to latest_ascii.ascii, overwriting existing links
    filename must be a absolute or relative path, not just the name of a file.
    """
    _,ext = os.path.splitext(filename)
    linkname = 'latest_%s%s' % (data_type, ext)
    tmpname = '.' + linkname
    tmppath = os.path.join( latest_dir, tmpname )
    linkpath = os.path.join( latest_dir, linkname )
    if os.path.islink(tmppath): os.unlink(tmppath)
    LOG.debug('symlink %s -> %s' % (linkpath, filename))
    os.symlink( rel(latest_dir, filename), tmppath )
    os.rename( tmppath, linkpath )

def link_latest_directory( source, latest_dir ):
    tmpname = os.path.join(latest_dir, ".latest_dir")
    linkname = os.path.join(latest_dir, "latest_dir")
    if os.path.islink(tmpname): os.unlink(tmpname)
    LOG.debug('softlinking latest directory %s -> %s' % (linkname, source ))
    os.symlink( rel(latest_dir, source), tmpname)
    os.rename(tmpname, linkname)

def unload_incoming(incoming_dir, praw_dir, cache_dir, latest_dir):
    """handle new images in incoming:
    check for/create raw, cache, and latest directories
    copy originals to raw directory
    move originals to cache directory
    create multiple scaled images
    update latest/ directory link
    update latest_w_h.jpg image links
    """
    if not os.path.exists(incoming_dir):
        LOG.warning("No incoming directory at: '%s'" % (incoming_dir))
        raise IOError
    if not os.path.exists(praw_dir):
        LOG.debug("Creating raw directory: '%s'" % praw_dir)
        os.makedirs(praw_dir)
    if not os.path.exists(cache_dir):
        LOG.debug("Creating cache directory: '%s'" % cache_dir)
        os.makedirs(cache_dir)
    if not os.path.exists(latest_dir):
        LOG.debug("Creating latest directory: '%s'" % latest_dir)
        os.makedirs(latest_dir)

    new_files = pending_incoming_files(incoming_dir)
    if not new_files:
        LOG.warning("no files found in %s" % incoming_dir)
        return

    raw_changes = [ c.rename_incoming(fn, site=site, description=description) for fn in new_files ]
    (praw_dirs, cache_dirs, renames, removes) = [ list(x) for x in zip(*raw_changes) ]
    raw_manager.daily_manage_raw(incoming_dir, praw_dirs, None, cache_dirs, new_files, renamed=renames, remove=removes)
    
    link_latest_file( latest_dir, os.path.join(cache_dir, renames[0]), data_type='ascii' )
    link_latest_directory( cache_dir, latest_dir )


def unload_incoming_all(when=None):
    nfo = file_dirs(when)
    try:
        unload_incoming(*nfo)
    except IOError:
        pass



def main():
    import optparse
    usage = """
%prog [options]
run "%prog help" to list commands
"""
    parser = optparse.OptionParser(usage)
    parser.add_option('-t', '--test', dest="self_test",
                    action="store_true", default=False, help="run unit tests")
    parser.add_option('-q', '--quiet', dest="quiet",
                    action="store_true", default=False, help="only error output")
    parser.add_option('-v', '--verbose', dest="verbose",
                    action="store_true", default=False, help="enable more informational output")
    parser.add_option('-w', '--debug', dest="debug",
                    action="store_true", default=False, help="enable debug output")
    parser.add_option('--date', dest="date", default=None,
                    help="specify date to use in specified command, MM/DD/YYYY")
    parser.add_option('--site', dest='site', default="rig",
                    help="used in product filename, see metobs.util for details")
    parser.add_option('--description', dest='description', default='',
                    help="used in product filename, see metobs.util for details")
    options, args = parser.parse_args()
    if options.self_test:
        import doctest
        doctest.testmod()
        sys.exit(2)

    lvl = logging.WARNING
    if options.debug: lvl = logging.DEBUG
    elif options.verbose: lvl = logging.INFO
    elif options.quiet: lvl = logging.ERROR
    logging.basicConfig(level = lvl)
    date = options.date
    globals()['site'] = options.site
    globals()['description'] = options.description

    if date:
        date = datetime.strptime(date, "%m/%d/%Y")

    commands = {}
    prior = None
    prior = dict(locals())

    def incoming(*args):
        """handle incoming images and directories
        Does standard image size conversions, moves data to raw, copies data to cache, sets up latest links.
        """
        unload_incoming_all(when=date)

    def quicklook(*args):
        """Create set of quicklooks for date specified or today.
        """
        create_quicklook(when=date)

    def nc(*args):
        """Create multiple nc files for (date - 2 days) to date.
        """
        create_nc(date)

    def help(command=None):
        """print command help or list of commands
        e.g. help help
        """
        if command is None:
            # print first line of docstring
            for cmd in commands:
                ds = commands[cmd].__doc__.split('\n')[0]
                print "%-16s %s" % (cmd,ds)
        else:
            print commands[command].__doc__

    def test():
        "run tests"
        #test1()
        raise NotImplementedError("No Test")

    commands.update(dict(x for x in locals().items() if x[0] not in prior))

    if (not args) or (args[0] not in commands):
        parser.print_help()
        help()
        return 9
    else:
        locals()[args[0]](*args[1:])

    return 0


if __name__=='__main__':
    sys.exit(main())