-
Geoff Cureton authoredGeoff Cureton authored
utils.py 6.86 KiB
#!/usr/bin/env python
# encoding: utf-8
"""
utils.py
* DESCRIPTION: This is the collection of utilities for parsing text, running external processes and
binaries, checking environments and whatever other mundane tasks aren't specific to this project.
Created by Geoff Cureton <geoff.cureton@ssec.wisc.edu> on 2016-04-11.
Copyright (c) 2011 University of Wisconsin Regents.
Licensed under GNU GPLv3.
"""
import os
import sys
import string
import traceback
import logging
import time
from glob import glob
from copy import copy
import shutil
from os.path import basename, dirname, abspath, isdir, isfile, exists, join as pjoin
import fileinput
from datetime import datetime
LOG = logging.getLogger(__name__)
def setup_logging(verbosity):
print("Verbosity is {}".format(verbosity))
levels = [logging.ERROR, logging.WARN, logging.INFO, logging.DEBUG]
level = levels[verbosity if verbosity < 4 else 3]
# Set up the logging
#console_logFormat = '%(asctime)s : %(name)-12s: %(levelname)-8s %(message)s'
#console_logFormat = '%(levelname)s:%(name)s:%(msg)s') # [%(filename)s:%(lineno)d]'
#console_logFormat = '%(asctime)s : %(funcName)s:%(lineno)d: %(message)s'
#console_logFormat = '%(message)s'
#console_logFormat = '(%(levelname)s):%(filename)s:%(funcName)s:%(lineno)d: %(message)s'
#console_logFormat = '%(asctime)s : (%(levelname)s):%(filename)s:%(funcName)s:%(lineno)d:' \
#' %(message)s'
if level == logging.DEBUG:
console_logFormat = '(%(levelname)s):%(filename)s:%(funcName)s:%(lineno)d: %(message)s'
else:
console_logFormat = '(%(levelname)s): %(message)s'
dateFormat = '%Y-%m-%d %H:%M:%S'
logging.basicConfig(
stream=sys.stderr,
level=level,
format=console_logFormat,
datefmt=dateFormat)
def _replaceAll(intputfile, searchExp, replaceExp):
'''
Replace all instances of 'searchExp' with 'replaceExp' in 'intputfile'
'''
for line in fileinput.input(intputfile, inplace=1):
if searchExp in line:
line = line.replace(searchExp, replaceExp)
sys.stdout.write(line)
fileinput.close()
def cleanup(work_dir, objs_to_remove):
"""
cleanup work directiory
remove evething except the products
"""
for file_obj in objs_to_remove:
try:
if isdir(file_obj):
LOG.debug('Removing directory: {}'.format(file_obj))
shutil.rmtree(file_obj)
elif isfile(file_obj):
LOG.debug('Removing file: {}'.format(file_obj))
os.unlink(file_obj)
except Exception:
LOG.warn(traceback.format_exc())
def link_files(dest_path, files):
'''
Link ancillary files into a destination directory.
'''
files_linked = 0
for src_file in files:
src = basename(src_file)
dest_file = pjoin(dest_path, src)
if not exists(dest_file):
LOG.debug("Link {0} -> {1}".format(src_file, dest_file))
os.symlink(src_file, dest_file)
files_linked += 1
else:
LOG.warn('link already exists: {}'.format(dest_file))
files_linked += 1
return files_linked
class SipsEnvironment(Exception):
def __init__(self, value):
self.value = value
def __str__(self):
return repr(self.value)
def make_time_stamp_d(timeObj):
'''
Returns a timestamp ending in deciseconds
'''
dateStamp = timeObj.strftime("%Y-%m-%d")
# seconds = repr(int(round(timeObj.second + float(timeObj.microsecond) / 1000000.)))
deciSeconds = int(round(float(timeObj.microsecond) / 100000.))
deciSeconds = repr(0 if deciSeconds > 9 else deciSeconds)
timeStamp = "{}.{}".format(timeObj.strftime("%H:%M:%S"), deciSeconds)
return "{} {}".format(dateStamp, timeStamp)
def make_time_stamp_m(timeObj):
"""
Returns a timestamp ending in milliseconds
"""
dateStamp = timeObj.strftime("%Y-%m-%d")
# seconds = repr(int(round(timeObj.second + float(timeObj.microsecond) / 1000000.)))
milliseconds = int(round(float(timeObj.microsecond) / 1000.))
milliseconds = repr(000 if milliseconds > 999 else milliseconds)
timeStamp = "{}.{}".format(timeObj.strftime("%H:%M:%S"), str(milliseconds).zfill(3))
return "{} {}".format(dateStamp, timeStamp)
def execution_time(startTime, endTime):
'''
Converts a time duration in seconds to days, hours, minutes etc...
'''
time_dict = {}
delta = endTime - startTime
days, remainder = divmod(delta, 86400.)
hours, remainder = divmod(remainder, 3600.)
minutes, seconds = divmod(remainder, 60.)
time_dict['delta'] = delta
time_dict['days'] = int(days)
time_dict['hours'] = int(hours)
time_dict['minutes'] = int(minutes)
time_dict['seconds'] = seconds
return time_dict
def create_dir(dir):
'''
Create a directory
'''
returned_dir = copy(dir)
LOG.debug("We want to create the dir {} ...".format(dir))
try:
if returned_dir is not None:
returned_dir_path = dirname(returned_dir)
returned_dir_base = basename(returned_dir)
LOG.debug("returned_dir_path = {}".format(returned_dir_path))
LOG.debug("returned_dir_base = {}".format(returned_dir_base))
returned_dir_path = '.' if returned_dir_path=="" else returned_dir_path
LOG.debug("returned_dir_path = {}".format(returned_dir_path))
# Check if a directory and has write permissions...
if not exists(returned_dir) and os.access(returned_dir_path, os.W_OK):
LOG.debug("Creating directory {} ...".format(returned_dir))
os.makedirs(returned_dir)
# Check if the created dir has write permissions
if not os.access(returned_dir, os.W_OK):
msg = "Created dir {} is not writable.".format(returned_dir)
raise SipsEnvironment(msg)
elif exists(returned_dir):
LOG.debug("Directory {} exists...".format(returned_dir))
if not(isdir(returned_dir) and os.access(returned_dir, os.W_OK)):
msg = "Existing dir {} is not writable.".format(returned_dir)
raise SipsEnvironment(msg)
else:
raise SipsEnvironment("Cannot create {}".format(returned_dir))
except SipsEnvironment:
LOG.debug("Unable to create {}".format(returned_dir))
LOG.debug(traceback.format_exc())
returned_dir = None
except OSError:
LOG.debug(
"Unable to create new dir '{}' in {}".format(returned_dir_base, returned_dir_path))
LOG.debug(traceback.format_exc())
returned_dir = None
except Exception:
LOG.warning("General error for {}".format(returned_dir))
LOG.debug(traceback.format_exc())
returned_dir = None
LOG.debug('Final returned_dir = {}'.format(returned_dir))
return returned_dir