Skip to content
Snippets Groups Projects
Commit 6d9360e7 authored by RH6B's avatar RH6B
Browse files

move to jpss-cloud

parent afa349a1
Branches
Tags
No related merge requests found
OUT = ../lib64/liblog_common_cb.so
default:
gcc -fPIC -g --shared log_common_cb.c -o ${OUT}
depend: dep
dep:
$(MAKEDEPEND) -Y${INCLUDE} $(INCLUDES) -- $(CCFLAGS) -- $(SRC)
clean:
rm -f $(OBJ) $(OUT) Makefile.bak *.d
# DO NOT DELETE
#!/usr/bin/env python
# encoding: utf-8
"""
$Id: adl_common.py 2196 2014-07-15 13:43:48Z scottm $
Purpose: Common routines for ADL XDR handling and ancillary data caching.
Requires: adl_asc
Created Oct 2011 by R.K.Garcia <rayg@ssec.wisc.edu>
Copyright (c) 2011 University of Wisconsin Regents.
Licensed under GNU GPLv3.
"""
import os
import sys
import logging
import log_common
import time
import types
import fileinput
from subprocess import Popen, CalledProcessError, call, PIPE
import datetime
LOG = logging.getLogger(__name__)
PROFILING_ENABLED = os.environ.get('CSPP_PROFILE', None) is not None
STRACE_ENABLED = os.environ.get('CSPP_STRACE', None) is not None
class SingleLevelFilter(logging.Filter):
"""
ref: http://stackoverflow.com/questions/1383254/logging-streamhandler-and-standard-streams
"""
def __init__(self, passlevels, reject):
"""
:rtype : object
:param passlevels:
:param reject:
"""
super(SingleLevelFilter, self).__init__()
self.passlevels = set(passlevels)
self.reject = reject
def filter(self, record):
"""
:param record:
:return:
"""
if self.reject:
return record.levelno not in self.passlevels
else:
return record.levelno in self.passlevels
def split_search_path(s):
"""break a colon-separated list of directories into a list of directories, else empty-list"""
if not s:
return []
back_list = []
for path in s.split(':'):
back_list.append(os.path.abspath(path))
return back_list
def _replaceAll(intputfile, searchExp, replaceExp):
"""
:param intputfile:
:param searchExp:
:param replaceExp:
"""
for line in fileinput.input(intputfile, inplace=1):
if searchExp in line:
line = line.replace(searchExp, replaceExp)
sys.stdout.write(line)
fileinput.close()
# ("RangeDateTime" DATETIMERANGE EQ "2014-01-13 11:22:39.900000" "2014-01-13 11:22:59.900000")
class AscLineParser(object):
def time_range(self, ascLine):
"""
:param ascLine:
:return:
"""
day, time = self.extract_time_range_tokens(ascLine)
return self.time_from_tokens(day, time)
def extract_time_range_tokens(self, ascLine):
return ascLine.split('"')[3:4][0].split(' ')
def time_from_tokens(self, day, time):
dt = datetime.datetime.strptime(day + time, '%Y-%m-%d%H:%M:%S.%f')
return dt
def _testParser():
"""
"""
dt = AscLineParser().time_range(
'("RangeDateTime" DATETIMERANGE EQ "2014-01-13 11:22:39.900000" "2014-01-13 11:22:59.900000")')
print (dt)
class CsppEnvironment(Exception):
def __init__(self, value):
self.value = value
def __str__(self):
return repr(self.value)
def check_and_convert_path(key, a_path, check_write=False):
"""
Make sure the path or paths specified exist
Return the path or list of absolute paths that exist
"""
try:
basestring
except NameError:
basestring = str
abs_locations = []
if ":" in a_path:
paths = a_path.split(":")
elif isinstance(a_path, basestring):
# elif isinstance(a_path, types.StringTypes):
paths = [a_path]
else:
paths = a_path
for path in paths:
if not os.path.exists(path):
if key:
msg = "Environment variable %s refers to a path that does not exists. %s=%s" % (key, key, path)
else:
msg = "Required path %s does not exist. " % (path)
raise CsppEnvironment(msg)
else:
LOG.debug("Found: %s at %s %s" % (key, path, os.path.abspath(path)))
abs_locations.append(os.path.abspath(path))
if check_write:
if not os.access(path, os.W_OK):
msg = "Path exists but is not writable %s=%s" % (key, path)
raise CsppEnvironment(msg)
sys.exit(2)
# return a string if only one and an array if more
if len(abs_locations) == 1:
return abs_locations[0]
else:
# return abs_locations
# return a :-joined string for use in an env variable
return ':'.join(abs_locations)
def check_existing_env_var(varname, default_value=None):
"""
Check for vaiable if it exists use vale otherwise use default
"""
if varname in os.environ:
value = os.environ.get(varname)
else:
if default_value is not None:
value = default_value
else:
print("ERROR: %s is not set, please update environment and re-try" % varname) >> sys.stderr
LOG.error("Environment variable missing. %s" % varname)
sys.exit(9)
return value
def check_and_convert_env_var(varname, check_write=False, default_value=None):
value = check_existing_env_var(varname, default_value=default_value)
path = check_and_convert_path(varname, value, check_write=check_write)
return path
def what_package_am_i():
path = os.path.dirname(os.path.abspath(__file__))
cspp_x = path.split("/common")
cspp_x_home = cspp_x[0]
return cspp_x_home
def _ldd_verify(exe):
"""check that a program is ready to run"""
rc = call(['ldd', exe], stdout=os.tmpfile(), stderr=os.tmpfile())
return rc == 0
def check_env():
""" Check that needed environment variables are set"""
for key in EXTERNAL_BINARY.iterkeys():
if not _ldd_verify(EXTERNAL_BINARY[key]):
LOG.warning("%r executable is unlikely to run, is LD_LIBRARY_PATH set?" % EXTERNAL_BINARY[key])
def env(**kv):
"""augment environment with new values"""
zult = dict(os.environ)
zult.update(kv)
return zult
def execute_binary_captured_io(work_dir, cmd, **kv):
"""
Execute the specifed ancillary script.
process the ouptut and return the file names.
"""
LOG.debug('executing %r with kv=%r' % (cmd, kv))
pop = Popen(cmd,
cwd=work_dir,
env=env(**kv),
shell=True,
stdin=PIPE,
stdout=PIPE,
stderr=PIPE,
close_fds=True)
anc_stdout, anc_stderr = pop.communicate()
rc = pop.returncode
if rc == 0:
LOG.debug("success " + cmd)
LOG.info(anc_stdout.strip())
elif rc == 1:
LOG.info(anc_stdout)
LOG.info(anc_stderr)
LOG.error("stderr:" + anc_stderr)
else:
LOG.warn("what " + cmd)
if rc != 0:
LOG.debug(anc_stdout)
LOG.error(anc_stderr)
return None
def simple_sh(cmd, log_execution=True, *args, **kwargs):
"""like subprocess.check_call, but returning the pid the process was given"""
if STRACE_ENABLED:
strace = open('strace.log', 'at')
print ("= " * 32) >> strace
print(repr(cmd)) >> strace
cmd = ['strace'] + list(cmd)
pop = Popen(cmd, *args, stderr=strace, **kwargs)
else:
pop = Popen(cmd, *args, **kwargs)
pid = pop.pid
startTime = time.time()
rc = pop.wait()
endTime = time.time()
delta = endTime - startTime
LOG.debug('statistics for "%s"' % ' '.join(cmd))
if log_execution:
log_common.status_line('Execution Time: %f Sec Cmd "%s"' % (delta, ' '.join(cmd)))
if rc != 0:
exc = CalledProcessError(rc, cmd)
exc.pid = pid
raise exc
return pid
def profiled_sh(cmd, log_execution=True, *args, **kwargs):
"""
like subprocess.check_call, but returning the pid the process was given and logging as
INFO the final content of /proc/PID/stat
:param cmd:
:param log_execution:
:param args:
:param kwargs:
"""
pop = Popen(cmd, *args, **kwargs)
pid = pop.pid
fn = '/proc/%d/status' % pid
LOG.debug('retrieving %s statistics to caller dictionary' % fn)
proc_stats = '-- no /proc/PID/status data --'
rc = 0
startTime = time.time()
while True:
time.sleep(1.0)
rc = pop.poll()
if rc is not None:
break
try:
proc = file(fn, 'rt')
proc_stats = proc.read()
proc.close()
del proc
except IOError:
LOG.warning('unable to get stats from %s' % fn)
endTime = time.time()
delta = endTime - startTime
LOG.debug('statistics for "%s"' % ' '.join(cmd))
if log_execution:
log_common.status_line('Execution Time: "%f" Sec Cmd "%s"' % (delta, ' '.join(cmd)))
LOG.debug(proc_stats)
if rc != 0:
exc = CalledProcessError(rc, cmd)
exc.pid = pid
raise exc
return pid
# default sh() is to profile on linux systems
if os.path.exists('/proc') and PROFILING_ENABLED:
sh = profiled_sh
else:
sh = simple_sh
# if __name__ == '__main__':
# logging.basicConfig(level=logging.DEBUG) we don't want basicConfig anymore
# log_common.configure_logging(level=logging.DEBUG, FILE="testlog.log")
#!/usr/bin/env python
# encoding: utf-8
"""
$Id: adl_common.py 1931 2014-03-17 18:43:26Z kathys $
Purpose: Common routines for ADL XDR handling and ancillary data caching.
Requires: adl_asc
Created Oct 2011 by R.K.Garcia <rayg@ssec.wisc.edu>
Copyright (c) 2011 University of Wisconsin Regents.
Licensed under GNU GPLv3.
"""
import os, sys, logging
import basics
import __main__
# from cffi import FFI
# ffi = FFI()
ffi = None
LOG = logging.getLogger(__name__)
logging_configured = False
# ref: http://stackoverflow.com/questions/1383254/logging-streamhandler-and-standard-streams
class SingleLevelFilter(logging.Filter):
def __init__(self, passlevels, reject):
self.passlevels = set(passlevels)
self.reject = reject
def filter(self, record):
if self.reject:
return (record.levelno not in self.passlevels)
else:
return (record.levelno in self.passlevels)
def configure_logging(level=logging.WARNING, FILE=None):
"""
route logging INFO and DEBUG to stdout instead of stderr, affects entire application
"""
global logging_configured
# create a formatter to be used across everything
# fm = logging.Formatter('%(levelname)s:%(name)s:%(msg)s') # [%(filename)s:%(lineno)d]')
fm = logging.Formatter('(%(levelname)s):%(filename)s:%(funcName)s:%(lineno)d:%(message)s')
rootLogger = logging.getLogger()
# set up the default logging
if logging_configured == False:
logging_configured = True
# create a handler which routes info and debug to stdout with std formatting
h1 = logging.StreamHandler(sys.stdout)
f1 = SingleLevelFilter([logging.INFO, logging.DEBUG], False)
h1.addFilter(f1)
h1.setFormatter(fm)
# create a second stream handler which sends everything else to stderr with std formatting
h2 = logging.StreamHandler(sys.stderr)
f2 = SingleLevelFilter([logging.INFO, logging.DEBUG], True)
h2.addFilter(f2)
h2.setFormatter(fm)
rootLogger.addHandler(h1)
rootLogger.addHandler(h2)
h3 = None
if FILE is not None:
work_dir = os.path.dirname(FILE)
basics.check_and_convert_path("WORKDIR", work_dir, check_write=True)
h3 = logging.FileHandler(filename=FILE)
# f3 = SingleLevelFilter([logging.INFO, logging.DEBUG], False)
# h3.addFilter(f3)
h3.setFormatter(fm)
rootLogger.addHandler(h3)
rootLogger.setLevel(level)
def status_line(status):
"""
Put out a special status line
"""
LOG.info("\n ( " + status + " )\n")
def log_from_C(in_type, in_msg):
type = ffi.string(in_type)
emsg = ffi.string(in_msg)
msg = emsg.decode()
if type == b'INFO':
LOG.info(msg)
elif type == b'WARN':
LOG.warn(msg)
elif type == b'ERROR':
LOG.error(msg)
elif type == b'DEBUG':
LOG.debug(msg)
else:
LOG.error('Bad TYPE %s %s' % (type, msg))
return int(0)
log_callback = None
log_lib = None
def C_log_support(ffi_in):
"""
Initalizer for C callbacks
"""
global ffi
global log_callback
global log_lib
ffi = ffi_in
ffi.cdef("""
void log_from_C(char * type, char * message );
void set_log( int (*callback)(char*,char*) );
int LOG_info( char *message);
int LOG_warn( char *message);
int LOG_debug( char *message);
int LOG_error( char *message);
""")
log_lib = ffi.dlopen("liblog_common_cb.so")
log_callback = ffi.callback("int(char*,char *)", log_from_C)
log_lib.set_log(log_callback)
def _test_logging():
LOG.debug('debug message')
LOG.info('info message')
LOG.warning('warning message')
LOG.error('error message')
LOG.critical('critical message')
def test_C_callbacks():
message = 'error'
arg1 = ffi.new("char[]", message)
log_lib.LOG_error(arg1)
message = 'warn'
arg2 = ffi.new("char[]", message)
log_lib.LOG_warn(arg2)
message = 'info'
arg3 = ffi.new("char[]", message)
log_lib.LOG_info(arg3)
message = 'debug'
arg4 = ffi.new("char[]", message)
log_lib.LOG_debug(arg4)
configure_logging()
if __name__ == '__main__':
# logging.basicConfig(level=logging.DEBUG) we don't want basicConfig anymore
configure_logging(level=logging.DEBUG, FILE="./testlog.log")
_test_logging()
C_log_support()
# test_C_callbacks()
#include<stdio.h>
static int (*log_back)(char*,char*)=0;
int set_log( int (*callback)(char*,char*) )
{
log_back=callback;
return 1;
}
int LOG_error( char *message)
{
if (log_back)
log_back( "ERROR" , message);
}
int LOG_warn( char * message)
{
if (log_back)
log_back( "WARN" , message);
}
int LOG_info( char * message)
{
if (log_back)
log_back( "INFO" , message);
}
int LOG_debug( char *message )
{
if (log_back)
log_back( "DEBUG" , message);
}
......@@ -16,23 +16,24 @@
"dereferenced_links": [
[
"apps/ql/quicklook/cspp_quicklooks.core-0.1.1-py2.7.egg",
"/home/RH6B/HCAST/apps/ql/quicklook/cspp_quicklooks.core-0.1.1-py2.7.egg",
"common/quicklook/cspp_quicklooks.core-0.1.1-py2.7.egg"
],
[
"apps/ql/quicklook/cspp_quicklooks.geo-0.1-py2.7.egg",
"/home/RH6B/HCAST/apps/ql/quicklook/cspp_quicklooks.geo-0.1-py2.7.egg",
"common/quicklook/cspp_quicklooks.geo-0.1-py2.7.egg"
],
[
"apps/ql/quicklook//mcarea-0.1-py2.7.egg",
"common/quicklook/mcarea-0.1-py2.7.egg"
],
[
"apps/ql/bin/ql_gvar.sh",
"l0/ql_gvar.sh"
]
],
"links_from_repos": [
[
"CSPP_GVAR/trunk/gvar/gvar_l0/cron_descriptor_runner.sh",
"l0/cron_descriptor_runner.sh"
],
[
"CSPP_GVAR/trunk/gvar/gvar_l0/descriptor.py",
"l0/descriptor.py"
......@@ -117,6 +118,10 @@
[
"CSPP_GVAR/trunk/gvar/gvar_l0/gvar_regions.json",
"anc/cache/gvar_regions.json"
],
[
"CSPP_GVAR/trunk/gvar/area/dist/mcarea-0.2-py2.7.egg",
"common/quicklook/mcarea-0.2-py2.7.egg"
]
],
"dirs_in_package": [
......
......
{
"a_home": "CSPP_QL/cspp_ql_1_B3",
"a_home": "CSPP_QL/cspp_ql_1_B4",
"a_repo_home": "repos",
"copy_from_app": [
[
......@@ -12,10 +12,25 @@
]
],
"links_in_package": [
],
"dereferenced_links": [
[
"apps/ql/quicklook/cspp_quicklooks.sdr-0.1.1-py2.7.egg",
"common/quicklook/cspp_quicklooks.sdr-0.1.1-py2.7.egg"
],
[
"apps/ql/quicklook/polar2grid.core-1.2.0-py2.7.egg",
"common/quicklook/polar2grid.core-1.2.0-py2.7.egg"
],
[
"apps/ql/quicklook/polar2grid.viirs-1.2.0-py2.7.egg",
"common/quicklook/polar2grid.viirs-1.2.0-py2.7.egg"
],
[
"apps/ql/bin/ql_viirs_sdr.sh",
"ql/ql_viirs_sdr.sh"
],
[
"apps/ql/quicklook/cspp_quicklooks.core-0.1.1-py2.7.egg",
......@@ -25,7 +40,6 @@
"apps/ql/quicklook/cspp_quicklooks.geo-0.1-py2.7.egg",
"common/quicklook/cspp_quicklooks.geo-0.1-py2.7.egg"
],
[
"apps/ql/bin/ql_grb.sh",
"ql/ql_grb.sh"
......@@ -53,14 +67,17 @@
],
"links_from_repos": [
[
"cspp_quicklooks/vendor_eggs/mcarea-0.1-py2.7.egg",
"common/mcarea-0.1-py2.7.egg"
"cspp_quicklooks/vendor_eggs/mcarea-0.2-py2.7.egg",
"common/mcarea-0.2-py2.7.egg"
],
[
"cspp_quicklooks/src/cspp_quicklooks/geo/b7_solar.so",
"common/local/lib/b7_solar.so"
],
[
"himawari/src/libHimawari.so",
"common/local/lib/libHimawari.so"
]
,
],
[
"util/ql_to_realearth.sh",
"ql/ql_to_realearth.sh"
......
......
{
"a_home": "CSPP_SDR/SDR_3_0_a1",
"a_repo_home": "repos",
"copy_from_app": [
[
"/home/RH6B/COTS_HOME/local",
"common/local"
],
[
"/home/RH6B/ShellB3/next_3/ShellB3/ShellB3/",
"common/ShellB3"
]
],
"links_in_package": [
],
"dereferenced_links": [
],
"links_from_repos": [
],
"dirs_in_package": [
"common/quicklook",
"viirs",
"atms",
"cris",
"anc/static",
"anc/cache"
],
"ignores": [
"common/ADL/data",
"anc/cache/*",
"anc/static/*",
"common/ShellB3/build",
"common/ShellB3/src",
"common/ShellB3/tests",
"common/ShellB3/trim",
"common/COTS",
"common/ADL"
],
"rpath_ignores": [
"common/cspp_cfg",
"common/ADL/data",
"anc/cache/*",
"anc/static/*",
"common/COTS",
"common/ADL/java",
"common/ADL/cfg",
"common/ADL/ANC/",
"common/ADL/SDR",
"common/ADL/xml",
"common/ADL/EDR",
"common/ADL/GIP",
"/home/RH6B/SDR/CSPP_SDR/SDR_2_2/common/ADL/perl",
"common/ADL/Geolocation",
"common/ADL/script",
"common/ADL/testdata",
"common/ADL/CMN"
],
"trace": [
"common/cspp_cfg",
"ADL/bin",
"ADL/cfg",
"ADL/lib",
"ADL/perl",
"ADL/script",
"ADL/tools",
"ADL/xml"
],
"rpaths": [
"common/ADL/lib",
"common/local/lib",
"common/local/lib64",
"common/ShellB3/lib"
],
"repo_urls": [
[
"https://svn.ssec.wisc.edu/repos/jpss_adl/trunk/ADL/4_2/I1_5_08_11",
"jpss_adl/trunk/ADL/4_2/I1_5_08_011"
],
[
"https://svn.ssec.wisc.edu/repos/jpss_adl/trunk/ADL/4_2/I1_5_08_10",
"jpss_adl/trunk/ADL/4_2/I1_5_08_10"
],
[
"https://svn.ssec.wisc.edu/repos/CSPP_INFRA/trunk/configs/SDR",
"configs/SDR"
],
[
"https://svn.ssec.wisc.edu/repos/jpss_adl/trunk/scripts",
"jpss_adl/trunk/scripts"
],
[
"https://svn.ssec.wisc.edu/repos/jpss_adl/trunk/util/build",
"jpss_adl/trunk/util/build"
],
[
"https://svn.ssec.wisc.edu/repos/jpss_adl/trunk/util/QC",
"jpss_adl/trunk/util/QC"
],
[
"https://svn.ssec.wisc.edu/repos/jpss_adl/trunk/util/cache_assembly",
"jpss_adl/trunk/util/cache_assembly"
],
[
"https://svn.ssec.wisc.edu/repos/jpss_adl/trunk/ancillary",
"jpss_adl/trunk/ancillary"
],
[
"https://svn.ssec.wisc.edu/repos/jpss_adl/trunk/util/system",
"jpss_adl/trunk/util/system"
],
[
"https://svn.ssec.wisc.edu/repos/jpss_adl/trunk/ancillary/scripts",
"jpss_adl/trunk/ancillary/scripts"
],
[
"https://svn.ssec.wisc.edu/repos/jpss_adl/trunk/ADL/4_2/mx7_2/cspp_cfg",
"cspp_cfg"
],
[
"https://svn.ssec.wisc.edu/repos/jpss_adl_distribution/trunk/httpsFiles/SSEC-Support",
"adl_support"
],
[
"ssh://gitlab.ssec.wisc.edu/scottm/cspp_quicklooks.git",
"cspp_quicklooks",
"develop"
],
[
"ssh://gitlab.ssec.wisc.edu/CSPP/cspp_ancil",
"cspp_ancil",
"master"
]
]
}
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please to comment