Skip to content
Snippets Groups Projects
Commit 2fb07470 authored by Scott Mindock's avatar Scott Mindock
Browse files

stuff

parent 828518b4
No related branches found
No related tags found
No related merge requests found
{
"a_home": "VIIRS_ASCI/VIIRS_ASCI_1_1",
"a_home": "VIIRS_ASCI/VIIRS_ASCI_1_2",
"a_repo_home": "repos",
"copy_from_app": [
[
"apps/py3/ShellB3/",
"common/ShellB3"
],
[
"dockerbuild/intel/swuser/local",
"common/local"
"podmanbuild/centos7intel19/swuser/common",
"common"
]
],
"links_in_package": [],
......@@ -16,73 +13,24 @@
],
"links_from_repos": [
[
"cspp-asci/scripts/viirs_asci_runtime.sh",
"cspp-asci/asci-env/viirs_asci_runtime.sh",
"viirs_asci_runtime.sh"
],
[
"cspp-asci/scripts/viirs_asci_env.sh",
"cspp-asci/asci-env/viirs_asci_env.sh",
"viirs_asci_env.sh"
],
[
"cspp-asci/scripts/viirs_asci.py",
"asci/viirs_asci.py"
],
[
"cspp-asci/scripts/viirs_lse.py",
"asci/viirs_lse.py"
],
[
"cspp-asci/scripts/viirs_asci.sh",
"asci/viirs_asci.sh"
],
[
"cspp-asci/scripts/sdr_inventory.py",
"asci/sdr_inventory.py"
],
[
"cspp-asci/scripts/opts_and_env.py",
"asci/opts_and_env.py"
],
[
"cspp-asci/scripts/ql_asci.sh",
"asci/ql_asci.sh"
],
[
"cspp-asci/scripts/asci_ancillary.py",
"asci/asci_ancillary.py"
"cspp-asci/scripts",
"asci"
],
[
"cspp-asci/scripts/viirs_lse.sh",
"asci/viirs_lse.sh"
"cspp-viirs-dap/cspp/scripts",
"dap/scripts"
],
[
"cspp-asci/scripts/asci_ancillary.sh",
"asci/asci_ancillary.sh"
],
[
"cspp-asci/scripts/delete_old_anc.sh",
"asci/delete_old_anc.sh"
],
[
"cspp-asci/scripts/gen_pcf.py",
"asci/gen_pcf.py"
],
[
"cspp-asci/scripts/nagg.py",
"asci/nagg.py"
],
[
"cspp-asci/scripts/status_report.py",
"asci/status_report.py"
],
[
"jrr_dap_src/code/OPS",
"common/OPS"
],
[
"jrr_dap_src/code/OPS/bin/run_wgrib.pl",
"common/local/bin/run_wgrib.pl"
"cspp-viirs-dap/cspp/SOURCE/Config",
"dap/config"
],
[
"nagg-builder/bin/centos7/nagg",
......@@ -90,9 +38,8 @@
]
],
"dirs_in_package": [
"asci",
"anc/cache",
"common"
"dap",
"anc/cache"
],
"rpaths": [
"common/local/lib",
......
{
"a_home": "VIIRS_ASCI/VIIRS_ASCI_1_1",
"a_home": "VIIRS_ASCI/VIIRS_ASCI_1_2",
"a_repo_home": "repos",
"copy_from_app": [
[
......@@ -8,12 +8,12 @@
],
[
"apps/dap_data/DATA/static_input/algorithm_ancillary",
"anc/static/static_input/algorithm_ancillary"
"apps/dap_data/DATA/ancillary/algorithm_ancillary",
"anc/static/algorithm_ancillary"
],
[
"apps/dap_data/DATA/static_input/framework_ancillary",
"anc/static/static_input/framework_ancillary"
"apps/dap_data/DATA/ancillary/framework_ancillary",
"anc/static/framework_ancillary"
]
,
[
......
......@@ -7,7 +7,7 @@
"common/ShellB3"
],
[
"dockerbuild/centos7gcc_git4.8/apps/local",
"podmanbuild/centos7gcc_git4.8/apps/local",
"common/local"
]
],
......@@ -62,7 +62,14 @@
"CSPP_SDR/common/adl_anc_retrieval.py",
"common/adl_anc_retrieval.py"
],
[
"CSPP_SDR/common/tile_retrieval.py",
"common/tile_retrieval.py"
],
[
"CSPP_SDR/common/sdr_tiles.sh",
"common/sdr_tiles.sh"
],
[
"CSPP_SDR/common/tle_rehab.py",
"common/tle_rehab.py"
......@@ -103,6 +110,10 @@
"CSPP_SDR/sdr/sdr_process.py",
"common/sdr_process.py"
],
[
"CSPP_SDR/common/sdr_config_paths.py",
"common/sdr_config_paths.py"
],
[
"CSPP_SDR/common/compress_and_aggregate.py",
"common/compress_and_aggregate.py"
......@@ -171,6 +182,10 @@
"CSPP_SDR/sdr/ql_sdr.sh",
"viirs/ql_sdr.sh"
],
[
"CSPP_SDR/sdr/sdr_quicklooks.sh",
"viirs/sdr_quicklooks.sh"
],
[
"CSPP_SDR/sdr/sdr_ancillary.sh",
"common/sdr_ancillary.sh"
......@@ -239,6 +254,7 @@
"ADL/perl",
"ADL/script",
"ADL/tools",
"ADL/tools/lib",
"ADL/xml"
],
"rpaths": [
......
......@@ -14,10 +14,10 @@
],
"dirs_in_package": [
"anc/static/shipped_luts",
"anc/static/SDR_3_2_DB/package"
"anc/static/SDR_3_3_DB/package"
],
"ignores": [
"ADL",
"common/ADL",
"omps_np",
"omps_tc",
"viirs",
......
......@@ -16,10 +16,13 @@
"anc/static/ecotiles"
],
"ignores": [
"common",
"viirs",
"atms",
"cris",
"common/ADL",
"omps_np",
"omps_tc",
"viirs",
"atms",
"cris",
"common",
"anc/cache/incoming_luts",
"anc/static/SDR_3_3_DB/package",
"anc/static/shipped_luts",
......
......@@ -16,7 +16,7 @@
"anc/static/stray_light_luts"
],
"ignores": [
"ADL",
"common/ADL",
"omps_np",
"omps_tc",
"viirs",
......@@ -25,7 +25,7 @@
"common",
"anc/static/ecotiles",
"anc/static/shipped_luts",
"anc/static/SDR_3_2_DB/package",
"anc/static/SDR_3_3_DB/package",
"anc/cache/incoming_luts"
],
"rpath_ignores": [
......
......@@ -23,9 +23,10 @@ class package(object):
def _decode_list(data):
rv = []
for item in data:
if isinstance(item, unicode):
item = item.encode('utf-8')
elif isinstance(item, list):
# if isinstance(item, unicode):
# item = item.encode('utf-8')
# el
if isinstance(item, list):
item = _decode_list(item)
elif isinstance(item, dict):
item = _decode_dict(item)
......@@ -34,15 +35,19 @@ def _decode_list(data):
def _decode_dict(data):
rv = {}
for key, value in data.iteritems():
if isinstance(key, unicode):
key = key.encode('utf-8')
if isinstance(value, unicode):
value = value.encode('utf-8')
elif isinstance(value, list):
#for key, value in data.iteritems():
for key, value in data.items():
print( key )
# if isinstance(key, unicode):
# key = key.encode('utf-8')
# if isinstance(value, unicode):
# value = value.encode('utf-8')
# el
if isinstance(value, list):
value = _decode_list(value)
elif isinstance(value, dict):
value = _decode_dict(value)
rv[key] = value
return rv
......@@ -91,8 +96,8 @@ def example_file():
infile = open(example_file_name, 'r')
input=json.load(infile, object_hook=_decode_dict)
print input['dirs_in_package']
print input['links_in_package']
print (input['dirs_in_package'])
print (input['links_in_package'])
def main():
import argparse
......
export INFRA_HOME=/data/users/scottm/dev/infra
export PATH=${INFRA_HOME}/tools:$PATH
preprocess()
{
TDIR=/data1/users/scottm/infra/trace_results/05-21-19
TDIR=/data/users/scottm/infra/trace_results/02-03-21/
rm ${TDIR}/*.t2
tfiles=`ls ${TDIR}`
echo $tfiles
for tfile in $tfiles
#for tfile in "atms_npp.sh.strace.log"
do
echo $tfile
cat ${TDIR}/${tfile} | grep SDR_3_2 | grep -v ENOE | grep open\( | grep -v ShellB3 2>&1 | grep -v '/anc/' | grep -v '.pyc' > ${TDIR}/${tfile}.t2
cat ${TDIR}/${tfile} | grep SDR_3_2 | grep -v ENOE | grep execve\( | grep -v ShellB3 2>&1 | grep -v '/anc/' | grep -v '.pyc' >> ${TDIR}/${tfile}.t2
cat ${TDIR}/${tfile} | grep SDR_3_3 | grep -v ENOE | grep openat\( | grep -v ShellB3 2>&1 | grep -v '/anc/' | grep -v '.pyc' > ${TDIR}/${tfile}.t2
cat ${TDIR}/${tfile} | grep SDR_3_3 | grep -v ENOE | grep execve\( | grep -v ShellB3 2>&1 | grep -v '/anc/' | grep -v '.pyc' >> ${TDIR}/${tfile}.t2
done
ls ${TDIR}/*.t2
......@@ -18,7 +21,7 @@ ls ${TDIR}/*.t2
#SDR_3_2/common/ADL/cfg/NPP/ProSdrAtmsVerifiedRDR_CFG.xml
preprocess
prj-trace.bash -r SDR_3_2 -H /data1/users/scottm/bootstrapped/viirs_sdr -c /data1/users/scottm/infra/infra/configs/SDR_MX/cspp_sdr_MX4.json -t ${TDIR}/*.t2 -o /data1/users/scottm/tracedev
prj-trace.bash -v -r SDR_3_3 -H /data/users/scottm/bootstraps/cspp_sdr_bootstrap -c /data/users/scottm/dev/infra/configs/cspp-sdr/cspp_sdr.json -t ${TDIR}/*.t2 -o /data/users/scottm/dev/cspp_sdr_alpha/trace_work
##SDR_3_2/common/easy-install.pth
##SDR_3_2/common/site.py
......
......@@ -180,7 +180,7 @@ def create_project(project,fout=None):
LOG.info('DIR: %s' % (os.path.join(home, dir)))
try:
if fout is None:
os.makedirs(os.path.join(home, dir), 0755)
os.makedirs(os.path.join(home, dir), 0x755)
else:
fout.write('mkdir -p ${AHOME}/%s\n'% dir)
except os.error:
......@@ -195,7 +195,7 @@ def check_and_create_directories(home,fout=None):
"""
directories = ("apps", "prj_scripts", "packages", "repos", "dockerbuild/buildbucket", "dockertest" )
directories = ("apps", "prj_scripts", "packages", "repos", "build", "test" )
for d in directories:
newd = os.path.join(home, d)
if not os.path.exists(newd):
......@@ -250,7 +250,7 @@ def start(config_name, home, outpath):
if mrepo is not None:
mrepo.close()
os.chmod(fname, 0775)
os.chmod(fname, 0x775)
return 0
......
......@@ -111,7 +111,7 @@ def start(config_name, home, output):
write_subversion_script(mrepo,home,project_dictionary)
write_git_script(mrepo,home,project_dictionary)
mrepo.close()
os.chmod(fname, 0775)
os.chmod(fname, 0x775)
......
......@@ -207,7 +207,7 @@ def tar_script(config_name, output_dir, tar_filename, home):
# Add actual files to the tarball
tar.write('# Add actual files to the tarball\n')
tar.write(
'''cmd="tar -v --exclude=.svn --exclude=.*.lock --exclude=*.pyc -c \\
'''cmd="tar -v --exclude=.svn --exclude=.*.lock --exclude=*.pyc --exclude=__pycache__ --exclude=*.log -c \\
-X ${CHOME}/%s \\
-X ${CHOME}/%s \\
-T ${CHOME}/%s \\
......@@ -217,7 +217,7 @@ def tar_script(config_name, output_dir, tar_filename, home):
tar.write('## Dereferenced files\n')
tar.write(
'''cmd="tar -v --exclude=.svn --exclude=.*.lock --exclude=*.pyc -rh \\
'''cmd="tar -v --exclude=.svn --exclude=.*.lock --exclude=*.pyc --exclude=__pycache__ --exclude=*.log -rh \\
-X ${CHOME}/%s \\
-T ${CHOME}/%s \\
-f ${PHOME}/%s"\n''' % (ignore_fname, dereference_links_fname, tar_filename)
......@@ -235,7 +235,7 @@ def tar_script(config_name, output_dir, tar_filename, home):
#tar.write('cmd=\"tar -v --exclude=.svn --exclude=.pyc -rh -X %s -T %s -f %s\";echo ${cmd};${cmd}\n' % (
# ignore_fname, trace_file, tar_filename))
tar.write('cmd=\"tar -v --exclude=.svn --exclude=.*.lock --exclude=*.pyc -rh -T %s -f ${PHOME}/%s\";echo ${cmd};${cmd}\n' % (
tar.write('cmd=\"tar -v --exclude=.svn --exclude=.*.lock --exclude=__pycache__ --exclude=*.pyc -rh -T %s -f ${PHOME}/%s\";echo ${cmd};${cmd}\n' % (
trace_file, tar_filename))
else :
LOG.warn('Trace file not present %s'%trace_file)
......@@ -246,7 +246,7 @@ def tar_script(config_name, output_dir, tar_filename, home):
tar.write('''cmd="gzip ${PHOME}/%s"\necho ${cmd}\n${cmd}\n''' % (tar_filename))
tar.close()
os.chmod(fname, 0775)
os.chmod(fname, 0x775)
return 0
......
......@@ -105,7 +105,8 @@ def _filter_to_next(infile, output_list, search_exp, excludes=None):
if line:
line = re.findall(search_exp, line)
if line:
output_list.append(line[0])
if not os.path.isdir(line[0]) :
output_list.append(line[0])
fileinput.close()
return output_list
......
......@@ -100,8 +100,9 @@ def _filter_to_next(infile, output_list, search_exp, excludes=None):
for line in fileinput.input(infile):
line2 = re.findall(search_exp, line)
if line2:
output_list.append(line2[0])
LOG.info(line2[0])
if not os.path.isdir(line2[0]):
output_list.append(line2[0])
LOG.info(line2[0])
fileinput.close()
return output_list
......@@ -114,7 +115,7 @@ def get_opened_files(trace_file, output_list):
search_expression = re.compile(r"""execve\("(.*?)",""")
output_list = _filter_to_next(trace_file, output_list, search_expression, excludes)
LOG.info('first list')
search_expression = re.compile(r"""open\("(.*?)",""")
search_expression = re.compile(r"""openat\(AT_FDCWD, "(.*?)",""")
LOG.info('second list')
output_list = _filter_to_next(trace_file, output_list, search_expression, excludes)
output_list=[e.replace("\'","") for e in output_list]
......
......@@ -55,7 +55,7 @@ export WORK_DIR=`pwd`
get_command_name $CMD
run_day=`date +"%m-%d-%y"`
TRACE_OUTPUTS=/data1/users/scottm/infra/trace_results/${run_day}
TRACE_OUTPUTS=/data/users/scottm/infra/trace_results/${run_day}
mkdir -p ${TRACE_OUTPUTS}
export TRACE=${TRACE_OUTPUTS}/${COMMAND_NAME}.strace.log
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment