Skip to content
GitLab
Commit 35ec9964 authored by Geoff Cureton's avatar Geoff Cureton
Browse files

Updated argparse implementation, general cleanup.

parent ecc58fa5
Hide whitespace changes
Inline Side-by-side
README.md
View file @ 35ec9964
......@@ -71,18 +71,16 @@ physical file locations.
- `trace`:
- `repo_urls`: A list of version control urls which contribute files to the package.
Once we have updated the json files, in `infra/tools`, the following files...
Once we have updated the json files, in `infra/tools` we have the following files...
prj-repos.bash - generate a script to pull the repos.
- `prj-repos.bash`: generate a script to pull the repos.
- `prj-create.bash` : This will create your project directory with links.
- `prj-rpath.bash` : do the rpathes
- `prj-trace.bash` : pull out binaries
- `prj-tar.bash` : generates script to create tarball.
prj-create.bash - This will create your project directory with links.
prj-rpath.bash - do the rpathes
prj-trace.bash - pull out binaries -
prj-tar.bash - generates script to create tarball.
take the command line args...
which take the command line args...
-H home, --home home : Home location to create package
-c config, --config config : The JSON configuration file.
......
tools/prj-create.py
View file @ 35ec9964
......@@ -235,7 +235,7 @@ def _argparse():
# Set up the logging
levels = [logging.ERROR, logging.WARN, logging.INFO, logging.DEBUG]
level = levels[args.verbosity if args.verbosity < 4 else 3]
level = levels[args.verbosity if args.verbosity < 3 else 3]
if level == logging.DEBUG :
console_logFormat = '%(asctime)s.%(msecs)03d (%(levelname)7s) : %(filename)s : %(funcName)s : %(lineno)d:%(message)s'
......
tools/prj-repos.py
View file @ 35ec9964
......@@ -75,7 +75,7 @@ def write_subversion_script(mrepo,home,project):
def check_and_create_directories(home):
"""
Check for standard project directorories based on home and create them
Check for standard project directories based on home and create them
:return:
"""
......
tools/prj-rpath.py
View file @ 35ec9964
#!/usr/bin/python
#!/usr/bin/env python
# encoding: utf-8
"""
prj-rpath.py
#######################
# University of Wisconsin Madison, Space Science Engineering Center (SSEC)
#file_Date = '$Date: 2014-06-18 15:41:57 -0500 (Wed, 18 Jun 2014) $'
#file_Revision = '$Rev: 2130 $'
#file_Author = '$Author: scottm $'
#file_HeadURL = '$HeadURL: https://svn.ssec.wisc.edu/repos/jpss_adl/trunk/util/build/patch_rpaths.py $'
#file_Id = '$Id: patch_rpaths.py 2130 2014-06-18 20:41:57Z scottm $'
# Copyright 2011-2012, University of Wisconsin Regents.
# Licensed under the GNU GPLv3.
######################
import os, sys, logging
Purpose: Set the correct RPATH in various executables.
Preconditions:
*
Optional:
*
Minimum commandline:
prj-rpath.py -H /path/to/pkgroot -c pkg_config.json
Created by Scott Mindock <scott.mindock@ssec.wisc.edu> on 2015-09-09.
Copyright (c) 2015 University of Wisconsin Regents. All rights reserved.
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
"""
import os, sys
import logging, traceback
from subprocess import Popen, PIPE,STDOUT
import contents
LOG = logging.getLogger(__name__)
LOG = logging.getLogger(__file__)
def env(**kv):
"augment environment with new values"
zult = dict(os.environ)
zult.update(kv)
return zult
#def env(**kv):
#"augment environment with new values"
#zult = dict(os.environ)
#zult.update(kv)
#return zult
# ref: http://stackoverflow.com/questions/1383254/logging-streamhandler-and-standard-streams
class SingleLevelFilter(logging.Filter):
def __init__(self, passlevels, reject):
self.passlevels = set(passlevels)
self.reject = reject
def filter(self, record):
if self.reject:
return (record.levelno not in self.passlevels)
else:
return (record.levelno in self.passlevels)
def configure_logging(level=logging.WARNING):
"route logging INFO and DEBUG to stdout instead of stderr, affects entire application"
# create a formatter to be used across everything
fm = logging.Formatter('%(levelname)s:%(name)s:%(msg)s') # [%(filename)s:%(lineno)d]')
# create a handler which routes info and debug to stdout with std formatting
h1 = logging.StreamHandler(sys.stdout)
f1 = SingleLevelFilter([logging.INFO, logging.DEBUG], False)
h1.addFilter(f1)
h1.setFormatter(fm)
# create a second stream handler which sends everything else to stderr with std formatting
h2 = logging.StreamHandler(sys.stderr)
f2 = SingleLevelFilter([logging.INFO, logging.DEBUG], True)
h2.addFilter(f2)
h2.setFormatter(fm)
# set up the default logging
rootLogger = logging.getLogger()
rootLogger.addHandler(h1)
rootLogger.addHandler(h2)
rootLogger.setLevel(level)
#class SingleLevelFilter(logging.Filter):
#def __init__(self, passlevels, reject):
#self.passlevels = set(passlevels)
#self.reject = reject
#def filter(self, record):
#if self.reject:
#return (record.levelno not in self.passlevels)
#else:
#return (record.levelno in self.passlevels)
#def configure_logging(level=logging.WARNING):
#"route logging INFO and DEBUG to stdout instead of stderr, affects entire application"
## create a formatter to be used across everything
#fm = logging.Formatter('%(levelname)s:%(name)s:%(msg)s') # [%(filename)s:%(lineno)d]')
## create a handler which routes info and debug to stdout with std formatting
#h1 = logging.StreamHandler(sys.stdout)
#f1 = SingleLevelFilter([logging.INFO, logging.DEBUG], False)
#h1.addFilter(f1)
#h1.setFormatter(fm)
## create a second stream handler which sends everything else to stderr with std formatting
#h2 = logging.StreamHandler(sys.stderr)
#f2 = SingleLevelFilter([logging.INFO, logging.DEBUG], True)
#h2.addFilter(f2)
#h2.setFormatter(fm)
## set up the default logging
#rootLogger = logging.getLogger()
#rootLogger.addHandler(h1)
#rootLogger.addHandler(h2)
#rootLogger.setLevel(level)
def simple_sh(cmd, *args, **kwargs):
......@@ -233,35 +258,67 @@ def start(config_name, home):
patch_me( project,install_home)
return 0
def main():
def _argparse():
'''
Method to encapsulate the option parsing and various setup tasks.
'''
import argparse
desc = """starter file
"""
parser = argparse.ArgumentParser(description=desc)
parser.add_argument('-v', '--verbosity', action="count", default=1,
help='each occurrence increases verbosity 1 level through ERROR-WARNING-INFO-DEBUG')
description = '''Set the correct RPATH in various executables.'''
usage = "usage: %prog [mandatory args] [options]"
version = 0.1
parser = argparse.ArgumentParser(
description=description
)
parser.add_argument('-c', '--config', metavar='config', default='.', required=True,
help='Configuration file')
#parser.add_argument('-I', '--install_home', metavar='home', default='.', required=True,
# help='Home location to create package')
parser.add_argument('-H', '--home', metavar='home', default='.', required=True,
help='Home location to create package')
parser.add_argument("-v", "--verbosity",
dest='verbosity',
action="count",
default=2,
help='''each occurrence increases verbosity 1 level from
ERROR: -v=WARNING -vv=INFO -vvv=DEBUG'''
)
args = parser.parse_args()
# Set up the logging
levels = [logging.ERROR, logging.WARN, logging.INFO, logging.DEBUG]
level = levels[args.verbosity if args.verbosity < 4 else 3]
logging.basicConfig(level=level)
if level == logging.DEBUG :
console_logFormat = '%(asctime)s.%(msecs)03d (%(levelname)7s) : %(filename)s : %(funcName)s : %(lineno)d:%(message)s'
date_format = '%Y-%m-%d %H:%M:%S'
else:
console_logFormat = '%(asctime)s.%(msecs)03d (%(levelname)7s) : %(message)s'
date_format = '%Y-%m-%d %H:%M:%S'
logging.basicConfig(level=level, format=console_logFormat, datefmt=date_format)
return args
def main():
# Read in the options
options = _argparse()
# Do the RPATH modifications
retval = start(options.config,options.home)
return retval
start(args.config,args.home)
if __name__=='__main__':
sys.exit(main())
......
tools/prj-tar.bash
View file @ 35ec9964
......@@ -15,7 +15,7 @@ fi
# python interpreter including numpy, h5py, pytables, scipy; used by CSPP scripts
export PY= ~/ShellB3/classic/ShellB3/ShellB3/bin/python
export PY=~/ShellB3/classic/ShellB3/ShellB3/bin/python
# common modules location used by CSPP scripts
export PYTHONPATH=${INFRA_HOME}/tools
export PATH=${PYTHONPATH}:${PATH}
......@@ -30,4 +30,4 @@ export LD_LIBRARY_PATH=${INFRA_HOME}/common/local/lib64:${LD_LIBRARY_PATH}
test -x "$QL_PYTHON" || QL_PYTHON="${INFRA_HOME}/common/ShellB3/bin/python"
python $MODULE -vv "$@"
\ No newline at end of file
python $MODULE "$@"
tools/prj-tar.py
View file @ 35ec9964
__author__ = 'scottm'
#!/usr/bin/env python
# encoding: utf-8
"""
prj-tar.py
Purpose: Create a series of packaging scripts.
Preconditions:
*
Optional:
*
Minimum commandline:
prj-tar.py -H /path/to/pkgroot -c pkg_config.json -t pkg.version
Created by Scott Mindock <scott.mindock@ssec.wisc.edu> on 2015-09-09.
Copyright (c) 2015 University of Wisconsin Regents. All rights reserved.
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
import os
import logging
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
"""
import os, sys
import logging, traceback
import contents
LOG = logging.getLogger(__name__)
LOG = logging.getLogger(__file__)
__author__ = 'scottm'
def with_links(project, output_dir, tar_filename):
"""
Create a list of directories to be added to the product tarball.
"""
fname = os.path.join(output_dir, os.path.basename(tar_filename) + "_links.bash")
......@@ -23,72 +56,91 @@ def with_links(project, output_dir, tar_filename):
LOG.error('missing %s value' % k)
return
LOG.info("\tAdding dirs/files from vendor applications (copy_from_app)...")
for links in project['copy_from_app']:
LOG.debug(links)
#LOG.debug("\t{0:}".format(links[1]))
line = os.path.join(os.path.basename(project['a_home']), links[1])
LOG.debug("\t\t{0:}".format(line))
tar.write(line + '\n')
LOG.info("\tAdding package symlinks (links_in_package)...")
for links in project['links_in_package']:
LOG.debug(links)
#LOG.debug("\t{0:}".format(links[1]))
line = os.path.join(os.path.basename(project['a_home']), links[1])
LOG.debug("\t\t{0:}".format(line))
tar.write(line + '\n')
LOG.info("\tAdding desired directories (dirs_in_package)...")
for links in project['dirs_in_package']:
LOG.debug(links)
#LOG.debug("\t{0:}".format(links))
line = os.path.join(os.path.basename(project['a_home']), links)
LOG.debug("\t\t{0:}".format(line))
tar.write(line + '\n')
tar.close()
return fname
def ignores(project, output_dir, tar_filename):
def dereference_links(project, output_dir, tar_filename):
"""
Create a list of directories/files (which are actually symlinks to other
locations) to be added to the product tarball. These links can point to
code repos ('links_from_repos'), or application binaries ('dereferenced_links').
"""
fname = os.path.join(output_dir, os.path.basename(tar_filename) + "_ignores.bash")
fname = os.path.join(output_dir, os.path.basename(tar_filename) + "_deref.bash")
LOG.info('Create %s'%fname)
tar = open(fname, 'w')
for k in ['a_home', 'ignores']:
for k in ['a_home', 'links_from_repos', 'dereferenced_links']:
if k not in project.keys():
LOG.error('missing %s value' % k)
return
for ignore in project['ignores']:
line = os.path.join(os.path.basename(project['a_home']), ignore)
LOG.info("\tAdding symlinks from code repositories (links_from_repos)...")
for links in project['links_from_repos']:
line = os.path.join(os.path.basename(project['a_home']), links[1])
LOG.debug("\t\t{0:}".format(line))
tar.write(line + '\n')
#LOG.info('Dereference %s'% line)
LOG.info("\tAdding other symlinks (dereferenced_links)...")
for links in project['dereferenced_links']:
line = os.path.join(os.path.basename(project['a_home']), links[1])
LOG.debug("\t\t{0:}".format(line))
tar.write(line + '\n')
#LOG.info('Dereference %s'% line)
tar.close()
return fname
def derefence_links(project, output_dir, tar_filename):
def ignores(project, output_dir, tar_filename):
"""
Create a list of directories and files which are to be excluded from the
tarball.
"""
fname = os.path.join(output_dir, os.path.basename(tar_filename) + "_deref.bash")
fname = os.path.join(output_dir, os.path.basename(tar_filename) + "_ignores.bash")
LOG.info('Create %s'%fname)
tar = open(fname, 'w')
for k in ['a_home', 'links_from_repos', 'dereferenced_links']:
for k in ['a_home', 'ignores']:
if k not in project.keys():
LOG.error('missing %s value' % k)
return
for links in project['links_from_repos'] + project['dereferenced_links']:
line = os.path.join(os.path.basename(project['a_home']), links[1])
#line = links[0]
LOG.info("\tAdding files/dirs to exclude from tar command (ignores)...")
for ignore in project['ignores']:
line = os.path.join(os.path.basename(project['a_home']), ignore)
LOG.debug("\t\t{0:}".format(line))
tar.write(line + '\n')
LOG.info('Dereference %s'% line)
tar.close()
return fname
def check_and_create_directories(home):
"""
Check for standard project directorories based on home and create them
......@@ -101,7 +153,12 @@ def check_and_create_directories(home):
if not os.path.exists(newd):
os.makedirs(newd)
def tar_script(config_name, output_dir, tar_filename, home):
"""
Create a series of file lists detailing how to construct a tarball, and
a script which runs tar using those file lists.
"""
project_dictionary = contents.project_dictionary(config_name)
check_and_create_directories(home)
......@@ -111,10 +168,12 @@ def tar_script(config_name, output_dir, tar_filename, home):
project_dictionary['a_home'] = os.path.join(home, project_dictionary['a_home'])
leave_links = with_links(project_dictionary, output_dir, tar_filename)
# Create a file containing the directories to add to the tarball, returning
# the filename
leave_links_fname = with_links(project_dictionary, output_dir, tar_filename)
derefenece_links = derefence_links(project_dictionary, output_dir, tar_filename)
ignore = ignores(project_dictionary, output_dir, tar_filename)
dereference_links_fname = dereference_links(project_dictionary, output_dir, tar_filename)
ignore_fname = ignores(project_dictionary, output_dir, tar_filename)
fname = os.path.join(output_dir, os.path.basename(tar_filename) + "_tar.bash")
......@@ -125,39 +184,70 @@ def tar_script(config_name, output_dir, tar_filename, home):
tar.write('# Contents before\n')
tar.write('cd %s; ls\n' % os.path.dirname(project_dictionary['a_home']))
tar.write('# Direct reference no de-reference\n')
tar.write(
'cmd=\"tar --exclude=.svn --exclude=.pyc -c -X %s -X %s -T %s -f %s\";echo ${cmd};${cmd}\n' % ( ignore,derefenece_links, leave_links, tar_filename))
tar.write('# Dereferenced files\n')
tar.write('cmd=\"tar --exclude=.svn --exclude=.pyc -rh -X %s -T %s -f %s\";echo ${cmd};${cmd}\n' % (
ignore, derefenece_links, tar_filename))
# Add actual files to the tarball
tar.write('# Add actual files to the tarball\n')
tar.write(
'''cmd="tar -v --exclude=.svn --exclude=*.pyc -c \\
-X %s \\
-X %s \\
-T %s \\
-f %s"\n''' % (ignore_fname, dereference_links_fname, leave_links_fname, tar_filename)
)
tar.write('''echo ${cmd}\n${cmd}\n\n''')
tar.write('## Dereferenced files\n')
tar.write(
'''cmd="tar -v --exclude=.svn --exclude=*.pyc -rh \\
-X %s \\
-T %s \\
-f %s"\n''' % (ignore_fname, dereference_links_fname, tar_filename)
)
tar.write('''echo ${cmd}\n${cmd}\n\n''')
tar.write('#Files added form trace\n')
trace_file = os.path.join(home,output_dir,os.path.basename(tar_filename))
trace_file= trace_file.replace('.tar', '.trc')
trace_file = trace_file.replace('.tar', '.trc')
trace_file = os.path.join(output_dir,project_dictionary['a_home'].split("/")[-1]+".trc")
#trace_file=os.path.join(output_dir,project_dictionary['a_home'].split("/")[-1]+".trc")
#tar.write('cmd=\"tar -v --exclude=.svn --exclude=.pyc -rh -X %s -T %s -f %s\";echo ${cmd};${cmd}\n' % (
# ignore, trace_file, tar_filename))
if os.path.exists(trace_file):
tar.write('# Files added form trace\n')
tar.write('cmd=\"tar -v --exclude=.svn --exclude=.pyc -rh -X %s -T %s -f %s\";echo ${cmd};${cmd}\n' % (
ignore_fname, trace_file, tar_filename))
tar.write('cmd=\"tar --exclude=.svn --exclude=.pyc -rh -T %s -f %s\";echo ${cmd};${cmd}\n' % (
trace_file, tar_filename))
tar.write('ls -al;cmd=\"gzip %s\";echo ${cmd};${cmd}\n' % (tar_filename))
tar.write('cmd=\"tar -v --exclude=.svn --exclude=*.pyc -rh -T %s -f %s\";echo ${cmd};${cmd}\n' % (
trace_file, tar_filename))
# gzip the tarball
tar.write('## gzip the tarball\n')
tar.write('''cmd="gzip %s"\necho ${cmd}\n${cmd}\n''' % (tar_filename))
tar.close()
os.chmod(fname, 0775)
return 0
def _argparse():
'''
Method to encapsulate the option parsing and various setup tasks.
'''
def main():
import argparse
desc = """Command to generate scripts for building tar file
"""
parser = argparse.ArgumentParser(description=desc)
description = '''Generate a series of package scripts.'''
parser.add_argument('-v', '--verbosity', action="count", default=0,
help='each occurrence increases verbosity 1 level through ERROR-WARNING-INFO-DEBUG')
usage = "usage: %prog [mandatory args] [options]"
version = 0.1
parser = argparse.ArgumentParser(
description=description
)
parser.add_argument('-c', '--config', metavar='config', default='.', required=True,
help='Configuration file')
parser.add_argument('-H', '--home', metavar='home', default='.', required=True,
help='Home location to create package')
parser.add_argument('-o', '--output', metavar='output', default='prj_scripts',
help='output directory for tar building scripts')
......@@ -165,21 +255,42 @@ def main():
parser.add_argument('-t', '--tar_filename', metavar='tar_filename', default='packages', required=True,
help='tar ball name without extension, name-version-A/B')
parser.add_argument('-c', '--config', metavar='config', default='.', required=True,
help='Configuration file')
parser.add_argument('-H', '--home', metavar='home', default='.', required=True,
help='Home location to create package home of project ')
parser.add_argument("-v", "--verbosity",
dest='verbosity',
action="count",
default=2,
help='''each occurrence increases verbosity 1 level from
ERROR: -v=WARNING -vv=INFO -vvv=DEBUG'''
)
args = parser.parse_args()
# Set up the logging
levels = [logging.ERROR, logging.WARN, logging.INFO, logging.DEBUG]
level = levels[args.verbosity if args.verbosity < 4 else 3]
level = levels[args.verbosity if args.verbosity < 3 else 3]
if level == logging.DEBUG :
console_logFormat = '%(asctime)s.%(msecs)03d (%(levelname)7s) : %(filename)s : %(funcName)s : %(lineno)d:%(message)s'
date_format = '%Y-%m-%d %H:%M:%S'
else:
console_logFormat = '%(asctime)s.%(msecs)03d (%(levelname)7s) : %(message)s'
date_format = '%Y-%m-%d %H:%M:%S'
logging.basicConfig(level=level, format=console_logFormat, datefmt=date_format)
return args
def main():
# Read in the options
options = _argparse()
logging.basicConfig(level=level)
# Generate the packaging scripts
retval = tar_script(options.config, options.output, options.tar_filename, options.home)
tar_script(args.config, args.output, args.tar_filename, args.home)
return retval
if __name__ == '__main__':
main()
if __name__=='__main__':
sys.exit(main())
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment