Updated argparse implementation, general cleanup.

README.md

@@ -71,18 +71,16 @@ physical file locations.
 - `trace`: 
 - `repo_urls`: A list of version control urls which contribute files to the package.
 
-Once we have updated the json files, in `infra/tools`, the following files...
+Once we have updated the json files, in `infra/tools` we have the following files...
 
-    prj-repos.bash - generate a script to pull the repos.
+- `prj-repos.bash`: generate a script to pull the repos.
+- `prj-create.bash` : This will create your project directory with links.
+- `prj-rpath.bash` : do the rpathes
+- `prj-trace.bash` : pull out binaries
+- `prj-tar.bash` : generates script to create tarball. 
 
-    prj-create.bash - This will create your project directory with links.
 
-    prj-rpath.bash - do the rpathes
-    prj-trace.bash - pull out binaries -
-
-    prj-tar.bash - generates script to create tarball. 
-
-take the command line args...
+which take the command line args...
 
     -H home, --home home       : Home location to create package
     -c config, --config config : The JSON configuration file.

tools/prj-create.py

@@ -235,7 +235,7 @@ def _argparse():
 
     # Set up the logging
     levels = [logging.ERROR, logging.WARN, logging.INFO, logging.DEBUG]
-    level = levels[args.verbosity if args.verbosity < 4 else 3]
+    level = levels[args.verbosity if args.verbosity < 3 else 3]
 
     if level == logging.DEBUG :
         console_logFormat = '%(asctime)s.%(msecs)03d (%(levelname)7s) : %(filename)s : %(funcName)s : %(lineno)d:%(message)s'

tools/prj-repos.py

@@ -75,7 +75,7 @@ def write_subversion_script(mrepo,home,project):
 
 def check_and_create_directories(home):
     """
-    Check for standard project directorories based on home and create them
+    Check for standard project directories based on home and create them
 
     :return:
     """

tools/prj-rpath.py

-#!/usr/bin/python
+#!/usr/bin/env python
 # encoding: utf-8
+"""
+prj-rpath.py
 
-#######################
-# University of Wisconsin Madison,  Space Science Engineering Center (SSEC)
-#file_Date = '$Date: 2014-06-18 15:41:57 -0500 (Wed, 18 Jun 2014) $'
-#file_Revision = '$Rev: 2130 $'
-#file_Author = '$Author: scottm $'
-#file_HeadURL = '$HeadURL: https://svn.ssec.wisc.edu/repos/jpss_adl/trunk/util/build/patch_rpaths.py $'
-#file_Id = '$Id: patch_rpaths.py 2130 2014-06-18 20:41:57Z scottm $'
-# Copyright 2011-2012, University of Wisconsin Regents.
-# Licensed under the GNU GPLv3.
-######################
-import os, sys, logging
+Purpose: Set the correct RPATH in various executables.
+
+Preconditions:
+    * 
+
+Optional:
+    * 
+
+Minimum commandline:
+
+    prj-rpath.py -H /path/to/pkgroot -c pkg_config.json
+
+Created by Scott Mindock <scott.mindock@ssec.wisc.edu> on 2015-09-09.
+Copyright (c) 2015 University of Wisconsin Regents. All rights reserved.
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+
+import os, sys
+import logging, traceback
 from subprocess import  Popen, PIPE,STDOUT
 import contents
-LOG = logging.getLogger(__name__)
 
+LOG = logging.getLogger(__file__)
 
-def env(**kv):
-    "augment environment with new values"
-    zult = dict(os.environ)
-    zult.update(kv)
-    return zult
+
+#def env(**kv):
+    #"augment environment with new values"
+    #zult = dict(os.environ)
+    #zult.update(kv)
+    #return zult
 
 
 # ref: http://stackoverflow.com/questions/1383254/logging-streamhandler-and-standard-streams
-class SingleLevelFilter(logging.Filter):
-    def __init__(self, passlevels, reject):
-        self.passlevels = set(passlevels)
-        self.reject = reject
-
-    def filter(self, record):
-        if self.reject:
-            return (record.levelno not in self.passlevels)
-        else:
-            return (record.levelno in self.passlevels)
-def configure_logging(level=logging.WARNING):
-    "route logging INFO and DEBUG to stdout instead of stderr, affects entire application"
-    # create a formatter to be used across everything
-    fm = logging.Formatter('%(levelname)s:%(name)s:%(msg)s') # [%(filename)s:%(lineno)d]')
-    # create a handler which routes info and debug to stdout with std formatting
-    h1 = logging.StreamHandler(sys.stdout)
-    f1 = SingleLevelFilter([logging.INFO, logging.DEBUG], False)
-    h1.addFilter(f1)
-    h1.setFormatter(fm)
-
-    # create a second stream handler which sends everything else to stderr with std formatting
-    h2 = logging.StreamHandler(sys.stderr)
-    f2 = SingleLevelFilter([logging.INFO, logging.DEBUG], True)
-    h2.addFilter(f2)
-    h2.setFormatter(fm)
-
-    # set up the default logging
-    rootLogger = logging.getLogger()
-    rootLogger.addHandler(h1)
-    rootLogger.addHandler(h2)
-    rootLogger.setLevel(level)
+#class SingleLevelFilter(logging.Filter):
+    #def __init__(self, passlevels, reject):
+        #self.passlevels = set(passlevels)
+        #self.reject = reject
+
+    #def filter(self, record):
+        #if self.reject:
+            #return (record.levelno not in self.passlevels)
+        #else:
+            #return (record.levelno in self.passlevels)
+
+
+#def configure_logging(level=logging.WARNING):
+    #"route logging INFO and DEBUG to stdout instead of stderr, affects entire application"
+    ## create a formatter to be used across everything
+    #fm = logging.Formatter('%(levelname)s:%(name)s:%(msg)s') # [%(filename)s:%(lineno)d]')
+    ## create a handler which routes info and debug to stdout with std formatting
+    #h1 = logging.StreamHandler(sys.stdout)
+    #f1 = SingleLevelFilter([logging.INFO, logging.DEBUG], False)
+    #h1.addFilter(f1)
+    #h1.setFormatter(fm)
+
+    ## create a second stream handler which sends everything else to stderr with std formatting
+    #h2 = logging.StreamHandler(sys.stderr)
+    #f2 = SingleLevelFilter([logging.INFO, logging.DEBUG], True)
+    #h2.addFilter(f2)
+    #h2.setFormatter(fm)
+
+    ## set up the default logging
+    #rootLogger = logging.getLogger()
+    #rootLogger.addHandler(h1)
+    #rootLogger.addHandler(h2)
+    #rootLogger.setLevel(level)
 
 
 def simple_sh(cmd, *args, **kwargs):
@@ -233,35 +258,67 @@ def start(config_name, home):
 
     patch_me( project,install_home)
 
+    return 0
 
-def main():
+
+def _argparse():
+    '''
+    Method to encapsulate the option parsing and various setup tasks.
+    '''
 
     import argparse
-    desc = """starter file
-        """
-    parser = argparse.ArgumentParser(description=desc)
 
-    parser.add_argument('-v', '--verbosity', action="count", default=1,
-                        help='each occurrence increases verbosity 1 level through ERROR-WARNING-INFO-DEBUG')
+    description = '''Set the correct RPATH in various executables.'''
+
+    usage = "usage: %prog [mandatory args] [options]"
+    version = 0.1
+
+    parser = argparse.ArgumentParser(
+                                     description=description
+                                     )
 
     parser.add_argument('-c', '--config', metavar='config', default='.', required=True,
                         help='Configuration file')
 
-    #parser.add_argument('-I', '--install_home', metavar='home', default='.', required=True,
-    #                    help='Home location to create package')
-
     parser.add_argument('-H', '--home', metavar='home', default='.', required=True,
                         help='Home location to create package')
 
+    parser.add_argument("-v", "--verbosity",
+                      dest='verbosity',
+                      action="count", 
+                      default=2,
+                      help='''each occurrence increases verbosity 1 level from 
+                      ERROR: -v=WARNING -vv=INFO -vvv=DEBUG'''
+                      )
 
     args = parser.parse_args()
 
+    # Set up the logging
     levels = [logging.ERROR, logging.WARN, logging.INFO, logging.DEBUG]
     level = levels[args.verbosity if args.verbosity < 4 else 3]
 
-    logging.basicConfig(level=level)
+    if level == logging.DEBUG :
+        console_logFormat = '%(asctime)s.%(msecs)03d (%(levelname)7s) : %(filename)s : %(funcName)s : %(lineno)d:%(message)s'
+        date_format = '%Y-%m-%d %H:%M:%S'
+    else:
+        console_logFormat = '%(asctime)s.%(msecs)03d (%(levelname)7s) : %(message)s'
+        date_format = '%Y-%m-%d %H:%M:%S'
+
+    logging.basicConfig(level=level, format=console_logFormat, datefmt=date_format)
+
+    return args
+
+
+def main():
+
+    # Read in the options
+    options = _argparse()
+
+    # Do the RPATH modifications
+    retval = start(options.config,options.home)
+
+    return retval
 
-    start(args.config,args.home)
 
 if __name__=='__main__':
     sys.exit(main())

tools/prj-tar.bash

@@ -15,7 +15,7 @@ fi
 
 
 # python interpreter including numpy, h5py, pytables, scipy; used by CSPP scripts
-export PY= ~/ShellB3/classic/ShellB3/ShellB3/bin/python
+export PY=~/ShellB3/classic/ShellB3/ShellB3/bin/python
 # common modules location used by CSPP scripts
 export PYTHONPATH=${INFRA_HOME}/tools
 export PATH=${PYTHONPATH}:${PATH}
@@ -30,4 +30,4 @@ export LD_LIBRARY_PATH=${INFRA_HOME}/common/local/lib64:${LD_LIBRARY_PATH}
 test -x "$QL_PYTHON" || QL_PYTHON="${INFRA_HOME}/common/ShellB3/bin/python"
 
 
-python  $MODULE -vv  "$@"
\ No newline at end of file
+python  $MODULE "$@"

tools/prj-tar.py

-__author__ = 'scottm'
+#!/usr/bin/env python
+# encoding: utf-8
+"""
+prj-tar.py
+
+Purpose: Create a series of packaging scripts.
+
+Preconditions:
+    * 
+
+Optional:
+    * 
+
+Minimum commandline:
+
+    prj-tar.py -H /path/to/pkgroot -c pkg_config.json -t pkg.version
+
+Created by Scott Mindock <scott.mindock@ssec.wisc.edu> on 2015-09-09.
+Copyright (c) 2015 University of Wisconsin Regents. All rights reserved.
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
 
-import os
-import logging
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+
+import os, sys
+import logging, traceback
 import contents
 
-LOG = logging.getLogger(__name__)
+LOG = logging.getLogger(__file__)
+__author__ = 'scottm'
 
 
 def with_links(project, output_dir, tar_filename):
     """
-
+    Create a list of directories to be added to the product tarball.
     """
 
     fname = os.path.join(output_dir, os.path.basename(tar_filename) + "_links.bash")
@@ -23,72 +56,91 @@ def with_links(project, output_dir, tar_filename):
             LOG.error('missing %s value' % k)
             return
 
+    LOG.info("\tAdding dirs/files from vendor applications (copy_from_app)...")
     for links in project['copy_from_app']:
-        LOG.debug(links)
+        #LOG.debug("\t{0:}".format(links[1]))
         line = os.path.join(os.path.basename(project['a_home']), links[1])
+        LOG.debug("\t\t{0:}".format(line))
         tar.write(line + '\n')
 
+    LOG.info("\tAdding package symlinks (links_in_package)...")
     for links in project['links_in_package']:
-        LOG.debug(links)
+        #LOG.debug("\t{0:}".format(links[1]))
         line = os.path.join(os.path.basename(project['a_home']), links[1])
+        LOG.debug("\t\t{0:}".format(line))
         tar.write(line + '\n')
 
+    LOG.info("\tAdding desired directories (dirs_in_package)...")
     for links in project['dirs_in_package']:
-        LOG.debug(links)
+        #LOG.debug("\t{0:}".format(links))
         line = os.path.join(os.path.basename(project['a_home']), links)
+        LOG.debug("\t\t{0:}".format(line))
         tar.write(line + '\n')
 
     tar.close()
     return fname
 
 
-def ignores(project, output_dir, tar_filename):
+def dereference_links(project, output_dir,  tar_filename):
     """
-
+    Create a list of directories/files (which are actually symlinks to other 
+    locations) to be added to the product tarball. These links can point to 
+    code repos ('links_from_repos'), or application binaries ('dereferenced_links').
     """
 
-    fname = os.path.join(output_dir, os.path.basename(tar_filename) + "_ignores.bash")
+    fname = os.path.join(output_dir, os.path.basename(tar_filename) + "_deref.bash")
     LOG.info('Create %s'%fname)
 
     tar = open(fname, 'w')
 
-    for k in ['a_home', 'ignores']:
+    for k in ['a_home', 'links_from_repos', 'dereferenced_links']:
         if k not in project.keys():
             LOG.error('missing %s value' % k)
             return
 
-    for ignore in project['ignores']:
-        line = os.path.join(os.path.basename(project['a_home']), ignore)
+    LOG.info("\tAdding symlinks from code repositories (links_from_repos)...")
+    for links in project['links_from_repos']:
+        line = os.path.join(os.path.basename(project['a_home']), links[1])
+        LOG.debug("\t\t{0:}".format(line))
+        tar.write(line + '\n')
+        #LOG.info('Dereference %s'% line)
+
+    LOG.info("\tAdding other symlinks (dereferenced_links)...")
+    for links in project['dereferenced_links']:
+        line = os.path.join(os.path.basename(project['a_home']), links[1])
+        LOG.debug("\t\t{0:}".format(line))
         tar.write(line + '\n')
+        #LOG.info('Dereference %s'% line)
 
     tar.close()
     return fname
 
-
-def derefence_links(project, output_dir,  tar_filename):
+def ignores(project, output_dir, tar_filename):
     """
-
+    Create a list of directories and files which are to be excluded from the 
+    tarball.
     """
 
-    fname = os.path.join(output_dir, os.path.basename(tar_filename) + "_deref.bash")
+    fname = os.path.join(output_dir, os.path.basename(tar_filename) + "_ignores.bash")
     LOG.info('Create %s'%fname)
 
     tar = open(fname, 'w')
 
-    for k in ['a_home', 'links_from_repos', 'dereferenced_links']:
+    for k in ['a_home', 'ignores']:
         if k not in project.keys():
             LOG.error('missing %s value' % k)
             return
 
-    for links in project['links_from_repos'] + project['dereferenced_links']:
-        line = os.path.join(os.path.basename(project['a_home']), links[1])
-        #line = links[0]
+    LOG.info("\tAdding files/dirs to exclude from tar command (ignores)...")
+    for ignore in project['ignores']:
+        line = os.path.join(os.path.basename(project['a_home']), ignore)
+        LOG.debug("\t\t{0:}".format(line))
         tar.write(line + '\n')
-        LOG.info('Dereference %s'% line)
 
     tar.close()
     return fname
 
+
 def check_and_create_directories(home):
     """
     Check for standard project directorories based on home and create them
@@ -101,7 +153,12 @@ def check_and_create_directories(home):
         if not os.path.exists(newd):
             os.makedirs(newd)
 
+
 def tar_script(config_name, output_dir, tar_filename, home):
+    """
+    Create a series of file lists detailing how to construct a tarball, and 
+    a script which runs tar using those file lists.
+    """
 
     project_dictionary = contents.project_dictionary(config_name)
     check_and_create_directories(home)
@@ -111,10 +168,12 @@ def tar_script(config_name, output_dir, tar_filename, home):
 
         project_dictionary['a_home'] = os.path.join(home, project_dictionary['a_home'])
 
-        leave_links = with_links(project_dictionary, output_dir, tar_filename)
+        # Create a file containing the directories to add to the tarball, returning
+        # the filename
+        leave_links_fname = with_links(project_dictionary, output_dir, tar_filename)
 
-        derefenece_links = derefence_links(project_dictionary, output_dir, tar_filename)
-        ignore = ignores(project_dictionary, output_dir, tar_filename)
+        dereference_links_fname = dereference_links(project_dictionary, output_dir, tar_filename)
+        ignore_fname = ignores(project_dictionary, output_dir, tar_filename)
 
         fname = os.path.join(output_dir, os.path.basename(tar_filename) + "_tar.bash")
 
@@ -125,39 +184,70 @@ def tar_script(config_name, output_dir, tar_filename, home):
 
         tar.write('# Contents before\n')
         tar.write('cd %s; ls\n' % os.path.dirname(project_dictionary['a_home']))
-        tar.write('# Direct reference no de-reference\n')
-        tar.write(
-            'cmd=\"tar  --exclude=.svn  --exclude=.pyc -c -X %s -X %s -T %s -f %s\";echo ${cmd};${cmd}\n' % ( ignore,derefenece_links, leave_links, tar_filename))
-        tar.write('# Dereferenced files\n')
-        tar.write('cmd=\"tar  --exclude=.svn  --exclude=.pyc  -rh -X %s -T %s -f %s\";echo ${cmd};${cmd}\n' % (
-            ignore, derefenece_links, tar_filename))
 
+        # Add actual files to the tarball
+        tar.write('# Add actual files to the tarball\n')
+        tar.write(
+            '''cmd="tar -v --exclude=.svn  --exclude=*.pyc -c \\
+            -X %s \\
+            -X %s \\
+            -T %s \\
+            -f %s"\n''' % (ignore_fname, dereference_links_fname, leave_links_fname, tar_filename)
+        )
+        tar.write('''echo ${cmd}\n${cmd}\n\n''')
+
+        tar.write('## Dereferenced files\n')
+        tar.write(
+            '''cmd="tar -v --exclude=.svn  --exclude=*.pyc -rh \\
+            -X %s \\
+            -T %s \\
+            -f %s"\n''' % (ignore_fname, dereference_links_fname, tar_filename)
+        )
+        tar.write('''echo ${cmd}\n${cmd}\n\n''')
 
-        tar.write('#Files added form trace\n')
         trace_file = os.path.join(home,output_dir,os.path.basename(tar_filename))
-        trace_file= trace_file.replace('.tar', '.trc')
+        trace_file = trace_file.replace('.tar', '.trc')
+        trace_file = os.path.join(output_dir,project_dictionary['a_home'].split("/")[-1]+".trc")
 
-        #trace_file=os.path.join(output_dir,project_dictionary['a_home'].split("/")[-1]+".trc")
-        #tar.write('cmd=\"tar  -v --exclude=.svn  --exclude=.pyc -rh  -X %s  -T %s -f %s\";echo ${cmd};${cmd}\n' % (
-        #    ignore,  trace_file, tar_filename))
+        if os.path.exists(trace_file):
+            tar.write('# Files added form trace\n')
+            tar.write('cmd=\"tar  -v --exclude=.svn  --exclude=.pyc -rh  -X %s  -T %s -f %s\";echo ${cmd};${cmd}\n' % (
+               ignore_fname,  trace_file, tar_filename))
 
-        tar.write('cmd=\"tar   --exclude=.svn  --exclude=.pyc -rh  -T %s -f %s\";echo ${cmd};${cmd}\n' % (
-              trace_file, tar_filename))
-        tar.write('ls -al;cmd=\"gzip  %s\";echo ${cmd};${cmd}\n' % (tar_filename))
+            tar.write('cmd=\"tar -v --exclude=.svn  --exclude=*.pyc -rh  -T %s -f %s\";echo ${cmd};${cmd}\n' % (
+                  trace_file, tar_filename))
+
+        # gzip the tarball
+        tar.write('## gzip the tarball\n')
+        tar.write('''cmd="gzip  %s"\necho ${cmd}\n${cmd}\n''' % (tar_filename))
 
         tar.close()
         os.chmod(fname, 0775)
 
+    return 0
+
+
+def _argparse():
+    '''
+    Method to encapsulate the option parsing and various setup tasks.
+    '''
 
-def main():
     import argparse
 
-    desc = """Command to generate scripts for building tar file
-        """
-    parser = argparse.ArgumentParser(description=desc)
+    description = '''Generate a series of package scripts.'''
 
-    parser.add_argument('-v', '--verbosity', action="count", default=0,
-                        help='each occurrence increases verbosity 1 level through ERROR-WARNING-INFO-DEBUG')
+    usage = "usage: %prog [mandatory args] [options]"
+    version = 0.1
+
+    parser = argparse.ArgumentParser(
+                                     description=description
+                                     )
+
+    parser.add_argument('-c', '--config', metavar='config', default='.', required=True,
+                        help='Configuration file')
+
+    parser.add_argument('-H', '--home', metavar='home', default='.', required=True,
+                        help='Home location to create package')
 
     parser.add_argument('-o', '--output', metavar='output', default='prj_scripts',
                         help='output directory for tar building scripts')
@@ -165,21 +255,42 @@ def main():
     parser.add_argument('-t', '--tar_filename', metavar='tar_filename', default='packages', required=True,
                         help='tar ball name without extension, name-version-A/B')
 
-    parser.add_argument('-c', '--config', metavar='config', default='.', required=True,
-                        help='Configuration file')
-
-    parser.add_argument('-H', '--home', metavar='home', default='.', required=True,
-                        help='Home location to create package home of project ')
+    parser.add_argument("-v", "--verbosity",
+                      dest='verbosity',
+                      action="count", 
+                      default=2,
+                      help='''each occurrence increases verbosity 1 level from 
+                      ERROR: -v=WARNING -vv=INFO -vvv=DEBUG'''
+                      )
 
     args = parser.parse_args()
 
+    # Set up the logging
     levels = [logging.ERROR, logging.WARN, logging.INFO, logging.DEBUG]
-    level = levels[args.verbosity if args.verbosity < 4 else 3]
+    level = levels[args.verbosity if args.verbosity < 3 else 3]
+
+    if level == logging.DEBUG :
+        console_logFormat = '%(asctime)s.%(msecs)03d (%(levelname)7s) : %(filename)s : %(funcName)s : %(lineno)d:%(message)s'
+        date_format = '%Y-%m-%d %H:%M:%S'
+    else:
+        console_logFormat = '%(asctime)s.%(msecs)03d (%(levelname)7s) : %(message)s'
+        date_format = '%Y-%m-%d %H:%M:%S'
+
+    logging.basicConfig(level=level, format=console_logFormat, datefmt=date_format)
+
+    return args
+
+
+def main():
+
+    # Read in the options
+    options = _argparse()
 
-    logging.basicConfig(level=level)
+    # Generate the packaging scripts
+    retval = tar_script(options.config, options.output, options.tar_filename, options.home)
 
-    tar_script(args.config, args.output, args.tar_filename, args.home)
+    return retval
 
 
-if __name__ == '__main__':
-    main()
+if __name__=='__main__':
+    sys.exit(main())