macos script using lipo to merge multiple platforms into a single deliverable

ShellB3/sb3bin/duolipo.py

+#!/usr/bin/env python
+import os, sys, stat
+import logging, re
+import subprocess as sp
+import platform
+import shutil
+
+LOG = logging.getLogger(__name__)
+import macholib.MachO
+import macholib.mach_o
+
+def acceptableMachoHeaders(path=None,machoobj=None,includeall=False):
+    if machoobj is None:
+        try:
+            machoobj=macholib.MachO.MachO(path)
+        except:
+            return
+    is64='64' in os.getenv('CPUTYPE',platform.machine())
+    headerclass = macholib.mach_o.mach_header_64 if is64 else macholib.mach_o.mach_header
+    for header in machoobj.headers:
+        if header.mach_header is not headerclass and not path.endswith('.in') and not includeall:
+            continue
+        yield header
+
+def twheismacho(path,linkok=False,bareObjectsToo=False,stubsToo=False,dsymsToo=False):
+    try:
+        #print path
+        if not linkok and os.path.islink(path):
+            return False
+        if path.endswith('.o') and not bareObjectsToo:
+            return False
+        m=macholib.MachO.MachO(path)
+        #print m.headers[0].size,path
+        if m==None:
+            return False
+        for h in acceptableMachoHeaders(path,m):
+            if h.filetype=='dylib_stub' and not stubsToo:
+                LOG.debug(path+" is a stub. can't rpath it")
+                return False#suspected stub
+            if h.filetype=="dsym" and not dsymsToo:
+                LOG.debug(path+" is a DWARF symbol table. can't rpath it")
+                return False
+        return True
+    except:
+        return False
+
+
+def copyperms(path,frompath):
+    perms=stat.S_IMODE(os.stat(frompath).st_mode)
+    os.chmod(path,perms)
+
+def recursivemkdir(path,matching=None):
+    if os.path.exists(path):
+        return
+    if not os.path.exists(os.path.dirname(path)):
+        recursivemkdir(os.path.dirname(path),matching=None if matching is None else os.path.dirname(matching))
+    os.mkdir(path)
+    if matching is not None:
+        copyperms(path,matching)
+
+def commandline(*args):
+    pid=sp.Popen(args, stdout=sp.PIPE, stderr=sp.PIPE)
+    out,err = pid.communicate()
+    return pid.returncode,out.decode('utf8'),err.decode('utf8')
+
+def inplacelipo(arch1,src1,arch2,src2):
+    os.rename(src1,src1+'.'+arch1)
+    r=lipo(src1,arch1,src1+'.'+arch1,arch2,src2)
+    if r==0:
+        os.unlink(src1+'.'+arch1)
+    else:
+        os.rename(src1+'.'+arch1,src1)
+    return r
+
+def lipo(output,arch1,src1,arch2,src2):
+    if not os.path.exists(os.path.dirname(output)):
+        recursivemkdir(os.path.dirname(output),matching=os.path.dirname(src1))
+    return commandline('lipo','-create','-arch',arch1,src1,'-arch',arch2,src2,'-output',output)[0]
+
+def rsync(src,dest,name=None):
+    if os.path.isdir(src):
+        src=src+'/'
+    LOG.warning("Syncing {} {}".format(name or dest,"" if not name else dest))
+    return commandline("rsync","-a","--delete",src,dest)[0]
+
+def ismacho(path):
+    ret=twheismacho(path,bareObjectsToo=True,stubsToo=True,dsymsToo=True)
+    #LOG.error("{} is{} macho".format(path,"" if ret else " NOT"))
+    return ret
+
+def sameFiles(p1,p2):
+    s1=os.stat(p1)
+    s2=os.stat(p2)
+    #if s1.st_size!=s2.st_size:
+    #    return False
+    f1=open(p1,"rb").read()
+    f2=open(p2,"rb").read()
+    return f1==f2
+
+def recurse(path,links=False,files=False,dirs=False,dorecurse=True):
+    dd=os.listdir(path)
+    dd.sort()
+    for f in dd:
+        if f.startswith("."):
+            continue
+        fp=os.path.join(path,f)
+        if os.path.islink(fp):
+            if links:
+                yield fp
+        elif os.path.isdir(fp):
+            if dirs:
+                yield fp
+            if dorecurse:
+                for y in recurse(fp,links=links,files=files,dirs=dirs):
+                    yield y
+        else:
+            if files:
+                yield fp
+
+def tryMergePaths(platform1,base1,platform2,base2,dry_run=False,skip=None):
+    syncedDirectories=list()
+    for p1 in recurse(base1,links=True,files=True,dirs=True):
+        if skip is not None and skip(p1):
+            LOG.info("Skipping {}".format(p1))
+            continue
+        if p1.startswith(tuple(syncedDirectories)):
+            continue
+        p2=p1.replace(base1,base2)
+        common=p1.replace(base1+"/","")
+        LOG.info("Comparing {} {}".format(common,p2))
+        if not os.path.exists(p2):
+            if os.path.basename(p2)=="__pycache__":
+                LOG.debug("{} no collision, but won't sync just a cache folder")
+                continue
+            LOG.debug("{} no collision".format(common))
+            if not dry_run:
+                recursivemkdir(os.path.dirname(p2),matching=os.path.dirname(p1))
+                rsync(p1,p2,name=common)
+            syncedDirectories.append(p1)
+            continue
+        if os.path.islink(p1) or os.path.islink(p2):
+            if not (os.path.islink(p1) and os.path.islink(p2)):
+                LOG.error("{} aren't both links. FAIL".format(common))
+                continue
+            l1=os.readlink(p1)
+            l2=os.readlink(p2)
+            if l1!=l2:
+                LOG.info("{} differ in link content {} and {}".format(common,l1,l2))
+                if os.path.basename(os.path.dirname(common)) in ("bin","MacOS"):
+                    #smart link
+                    LOG.warning("link {} to {}:{} and {}:{}".format(common,platform1,l1,platform2,l2))
+                    if not dry_run:
+                        os.unlink(p2)
+                        f=open(p2,"w")
+                        f.write("#!/usr/bin/python\n")
+                        f.write("import platform\n")
+                        f.write("import os,sys\n")
+                        f.write("runoptions={\n")
+                        f.write("   '{}':'{}',\n".format(platform1,l1))
+                        f.write("   '{}':'{}'\n".format(platform2,l2))
+                        f.write("}\n")
+                        f.write("bn=os.path.dirname(sys.argv[0])\n")
+                        f.write("mach=platform.machine()\n")
+                        f.write("if mach not in runoptions:")
+                        f.write("    raise RuntimeError('Unsupported platform '+mach)\n")
+                        f.write("binval=runoptions[mach]\n")
+                        f.write("os.execv(os.path.join(bn,binval),[os.path.join(bn,binval)]+list(sys.argv[1:]))\n")
+                        del f
+                        os.chmod(p2,0o755)
+                elif os.path.basename(common)=="Current":
+                    #current link. remove it
+                    if not dry_run:
+                        os.unlink(p2)
+                elif common.endswith(".dylib"):
+                    #symlink to a dylib. drop
+                    LOG.info("Would remove {} for compiletime library".format(common))
+                    if not dry_run:
+                        os.unlink(p2)
+                else:
+                    LOG.error("Can't fix link {} to {} or {}".format(common,l1,l2))
+                continue
+            LOG.debug("{} match link content {}".format(common,l1))
+            continue
+        if os.path.isdir(p1) or os.path.isdir(p2):
+            if not (os.path.isdir(p1) and os.path.isdir(p2)):
+                LOG.error("{} aren't both directories. FAIL".format(common))
+                continue
+            continue
+        if ismacho(p1) or ismacho(p2):
+            if not (ismacho(p1) and ismacho(p2)):
+                LOG.error("{} aren't both macho. FAIL".format(common))
+                continue
+            if dry_run:
+                LOG.info("Would lipo {}".format(common))
+            elif inplacelipo(platform2,p2,platform1,p1)==0:
+                LOG.debug("Merged {}".format(common))
+            else:
+                LOG.error("Failed to merge {}".format(common))
+        elif sameFiles(p1,p2):
+            LOG.debug("{} match content".format(common))
+        else:
+            #pass
+
+            if common.startswith("doc/") or "/man/" in common or "/info/" in common: 
+                LOG.info("Skipping documentation collision {}".format(common))
+            elif common.endswith(('.cmake','.pc','.h','.mod',"Config.sh",".prl",".prf",".pri")) or "/Headers/" in common or "/include/" in common or common.startswith("include/"):
+                LOG.info("Skipping collision of compile-time file {}".format(common))
+            elif common.endswith(('.qml','.qmltypes','/qmldir','.metainfo','.ttf',".plist",".icns",".qm")):
+                LOG.error("Version collision on file {}".format(common))
+            elif common in ("trim",):
+                if not dry_run:
+                    os.rename(p2,p2+"."+platform2)
+                    rsync(p1,p2+"."+platform1,name=common+"."+platform1)                
+            elif os.path.basename(os.path.dirname(common)) in ("bin","MacOS"):
+                LOG.info("Collision of {} to be replaced with a script".format(common))
+                if not dry_run:
+                    os.rename(p2,p2+"."+platform2)
+                    rsync(p1,p2+"."+platform1,name=common+"."+platform1)
+                    if True:
+                        f=open(p2,"w")
+                        f.write("#!/usr/bin/python\n")
+                        f.write("import platform\n")
+                        f.write("import os,sys\n")
+                        f.write("os.execv(sys.argv[0]+'.'+platform.machine(),sys.argv)\n")
+                        del f
+                        os.chmod(p2,0o755)
+            elif '/__pycache__/' in common and common.endswith(".pyc"): #is part of a pycache. should purge because one won't like it
+                LOG.info("Collision of {} is not critical, but bad for both. Removing from merged".format(common))
+                if not dry_run:
+                    os.unlink(p2)
+            elif common.endswith('.py'):
+                LOG.warning("Python module Collision of {} to be replaced with a python module proxy".format(common))
+                if not dry_run:
+                    p2base=os.path.dirname(p2)
+                    modulename=os.path.basename(p2)[:-3]
+                    while modulename.startswith('_'):
+                        modulename=modulename[1:]
+                    modulename='lipo_'+modulename
+                    module1=modulename+"_"+platform1
+                    module2=modulename+"_"+platform2
+                    form=dict(platform1=platform1,platform2=platform2,module1=module1,module2=module2)
+                    os.rename(p2,os.path.join(p2base,module2+".py"))
+                    rsync(p1,os.path.join(p2base,module1+".py"),name=os.path.join(os.path.dirname(common),module1+".py"))
+                    if True:
+                        f=open(p2,"w")
+                        f.write("import sys\n")
+                        f.write("import platform\n")
+                        f.write("import logging\n")
+                        f.write("logger=logging.getLogger(__name__)\n")
+                        f.write("self=sys.modules[__name__]\n")
+                        f.write("if platform.machine() == '{platform1}':\n".format(**form))
+                        f.write("    logger.debug('Will proxy load {module1} version '+platform.machine()+' in place of '+__name__)\n".format(**form))
+                        f.write("    try:\n")
+                        f.write("        from . import {module1} as lipod\n".format(**form))
+                        f.write("    except ImportError:\n")
+                        f.write("        import {module1} as lipod\n".format(**form))
+                        f.write("elif platform.machine() == '{platform2}':\n".format(**form))
+                        f.write("    logger.debug('Will proxy load {module2} version '+platform.machine()+' in place of '+__name__)\n".format(**form))
+                        f.write("    try:\n")
+                        f.write("        from . import {module2} as lipod\n".format(**form))
+                        f.write("    except ImportError:\n")
+                        f.write("        import {module2} as lipod\n".format(**form))
+                        f.write("else:\n")
+                        f.write("    raise RuntimeError('Unknown platform '+platform.machine())\n")
+                        f.write("lipod.__name__=__name__\n")
+                        f.write("sys.modules[__name__]=lipod\n")
+                        f.write("\n")
+                        del f
+                        os.chmod(p2,0o755)
+            else:
+                LOG.error("{} contents differ".format(common))
+                #r=["","Would compare {}".format(common)]
+                #r=commandline("diff","-u",p1,p2)
+                #print(r[1])
+
+def skipLameFiles(f):
+    if os.path.basename(f)==".DS_Store" or os.path.basename(f).startswith('.'):
+        return True
+    return False
+
+def main():
+    import optparse
+    usage = """
+%prog [options] [platform1 base1] [platform2 base2] newbase
+example:
+python duolipo.py x86_64 /somewhere/ShellB3-x86path/ arm64 /somewhere/ShellB3-arm64path/ /somewhere/ShellB3-mergetargetpath/
+"""
+    parser = optparse.OptionParser(usage)
+    parser.add_option('-d', '--dry-run', dest="dry_run",
+                    action="store_true", default=False, help="show what commands would be done") 
+    parser.add_option('-v', '--verbose', dest='verbosity', action="count", default=0,
+                    help='each occurrence increases verbosity 1 level through ERROR-WARNING-INFO-DEBUG')
+    # parser.add_option('-I', '--include-path', dest="includes",
+    #                 action="append", help="include path to append to GCCXML call")                           
+
+    if len(sys.argv)==1:
+        parser.print_help()
+        return 0
+    (options, args) = parser.parse_args()
+    #print args
+
+    levels = [logging.ERROR, logging.WARN, logging.INFO, logging.DEBUG]
+    logging.basicConfig(level = levels[min(3,options.verbosity)])
+
+    # make options a globally accessible structure, e.g. OPTS.
+    global OPTS
+    OPTS = options
+
+    platform1=args[0]
+    base1=args[1]
+    platform2=args[2]
+    base2=args[3]
+    if len(args)>4:
+        newbase=args[4]
+        assert(len(args)==5)
+    else:
+        options.dry_run=True
+
+    if options.dry_run:
+        tryMergePaths(platform2,base2,platform1,base1,dry_run=True,skip=skipLameFiles)
+        # FIXME - run any self-tests
+        # import doctest
+        # doctest.testmod()
+        sys.exit(0)
+
+    rsync(base1,newbase,name="base")
+    tryMergePaths(platform2,base2,platform1,newbase,skip=skipLameFiles)
+
+    return 0
+
+if __name__=='__main__':
+    sys.exit(main())