Annual spring clean and apache version bump

Author: ianamason

.travis.yml

@@ -14,7 +14,7 @@ python:
 install:
   - sudo apt-get update
   # apache prerequisites
-  - sudo apt-get install  -y libapr1-dev libaprutil1-dev
+  - sudo apt-get install  -y libapr1-dev libaprutil1-dev libpcre3-dev
   # for the clang build
   - sudo apt-get install  -y llvm-3.5 clang-3.5
   # dragonegg prereqs. dragonegg and llvm-gcc use llvm 3.3
@@ -34,7 +34,7 @@ script:
   - export WLLVM_HOME=`pwd`
   - ${WLLVM_HOME}/.travis/store.sh
   - export APACHE_URL=https://www-us.apache.org/dist/httpd/
-  - export APACHE_VER=2.4.43
+  - export APACHE_VER=2.4.46
   # build apache with clang
   - ${WLLVM_HOME}/.travis/apache_clang.sh
   # build apache with gcc and dragonegg

Makefile

@@ -43,10 +43,10 @@ install:
 	pip install
 
 check_clang:
-	cd test; python -m unittest -v test_base_driver test_clang_driver
+	cd test; python3 -m unittest -v test_base_driver test_clang_driver
 
 check_dragonegg:
-	cd test; python -m unittest -v test_base_driver test_dragonegg_driver
+	cd test; python3 -m unittest -v test_base_driver test_dragonegg_driver
 
 clean:
 	rm -f  wllvm/*.pyc wllvm/*~

setup.py

@@ -17,6 +17,7 @@
 setup(
     name='wllvm',
     version=wllvm_version,
+    python_requires='>=3.6',
     description='Whole Program LLVM',
     long_description=long_description,
     url='https://github.com/SRI-CSL/whole-program-llvm',

wllvm/arglistfilter.py

@@ -27,7 +27,7 @@
 # recognized by regular expressions.  All regular expressions must be
 # tried, obviously.  The first one that matches is taken, and no order
 # is specified.  Try to avoid overlapping patterns.
-class ArgumentListFilter(object):
+class ArgumentListFilter:
     def __init__(self, inputList, exactMatches={}, patternMatches={}):
         defaultArgExactMatches = {
 
@@ -465,16 +465,16 @@ def compileLinkUnaryCallback(self, flag):
     def getOutputFilename(self):
         if self.outputFilename is not None:
             return self.outputFilename
-        elif self.isCompileOnly:
+        if self.isCompileOnly:
             #iam: -c but no -o, therefore the obj should end up in the cwd.
             (_, base) = os.path.split(self.inputFiles[0])
             (root, _) = os.path.splitext(base)
-            return '{0}.o'.format(root)
+            return f'{root}.o'
         return 'a.out'
 
     def getBitcodeFileName(self):
         (dirs, baseFile) = os.path.split(self.getOutputFilename())
-        bcfilename = os.path.join(dirs, '.{0}.bc'.format(baseFile))
+        bcfilename = os.path.join(dirs, f'.{baseFile}.bc')
         return bcfilename
 
     # iam: returns a pair [objectFilename, bitcodeFilename] i.e .o and .bc.
@@ -485,18 +485,18 @@ def getArtifactNames(self, srcFile, hidden=False):
         (_, srcbase) = os.path.split(srcFile)
         (srcroot, _) = os.path.splitext(srcbase)
         if hidden:
-            objbase = '.{0}.o'.format(srcroot)
+            objbase = f'.{srcroot}.o'
         else:
-            objbase = '{0}.o'.format(srcroot)
-        bcbase = '.{0}.o.bc'.format(srcroot)
+            objbase = f'{srcroot}.o'
+        bcbase = f'.{srcroot}.o.bc'
         return [objbase, bcbase]
 
     #iam: for printing our partitioning of the args
     def dump(self):
         efn = sys.stderr.write
-        efn('\ncompileArgs: {0}\ninputFiles: {1}\nlinkArgs: {2}\n'.format(self.compileArgs, self.inputFiles, self.linkArgs))
-        efn('\nobjectFiles: {0}\noutputFilename: {1}\n'.format(self.objectFiles, self.outputFilename))
+        efn(f'\ncompileArgs: {self.compileArgs}\ninputFiles: {self.inputFiles}\nlinkArgs: {self.linkArgs}\n')
+        efn(f'\nobjectFiles: {self.objectFiles}\noutputFilename: {self.outputFilename}\n')
         for srcFile in self.inputFiles:
-            efn('\nsrcFile: {0}\n'.format(srcFile))
+            efn(f'\nsrcFile: {srcFile}\n')
             (objFile, bcFile) = self.getArtifactNames(srcFile)
-            efn('\n{0} ===> ({1}, {2})\n'.format(srcFile, objFile, bcFile))
+            efn(f'\n{srcFile} ===> ({objFile}, {bcFile})\n')

wllvm/as.py

@@ -53,7 +53,7 @@ def __init__(self, arglist):
         self.bcName = None
         self.outFileName = None
         localCallbacks = {'-o' : (1, BCFilter.outFileCallback)}
-        super(BCFilter, self).__init__(arglist, exactMatches=localCallbacks)
+        super().__init__(arglist, exactMatches=localCallbacks)
 
     def outFileCallback(self, flag, name):
         """ Callback for the -o flag.

wllvm/checker.py

@@ -75,7 +75,7 @@
 
 """
 
-class Checker(object):
+class Checker:
     def __init__(self):
         path = os.getenv('LLVM_COMPILER_PATH')
 
@@ -112,17 +112,17 @@ def check(self):
         return 0 if success else 1
 
     def checkSelf(self):
-        print('wllvm version: {0}'.format(wllvm_version))
-        print('wllvm released: {0}\n'.format(wllvm_date))
+        print(f'wllvm version: {wllvm_version}')
+        print(f'wllvm released: {wllvm_date}\n')
 
 
     def checkLogging(self):
         (destination, level) = loggingConfiguration()
-        print('Logging output to {0}.'.format(destination if destination else 'standard error'))
+        print(f'Logging output to {destination if destination else "standard error"}.')
         if not level:
-            print('Logging level not set, defaulting to WARNING\n')
+            print('Logging level not set, defaulting to WARNING.')
         else:
-            print('Logging level set to {0}.\n'.format(level))
+            print(f'Logging level set to {level}.')
 
 
     def checkOS(self):
@@ -137,7 +137,7 @@ def checkSwitch(self):
         compiler_type = os.getenv('LLVM_COMPILER')
         if compiler_type == 'clang':
             return (1, '\nWe are using clang.\n')
-        elif compiler_type == 'dragonegg':
+        if compiler_type == 'dragonegg':
             return (2, '\nWe are using dragonegg.\n')
         return (0, explain_LLVM_COMPILER)
 
@@ -147,8 +147,8 @@ def checkClang(self):
         cc_name = os.getenv('LLVM_CC_NAME')
         cxx_name = os.getenv('LLVM_CXX_NAME')
 
-        cc = '{0}{1}'.format(self.path, cc_name if cc_name else 'clang')
-        cxx = '{0}{1}'.format(self.path, cxx_name if cxx_name else 'clang++')
+        cc = f'{self.path}{cc_name if cc_name else "clang"}'
+        cxx = f'{self.path}{cxx_name if cxx_name else "clang++"}'
 
         return self.checkCompilers(cc, cxx)
 
@@ -162,8 +162,8 @@ def checkDragonegg(self):
         if os.getenv('LLVM_GCC_PREFIX') is not None:
             pfx = os.getenv('LLVM_GCC_PREFIX')
 
-        cc = '{0}{1}gcc'.format(self.path, pfx)
-        cxx = '{0}{1}g++'.format(self.path, pfx)
+        cc = f'{self.path}{pfx}gcc'
+        cxx = f'{self.path}{pfx}g++'
 
         return self.checkCompilers(cc, cxx)
 
@@ -180,11 +180,11 @@ def checkDragoneggPlugin(self):
             try:
                 open(plugin)
             except IOError as e:
-                print("Unable to open {0}: {1}".format(plugin, str(e)))
+                print(f'Unable to open {plugin}: {str(e)}')
             else:
                 return True
         else:
-            print("Could not find {0}".format(plugin))
+            print(f'Could not find {plugin}')
             return False
 
 
@@ -195,10 +195,10 @@ def checkCompiler(self):
         if code == 0:
             print(comment)
             return False
-        elif code == 1:
+        if code == 1:
             print(comment)
             return self.checkClang()
-        elif code == 2:
+        if code == 2:
             print(comment)
             return self.checkDragonegg()
         print('Insane')
@@ -212,14 +212,14 @@ def checkCompilers(self, cc, cxx):
         (cxxOk, cxxVersion) = self.checkExecutable(cxx)
 
         if not ccOk:
-            print('The C compiler {0} was not found or not executable.\nBetter not try using wllvm!\n'.format(cc))
+            print(f'The C compiler {cc} was not found or not executable.\nBetter not try using wllvm!\n')
         else:
-            print('The C compiler {0} is:\n\n\t{1}\n'.format(cc, extractLine(ccVersion, 0)))
+            print(f'The C compiler {cc} is:\n\n\t{extractLine(ccVersion, 0)}\n')
 
         if not cxxOk:
-            print('The CXX compiler {0} was not found or not executable.\nBetter not try using wllvm++!\n'.format(cxx))
+            print(f'The CXX compiler {cxx} was not found or not executable.\nBetter not try using wllvm++!\n')
         else:
-            print('The C++ compiler {0} is:\n\n\t{1}\n'.format(cxx, extractLine(cxxVersion, 0)))
+            print(f'The C++ compiler {cxx} is:\n\n\t{extractLine(cxxVersion, 0)}\n')
 
         if not ccOk or  not cxxOk:
             print(explain_LLVM_COMPILER_PATH)
@@ -239,13 +239,13 @@ def checkExecutable(self, exe, version_switch='-v'):
         try:
             compiler = sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.PIPE)
             output = compiler.communicate()
-            compilerOutput = '{0}{1}'.format(output[0], output[1])
+            compilerOutput = f'{output[0].decode()}{output[1].decode()}'
         except OSError as e:
             if e.errno == errno.EPERM:
-                return (False, '{0} not executable'.format(exe))
-            elif e.errno == errno.ENOENT:
-                return (False, '{0} not found'.format(exe))
-            return (False, '{0} not sure why, errno is {1}'.format(exe, e.errno))
+                return (False, f'{exe} not executable')
+            if e.errno == errno.ENOENT:
+                return (False, f'{exe} not found')
+            return (False, f'{exe} not sure why, errno is {e.errno}')
         else:
             return (True, compilerOutput)
 
@@ -262,34 +262,34 @@ def checkAuxiliaries(self):
         if not ar_name:
             ar_name = 'llvm-ar'
 
-        link = '{0}{1}'.format(self.path, link_name) if self.path else link_name
-        ar = '{0}{1}'.format(self.path, ar_name) if self.path else ar_name
+        link = f'{self.path}{link_name}' if self.path else link_name
+        ar = f'{self.path}{ar_name}' if self.path else ar_name
 
         (linkOk, linkVersion) = self.checkExecutable(link, '-version')
 
         (arOk, arVersion) = self.checkExecutable(ar, '-version')
 
         if not linkOk:
-            print('The bitcode linker {0} was not found or not executable.\nBetter not try using extract-bc!\n'.format(link))
+            print(f'The bitcode linker {link} was not found or not executable.\nBetter not try using extract-bc!\n')
             print(explain_LLVM_LINK_NAME)
         else:
-            print('The bitcode linker {0} is:\n\n\t{1}\n'.format(link, extractLine(linkVersion, 1)))
+            print(f'The bitcode linker {link} is:\n\n\t{extractLine(linkVersion, 1)}\n')
 
         if not arOk:
-            print('The bitcode archiver {0} was not found or not executable.\nBetter not try using extract-bc!\n'.format(ar))
+            print(f'The bitcode archiver {ar} was not found or not executable.\nBetter not try using extract-bc!\n')
             print(explain_LLVM_AR_NAME)
         else:
-            print('The bitcode archiver {0} is:\n\n\t{1}\n'.format(ar, extractLine(arVersion, 1)))
+            print(f'The bitcode archiver {ar} is:\n\n\t{extractLine(arVersion, 1)}\n')
 
 
     def checkStore(self):
         """Checks that the bitcode store, if set, makes sense."""
         store_dir = os.getenv('WLLVM_BC_STORE')
         if store_dir:
             if os.path.exists(store_dir) and os.path.isdir(store_dir) and os.path.isabs(store_dir):
-                print('Using the bitcode store:\n\n\t{0}\n\n'.format(store_dir))
+                print(f'Using the bitcode store:\n\n\t{store_dir}\n\n')
             else:
-                print('The bitcode store:\n\n\t{0}\n\nis either not absolute, does not exist, or is not a directory.\n\n'.format(store_dir))
+                print(f'The bitcode store:\n\n\t{store_dir}\n\nis either not absolute, does not exist, or is not a directory.\n\n')
         else:
             print('Not using a bitcode store.\n\n')
 

wllvm/compilers.py

@@ -92,7 +92,8 @@ def wcompile(mode):
 class ClangBitcodeArgumentListFilter(ArgumentListFilter):
     def __init__(self, arglist):
         localCallbacks = {'-o' : (1, ClangBitcodeArgumentListFilter.outputFileCallback)}
-        super(ClangBitcodeArgumentListFilter, self).__init__(arglist, exactMatches=localCallbacks)
+        #super(ClangBitcodeArgumentListFilter, self).__init__(arglist, exactMatches=localCallbacks)
+        super().__init__(arglist, exactMatches=localCallbacks)
 
     def outputFileCallback(self, flag, filename):
         self.outputFilename = filename
@@ -138,11 +139,11 @@ def attachBitcodePathToObject(bcPath, outFileName):
 
     # Now write our bitcode section
     if sys.platform.startswith('darwin'):
-        objcopyBin = '{}-{}'.format(binUtilsTargetPrefix, 'ld') if binUtilsTargetPrefix else 'ld'
+        objcopyBin = f'{binUtilsTargetPrefix}-{"ld"}' if binUtilsTargetPrefix else 'ld'
         objcopyCmd = [objcopyBin, '-r', '-keep_private_externs', outFileName, '-sectcreate', darwinSegmentName, darwinSectionName, f.name, '-o', outFileName]
     else:
-        objcopyBin = '{}-{}'.format(binUtilsTargetPrefix, 'objcopy') if binUtilsTargetPrefix else 'objcopy'
-        objcopyCmd = [objcopyBin, '--add-section', '{0}={1}'.format(elfSectionName, f.name), outFileName]
+        objcopyBin = f'{binUtilsTargetPrefix}-{"objcopy"}' if binUtilsTargetPrefix else 'objcopy'
+        objcopyCmd = [objcopyBin, '--add-section', f'{elfSectionName}={f.name}', outFileName]
     orc = 0
 
     # loicg: If the environment variable WLLVM_BC_STORE is set, copy the bitcode
@@ -168,7 +169,7 @@ def attachBitcodePathToObject(bcPath, outFileName):
         _logger.error('objcopy failed with %s', orc)
         sys.exit(-1)
 
-class BuilderBase(object):
+class BuilderBase:
     def __init__(self, cmd, mode, prefixPath=None):
         self.af = None     #memoize the arglist filter
         self.cmd = cmd
@@ -221,8 +222,8 @@ def getCompiler(self):
         elif self.mode == "wfortran":
             env, prog = 'LLVM_F77_NAME', 'flang'
         else:
-            raise Exception("Unknown mode {0}".format(self.mode))
-        return ['{0}{1}'.format(self.prefixPath, os.getenv(env) or prog)]
+            raise Exception(f'Unknown mode {self.mode}')
+        return [f'{self.prefixPath}{os.getenv(env) or prog}']
 
     def getBitcodeArglistFilter(self):
         if self.af is None:
@@ -236,7 +237,7 @@ def getBitcodeCompiler(self):
         # We use '-B' to tell gcc where to look for an assembler.
         # When we build LLVM bitcode we do not want to use the GNU assembler,
         # instead we want gcc to use our own assembler (see as.py).
-        cmd = cc + ['-B', asDir, '-fplugin={0}'.format(pth), '-fplugin-arg-dragonegg-emit-ir']
+        cmd = cc + ['-B', asDir, f'-fplugin={pth}', '-fplugin-arg-dragonegg-emit-ir']
         _logger.debug(cmd)
         return cmd
 
@@ -252,8 +253,8 @@ def getCompiler(self):
         elif self.mode == "wfortran":
             mode = 'gfortran'
         else:
-            raise Exception("Unknown mode {0}".format(self.mode))
-        return ['{0}{1}{2}'.format(self.prefixPath, pfx, mode)]
+            raise Exception(f'Unknown mode {self.mode}')
+        return [f'{self.prefixPath}{pfx}{mode}']
 
     def getBitcodeArglistFilter(self):
         if self.af is None:
@@ -271,16 +272,15 @@ def getBuilder(cmd, mode):
 
     if cstring == 'clang':
         return ClangBuilder(cmd, mode, pathPrefix)
-    elif cstring == 'dragonegg':
+    if cstring == 'dragonegg':
         return DragoneggBuilder(cmd, mode, pathPrefix)
-    elif cstring is None:
+    if cstring is None:
         errorMsg = ' No compiler set. Please set environment variable %s'
         _logger.critical(errorMsg, compilerEnv)
         raise Exception(errorMsg)
-    else:
-        errorMsg = '%s = %s : Invalid compiler type'
-        _logger.critical(errorMsg, compilerEnv, str(cstring))
-        raise Exception(errorMsg)
+    errorMsg = '%s = %s : Invalid compiler type'
+    _logger.critical(errorMsg, compilerEnv, str(cstring))
+    raise Exception(errorMsg)
 
 def buildObject(builder):
     objCompiler = builder.getCompiler()