Fixes an issue regarding blacklist and bytecode compile + some cleanup (kivy#2693)

Author: misl6

doc/source/buildoptions.rst

@@ -89,7 +89,7 @@ options (this list may not be exhaustive):
 - ``--service``: A service name and the Python script it should
   run. See :ref:`arbitrary_scripts_services`.
 - ``--add-source``: Add a source directory to the app's Java code.
-- ``--no-compile-pyo``: Do not optimise .py files to .pyo.
+- ``--no-byte-compile-python``: Skip byte compile for .py files.
 - ``--enable-androidx``: Enable AndroidX support library.
 
 

doc/source/launcher.rst

@@ -48,7 +48,7 @@ grab an old (cached) package instead of a fresh one.
 .. warning::
 
     Do not use any of `--private`, `--public`, `--dir` or other arguments for
-    adding `main.py` or `main.pyo` to the app. The argument `--launcher` is
+    adding `main.py` or `main.pyc` to the app. The argument `--launcher` is
     above them and tells the p4a to build the launcher version of the APK.
 
 Usage

pythonforandroid/bdistapk.py

@@ -108,7 +108,7 @@ def prepare_build_dir(self):
                     makedirs(new_dir)
                 print('Including {}'.format(filen))
                 copyfile(filen, join(bdist_dir, filen))
-                if basename(filen) in ('main.py', 'main.pyo'):
+                if basename(filen) in ('main.py', 'main.pyc'):
                     main_py_dirs.append(filen)
 
         # This feels ridiculous, but how else to define the main.py dir?

pythonforandroid/bootstraps/common/build/build.py

@@ -40,10 +40,6 @@ def get_hostpython():
     return get_dist_info_for('hostpython')
 
 
-def get_python_version():
-    return get_dist_info_for('python_version')
-
-
 def get_bootstrap_name():
     return get_dist_info_for('bootstrap')
 
@@ -58,7 +54,6 @@ def get_bootstrap_name():
 curdir = dirname(__file__)
 
 PYTHON = get_hostpython()
-PYTHON_VERSION = get_python_version()
 if PYTHON is not None and not exists(PYTHON):
     PYTHON = None
 
@@ -73,17 +68,16 @@ def get_bootstrap_name():
     '~',
     '*.bak',
     '*.swp',
+
+    # Android artifacts
+    '*.apk',
+    '*.aab',
 ]
-# pyc/py
-if PYTHON is not None:
-    BLACKLIST_PATTERNS.append('*.py')
 
 WHITELIST_PATTERNS = []
 if get_bootstrap_name() in ('sdl2', 'webview', 'service_only'):
     WHITELIST_PATTERNS.append('pyconfig.h')
 
-python_files = []
-
 
 environment = jinja2.Environment(loader=jinja2.FileSystemLoader(
     join(curdir, 'templates')))
@@ -150,23 +144,11 @@ def listfiles(d):
             yield fn
 
 
-def make_tar(tfn, source_dirs, ignore_path=[], optimize_python=True):
+def make_tar(tfn, source_dirs, byte_compile_python=False, optimize_python=True):
     '''
     Make a zip file `fn` from the contents of source_dis.
     '''
 
-    # selector function
-    def select(fn):
-        rfn = realpath(fn)
-        for p in ignore_path:
-            if p.endswith('/'):
-                p = p[:-1]
-            if rfn.startswith(p):
-                return False
-        if rfn in python_files:
-            return False
-        return not is_blacklist(fn)
-
     def clean(tinfo):
         """cleaning function (for reproducible builds)"""
         tinfo.uid = tinfo.gid = 0
@@ -178,9 +160,12 @@ def clean(tinfo):
     files = []
     for sd in source_dirs:
         sd = realpath(sd)
-        compile_dir(sd, optimize_python=optimize_python)
-        files += [(x, relpath(realpath(x), sd)) for x in listfiles(sd)
-                  if select(x)]
+        for fn in listfiles(sd):
+            if is_blacklist(fn):
+                continue
+            if fn.endswith('.py') and byte_compile_python:
+                fn = compile_py_file(fn, optimize_python=optimize_python)
+            files.append((fn, relpath(realpath(fn), sd)))
     files.sort()  # deterministic
 
     # create tar.gz of thoses files
@@ -210,18 +195,15 @@ def clean(tinfo):
     gf.close()
 
 
-def compile_dir(dfn, optimize_python=True):
+def compile_py_file(python_file, optimize_python=True):
     '''
-    Compile *.py in directory `dfn` to *.pyo
+    Compile python_file to *.pyc and return the filename of the *.pyc file.
     '''
 
     if PYTHON is None:
         return
 
-    if int(PYTHON_VERSION[0]) >= 3:
-        args = [PYTHON, '-m', 'compileall', '-b', '-f', dfn]
-    else:
-        args = [PYTHON, '-m', 'compileall', '-f', dfn]
+    args = [PYTHON, '-m', 'compileall', '-b', '-f', python_file]
     if optimize_python:
         # -OO = strip docstrings
         args.insert(1, '-OO')
@@ -233,16 +215,18 @@ def compile_dir(dfn, optimize_python=True):
               'error, see logs above')
         exit(1)
 
+    return ".".join([os.path.splitext(python_file)[0], "pyc"])
+
 
 def make_package(args):
-    # If no launcher is specified, require a main.py/main.pyo:
+    # If no launcher is specified, require a main.py/main.pyc:
     if (get_bootstrap_name() != "sdl" or args.launcher is None) and \
             get_bootstrap_name() not in ["webview", "service_library"]:
         # (webview doesn't need an entrypoint, apparently)
         if args.private is None or (
                 not exists(join(realpath(args.private), 'main.py')) and
-                not exists(join(realpath(args.private), 'main.pyo'))):
-            print('''BUILD FAILURE: No main.py(o) found in your app directory. This
+                not exists(join(realpath(args.private), 'main.pyc'))):
+            print('''BUILD FAILURE: No main.py(c) found in your app directory. This
 file must exist to act as the entry point for you app. If your app is
 started by a file with a different name, rename it to main.py or add a
 main.py that loads it.''')
@@ -290,7 +274,6 @@ def make_package(args):
                     variants = [
                         copy_path,
                         copy_path.partition(".")[0] + ".pyc",
-                        copy_path.partition(".")[0] + ".pyo",
                     ]
                     # Check in all variants with all possible endings:
                     for variant in variants:
@@ -326,11 +309,17 @@ def make_package(args):
             for arch in get_dist_info_for("archs"):
                 libs_dir = f"libs/{arch}"
                 make_tar(
-                    join(libs_dir, 'libpybundle.so'), [f'_python_bundle__{arch}'], args.ignore_path,
-                    optimize_python=args.optimize_python)
+                    join(libs_dir, "libpybundle.so"),
+                    [f"_python_bundle__{arch}"],
+                    byte_compile_python=args.byte_compile_python,
+                    optimize_python=args.optimize_python,
+                )
             make_tar(
-                join(assets_dir, 'private.tar'), private_tar_dirs, args.ignore_path,
-                optimize_python=args.optimize_python)
+                join(assets_dir, "private.tar"),
+                private_tar_dirs,
+                byte_compile_python=args.byte_compile_python,
+                optimize_python=args.optimize_python,
+            )
     finally:
         for directory in _temp_dirs_to_clean:
             shutil.rmtree(directory)
@@ -824,8 +813,6 @@ def parse_args_and_make_package(args=None):
     ap.add_argument('--try-system-python-compile', dest='try_system_python_compile',
                     action='store_true',
                     help='Use the system python during compileall if possible.')
-    ap.add_argument('--no-compile-pyo', dest='no_compile_pyo', action='store_true',
-                    help='Do not optimise .py files to .pyo.')
     ap.add_argument('--sign', action='store_true',
                     help=('Try to sign the APK with your credentials. You must set '
                           'the appropriate environment variables.'))
@@ -844,9 +831,12 @@ def parse_args_and_make_package(args=None):
                           'files (containing your main.py entrypoint). '
                           'See https://developer.android.com/guide/topics/data/'
                           'autobackup#IncludingFiles for more information'))
+    ap.add_argument('--no-byte-compile-python', dest='byte_compile_python',
+                    action='store_false', default=True,
+                    help='Skip byte compile for .py files.')
     ap.add_argument('--no-optimize-python', dest='optimize_python',
                     action='store_false', default=True,
-                    help=('Whether to compile to optimised .pyo files, using -OO '
+                    help=('Whether to compile to optimised .pyc files, using -OO '
                           '(strips docstrings and asserts)'))
     ap.add_argument('--extra-manifest-xml', default='',
                     help=('Extra xml to write directly inside the <manifest> element of'
@@ -881,8 +871,6 @@ def _read_configuration():
 
     args = ap.parse_args(args)
 
-    args.ignore_path = []
-
     if args.name and args.name[0] == '"' and args.name[-1] == '"':
         args.name = args.name[1:-1]
 
@@ -925,10 +913,6 @@ def _read_configuration():
         else:
             PYTHON = python_executable
 
-    if args.no_compile_pyo:
-        PYTHON = None
-        BLACKLIST_PATTERNS.remove('*.py')
-
     if args.blacklist:
         with open(args.blacklist) as fd:
             patterns = [x.strip() for x in fd.read().splitlines()

pythonforandroid/bootstraps/common/build/jni/application/src/start.c

@@ -251,14 +251,10 @@ int main(int argc, char *argv[]) {
    */
   LOGP("Run user program, change dir and execute entrypoint");
 
-  /* Get the entrypoint, search the .pyo then .py
+  /* Get the entrypoint, search the .pyc then .py
    */
   char *dot = strrchr(env_entrypoint, '.');
-#if PY_MAJOR_VERSION > 2
   char *ext = ".pyc";
-#else
-  char *ext = ".pyo";
-#endif
   if (dot <= 0) {
     LOGP("Invalid entrypoint, abort.");
     return -1;
@@ -281,14 +277,10 @@ int main(int argc, char *argv[]) {
       strcpy(entrypoint, env_entrypoint);
     }
   } else if (!strcmp(dot, ".py")) {
-    /* if .py is passed, check the pyo version first */
+    /* if .py is passed, check the pyc version first */
     strcpy(entrypoint, env_entrypoint);
     entrypoint[strlen(env_entrypoint) + 1] = '\0';
-#if PY_MAJOR_VERSION > 2
     entrypoint[strlen(env_entrypoint)] = 'c';
-#else
-    entrypoint[strlen(env_entrypoint)] = 'o';
-#endif
     if (!file_exists(entrypoint)) {
       /* fallback on pure python version */
       if (!file_exists(env_entrypoint)) {

pythonforandroid/bootstraps/sdl2/build/src/main/java/org/kivy/android/PythonActivity.java

@@ -419,11 +419,10 @@ public void run() {
     }
 
     public String getEntryPoint(String search_dir) {
-        /* Get the main file (.pyc|.pyo|.py) depending on if we
+        /* Get the main file (.pyc|.py) depending on if we
          * have a compiled version or not.
         */
         List<String> entryPoints = new ArrayList<String>();
-        entryPoints.add("main.pyo");  // python 2 compiled files
         entryPoints.add("main.pyc");  // python 3 compiled files
 		for (String value : entryPoints) {
             File mainFile = new File(search_dir + "/" + value);

pythonforandroid/bootstraps/service_only/build/src/main/java/org/kivy/android/PythonActivity.java

@@ -46,11 +46,10 @@ public String getAppRoot() {
     }
 
     public String getEntryPoint(String search_dir) {
-        /* Get the main file (.pyc|.pyo|.py) depending on if we
+        /* Get the main file (.pyc|.py) depending on if we
          * have a compiled version or not.
         */
         List<String> entryPoints = new ArrayList<String>();
-        entryPoints.add("main.pyo");  // python 2 compiled files
         entryPoints.add("main.pyc");  // python 3 compiled files
 		for (String value : entryPoints) {
             File mainFile = new File(search_dir + "/" + value);

pythonforandroid/bootstraps/webview/build/src/main/java/org/kivy/android/PythonActivity.java

@@ -68,11 +68,10 @@ public String getAppRoot() {
     }
 
     public String getEntryPoint(String search_dir) {
-        /* Get the main file (.pyc|.pyo|.py) depending on if we
+        /* Get the main file (.pyc|.py) depending on if we
          * have a compiled version or not.
         */
         List<String> entryPoints = new ArrayList<String>();
-        entryPoints.add("main.pyo");  // python 2 compiled files
         entryPoints.add("main.pyc");  // python 3 compiled files
         for (String value : entryPoints) {
             File mainFile = new File(search_dir + "/" + value);

pythonforandroid/build.py

@@ -456,7 +456,6 @@ def has_package(self, name, arch=None):
         return (exists(join(site_packages_dir, name)) or
                 exists(join(site_packages_dir, name + '.py')) or
                 exists(join(site_packages_dir, name + '.pyc')) or
-                exists(join(site_packages_dir, name + '.pyo')) or
                 exists(join(site_packages_dir, name + '.so')) or
                 glob.glob(join(site_packages_dir, name + '-*.egg')))
 

pythonforandroid/recipes/python3/__init__.py

@@ -364,11 +364,11 @@ def create_python_bundle(self, dirn, arch):
                 self.major_minor_version_string
             ))
 
-        # Compile to *.pyc/*.pyo the python modules
+        # Compile to *.pyc the python modules
         self.compile_python_files(modules_build_dir)
-        # Compile to *.pyc/*.pyo the standard python library
+        # Compile to *.pyc the standard python library
         self.compile_python_files(join(self.get_build_dir(arch.arch), 'Lib'))
-        # Compile to *.pyc/*.pyo the other python packages (site-packages)
+        # Compile to *.pyc the other python packages (site-packages)
         self.compile_python_files(self.ctx.get_python_install_dir(arch.arch))
 
         # Bundle compiled python modules to a folder