Merge pull request #3074 from autosportlabs/brent/private_github_repos

Add ability to use private github repos for recipes

Author: AndreMiras

.github/workflows/push.yml

@@ -8,6 +8,10 @@ env:
   AAR_ARTIFACT_FILENAME: bdist_unit_tests_app-release-1.1.aar
   PYTHONFORANDROID_PREREQUISITES_INSTALL_INTERACTIVE: 0
 
+concurrency:
+  group: build-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
 
   flake8:

doc/source/recipes.rst

@@ -54,10 +54,28 @@ omitted if the source is somehow loaded from elsewhere.
 You must include ``recipe = YourRecipe()``. This variable is accessed
 when the recipe is imported.
 
+Specifying the URL
+------------------
+
 .. note:: The url includes the ``{version}`` tag. You should only
           access the url with the ``versioned_url`` property, which
           replaces this with the version attribute.
 
+.. note:: you may need to specify additional headers to allow python-for-android
+          to download the archive. Specify your additional headers by setting the
+          download_headers property.
+
+For example, when downloading from a private github repository, you can specify the following:
+
+(For the download_headers property in your recipe)
+```
+[('Authorization', 'token <your personal access token>'), ('Accept', 'application/vnd.github+json')]
+```
+
+(For the DOWNLOAD_HEADERS_my-package-name environment variable - specify as a JSON formatted set of values)
+```
+    [["Authorization","token <your personal access token>"],["Accept", "application/vnd.github+json"]]
+```
 The actual build process takes place via three core methods::
 
       def prebuild_arch(self, arch):

pythonforandroid/recipe.py

@@ -1,7 +1,7 @@
 from os.path import basename, dirname, exists, isdir, isfile, join, realpath, split
 import glob
-
 import hashlib
+import json
 from re import match
 
 import sh
@@ -59,6 +59,21 @@ class Recipe(metaclass=RecipeMeta):
               if you want.
     '''
 
+    _download_headers = None
+    '''Add additional headers used when downloading the package, typically
+    for authorization purposes.
+
+    Specified as an array of tuples:
+    [("header1", "foo"), ("header2", "bar")]
+
+    When specifying as an environment variable (DOWNLOAD_HEADER_my-package-name), use a JSON formatted fragement:
+    [["header1","foo"],["header2", "bar"]]
+
+    For example, when downloading from a private
+    github repository, you can specify the following:
+    [('Authorization', 'token <your personal access token>'), ('Accept', 'application/vnd.github+json')]
+    '''
+
     _version = None
     '''A string giving the version of the software the recipe describes,
     e.g. ``2.0.3`` or ``master``.'''
@@ -170,6 +185,18 @@ def versioned_url(self):
             return None
         return self.url.format(version=self.version)
 
+    @property
+    def download_headers(self):
+        key = "DOWNLOAD_HEADERS_" + self.name
+        env_headers = environ.get(key)
+        if env_headers:
+            try:
+                return [tuple(h) for h in json.loads(env_headers)]
+            except Exception as ex:
+                raise ValueError(f'Invalid Download headers for {key} - must be JSON formatted as [["header1","foo"],["header2","bar"]]: {ex}')
+
+        return environ.get(key, self._download_headers)
+
     def download_file(self, url, target, cwd=None):
         """
         (internal) Download an ``url`` to a ``target``.
@@ -205,6 +232,8 @@ def report_hook(index, blksize, size):
                     # jqueryui.com returns a 403 w/ the default user agent
                     # Mozilla/5.0 does not handle redirection for liblzma
                     url_opener.addheaders = [('User-agent', 'Wget/1.0')]
+                    if self.download_headers:
+                        url_opener.addheaders += self.download_headers
                     urlretrieve(url, target, report_hook)
                 except OSError as e:
                     attempts += 1

pythonforandroid/toolchain.py

@@ -1023,7 +1023,7 @@ def _build_package(self, args, package_type):
                 # .../build/bootstrap_builds/sdl2-python3/gradlew
                 # if docker on windows, gradle contains CRLF
                 output = shprint(
-                    sh.Command('dos2unix'), gradlew._path.decode('utf8'),
+                    sh.Command('dos2unix'), gradlew._path,
                     _tail=20, _critical=True, _env=env
                 )
             if args.build_mode == "debug":

tests/test_recipe.py

@@ -326,3 +326,10 @@ def test_postarch_build(self, mock_install_stl_lib):
         assert recipe.need_stl_shared, True
         recipe.postbuild_arch(arch)
         mock_install_stl_lib.assert_called_once_with(arch)
+
+    def test_recipe_download_headers(self):
+        """Download header can be created on the fly using environment variables."""
+        recipe = DummyRecipe()
+        with mock.patch.dict(os.environ, {f'DOWNLOAD_HEADERS_{recipe.name}': '[["header1","foo"],["header2", "bar"]]'}):
+            download_headers = recipe.download_headers
+        assert download_headers == [("header1", "foo"), ("header2", "bar")]