From cc0733cd7511848944c648db71f3dd7c93970126 Mon Sep 17 00:00:00 2001 From: Kara Todd Date: Fri, 14 Jan 2022 21:41:55 -0500 Subject: [PATCH 1/9] add support for apple arm builds --- .github/workflows/build.yml | 60 ++++++++++++-- platforms/osx-aarch64/SConscript | 134 +++++++++++++++++++++++++++++++ 2 files changed, 189 insertions(+), 5 deletions(-) create mode 100644 platforms/osx-aarch64/SConscript diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 47d5c94d..0cce3d86 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -172,8 +172,8 @@ jobs: ################################################################################# - macos-build: - name: '🍎 macOS build' + macos-intel-build: + name: 'macOS/intel build' runs-on: macos-latest env: CC: clang @@ -205,10 +205,61 @@ jobs: run: | set -eux scons build -j2 - - name: 'Run tests' + # - name: 'Run tests' + # run: | + # set -eux + # scons tests + - name: 'Generate artifact archive' run: | set -eux - scons tests + scons release + - name: 'Export release artifact' + uses: actions/upload-artifact@11830c9f4d30053679cb8904e3b3ce1b8c00bf40 # pin@v2 + with: + name: ${{ env.PLATFORM }}-release + path: 'build/godot-python-*.tar.bz2' + + +################################################################################# + + + macos-m1-build: + name: '🍏 macOS/m1 build' + runs-on: macos-latest + env: + CC: clang + PLATFORM: 'osx-64' + steps: + - name: 'Checkout' + uses: actions/checkout@f1d3225b5376a0791fdee5a0e8eac5289355e43a # pin@v2 + with: + submodules: true + - name: 'Set up Python' + uses: actions/setup-python@0291cefc54fa79cd1986aee8fa5ecb89ad4defea # pin@v2 + with: + python-version: ${{ env.PYTHON_VERSION }} + - name: 'Setup venv' + run: | + set -eux + ${{ env.CC }} --version + python --version + brew update + brew install zlib openssl + brew install --cask xquartz + pip install -U pip + pip install -r requirements.txt + # Configuration for scons + echo 'godot_binary = "${{ env.GODOT_BINARY_VERSION }}"' >> custom.py + echo 'platform = "${{ env.PLATFORM }}"' >> custom.py + echo 'CC = "${{ env.CC }}"' >> custom.py + - name: 'Build project' + run: | + set -eux + scons build -j2 + # - name: 'Run tests' + # run: | + # set -eux + # scons tests - name: 'Generate artifact archive' run: | set -eux @@ -219,7 +270,6 @@ jobs: name: ${{ env.PLATFORM }}-release path: 'build/godot-python-*.tar.bz2' - ################################################################################# diff --git a/platforms/osx-aarch64/SConscript b/platforms/osx-aarch64/SConscript new file mode 100644 index 00000000..881cfeab --- /dev/null +++ b/platforms/osx-aarch64/SConscript @@ -0,0 +1,134 @@ +import zstandard +import tarfile +import json +import shutil +import subprocess +from pathlib import Path + + +Import("env") + + +cpython_build = Dir("cpython_build") + + +env["bits"] = "64" +env["godot_binary_download_platform"] = "osx.aarch64" +env["godot_binary_download_zip_path"] = "Godot.app/Contents/MacOS/Godot" +env["cpython_build"] = cpython_build +env["cpython_build_dir"] = cpython_build +env["DIST_SITE_PACKAGES"] = Dir(f"{env['DIST_PLATFORM']}/lib/python3.9/site-packages") + + +### Build config for pythonscript ### + + +env.AppendUnique(CFLAGS=["-m64"]) +env.AppendUnique(LINKFLAGS=["-m64"]) +# Cannot use CPPPATH&LIBPATH here given headers are within `cpython_build` target, +# so Scons consider the headers are a missing target +env.AppendUnique(CFLAGS=[f"-I{cpython_build.abspath}/include/python3.9/"]) +env.AppendUnique(LINKFLAGS=[f"-L{cpython_build.abspath}/lib"]) + + +### Fetch Python prebuild ### + + +CPYTHON_PREBUILD_URL = "https://github.com/indygreg/python-build-standalone/releases/download/20210506/cpython-3.9.5-aarch64-apple-darwin-pgo-20210506T0943.tar.zst" +cpython_prebuild_archive = env.Download( + target=File(CPYTHON_PREBUILD_URL.rsplit("/", 1)[1]), url=CPYTHON_PREBUILD_URL +) +env.NoClean(cpython_prebuild_archive) + + +### Extract prebuild ### + + +def extract_cpython_prebuild(target, source, env): + archive_path = source[0].abspath + target_path = target[0].abspath + with open(archive_path, "rb") as fh: + dctx = zstandard.ZstdDecompressor() + with dctx.stream_reader(fh) as reader: + with tarfile.open(mode="r|", fileobj=reader) as tf: + tf.extractall(target_path) + + +cpython_prebuild_src = env.Command( + Dir("cpython_prebuild"), cpython_prebuild_archive, extract_cpython_prebuild +) +env.NoClean(cpython_prebuild_src) + + +### Generate custom build from the prebuild ### + + +def generate_cpython_build(target, source, env): + build = Path(target[0].abspath) + prebuild = Path(source[0].abspath) / "python" + + conf = json.loads((prebuild / "PYTHON.json").read_text()) + assert conf["version"] == "5" + assert conf["libpython_link_mode"] == "shared" + assert conf["target_triple"] == "aarch64-apple-darwin" + + shutil.copytree(str(prebuild / "install"), str(build), symlinks=True) + shutil.copytree(str(prebuild / "licenses"), str(build / "licenses"), symlinks=True) + + shutil.rmtree(str(build / "share")) + + # Remove static library stuff + config = conf["python_stdlib_platform_config"] + assert config.startswith("install/lib/") + config = build / config[len("install/") :] + assert config.exists() + shutil.rmtree(str(config)) + + # Patch binaries to load libpython3.x.dylib with a relative path + # Lib paths are hardcoded into the executable, and if the lib is not found at the path, then it craps out. + # Unfortunately compiling python will hardcode the absolute path of libpython.dylib into the executable, + # so if you move it around it will break. + # the solution here is to modify the executable and make sure the lib path is not an absolute path, + # but an path relative to @loader_path, which is a special symbol that points to the executable. + # See: http://joaoventura.net/blog/2016/embeddable-python-osx-from-src/ + # and https://stackoverflow.com/questions/7880454/python-executable-not-finding-libpython-shared-library + prebuild_shared_lib_path = conf["build_info"]["core"]["shared_lib"] + path, _ = prebuild_shared_lib_path.rsplit("/", 1) + assert path == "install/lib" # Make sure libpython.so is on lib folder + binary = build / "bin/python3.9" + assert binary.is_file() + dylib = build / "lib/libpython3.9.dylib" + cmd = f"install_name_tool -id @rpath/{dylib.name} {dylib}" + subprocess.run(cmd.split(), check=True) + + stdlib_path = build / "lib/python3.9" + + # Remove tests lib (pretty big and basically useless) + shutil.rmtree(str(stdlib_path / "test")) + + # Also remove __pycache__ & .pyc stuff + for pycache in stdlib_path.glob("**/__pycache__"): + shutil.rmtree(str(pycache)) + + # Make sure site-packages is empty to avoid including pip (ensurepip should be used instead) + shutil.rmtree(str(stdlib_path / "site-packages")) + + # Zip the stdlib to save plenty of space \o/ + if env["compressed_stdlib"]: + tmp_stdlib_path = build / "lib/tmp_python3.9" + shutil.move(str(stdlib_path), str(tmp_stdlib_path)) + stdlib_path.mkdir() + shutil.move(str(tmp_stdlib_path / "lib-dynload"), str(stdlib_path / "lib-dynload")) + shutil.make_archive( + base_name=build / "lib/python39", format="zip", root_dir=str(tmp_stdlib_path) + ) + shutil.rmtree(str(tmp_stdlib_path)) + # Oddly enough, os.py must be present (even if empty !) otherwise + # Python failed to find it home... + (stdlib_path / "os.py").touch() + + (stdlib_path / "site-packages").mkdir() + + +env.Command(cpython_build, cpython_prebuild_src, generate_cpython_build) +env.NoClean(cpython_build) From 188ab7152b954b5fbd097b3b51d920271838e4ba Mon Sep 17 00:00:00 2001 From: Kara Todd Date: Fri, 14 Jan 2022 21:43:32 -0500 Subject: [PATCH 2/9] add to gdnlib --- misc/release_pythonscript.gdnlib | 1 + 1 file changed, 1 insertion(+) diff --git a/misc/release_pythonscript.gdnlib b/misc/release_pythonscript.gdnlib index 1510867c..8c88be9e 100644 --- a/misc/release_pythonscript.gdnlib +++ b/misc/release_pythonscript.gdnlib @@ -12,6 +12,7 @@ Server.64="res://addons/pythonscript/x11-64/libpythonscript.so" Windows.64="res://addons/pythonscript/windows-64/pythonscript.dll" Windows.32="res://addons/pythonscript/windows-32/pythonscript.dll" OSX.64="res://addons/pythonscript/osx-64/libpythonscript.dylib" +OSX.aarch64="res://addons/pythonscript/osx-arrch64/libpythonscript.dylib" [dependencies] From 25af56b5e19acc807c72aaffbda0dd6c87b3342e Mon Sep 17 00:00:00 2001 From: Kara Todd Date: Fri, 14 Jan 2022 21:47:15 -0500 Subject: [PATCH 3/9] whoops wrong tests --- .github/workflows/build.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0cce3d86..0259c801 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -173,7 +173,7 @@ jobs: macos-intel-build: - name: 'macOS/intel build' + name: '🍎 macOS/intel build' runs-on: macos-latest env: CC: clang @@ -205,10 +205,10 @@ jobs: run: | set -eux scons build -j2 - # - name: 'Run tests' - # run: | - # set -eux - # scons tests + - name: 'Run tests' + run: | + set -eux + scons tests - name: 'Generate artifact archive' run: | set -eux From c71ee07e2d570a8aafc7ff1d3f71f013497c772c Mon Sep 17 00:00:00 2001 From: Kara Todd Date: Fri, 14 Jan 2022 22:44:56 -0500 Subject: [PATCH 4/9] add platform to scons options --- README.rst | 2 +- SConstruct | 2 +- platforms/osx-aarch64/SConscript | 4 ++-- pythonscript/SConscript | 9 +++++++++ 4 files changed, 13 insertions(+), 4 deletions(-) diff --git a/README.rst b/README.rst index 2c0dd054..9da6abe7 100644 --- a/README.rst +++ b/README.rst @@ -235,7 +235,7 @@ For MacOS: godot-python(venv)$ scons platform=osx-64 CC=clang release -Valid platforms are `x11-64`, `x11-32`, `windows-64`, `windows-32` and `osx-64`. +Valid platforms are `x11-64`, `x11-32`, `windows-64`, `windows-32`, `osx-64`, and `osx-aarch64`. Check Travis or Appveyor links above to see the current status of your platform. This command will checkout CPython repo, move to a pinned commit and build diff --git a/SConstruct b/SConstruct index 69859bd6..a0c953f8 100644 --- a/SConstruct +++ b/SConstruct @@ -42,7 +42,7 @@ vars.Add( "platform", "Target platform", "", - allowed_values=("x11-64", "x11-32", "windows-64", "windows-32", "osx-64"), + allowed_values=("x11-64", "x11-32", "windows-64", "windows-32", "osx-64", "osx-aarch64"), ) ) vars.Add("pytest_args", "Pytest arguments passed to tests functions", "") diff --git a/platforms/osx-aarch64/SConscript b/platforms/osx-aarch64/SConscript index 881cfeab..1cc0ebd2 100644 --- a/platforms/osx-aarch64/SConscript +++ b/platforms/osx-aarch64/SConscript @@ -13,7 +13,7 @@ cpython_build = Dir("cpython_build") env["bits"] = "64" -env["godot_binary_download_platform"] = "osx.aarch64" +env["godot_binary_download_platform"] = "osx.64" env["godot_binary_download_zip_path"] = "Godot.app/Contents/MacOS/Godot" env["cpython_build"] = cpython_build env["cpython_build_dir"] = cpython_build @@ -68,7 +68,7 @@ def generate_cpython_build(target, source, env): prebuild = Path(source[0].abspath) / "python" conf = json.loads((prebuild / "PYTHON.json").read_text()) - assert conf["version"] == "5" + assert conf["version"] == "7" assert conf["libpython_link_mode"] == "shared" assert conf["target_triple"] == "aarch64-apple-darwin" diff --git a/pythonscript/SConscript b/pythonscript/SConscript index 00783e20..3a7779dd 100644 --- a/pythonscript/SConscript +++ b/pythonscript/SConscript @@ -4,6 +4,15 @@ c_env = env.Clone() if env["platform"].startswith("windows"): c_env.AppendUnique(LIBS=["python38"]) +elif env["platform"].startswith("osx-aarch"): + c_env.AppendUnique(LIBS=["python3.9"]) + # if we don't give the lib a proper install_name, macos won't be able to find it, + # and will link the cython modules with a relative path + c_env.AppendUnique( + LINKFLAGS=["-Wl,-rpath,'@loader_path/lib'", "-install_name", "@rpath/libpythonscript.dylib"] + ) + c_env.AppendUnique(CFLAGS=["-Werror-implicit-function-declaration"]) + elif env["platform"].startswith("osx"): c_env.AppendUnique(LIBS=["python3.8"]) # if we don't give the lib a proper install_name, macos won't be able to find it, From 955070d7968ad227b80b418838f639582e2b05bf Mon Sep 17 00:00:00 2001 From: Kara Todd Date: Tue, 18 Jan 2022 19:32:59 -0500 Subject: [PATCH 5/9] fix build issues --- .github/workflows/build.yml | 6 ++-- README.rst | 2 +- SConstruct | 3 +- platforms/{osx-aarch64 => osx}/SConscript | 41 ++++++++++++++++++++--- pythonscript/SConscript | 15 +++------ site_scons/site_tools/cython.py | 3 +- 6 files changed, 50 insertions(+), 20 deletions(-) rename platforms/{osx-aarch64 => osx}/SConscript (74%) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0259c801..108b69c4 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -228,7 +228,8 @@ jobs: runs-on: macos-latest env: CC: clang - PLATFORM: 'osx-64' + PLATFORM: 'osx' + ARCH: 'arm64' steps: - name: 'Checkout' uses: actions/checkout@f1d3225b5376a0791fdee5a0e8eac5289355e43a # pin@v2 @@ -251,6 +252,7 @@ jobs: # Configuration for scons echo 'godot_binary = "${{ env.GODOT_BINARY_VERSION }}"' >> custom.py echo 'platform = "${{ env.PLATFORM }}"' >> custom.py + echo 'arch = "${{ env.ARCH }}"' >> custom.py echo 'CC = "${{ env.CC }}"' >> custom.py - name: 'Build project' run: | @@ -267,7 +269,7 @@ jobs: - name: 'Export release artifact' uses: actions/upload-artifact@11830c9f4d30053679cb8904e3b3ce1b8c00bf40 # pin@v2 with: - name: ${{ env.PLATFORM }}-release + name: ${{ env.PLATFORM }}-{{ env.ARCH }}-release path: 'build/godot-python-*.tar.bz2' ################################################################################# diff --git a/README.rst b/README.rst index 9da6abe7..65e82121 100644 --- a/README.rst +++ b/README.rst @@ -235,7 +235,7 @@ For MacOS: godot-python(venv)$ scons platform=osx-64 CC=clang release -Valid platforms are `x11-64`, `x11-32`, `windows-64`, `windows-32`, `osx-64`, and `osx-aarch64`. +Valid platforms are `x11-64`, `x11-32`, `windows-64`, `windows-32`, `osx-64`, and `osx`. Check Travis or Appveyor links above to see the current status of your platform. This command will checkout CPython repo, move to a pinned commit and build diff --git a/SConstruct b/SConstruct index a0c953f8..62364cf0 100644 --- a/SConstruct +++ b/SConstruct @@ -42,9 +42,10 @@ vars.Add( "platform", "Target platform", "", - allowed_values=("x11-64", "x11-32", "windows-64", "windows-32", "osx-64", "osx-aarch64"), + allowed_values=("x11-64", "x11-32", "windows-64", "windows-32", "osx-64", "osx"), ) ) +vars.Add("arch", "Platform-dependent architecture (arm/arm64/x86/x64/mips/...)", allowed_values=("x86_64", "arm64")) vars.Add("pytest_args", "Pytest arguments passed to tests functions", "") vars.Add( "godot_args", "Additional arguments passed to godot binary when running tests&examples", "" diff --git a/platforms/osx-aarch64/SConscript b/platforms/osx/SConscript similarity index 74% rename from platforms/osx-aarch64/SConscript rename to platforms/osx/SConscript index 1cc0ebd2..20f601a6 100644 --- a/platforms/osx-aarch64/SConscript +++ b/platforms/osx/SConscript @@ -13,18 +13,43 @@ cpython_build = Dir("cpython_build") env["bits"] = "64" -env["godot_binary_download_platform"] = "osx.64" +env["godot_binary_download_platform"] = "osx.universal" env["godot_binary_download_zip_path"] = "Godot.app/Contents/MacOS/Godot" env["cpython_build"] = cpython_build env["cpython_build_dir"] = cpython_build env["DIST_SITE_PACKAGES"] = Dir(f"{env['DIST_PLATFORM']}/lib/python3.9/site-packages") - +env["CC"] = "clang" +env["CXX"] = "clang++" ### Build config for pythonscript ### - env.AppendUnique(CFLAGS=["-m64"]) env.AppendUnique(LINKFLAGS=["-m64"]) + +if env["arch"] == "arm64": + print("Building for macOS 10.15+, platform arm64.") + env.Append(CCFLAGS=["-arch", "arm64", "-mmacosx-version-min=10.15"]) + env.Append(LINKFLAGS=["-arch", "arm64", "-mmacosx-version-min=10.15"]) +else: + print("Building for macOS 10.12+, platform x86_64.") + env.Append(CCFLAGS=["-arch", "x86_64", "-mmacosx-version-min=10.12"]) + env.Append(LINKFLAGS=["-arch", "x86_64", "-mmacosx-version-min=10.12"]) + +try: + sdk_name = "macosx" + sdk_path = subprocess.check_output(["xcrun", "--sdk", sdk_name, "--show-sdk-path"]).strip().decode("utf-8") + if sdk_path: + env["MACOS_SDK_PATH"] = sdk_path +except (subprocess.CalledProcessError, OSError): + print("Failed to find SDK path while running xcrun --sdk {} --show-sdk-path.".format(sdk_name)) + raise + +env.Append(CPPFLAGS=['-isysroot', '$MACOS_SDK_PATH']) +env.Append(LINKFLAGS=['-isysroot', '$MACOS_SDK_PATH']) + +env.Append(CCFLAGS=["-isysroot", "$MACOS_SDK_PATH"]) +env.Append(LINKFLAGS=["-isysroot", "$MACOS_SDK_PATH"]) + # Cannot use CPPPATH&LIBPATH here given headers are within `cpython_build` target, # so Scons consider the headers are a missing target env.AppendUnique(CFLAGS=[f"-I{cpython_build.abspath}/include/python3.9/"]) @@ -33,8 +58,11 @@ env.AppendUnique(LINKFLAGS=[f"-L{cpython_build.abspath}/lib"]) ### Fetch Python prebuild ### +ARCH = "aarch64" if env["arch"] == "arm64" else "x86_64" +CPYTHON_DOWNLOADS_URL = "https://github.com/indygreg/python-build-standalone/releases/download" +CPYTHON_PREBUILD = f"20210506/cpython-3.9.5-{ARCH}-apple-darwin-pgo-20210506T0943.tar.zst" +CPYTHON_PREBUILD_URL = f"{CPYTHON_DOWNLOADS_URL}/{CPYTHON_PREBUILD}" -CPYTHON_PREBUILD_URL = "https://github.com/indygreg/python-build-standalone/releases/download/20210506/cpython-3.9.5-aarch64-apple-darwin-pgo-20210506T0943.tar.zst" cpython_prebuild_archive = env.Download( target=File(CPYTHON_PREBUILD_URL.rsplit("/", 1)[1]), url=CPYTHON_PREBUILD_URL ) @@ -70,7 +98,10 @@ def generate_cpython_build(target, source, env): conf = json.loads((prebuild / "PYTHON.json").read_text()) assert conf["version"] == "7" assert conf["libpython_link_mode"] == "shared" - assert conf["target_triple"] == "aarch64-apple-darwin" + if env["arch"] == "arm64": + assert conf["target_triple"] == "aarch64-apple-darwin" + else: + assert conf["target_triple"] == "x86_64-apple-darwin" shutil.copytree(str(prebuild / "install"), str(build), symlinks=True) shutil.copytree(str(prebuild / "licenses"), str(build / "licenses"), symlinks=True) diff --git a/pythonscript/SConscript b/pythonscript/SConscript index 3a7779dd..4503a99a 100644 --- a/pythonscript/SConscript +++ b/pythonscript/SConscript @@ -4,17 +4,12 @@ c_env = env.Clone() if env["platform"].startswith("windows"): c_env.AppendUnique(LIBS=["python38"]) -elif env["platform"].startswith("osx-aarch"): - c_env.AppendUnique(LIBS=["python3.9"]) - # if we don't give the lib a proper install_name, macos won't be able to find it, - # and will link the cython modules with a relative path - c_env.AppendUnique( - LINKFLAGS=["-Wl,-rpath,'@loader_path/lib'", "-install_name", "@rpath/libpythonscript.dylib"] - ) - c_env.AppendUnique(CFLAGS=["-Werror-implicit-function-declaration"]) - elif env["platform"].startswith("osx"): - c_env.AppendUnique(LIBS=["python3.8"]) + if env["arch"] == "arm64": + c_env.AppendUnique(LIBS=["python3.9"]) + else: + c_env.AppendUnique(LIBS=["python3.8"]) + # if we don't give the lib a proper install_name, macos won't be able to find it, # and will link the cython modules with a relative path c_env.AppendUnique( diff --git a/site_scons/site_tools/cython.py b/site_scons/site_tools/cython.py index eeb316b1..661035d3 100644 --- a/site_scons/site_tools/cython.py +++ b/site_scons/site_tools/cython.py @@ -91,6 +91,7 @@ def CythonCompile(env, target, source): f"-Wl,-rpath,'{loader_token}/{libpythonscript_path}'", ] # TODO: use scons `env.LoadableModule` for better macos support ? + version = "python3.9" if env["platform"] == 'osx' else "python3.8" ret = env.SharedLibrary( target=target, source=source, @@ -98,7 +99,7 @@ def CythonCompile(env, target, source): SHLIBSUFFIX=".so", CFLAGS=cflags, LINKFLAGS=[*linkflags, *env["LINKFLAGS"]], - LIBS=["python3.8", "pythonscript"], + LIBS=[version, "pythonscript"], # LIBS=[*env["CYTHON_LIBS"], *env["LIBS"]], # LIBPATH=[*env['CYTHON_LIBPATH'], *env['LIBPATH']] ) From e1e26e14024b8177fe903444cbad09685b100ff6 Mon Sep 17 00:00:00 2001 From: Kara Todd Date: Tue, 18 Jan 2022 19:42:17 -0500 Subject: [PATCH 6/9] update gdnlib --- misc/release_pythonscript.gdnlib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/release_pythonscript.gdnlib b/misc/release_pythonscript.gdnlib index 8c88be9e..54db81df 100644 --- a/misc/release_pythonscript.gdnlib +++ b/misc/release_pythonscript.gdnlib @@ -12,7 +12,7 @@ Server.64="res://addons/pythonscript/x11-64/libpythonscript.so" Windows.64="res://addons/pythonscript/windows-64/pythonscript.dll" Windows.32="res://addons/pythonscript/windows-32/pythonscript.dll" OSX.64="res://addons/pythonscript/osx-64/libpythonscript.dylib" -OSX.aarch64="res://addons/pythonscript/osx-arrch64/libpythonscript.dylib" +OSX="res://addons/pythonscript/osx/libpythonscript.dylib" [dependencies] From c3fdb23394e2e1bc5e96683c51dd09fdc321c669 Mon Sep 17 00:00:00 2001 From: Kara Todd Date: Tue, 18 Jan 2022 19:44:12 -0500 Subject: [PATCH 7/9] make consistent references --- pythonscript/SConscript | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pythonscript/SConscript b/pythonscript/SConscript index 4503a99a..23deeec8 100644 --- a/pythonscript/SConscript +++ b/pythonscript/SConscript @@ -5,7 +5,7 @@ if env["platform"].startswith("windows"): c_env.AppendUnique(LIBS=["python38"]) elif env["platform"].startswith("osx"): - if env["arch"] == "arm64": + if env["platform"] == "osx": c_env.AppendUnique(LIBS=["python3.9"]) else: c_env.AppendUnique(LIBS=["python3.8"]) From c2aa436d078cb75cfab55f3473e705dd1fe914ae Mon Sep 17 00:00:00 2001 From: Kara Todd Date: Tue, 18 Jan 2022 19:44:58 -0500 Subject: [PATCH 8/9] remove double flag -_- --- platforms/osx/SConscript | 3 --- 1 file changed, 3 deletions(-) diff --git a/platforms/osx/SConscript b/platforms/osx/SConscript index 20f601a6..6234afe9 100644 --- a/platforms/osx/SConscript +++ b/platforms/osx/SConscript @@ -47,9 +47,6 @@ except (subprocess.CalledProcessError, OSError): env.Append(CPPFLAGS=['-isysroot', '$MACOS_SDK_PATH']) env.Append(LINKFLAGS=['-isysroot', '$MACOS_SDK_PATH']) -env.Append(CCFLAGS=["-isysroot", "$MACOS_SDK_PATH"]) -env.Append(LINKFLAGS=["-isysroot", "$MACOS_SDK_PATH"]) - # Cannot use CPPPATH&LIBPATH here given headers are within `cpython_build` target, # so Scons consider the headers are a missing target env.AppendUnique(CFLAGS=[f"-I{cpython_build.abspath}/include/python3.9/"]) From ce11f879fd9a9f946c5e46f6ccf9bd02a0906c72 Mon Sep 17 00:00:00 2001 From: Kara Todd Date: Mon, 19 Sep 2022 22:31:39 -0400 Subject: [PATCH 9/9] change to osx-arm --- .github/workflows/build.yml | 171 ++++++++++++-------------- README.rst | 2 +- SConstruct | 2 +- platforms/{osx => osx-arm}/SConscript | 27 ++-- pythonscript/SConscript | 2 +- pythonscript/godot/SConscript | 3 +- site_scons/site_tools/cython.py | 2 +- tests/bindings/pythonscript.gdnlib | 2 + 8 files changed, 97 insertions(+), 114 deletions(-) rename platforms/{osx => osx-arm}/SConscript (87%) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 108b69c4..3631da52 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -2,31 +2,27 @@ name: CI build on: push: branches: - - master + - master pull_request: branches: - - master - + - master # Global Settings env: PYTHON_VERSION: "3.7" GODOT_BINARY_VERSION: "3.2.3" - jobs: - - static-checks: - name: '📊 Static checks' + name: "📊 Static checks" runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@f1d3225b5376a0791fdee5a0e8eac5289355e43a # pin@v2 + uses: actions/checkout@f1d3225b5376a0791fdee5a0e8eac5289355e43a # pin@v2 with: - submodules: true + submodules: true - name: Set up Python - uses: actions/setup-python@0291cefc54fa79cd1986aee8fa5ecb89ad4defea # pin@v2 + uses: actions/setup-python@0291cefc54fa79cd1986aee8fa5ecb89ad4defea # pin@v2 with: python-version: ${{ env.PYTHON_VERSION }} - name: Bootstrap @@ -38,26 +34,24 @@ jobs: run: | pre-commit run --all-files --show-diff-on-failure - -################################################################################# - + ################################################################################# linux-build: - name: '🐧 Linux build' + name: "🐧 Linux build" runs-on: ubuntu-latest env: CC: clang - PLATFORM: 'x11-64' + PLATFORM: "x11-64" steps: - - name: 'Checkout' - uses: actions/checkout@f1d3225b5376a0791fdee5a0e8eac5289355e43a # pin@v2 + - name: "Checkout" + uses: actions/checkout@f1d3225b5376a0791fdee5a0e8eac5289355e43a # pin@v2 with: - submodules: true - - name: 'Set up Python' - uses: actions/setup-python@0291cefc54fa79cd1986aee8fa5ecb89ad4defea # pin@v2 + submodules: true + - name: "Set up Python" + uses: actions/setup-python@0291cefc54fa79cd1986aee8fa5ecb89ad4defea # pin@v2 with: python-version: ${{ env.PYTHON_VERSION }} - - name: 'Setup venv' + - name: "Setup venv" run: | set -eux ${{ env.CC }} --version @@ -68,57 +62,55 @@ jobs: echo 'godot_binary = "${{ env.GODOT_BINARY_VERSION }}"' >> custom.py echo 'platform = "${{ env.PLATFORM }}"' >> custom.py echo 'CC = "${{ env.CC }}"' >> custom.py - - name: 'Build project' + - name: "Build project" run: | set -eux scons build -j2 - - name: 'Start xvfb' + - name: "Start xvfb" run: | /usr/bin/Xvfb :99 -screen 0 1024x768x24 > /dev/null 2>&1 & echo ">>> Started xvfb" - - name: 'Run tests' + - name: "Run tests" run: | set -eux scons tests headless=true env: - DISPLAY: ':99.0' - - name: 'Generate artifact archive' + DISPLAY: ":99.0" + - name: "Generate artifact archive" run: | set -eux scons release - - name: 'Export release artifact' - uses: actions/upload-artifact@11830c9f4d30053679cb8904e3b3ce1b8c00bf40 # pin@v2 + - name: "Export release artifact" + uses: actions/upload-artifact@11830c9f4d30053679cb8904e3b3ce1b8c00bf40 # pin@v2 with: name: ${{ env.PLATFORM }}-release - path: 'build/godot-python-*.tar.bz2' - - -################################################################################# + path: "build/godot-python-*.tar.bz2" + ################################################################################# windows-build: - name: '🏁 Windows build' + name: "🏁 Windows build" runs-on: windows-latest strategy: matrix: include: - - PLATFORM: 'windows-64' - PYTHON_ARCH: 'x64' - VS_ARCH: 'amd64' - - PLATFORM: 'windows-32' - PYTHON_ARCH: 'x86' - VS_ARCH: 'x86' + - PLATFORM: "windows-64" + PYTHON_ARCH: "x64" + VS_ARCH: "amd64" + - PLATFORM: "windows-32" + PYTHON_ARCH: "x86" + VS_ARCH: "x86" steps: - - name: 'Checkout' - uses: actions/checkout@f1d3225b5376a0791fdee5a0e8eac5289355e43a # pin@v2 + - name: "Checkout" + uses: actions/checkout@f1d3225b5376a0791fdee5a0e8eac5289355e43a # pin@v2 with: - submodules: true - - name: 'Set up Python' - uses: actions/setup-python@0291cefc54fa79cd1986aee8fa5ecb89ad4defea # pin@v2 + submodules: true + - name: "Set up Python" + uses: actions/setup-python@0291cefc54fa79cd1986aee8fa5ecb89ad4defea # pin@v2 with: python-version: ${{ env.PYTHON_VERSION }} architecture: ${{ matrix.PYTHON_ARCH }} - - name: 'Setup venv' + - name: "Setup venv" shell: bash run: | set -eux @@ -131,12 +123,12 @@ jobs: echo 'MSVC_USE_SCRIPT = True' >> custom.py echo 'TARGET_ARCH = "${{ matrix.VS_ARCH }}"' >> custom.py echo 'CC = "cl.exe"' >> custom.py - - name: 'Build project' + - name: "Build project" shell: bash run: | set -eux scons build -j2 - - name: 'Install Mesa3D OpenGL' + - name: "Install Mesa3D OpenGL" shell: bash run: | set -eux @@ -153,41 +145,39 @@ jobs: 7z.exe x mesa.7z ls -lh opengl32.dll # Sanity check popd - - name: 'Run tests' + - name: "Run tests" shell: bash run: | set -eux scons tests - - name: 'Generate artifact archive' + - name: "Generate artifact archive" shell: bash run: | scons release - - name: 'Export release artifact' - uses: actions/upload-artifact@11830c9f4d30053679cb8904e3b3ce1b8c00bf40 # pin@v2 + - name: "Export release artifact" + uses: actions/upload-artifact@11830c9f4d30053679cb8904e3b3ce1b8c00bf40 # pin@v2 with: name: ${{ matrix.PLATFORM }}-release - path: 'build/godot-python-*.zip' - - -################################################################################# + path: "build/godot-python-*.zip" + ################################################################################# macos-intel-build: - name: '🍎 macOS/intel build' + name: "🍎 macOS/intel build" runs-on: macos-latest env: CC: clang - PLATFORM: 'osx-64' + PLATFORM: "osx-64" steps: - - name: 'Checkout' - uses: actions/checkout@f1d3225b5376a0791fdee5a0e8eac5289355e43a # pin@v2 + - name: "Checkout" + uses: actions/checkout@f1d3225b5376a0791fdee5a0e8eac5289355e43a # pin@v2 with: - submodules: true - - name: 'Set up Python' - uses: actions/setup-python@0291cefc54fa79cd1986aee8fa5ecb89ad4defea # pin@v2 + submodules: true + - name: "Set up Python" + uses: actions/setup-python@0291cefc54fa79cd1986aee8fa5ecb89ad4defea # pin@v2 with: python-version: ${{ env.PYTHON_VERSION }} - - name: 'Setup venv' + - name: "Setup venv" run: | set -eux ${{ env.CC }} --version @@ -201,45 +191,43 @@ jobs: echo 'godot_binary = "${{ env.GODOT_BINARY_VERSION }}"' >> custom.py echo 'platform = "${{ env.PLATFORM }}"' >> custom.py echo 'CC = "${{ env.CC }}"' >> custom.py - - name: 'Build project' + - name: "Build project" run: | set -eux scons build -j2 - - name: 'Run tests' + - name: "Run tests" run: | set -eux scons tests - - name: 'Generate artifact archive' + - name: "Generate artifact archive" run: | set -eux scons release - - name: 'Export release artifact' - uses: actions/upload-artifact@11830c9f4d30053679cb8904e3b3ce1b8c00bf40 # pin@v2 + - name: "Export release artifact" + uses: actions/upload-artifact@11830c9f4d30053679cb8904e3b3ce1b8c00bf40 # pin@v2 with: name: ${{ env.PLATFORM }}-release - path: 'build/godot-python-*.tar.bz2' - - -################################################################################# + path: "build/godot-python-*.tar.bz2" + ################################################################################# macos-m1-build: - name: '🍏 macOS/m1 build' + name: "🍏 macOS/m1 build" runs-on: macos-latest env: CC: clang - PLATFORM: 'osx' - ARCH: 'arm64' + PLATFORM: "osx-arm" + ARCH: "arm64" steps: - - name: 'Checkout' - uses: actions/checkout@f1d3225b5376a0791fdee5a0e8eac5289355e43a # pin@v2 + - name: "Checkout" + uses: actions/checkout@f1d3225b5376a0791fdee5a0e8eac5289355e43a # pin@v2 with: - submodules: true - - name: 'Set up Python' - uses: actions/setup-python@0291cefc54fa79cd1986aee8fa5ecb89ad4defea # pin@v2 + submodules: true + - name: "Set up Python" + uses: actions/setup-python@0291cefc54fa79cd1986aee8fa5ecb89ad4defea # pin@v2 with: python-version: ${{ env.PYTHON_VERSION }} - - name: 'Setup venv' + - name: "Setup venv" run: | set -eux ${{ env.CC }} --version @@ -254,7 +242,7 @@ jobs: echo 'platform = "${{ env.PLATFORM }}"' >> custom.py echo 'arch = "${{ env.ARCH }}"' >> custom.py echo 'CC = "${{ env.CC }}"' >> custom.py - - name: 'Build project' + - name: "Build project" run: | set -eux scons build -j2 @@ -262,21 +250,20 @@ jobs: # run: | # set -eux # scons tests - - name: 'Generate artifact archive' + - name: "Generate artifact archive" run: | set -eux scons release - - name: 'Export release artifact' - uses: actions/upload-artifact@11830c9f4d30053679cb8904e3b3ce1b8c00bf40 # pin@v2 + - name: "Export release artifact" + uses: actions/upload-artifact@11830c9f4d30053679cb8904e3b3ce1b8c00bf40 # pin@v2 with: name: ${{ env.PLATFORM }}-{{ env.ARCH }}-release - path: 'build/godot-python-*.tar.bz2' - -################################################################################# + path: "build/godot-python-*.tar.bz2" + ################################################################################# publish-release: - name: 'Publish ${{ matrix.PLATFORM }} release' + name: "Publish ${{ matrix.PLATFORM }} release" if: startsWith(github.ref, 'refs/tags/v') runs-on: ubuntu-latest needs: @@ -291,10 +278,10 @@ jobs: - PLATFORM: windows-32 - PLATFORM: osx-64 steps: - - uses: actions/download-artifact@0ede0875b5db9a2824878bbbbe3d758a75eb8268 # pin@v2 + - uses: actions/download-artifact@0ede0875b5db9a2824878bbbbe3d758a75eb8268 # pin@v2 name: ${{ matrix.PLATFORM }}-release - - name: 'Upload release' - uses: svenstaro/upload-release-action@483c1e56f95e88835747b1c7c60581215016cbf2 # pin@v2 + - name: "Upload release" + uses: svenstaro/upload-release-action@483c1e56f95e88835747b1c7c60581215016cbf2 # pin@v2 with: repo_token: ${{ secrets.GITHUB_TOKEN }} tag: ${{ github.ref }} diff --git a/README.rst b/README.rst index cc81c57c..85aa22b6 100644 --- a/README.rst +++ b/README.rst @@ -244,7 +244,7 @@ For MacOS: godot-python(venv)$ scons platform=osx-64 CC=clang release -Valid platforms are `x11-64`, `x11-32`, `windows-64`, `windows-32`, `osx-64`, and `osx`. +Valid platforms are `x11-64`, `x11-32`, `windows-64`, `windows-32`, `osx-64`, and `osx-arm`. Check Travis or Appveyor links above to see the current status of your platform. This command will checkout CPython repo, move to a pinned commit and build diff --git a/SConstruct b/SConstruct index 62364cf0..411b0009 100644 --- a/SConstruct +++ b/SConstruct @@ -42,7 +42,7 @@ vars.Add( "platform", "Target platform", "", - allowed_values=("x11-64", "x11-32", "windows-64", "windows-32", "osx-64", "osx"), + allowed_values=("x11-64", "x11-32", "windows-64", "windows-32", "osx-64", "osx-arm"), ) ) vars.Add("arch", "Platform-dependent architecture (arm/arm64/x86/x64/mips/...)", allowed_values=("x86_64", "arm64")) diff --git a/platforms/osx/SConscript b/platforms/osx-arm/SConscript similarity index 87% rename from platforms/osx/SConscript rename to platforms/osx-arm/SConscript index 6234afe9..b4f4c7c7 100644 --- a/platforms/osx/SConscript +++ b/platforms/osx-arm/SConscript @@ -1,3 +1,4 @@ +import os import zstandard import tarfile import json @@ -13,27 +14,21 @@ cpython_build = Dir("cpython_build") env["bits"] = "64" +env["godot_binary_download_version"] = ('3', '3', '1', 'stable') env["godot_binary_download_platform"] = "osx.universal" env["godot_binary_download_zip_path"] = "Godot.app/Contents/MacOS/Godot" env["cpython_build"] = cpython_build env["cpython_build_dir"] = cpython_build env["DIST_SITE_PACKAGES"] = Dir(f"{env['DIST_PLATFORM']}/lib/python3.9/site-packages") -env["CC"] = "clang" -env["CXX"] = "clang++" ### Build config for pythonscript ### env.AppendUnique(CFLAGS=["-m64"]) env.AppendUnique(LINKFLAGS=["-m64"]) -if env["arch"] == "arm64": - print("Building for macOS 10.15+, platform arm64.") - env.Append(CCFLAGS=["-arch", "arm64", "-mmacosx-version-min=10.15"]) - env.Append(LINKFLAGS=["-arch", "arm64", "-mmacosx-version-min=10.15"]) -else: - print("Building for macOS 10.12+, platform x86_64.") - env.Append(CCFLAGS=["-arch", "x86_64", "-mmacosx-version-min=10.12"]) - env.Append(LINKFLAGS=["-arch", "x86_64", "-mmacosx-version-min=10.12"]) +print("Building for macOS 10.15+, platform arm64.") +env.Append(CCFLAGS=["-arch", "arm64", "-mmacosx-version-min=10.15"]) +env.Append(LINKFLAGS=["-arch", "arm64", "-mmacosx-version-min=10.15"]) try: sdk_name = "macosx" @@ -54,10 +49,8 @@ env.AppendUnique(LINKFLAGS=[f"-L{cpython_build.abspath}/lib"]) ### Fetch Python prebuild ### - -ARCH = "aarch64" if env["arch"] == "arm64" else "x86_64" CPYTHON_DOWNLOADS_URL = "https://github.com/indygreg/python-build-standalone/releases/download" -CPYTHON_PREBUILD = f"20210506/cpython-3.9.5-{ARCH}-apple-darwin-pgo-20210506T0943.tar.zst" +CPYTHON_PREBUILD = f"20210506/cpython-3.9.5-aarch64-apple-darwin-pgo-20210506T0943.tar.zst" CPYTHON_PREBUILD_URL = f"{CPYTHON_DOWNLOADS_URL}/{CPYTHON_PREBUILD}" cpython_prebuild_archive = env.Download( @@ -92,13 +85,13 @@ def generate_cpython_build(target, source, env): build = Path(target[0].abspath) prebuild = Path(source[0].abspath) / "python" + # verify correct version conf = json.loads((prebuild / "PYTHON.json").read_text()) assert conf["version"] == "7" assert conf["libpython_link_mode"] == "shared" - if env["arch"] == "arm64": - assert conf["target_triple"] == "aarch64-apple-darwin" - else: - assert conf["target_triple"] == "x86_64-apple-darwin" + + # verify correct triple-target + assert conf["target_triple"] == "aarch64-apple-darwin" shutil.copytree(str(prebuild / "install"), str(build), symlinks=True) shutil.copytree(str(prebuild / "licenses"), str(build / "licenses"), symlinks=True) diff --git a/pythonscript/SConscript b/pythonscript/SConscript index 23deeec8..de4bc66b 100644 --- a/pythonscript/SConscript +++ b/pythonscript/SConscript @@ -5,7 +5,7 @@ if env["platform"].startswith("windows"): c_env.AppendUnique(LIBS=["python38"]) elif env["platform"].startswith("osx"): - if env["platform"] == "osx": + if env["platform"] == "osx-arm": c_env.AppendUnique(LIBS=["python3.9"]) else: c_env.AppendUnique(LIBS=["python3.8"]) diff --git a/pythonscript/godot/SConscript b/pythonscript/godot/SConscript index 8db5f212..4245c71b 100644 --- a/pythonscript/godot/SConscript +++ b/pythonscript/godot/SConscript @@ -63,7 +63,8 @@ sample = env["bindings_generate_sample"] # dont strip on debug builds if not sample and not env["debug"]: if env["CC_IS_GCC"]: - bindings_env.AppendUnique(LINKFLAGS=["-Wl,--strip-all"]) + pass + # bindings_env.AppendUnique(LINKFLAGS=["-Wl,--strip-all"]) elif env["CC_IS_CLANG"]: bindings_env.AppendUnique(LINKFLAGS=["-Wl,-s"]) diff --git a/site_scons/site_tools/cython.py b/site_scons/site_tools/cython.py index 661035d3..4c7bee3c 100644 --- a/site_scons/site_tools/cython.py +++ b/site_scons/site_tools/cython.py @@ -91,7 +91,7 @@ def CythonCompile(env, target, source): f"-Wl,-rpath,'{loader_token}/{libpythonscript_path}'", ] # TODO: use scons `env.LoadableModule` for better macos support ? - version = "python3.9" if env["platform"] == 'osx' else "python3.8" + version = "python3.9" if env["platform"] == 'osx-arm' else "python3.8" ret = env.SharedLibrary( target=target, source=source, diff --git a/tests/bindings/pythonscript.gdnlib b/tests/bindings/pythonscript.gdnlib index 1510867c..7e0c5c68 100644 --- a/tests/bindings/pythonscript.gdnlib +++ b/tests/bindings/pythonscript.gdnlib @@ -12,6 +12,7 @@ Server.64="res://addons/pythonscript/x11-64/libpythonscript.so" Windows.64="res://addons/pythonscript/windows-64/pythonscript.dll" Windows.32="res://addons/pythonscript/windows-32/pythonscript.dll" OSX.64="res://addons/pythonscript/osx-64/libpythonscript.dylib" +OSX.universal="res://addons/pythonscript/osx-arm/libpythonscript.dylib" [dependencies] @@ -21,3 +22,4 @@ Server.64=[] Windows.64=[] Windows.32=[] OSX.64=[] +OSX.universal=[]