From a3d5966718677385e739f0b23e9bd29bcb62af92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mos=C3=A8=20Giordano?= Date: Fri, 15 Mar 2024 10:40:20 +0000 Subject: [PATCH 1/4] Build the libpoplar wrapper in CI --- .github/workflows/build_libpoplar.yml | 58 +++++++++++++++++++++++++++ .github/workflows/ci.yml | 2 +- deps/Project.toml | 16 ++++++++ deps/bb_recipe/Project.toml | 3 ++ deps/bb_recipe/build_tarballs.jl | 58 +++++++++++++++++++++++++++ 5 files changed, 136 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/build_libpoplar.yml create mode 100644 deps/Project.toml create mode 100644 deps/bb_recipe/Project.toml create mode 100644 deps/bb_recipe/build_tarballs.jl diff --git a/.github/workflows/build_libpoplar.yml b/.github/workflows/build_libpoplar.yml new file mode 100644 index 0000000..8ffb02e --- /dev/null +++ b/.github/workflows/build_libpoplar.yml @@ -0,0 +1,58 @@ +name: Build libpoplar wrapper + +on: + push: + branches: "main" + tags: ["*"] + pull_request: + release: + +jobs: + build-libpoplar: + runs-on: ubuntu-latest + container: + image: ${{ matrix.docker_image }} + options: --privileged + strategy: + fail-fast: false + matrix: + include: + - docker_image: graphcore/poplar:3.2.0-ubuntu-20.04-20230314 + steps: + - name: "Install, g++, jq" + # g++ is needed to build the wrapper, jq for + # the `julia-actions/cache` workflow + run: | + apt-get update -y + apt-get install -y g++ jq + env: + DEBIAN_FRONTEND: noninteractive + - uses: actions/checkout@v4 + - uses: julia-actions/setup-julia@v1 + with: + version: "1.7" + - uses: julia-actions/cache@v1 + - name: Instantiate wrapper generation environment + working-directory: ${{ github.workspace }}/deps + run: | + julia --project=. --color=yes -e 'using Pkg; Pkg.instantiate(); Pkg.precompile()' + - name: Generate source code of libpoplar wrapper + working-directory: ${{ github.workspace }}/deps + shell: julia --project=. --color=yes {0} + run: | + include(joinpath(pwd(), "generate_wrapper.jl")) + generate_wrapper() + env: + CXX: g++ + - name: Instantiate build environment + working-directory: ${{ github.workspace }}/deps/bb_recipe + run: | + julia --project=. --color=yes -e 'using Pkg; Pkg.instantiate(); Pkg.precompile()' + - name: Compile libpoplar wrapper + working-directory: ${{ github.workspace }}/deps/bb_recipe + run: | + julia --project=. --color=yes build_tarballs.jl --verbose + - uses: actions/upload-artifact@v4 + with: + name: products + path: ${{ github.workspace }}/deps/bb_recipe/products diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 86d4df4..19a169f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,7 +4,7 @@ on: push: branches: "main" tags: ["*"] - pull_request: + # pull_request: release: concurrency: diff --git a/deps/Project.toml b/deps/Project.toml new file mode 100644 index 0000000..21edc42 --- /dev/null +++ b/deps/Project.toml @@ -0,0 +1,16 @@ +[deps] +Clang = "40e3b903-d033-50b4-a0cc-940c62c95e31" +JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" +Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" +Scratch = "6c6a2e73-6563-6170-7368-637461726353" +UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" +libcxxwrap_julia_jll = "3eaa8342-bff7-56a5-9981-c04077f7cee7" + +[compat] +Clang = "0.14, 0.17.4" +JSON = "0.21.4" +Pkg = "1" +Scratch = "1.1" +UUIDs = "1" +libcxxwrap_julia_jll = "0.11" +julia = "1.7" diff --git a/deps/bb_recipe/Project.toml b/deps/bb_recipe/Project.toml new file mode 100644 index 0000000..c70d458 --- /dev/null +++ b/deps/bb_recipe/Project.toml @@ -0,0 +1,3 @@ +[deps] +BinaryBuilder = "12aac903-9f7c-5d81-afc2-d9565ea332ae" +Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" diff --git a/deps/bb_recipe/build_tarballs.jl b/deps/bb_recipe/build_tarballs.jl new file mode 100644 index 0000000..566d8f1 --- /dev/null +++ b/deps/bb_recipe/build_tarballs.jl @@ -0,0 +1,58 @@ +using BinaryBuilder +using Pkg + +name = "libpoplar_wrapper" +version = v"1.0" + +# `POPLAR_SDK_ENABLED` should point to where the Poplar SDK is installed, +# otherwise fallback to `/opt/poplar` (default dir in Graphcore Docker +# containers). +poplar_dir = get(ENV, "POPLAR_SDK_ENABLED", "/opt/poplar") + +sources = [ + # The generated wrappers + DirectorySource("../wrapper"), + DirectorySource(dirname(poplar_dir)), +] + +script = "POPLAR_DIR=$(basename(poplar_dir))\n" * raw""" +cd ${WORKSPACE}/srcdir/ + +mkdir -p "${libdir}" + +c++ \ + -O0 \ + -std=c++17 \ + -fPIC \ + -shared \ + -I${POPLAR_DIR}/include \ + -I${includedir} \ + -I${includedir}/julia \ + -I. \ + -o "${libdir}/libpoplar_julia.${dlext}" \ + template.cpp \ + -L${POPLAR_DIR}/lib \ + -L${libdir} \ + -lpopops \ + -lpoplar \ + -lcxxwrap_julia \ + -ljulia + +install_license /usr/share/licenses/MIT +""" + +platforms = [ + Platform("x86_64", "linux"; libc="glibc", cxxstring_abi="cxx11", julia_version="1.10.0"), +] + +products = [ + LibraryProduct("libpoplar_julia", :libpoplar_julia; dont_dlopen=true), +] + +dependencies = [ + BuildDependency(PackageSpec(;name="libjulia_jll", version=v"1.10.9")), + Dependency("libcxxwrap_julia_jll"; compat = "~0.11.2"), +] + +build_tarballs(ARGS, name, version, sources, script, platforms, products, dependencies; + julia_compat="1.9", preferred_gcc_version=v"10", dont_dlopen=true) From b3d12390ff6335d8c0288dbf0ded862da8e821af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mos=C3=A8=20Giordano?= Date: Tue, 9 Apr 2024 02:45:03 +0100 Subject: [PATCH 2/4] Build also for Poplar SDK v3.3 --- .github/workflows/build_libpoplar.yml | 18 +++++++++++++++++- deps/Project.toml | 2 +- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_libpoplar.yml b/.github/workflows/build_libpoplar.yml index 8ffb02e..9af5091 100644 --- a/.github/workflows/build_libpoplar.yml +++ b/.github/workflows/build_libpoplar.yml @@ -18,6 +18,9 @@ jobs: matrix: include: - docker_image: graphcore/poplar:3.2.0-ubuntu-20.04-20230314 + artifact_name: poplar-3.2.0 + - docker_image: graphcore/poplar:3.3.0-ubuntu-20.04-20230703 + artifact_name: poplar-3.3.0 steps: - name: "Install, g++, jq" # g++ is needed to build the wrapper, jq for @@ -32,6 +35,19 @@ jobs: with: version: "1.7" - uses: julia-actions/cache@v1 + - name: "Export environment variables" + # Starting from graphcore/poplar:3.3.0 some environment variables like `CPATH` and + # `LD_LIBRARY_PATH`, which we need for building the bindings, are only set with the + # `bash` shell, while GHA uses by default `sh` for steps inside containers. So we + # need to start a step with `shell: bash` and re-export the relevant variables for + # the following steps. + shell: bash + run: | + echo "PATH=${PATH}" >> "${GITHUB_ENV}" + echo "CPATH=${CPATH}" >> "${GITHUB_ENV}" + echo "LIBRARY_PATH=${LIBRARY_PATH}" >> "${GITHUB_ENV}" + echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH}" >> "${GITHUB_ENV}" + echo "POPLAR_SDK_ENABLED=${POPLAR_SDK_ENABLED}" >> "${GITHUB_ENV}" - name: Instantiate wrapper generation environment working-directory: ${{ github.workspace }}/deps run: | @@ -54,5 +70,5 @@ jobs: julia --project=. --color=yes build_tarballs.jl --verbose - uses: actions/upload-artifact@v4 with: - name: products + name: products-${{ matrix.artifact_name }} path: ${{ github.workspace }}/deps/bb_recipe/products diff --git a/deps/Project.toml b/deps/Project.toml index 21edc42..e5949b7 100644 --- a/deps/Project.toml +++ b/deps/Project.toml @@ -7,7 +7,7 @@ UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" libcxxwrap_julia_jll = "3eaa8342-bff7-56a5-9981-c04077f7cee7" [compat] -Clang = "0.14, 0.17.4" +Clang = "0.14, 0.17.4, 0.18" JSON = "0.21.4" Pkg = "1" Scratch = "1.1" From 6b8e15ab736cd075ae6c7642df6766b978501262 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mos=C3=A8=20Giordano?= Date: Tue, 9 Apr 2024 17:35:48 +0100 Subject: [PATCH 3/4] Build library for different versions of Julia --- deps/bb_recipe/build_tarballs.jl | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/deps/bb_recipe/build_tarballs.jl b/deps/bb_recipe/build_tarballs.jl index 566d8f1..e6d5cb5 100644 --- a/deps/bb_recipe/build_tarballs.jl +++ b/deps/bb_recipe/build_tarballs.jl @@ -41,10 +41,18 @@ c++ \ install_license /usr/share/licenses/MIT """ -platforms = [ - Platform("x86_64", "linux"; libc="glibc", cxxstring_abi="cxx11", julia_version="1.10.0"), +julia_versions = if VersionNumber(ENV["SDK_VERSION"]) < v"3.3" + ["1.9.0", "1.10.0"] +else + ["1.11.0"] +end + +base_platforms = [ + Platform("x86_64", "linux"; libc="glibc", cxxstring_abi="cxx11") ] +platforms = vcat([(p = deepcopy(platform); p["julia_version"] = julia_version; p) for platform in base_platforms, julia_version in julia_versions]...) + products = [ LibraryProduct("libpoplar_julia", :libpoplar_julia; dont_dlopen=true), ] From dd2d3c5675a4dc524e4b0f37b8540e403d9d4496 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mos=C3=A8=20Giordano?= Date: Mon, 1 Sep 2025 00:58:11 +0100 Subject: [PATCH 4/4] [CI] Update workflows and Docker images --- .github/workflows/build_libpoplar.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build_libpoplar.yml b/.github/workflows/build_libpoplar.yml index 9af5091..9d8e848 100644 --- a/.github/workflows/build_libpoplar.yml +++ b/.github/workflows/build_libpoplar.yml @@ -17,9 +17,9 @@ jobs: fail-fast: false matrix: include: - - docker_image: graphcore/poplar:3.2.0-ubuntu-20.04-20230314 + - docker_image: ghcr.io/juliaipu/poplar:3.2.0-ubuntu-20.04-20230314 artifact_name: poplar-3.2.0 - - docker_image: graphcore/poplar:3.3.0-ubuntu-20.04-20230703 + - docker_image: ghcr.io/juliaipu/poplar:3.3.0-ubuntu-20.04-20230703 artifact_name: poplar-3.3.0 steps: - name: "Install, g++, jq" @@ -30,11 +30,11 @@ jobs: apt-get install -y g++ jq env: DEBIAN_FRONTEND: noninteractive - - uses: actions/checkout@v4 - - uses: julia-actions/setup-julia@v1 + - uses: actions/checkout@v5 + - uses: julia-actions/setup-julia@v2 with: version: "1.7" - - uses: julia-actions/cache@v1 + - uses: julia-actions/cache@v2 - name: "Export environment variables" # Starting from graphcore/poplar:3.3.0 some environment variables like `CPATH` and # `LD_LIBRARY_PATH`, which we need for building the bindings, are only set with the