Add support for macos arm64 build (#144)

* various fixes on the package index generation

* uniform `DistTask.yml` to the assets repository:
- add macos arm64
- differentiate linux arm into linux armv6, and linux armv7

* uniform `publish-go-tester-task` to the assets repo:
- add macos arm64
- differentiate linux arm into linux armv6, and linux armv7
- bump version of GO to 1.17
- bump upload/artifact action from 2 to 3
- add checksum upload
- remove unnecessary line continuation escaping from workflow

* uniform `release-go-crosscompile-task` to the assets repo:
- add macos arm64
- differentiate linux arm into linux armv6, and linux armv7
- bump version of GO to 1.17
- bump upload/artifact action from 2 to 3
- add checksum upload
- remove unnecessary line continuation escaping from workflow
- rename workflow
- remove `gon.config.hcl` since now is hardcoded in the workflow

* add support for darwin arm64 package index generation

* update doc

* disable s3 upload (for testing only)

* Revert "disable s3 upload (for testing only)"

This reverts commit 5ddd847.

Author: umbynos

.github/workflows/publish-go-tester-task.yml

@@ -1,14 +1,7 @@
 # Source: https://github.com/arduino/tooling-project-assets/blob/main/workflow-templates/publish-go-tester-task.md
 name: Publish Tester Build
 
-env:
-  # See: https://github.com/actions/setup-go/tree/v2#readme
-  GO_VERSION: ^1.16.2
-  # As defined by the Taskfile's DIST_DIR variable
-  DIST_DIR: dist
-  BUILDS_ARTIFACT: build-artifacts
-
-# See: https://docs.github.com/en/free-pro-team@latest/actions/reference/events-that-trigger-workflows
+# See: https://docs.github.com/actions/using-workflows/events-that-trigger-workflows
 on:
   create:
   push:
@@ -30,6 +23,13 @@ on:
   workflow_dispatch:
   repository_dispatch:
 
+env:
+  # See: https://github.com/actions/setup-go/tree/main#supported-version-syntax
+  GO_VERSION: ^1.16.2
+  # As defined by the Taskfile's DIST_DIR variable
+  DIST_DIR: dist
+  BUILDS_ARTIFACT: build-artifacts
+
 jobs:
   run-determination:
     runs-on: ubuntu-latest
@@ -41,9 +41,9 @@ jobs:
         run: |
           RELEASE_BRANCH_REGEX="refs/heads/[0-9]+.[0-9]+.x"
           # The `create` event trigger doesn't support `branches` filters, so it's necessary to use Bash instead.
-          if [[ \
-            "${{ github.event_name }}" != "create" || \
-            "${{ github.ref }}" =~ $RELEASE_BRANCH_REGEX \
+          if [[
+            "${{ github.event_name }}" != "create" ||
+            "${{ github.ref }}" =~ $RELEASE_BRANCH_REGEX
           ]]; then
             # Run the other jobs.
             RESULT="true"
@@ -86,7 +86,7 @@ jobs:
 
       # Transfer builds to artifacts job
       - name: Upload combined builds artifact
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         with:
           path: ${{ env.DIST_DIR }}
           name: ${{ env.BUILDS_ARTIFACT }}
@@ -99,16 +99,22 @@ jobs:
     strategy:
       matrix:
         artifact:
+          - path: "*checksums.txt"
+            name: checksums
           - path: "*Linux_32bit.tar.gz"
             name: Linux_X86-32
           - path: "*Linux_64bit.tar.gz"
             name: Linux_X86-64
-          - path: "*Linux_ARM.tar.gz"
-            name: Linux_ARM
           - path: "*Linux_ARM64.tar.gz"
             name: Linux_ARM64
+          - path: "*Linux_ARMv6.tar.gz"
+            name: Linux_ARMv6
+          - path: "*Linux_ARMv7.tar.gz"
+            name: Linux_ARMv7
           - path: "*macOS_64bit.tar.gz"
             name: macOS_64
+          - path: "*macOS_ARM64.tar.gz"
+            name: macOS_ARM64
           - path: "*Windows_32bit.zip"
             name: Windows_X86-32
           - path: "*Windows_64bit.zip"
@@ -122,7 +128,7 @@ jobs:
           path: ${{ env.BUILDS_ARTIFACT }}
 
       - name: Upload individual build artifact
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         with:
           path: ${{ env.BUILDS_ARTIFACT }}/${{ matrix.artifact.path }}
           name: ${{ matrix.artifact.name }}

.github/workflows/release-go-task.yml renamed to .github/workflows/release-go-crosscompile-task.yml

@@ -1,4 +1,4 @@
-# Source: https://github.com/arduino/tooling-project-assets/blob/main/workflow-templates/release-go-task.md
+# Source: https://github.com/arduino/tooling-project-assets/blob/main/workflow-templates/release-go-crosscompile-task.md
 name: Release
 
 env:
@@ -9,7 +9,7 @@ env:
   # The project's folder on Arduino's download server for uploading builds
   AWS_PLUGIN_TARGET: /arduino-fwuploader/
   ARTIFACT_NAME: dist
-  # See: https://github.com/actions/setup-go/tree/v2#readme
+  # See: https://github.com/actions/setup-go/tree/main#supported-version-syntax
   GO_VERSION: ^1.16.2
 
 on:
@@ -50,15 +50,30 @@ jobs:
         run: task dist:all
 
       - name: Upload artifacts
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         with:
           if-no-files-found: error
           name: ${{ env.ARTIFACT_NAME }}
           path: ${{ env.DIST_DIR }}
 
   notarize-macos:
+    name: Notarize ${{ matrix.artifact.name }}
     runs-on: macos-latest
     needs: create-release-artifacts
+    outputs:
+      checksum-darwin_amd64: ${{ steps.re-package.outputs.checksum-darwin_amd64 }}
+      checksum-darwin_arm64: ${{ steps.re-package.outputs.checksum-darwin_arm64 }}
+
+    env:
+      GON_CONFIG_PATH: gon.config.hcl
+
+    strategy:
+      matrix:
+        artifact:
+          - name: darwin_amd64
+            path: "macOS_64bit.tar.gz"
+          - name: darwin_arm64
+            path: "macOS_ARM64.tar.gz"
 
     steps:
       - name: Checkout repository
@@ -98,38 +113,66 @@ jobs:
           wget -q https://github.com/mitchellh/gon/releases/download/v0.2.3/gon_macos.zip
           unzip gon_macos.zip -d /usr/local/bin
 
+      - name: Write gon config to file
+        # gon does not allow env variables in config file (https://github.com/mitchellh/gon/issues/20)
+        run: |
+          cat > "${{ env.GON_CONFIG_PATH }}" <<EOF
+          # See: https://github.com/mitchellh/gon#configuration-file
+          source = ["${{ env.DIST_DIR }}/${{ env.PROJECT_NAME }}_osx_${{ matrix.artifact.name }}/${{ env.PROJECT_NAME }}"]
+          bundle_id = "cc.arduino.${{ env.PROJECT_NAME }}"
+
+          sign {
+            application_identity = "Developer ID Application: ARDUINO SA (7KT7ZWMCJT)"
+          }
+
+          # Ask Gon for zip output to force notarization process to take place.
+          # The CI will ignore the zip output, using the signed binary only.
+          zip {
+            output_path = "unused.zip"
+          }
+          EOF
+
       - name: Sign and notarize binary
         env:
           AC_USERNAME: ${{ secrets.AC_USERNAME }}
           AC_PASSWORD: ${{ secrets.AC_PASSWORD }}
         run: |
-          gon gon.config.hcl
+          gon "${{ env.GON_CONFIG_PATH }}"
 
-      - name: Re-package binary
+      - name: Re-package binary and output checksum
+        id: re-package
+        working-directory: ${{ env.DIST_DIR }}
         # This step performs the following:
         # 1. Repackage the signed binary replaced in place by Gon (ignoring the output zip file)
+        # 2. Recalculate package checksum
+        # 3. Output the new checksum to include in the nnnnnn-checksums.txt file
+        #    (it cannot be done there because of workflow job parallelization)
         run: |
-          # GitHub's upload/download-artifact@v2 actions don't preserve file permissions,
+          # GitHub's upload/download-artifact actions don't preserve file permissions,
           # so we need to add execution permission back until the action is made to do this.
-          chmod +x ${{ env.DIST_DIR }}/macos64/${{ env.PROJECT_NAME }}
+          chmod +x "${{ env.PROJECT_NAME }}_osx_${{ matrix.artifact.name }}/${{ env.PROJECT_NAME }}"
           TAG="${GITHUB_REF/refs\/tags\//}"
-          tar -czvf "${{ env.DIST_DIR }}/${{ env.PROJECT_NAME }}_${TAG}_macOS_64bit.tar.gz" \
-            LICENSE.txt \
-            -C ${{ env.DIST_DIR }}/macos64/ ${{ env.PROJECT_NAME }}
+          PACKAGE_FILENAME="${{ env.PROJECT_NAME }}_${TAG}_${{ matrix.artifact.path }}"
+          tar -czvf "$PACKAGE_FILENAME" \
+          -C "${{ env.PROJECT_NAME }}_osx_${{ matrix.artifact.name }}/" "${{ env.PROJECT_NAME }}" \
+          -C ../../ LICENSE.txt
+          CHECKSUM_LINE="$(shasum -a 256 $PACKAGE_FILENAME)"
+          echo "PACKAGE_FILENAME=$PACKAGE_FILENAME" >> $GITHUB_ENV
+          echo "::set-output name=checksum-${{ matrix.artifact.name }}::$CHECKSUM_LINE"
 
       - name: Upload artifacts
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         with:
           if-no-files-found: error
           name: ${{ env.ARTIFACT_NAME }}
-          path: ${{ env.DIST_DIR }}
+          path: ${{ env.DIST_DIR }}/${{ env.PACKAGE_FILENAME }}
 
   create-release:
     runs-on: ubuntu-latest
     needs: notarize-macos
 
     steps:
-      - name: Checkout
+      - name: Checkout # we need package_index.template
         uses: actions/checkout@v3
 
       - name: Download artifact
@@ -149,13 +192,23 @@ jobs:
         # would be calculated since the binary is modified during notarization
         run: task dist:generate-index-data
 
+      - name: Update checksum
+        run: |
+          declare -a checksum_lines=("${{ needs.notarize-macos.outputs.checksum-darwin_amd64 }}" "${{ needs.notarize-macos.outputs.checksum-darwin_arm64 }}")
+          for checksum_line in "${checksum_lines[@]}"
+          do
+            CHECKSUM=$(echo ${checksum_line} | cut -d " " -f 1)
+            PACKAGE_FILENAME=$(echo ${checksum_line} | cut -d " " -f 2)
+            perl -pi -w -e "s/.*${PACKAGE_FILENAME}/${CHECKSUM}  ${PACKAGE_FILENAME}/g;" ${{ env.DIST_DIR }}/*-checksums.txt
+          done
+
       - name: Identify Prerelease
         # This is a workaround while waiting for create-release action
         # to implement auto pre-release based on tag
         id: prerelease
         run: |
-          wget -q -P /tmp https://github.com/fsaintjacques/semver-tool/archive/3.0.0.zip
-          unzip -p /tmp/3.0.0.zip semver-tool-3.0.0/src/semver >/tmp/semver && chmod +x /tmp/semver
+          wget -q -P /tmp https://github.com/fsaintjacques/semver-tool/archive/3.2.0.zip
+          unzip -p /tmp/3.2.0.zip semver-tool-3.2.0/src/semver >/tmp/semver && chmod +x /tmp/semver
           if [[ "$(/tmp/semver get prerel "${GITHUB_REF/refs\/tags\//}")" ]]; then echo "::set-output name=IS_PRE::true"; fi
 
       - name: Create Github Release and upload artifacts
@@ -165,7 +218,9 @@ jobs:
           bodyFile: ${{ env.DIST_DIR }}/CHANGELOG.md
           draft: false
           prerelease: ${{ steps.prerelease.outputs.IS_PRE }}
-          artifacts: ${{ env.DIST_DIR }}/arduino-fwuploader*,${{ env.DIST_DIR }}/package_index.json
+          # NOTE: "Artifact is a directory" warnings are expected and don't indicate a problem
+          # (all the files we need are in the DIST_DIR root)
+          artifacts: ${{ env.DIST_DIR }}/*
 
       - name: Upload release files on Arduino downloads servers
         uses: docker://plugins/s3