From c85279e4befef2a805805910c398d0964190323a Mon Sep 17 00:00:00 2001 From: Meng-Hung Chen Date: Mon, 10 Feb 2025 15:58:57 +0800 Subject: [PATCH] CI: Reduce HTTP requests Check prebuilt tarballs are existing or not first. If so, skip the downloading of releases tags and tarballs. In the CI, we first call "make artifact" to prepare the tarballs, so the GitHub API for fetching releases tags would not be called and be skipped when "make" is called next time. After this commit, wget in the CI would not complain about error code 403 (rate limit exceeded) anymore. However, the build on Aarch64 still failed because of other reansons, e.g. software bugs of GCC or QEMU. Close #536 --- .github/workflows/main.yml | 13 +++++++- Makefile | 3 -- mk/artifact.mk | 66 ++++++++++++++++++++++++++++---------- 3 files changed, 61 insertions(+), 21 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 222f04dc9..4614f7258 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -58,10 +58,20 @@ jobs: uses: rlalik/setup-cpp-compiler@master with: compiler: ${{ matrix.compiler }} + - name: fetch artifact first to reduce HTTP requests + env: + CC: ${{ steps.install_cc.outputs.cc }} + run: | + make artifact + make ENABLE_SYSTEM=1 artifact + make ENABLE_ARCH_TEST=1 artifact + if: ${{ always() }} - name: default build with -g env: CC: ${{ steps.install_cc.outputs.cc }} - run: make OPT_LEVEL=-g -j$(nproc) + run: | + make distclean + make OPT_LEVEL=-g -j$(nproc) if: ${{ always() }} - name: default build with -Og env: @@ -268,6 +278,7 @@ jobs: ./llvm.sh 18 # Append custom commands here run: | + make artifact make -j$(nproc) make check -j$(nproc) make ENABLE_JIT=1 clean && make ENABLE_JIT=1 check -j$(nproc) diff --git a/Makefile b/Makefile index 755579fe0..29e6dd1a4 100644 --- a/Makefile +++ b/Makefile @@ -416,9 +416,6 @@ endif clean: $(RM) $(BIN) $(OBJS) $(DEV_OBJS) $(BUILD_DTB) $(BUILD_DTB2C) $(HIST_BIN) $(HIST_OBJS) $(deps) $(WEB_FILES) $(CACHE_OUT) src/rv32_jit.c distclean: clean - -$(RM) $(DOOM_DATA) $(QUAKE_DATA) $(BUILDROOT_DATA) $(LINUX_DATA) - $(RM) -r $(OUT)/linux-image - $(RM) -r $(TIMIDITY_DATA) $(RM) -r $(OUT)/id1 $(RM) -r $(DEMO_DIR) $(RM) *.zip diff --git a/mk/artifact.mk b/mk/artifact.mk index 4c24556b3..e961da135 100644 --- a/mk/artifact.mk +++ b/mk/artifact.mk @@ -36,15 +36,33 @@ SCIMARK2_SHA1 := de278c5b8cef84ab6dda41855052c7bfef919e36 SHELL_HACK := $(shell mkdir -p $(BIN_DIR)/linux-x86-softfp $(BIN_DIR)/riscv32 $(BIN_DIR)/linux-image) +# $(1): tag of GitHub releases +# $(2): name of GitHub releases +# $(3): name showing in terminal +define fetch-releases-tag + $(if $(wildcard $(BIN_DIR)/$(2)), \ + $(info $(3) is found. Skipping downloading.), \ + $(eval LATEST_RELEASE := $(shell wget -q https://api.github.com/repos/sysprog21/rv32emu-prebuilt/releases -O- \ + | grep '"tag_name"' \ + | grep "$(1)" \ + | head -n 1 \ + | sed -E 's/.*"tag_name": "([^"]+)".*/\1/')) \ + $(if $(LATEST_RELEASE),, \ + $(error Fetching tag of latest releases failed) \ + ) \ + ) +endef + ifeq ($(call has, PREBUILT), 1) -ifeq ($(call has, SYSTEM), 1) - LATEST_RELEASE := $(shell wget -q https://api.github.com/repos/sysprog21/rv32emu-prebuilt/releases -O- | grep '"tag_name"' | grep "Linux-Image" | head -n 1 | sed -E 's/.*"tag_name": "([^"]+)".*/\1/') -else ifeq ($(call has, ARCH_TEST), 1) - LATEST_RELEASE := $(shell wget -q https://api.github.com/repos/sysprog21/rv32emu-prebuilt/releases -O- | grep '"tag_name"' | grep "sail" | head -n 1 | sed -E 's/.*"tag_name": "([^"]+)".*/\1/') -else - LATEST_RELEASE := $(shell wget -q https://api.github.com/repos/sysprog21/rv32emu-prebuilt/releases -O- | grep '"tag_name"' | grep "ELF" | head -n 1 | sed -E 's/.*"tag_name": "([^"]+)".*/\1/') -endif - PREBUILT_BLOB_URL = https://github.com/sysprog21/rv32emu-prebuilt/releases/download/$(LATEST_RELEASE) + ifeq ($(call has, SYSTEM), 1) + $(call fetch-releases-tag,Linux-Image,rv32emu-linux-image-prebuilt.tar.gz,Linux image) + else ifeq ($(call has, ARCH_TEST), 1) + $(call fetch-releases-tag,sail,rv32emu-prebuilt-sail-$(HOST_PLATFORM),Sail model) + else + $(call fetch-releases-tag,ELF,rv32emu-prebuilt.tar.gz,Prebuilt benchmark) + endif + + PREBUILT_BLOB_URL = https://github.com/sysprog21/rv32emu-prebuilt/releases/download/$(LATEST_RELEASE) else # Since rv32emu only supports the dynamic binary translation of integer instruction in tiered compilation currently, # we disable the hardware floating-point and the related SIMD operation of x86. @@ -95,14 +113,15 @@ endif ifeq ($(call has, ARCH_TEST), 1) $(Q)if [ "$(RES)" = "1" ]; then \ $(PRINTF) "\n$(YELLOW)SHA-1 verification failed! Re-fetching prebuilt binaries from \"rv32emu-prebuilt\" ...\n$(NO_COLOR)"; \ - wget -q --show-progress $(PREBUILT_BLOB_URL)/$(RV32EMU_PREBUILT_TARBALL) -O build/$(RV32EMU_PREBUILT_TARBALL);\ + wget -q --show-progress $(PREBUILT_BLOB_URL)/$(RV32EMU_PREBUILT_TARBALL) -O build/$(RV32EMU_PREBUILT_TARBALL); \ else \ $(call notice, [OK]); \ fi else $(Q)if [ "$(RES)" = "1" ]; then \ $(PRINTF) "\n$(YELLOW)SHA-1 verification failed! Re-fetching prebuilt binaries from \"rv32emu-prebuilt\" ...\n$(NO_COLOR)"; \ - wget -q --show-progress $(PREBUILT_BLOB_URL)/$(RV32EMU_PREBUILT_TARBALL) -O- | tar -C build --strip-components=1 -xz; \ + wget -q --show-progress $(PREBUILT_BLOB_URL)/$(RV32EMU_PREBUILT_TARBALL) -O build/$(RV32EMU_PREBUILT_TARBALL); \ + tar --strip-components=1 -zxf build/$(RV32EMU_PREBUILT_TARBALL) -C build; \ else \ $(call notice, [OK]); \ fi @@ -147,16 +166,29 @@ endif fetch-checksum: ifeq ($(call has, PREBUILT), 1) - $(Q)$(PRINTF) "Fetching SHA-1 of prebuilt binaries ...\n" + $(Q)$(PRINTF) "Fetching SHA-1 of prebuilt binaries ... " ifeq ($(call has, SYSTEM), 1) - $(Q)wget -q -O $(BIN_DIR)/sha1sum-linux-image $(PREBUILT_BLOB_URL)/sha1sum-linux-image - $(Q)$(call notice, [OK]) + ifeq ($(wildcard $(BIN_DIR)/rv32emu-linux-image-prebuilt.tar.gz),) + $(Q)wget -q -O $(BIN_DIR)/sha1sum-linux-image $(PREBUILT_BLOB_URL)/sha1sum-linux-image + $(Q)$(call notice, [OK]) + else + $(Q)$(PRINTF) "Skipped\n" + endif else ifeq ($(call has, ARCH_TEST), 1) - $(Q)wget -q -O $(BIN_DIR)/rv32emu-prebuilt-sail-$(HOST_PLATFORM).sha $(PREBUILT_BLOB_URL)/rv32emu-prebuilt-sail-$(HOST_PLATFORM).sha + ifeq ($(wildcard $(BIN_DIR)/rv32emu-prebuilt-sail-$(HOST_PLATFORM)),) + $(Q)wget -q -O $(BIN_DIR)/rv32emu-prebuilt-sail-$(HOST_PLATFORM).sha $(PREBUILT_BLOB_URL)/rv32emu-prebuilt-sail-$(HOST_PLATFORM).sha + $(Q)$(call notice, [OK]) + else + $(Q)$(PRINTF) "Skipped\n" + endif else - $(Q)wget -q -O $(BIN_DIR)/sha1sum-linux-x86-softfp $(PREBUILT_BLOB_URL)/sha1sum-linux-x86-softfp - $(Q)wget -q -O $(BIN_DIR)/sha1sum-riscv32 $(PREBUILT_BLOB_URL)/sha1sum-riscv32 - $(Q)$(call notice, [OK]) + ifeq ($(wildcard $(BIN_DIR)/rv32emu-prebuilt.tar.gz),) + $(Q)wget -q -O $(BIN_DIR)/sha1sum-linux-x86-softfp $(PREBUILT_BLOB_URL)/sha1sum-linux-x86-softfp + $(Q)wget -q -O $(BIN_DIR)/sha1sum-riscv32 $(PREBUILT_BLOB_URL)/sha1sum-riscv32 + $(Q)$(call notice, [OK]) + else + $(Q)$(PRINTF) "Skipped\n" + endif endif endif