Skip to content

Commit c8c8263

Browse files
vacantronvacantron
committed
CI: Reduce HTTP requests
Check prebuilt tarballs are existing or not first. If so, skip the downloading of releases tags and tarballs. In the CI, we first call "make artifact" to prepare the tarballs, so the GitHub API for fetching releases tags would not be called and be skipped when "make" is called next time. After this commit, wget in the CI would not complain about error code 403 (rate limit exceeded) anymore. However, the build on Aarch64 still failed because of other reansons, e.g. software bugs of GCC or QEMU. Close #536
1 parent d1ed003 commit c8c8263

File tree

3 files changed

+57
-21
lines changed

3 files changed

+57
-21
lines changed

‎.github/workflows/main.yml‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -58,10 +58,20 @@ jobs:
5858
uses: rlalik/setup-cpp-compiler@master
5959
with:
6060
compiler: ${{ matrix.compiler }}
61+
- name: fetch artifact first to reduce HTTP requests
62+
env:
63+
CC: ${{ steps.install_cc.outputs.cc }}
64+
run: |
65+
make artifact
66+
make ENABLE_SYSTEM=1 artifact
67+
make ENABLE_ARCH_TEST=1 artifact
68+
if: ${{ always() }}
6169
- name: default build with -g
6270
env:
6371
CC: ${{ steps.install_cc.outputs.cc }}
64-
run: make OPT_LEVEL=-g -j$(nproc)
72+
run: |
73+
make distclean
74+
make OPT_LEVEL=-g -j$(nproc)
6575
if: ${{ always() }}
6676
- name: default build with -Og
6777
env:
@@ -268,6 +278,7 @@ jobs:
268278
./llvm.sh 18
269279
# Append custom commands here
270280
run: |
281+
make artifact
271282
make -j$(nproc)
272283
make check -j$(nproc)
273284
make ENABLE_JIT=1 clean && make ENABLE_JIT=1 check -j$(nproc)

‎Makefile‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -416,9 +416,6 @@ endif
416416
clean:
417417
$(RM) $(BIN) $(OBJS) $(DEV_OBJS) $(BUILD_DTB) $(BUILD_DTB2C) $(HIST_BIN) $(HIST_OBJS) $(deps) $(WEB_FILES) $(CACHE_OUT) src/rv32_jit.c
418418
distclean: clean
419-
-$(RM) $(DOOM_DATA) $(QUAKE_DATA) $(BUILDROOT_DATA) $(LINUX_DATA)
420-
$(RM) -r $(OUT)/linux-image
421-
$(RM) -r $(TIMIDITY_DATA)
422419
$(RM) -r $(OUT)/id1
423420
$(RM) -r $(DEMO_DIR)
424421
$(RM) *.zip

‎mk/artifact.mk‎

Lines changed: 45 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -36,15 +36,30 @@ SCIMARK2_SHA1 := de278c5b8cef84ab6dda41855052c7bfef919e36
3636

3737
SHELL_HACK := $(shell mkdir -p $(BIN_DIR)/linux-x86-softfp $(BIN_DIR)/riscv32 $(BIN_DIR)/linux-image)
3838

39+
# $(1): tag of GitHub releases
40+
# $(2): name of GitHub releases
41+
# $(3): name showing in terminal
42+
define fetch-sha1
43+
$(if $(wildcard $(BIN_DIR)/$(2)), \
44+
$(eval $(info $(3) is found. Skipping downloading.)), \
45+
$(eval LATEST_RELEASE := $(shell wget -q https://api.github.com/repos/sysprog21/rv32emu-prebuilt/releases -O- \
46+
| grep '"tag_name"' \
47+
| grep "$(1)" \
48+
| head -n 1 \
49+
| sed -E 's/.*"tag_name": "([^"]+)".*/\1/')) \
50+
)
51+
endef
52+
3953
ifeq ($(call has, PREBUILT), 1)
40-
ifeq ($(call has, SYSTEM), 1)
41-
LATEST_RELEASE := $(shell wget -q https://api.github.com/repos/sysprog21/rv32emu-prebuilt/releases -O- | grep '"tag_name"' | grep "Linux-Image" | head -n 1 | sed -E 's/.*"tag_name": "([^"]+)".*/\1/')
42-
else ifeq ($(call has, ARCH_TEST), 1)
43-
LATEST_RELEASE := $(shell wget -q https://api.github.com/repos/sysprog21/rv32emu-prebuilt/releases -O- | grep '"tag_name"' | grep "sail" | head -n 1 | sed -E 's/.*"tag_name": "([^"]+)".*/\1/')
44-
else
45-
LATEST_RELEASE := $(shell wget -q https://api.github.com/repos/sysprog21/rv32emu-prebuilt/releases -O- | grep '"tag_name"' | grep "ELF" | head -n 1 | sed -E 's/.*"tag_name": "([^"]+)".*/\1/')
46-
endif
47-
PREBUILT_BLOB_URL = https://github.com/sysprog21/rv32emu-prebuilt/releases/download/$(LATEST_RELEASE)
54+
ifeq ($(call has, SYSTEM), 1)
55+
$(call fetch-releases-tag,Linux-Image,rv32emu-linux-image-prebuilt.tar.gz,Linux image)
56+
else ifeq ($(call has, ARCH_TEST), 1)
57+
$(call fetch-releases-tag,sail,rv32emu-prebuilt-sail-$(HOST_PLATFORM),Sail model)
58+
else
59+
$(call fetch-releases-tag,ELF,rv32emu-prebuilt.tar.gz,Prebuilt benchmark)
60+
endif
61+
62+
PREBUILT_BLOB_URL = https://github.com/sysprog21/rv32emu-prebuilt/releases/download/$(LATEST_RELEASE)
4863
else
4964
# Since rv32emu only supports the dynamic binary translation of integer instruction in tiered compilation currently,
5065
# we disable the hardware floating-point and the related SIMD operation of x86.
@@ -95,14 +110,15 @@ endif
95110
ifeq ($(call has, ARCH_TEST), 1)
96111
$(Q)if [ "$(RES)" = "1" ]; then \
97112
$(PRINTF) "\n$(YELLOW)SHA-1 verification failed! Re-fetching prebuilt binaries from \"rv32emu-prebuilt\" ...\n$(NO_COLOR)"; \
98-
wget -q --show-progress $(PREBUILT_BLOB_URL)/$(RV32EMU_PREBUILT_TARBALL) -O build/$(RV32EMU_PREBUILT_TARBALL);\
113+
wget -q --show-progress $(PREBUILT_BLOB_URL)/$(RV32EMU_PREBUILT_TARBALL) -O build/$(RV32EMU_PREBUILT_TARBALL); \
99114
else \
100115
$(call notice, [OK]); \
101116
fi
102117
else
103118
$(Q)if [ "$(RES)" = "1" ]; then \
104119
$(PRINTF) "\n$(YELLOW)SHA-1 verification failed! Re-fetching prebuilt binaries from \"rv32emu-prebuilt\" ...\n$(NO_COLOR)"; \
105-
wget -q --show-progress $(PREBUILT_BLOB_URL)/$(RV32EMU_PREBUILT_TARBALL) -O- | tar -C build --strip-components=1 -xz; \
120+
wget -q --show-progress $(PREBUILT_BLOB_URL)/$(RV32EMU_PREBUILT_TARBALL) -O build/$(RV32EMU_PREBUILT_TARBALL); \
121+
tar --strip-components=1 -zxf build/$(RV32EMU_PREBUILT_TARBALL) -C build; \
106122
else \
107123
$(call notice, [OK]); \
108124
fi
@@ -147,16 +163,28 @@ endif
147163

148164
fetch-checksum:
149165
ifeq ($(call has, PREBUILT), 1)
150-
$(Q)$(PRINTF) "Fetching SHA-1 of prebuilt binaries ...\n"
166+
$(Q)$(PRINTF) "Fetching SHA-1 of prebuilt binaries ... "
151167
ifeq ($(call has, SYSTEM), 1)
152-
$(Q)wget -q -O $(BIN_DIR)/sha1sum-linux-image $(PREBUILT_BLOB_URL)/sha1sum-linux-image
153-
$(Q)$(call notice, [OK])
168+
ifeq ($(wildcard $(BIN_DIR)/rv32emu-linux-image-prebuilt.tar.gz),)
169+
$(Q)wget -q -O $(BIN_DIR)/sha1sum-linux-image $(PREBUILT_BLOB_URL)/sha1sum-linux-image
170+
$(Q)$(call notice, [OK])
171+
else
172+
$(Q)$(PRINTF) "Skipped\n"
173+
endif
154174
else ifeq ($(call has, ARCH_TEST), 1)
155-
$(Q)wget -q -O $(BIN_DIR)/rv32emu-prebuilt-sail-$(HOST_PLATFORM).sha $(PREBUILT_BLOB_URL)/rv32emu-prebuilt-sail-$(HOST_PLATFORM).sha
175+
ifeq ($(wildcard $(BIN_DIR)/rv32emu-prebuilt-sail-$(HOST_PLATFORM)),)
176+
wget -q -O $(BIN_DIR)/rv32emu-prebuilt-sail-$(HOST_PLATFORM).sha $(PREBUILT_BLOB_URL)/rv32emu-prebuilt-sail-$(HOST_PLATFORM).sha
177+
else
178+
$(Q)$(PRINTF) "Skipped\n"
179+
endif
156180
else
157-
$(Q)wget -q -O $(BIN_DIR)/sha1sum-linux-x86-softfp $(PREBUILT_BLOB_URL)/sha1sum-linux-x86-softfp
158-
$(Q)wget -q -O $(BIN_DIR)/sha1sum-riscv32 $(PREBUILT_BLOB_URL)/sha1sum-riscv32
159-
$(Q)$(call notice, [OK])
181+
ifeq ($(wildcard $(BIN_DIR)/rv32emu-prebuilt.tar.gz),)
182+
$(Q)wget -q -O $(BIN_DIR)/sha1sum-linux-x86-softfp $(PREBUILT_BLOB_URL)/sha1sum-linux-x86-softfp
183+
$(Q)wget -q -O $(BIN_DIR)/sha1sum-riscv32 $(PREBUILT_BLOB_URL)/sha1sum-riscv32
184+
$(Q)$(call notice, [OK])
185+
else
186+
$(Q)$(PRINTF) "Skipped\n"
187+
endif
160188
endif
161189
endif
162190

0 commit comments

Comments
 (0)