Skip to content

Commit 07b778e

Browse files
nodecenodece
committed
[feat][ci] Add build and unit tests ci
1 parent fb78ce0 commit 07b778e

File tree

9 files changed

+1658
-0
lines changed

9 files changed

+1658
-0
lines changed

‎.github/actions/clean-disk/action.yml‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
#
2+
# Licensed to the Apache Software Foundation (ASF) under one
3+
# or more contributor license agreements. See the NOTICE file
4+
# distributed with this work for additional information
5+
# regarding copyright ownership. The ASF licenses this file
6+
# to you under the Apache License, Version 2.0 (the
7+
# "License"); you may not use this file except in compliance
8+
# with the License. You may obtain a copy of the License at
9+
#
10+
# http://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# Unless required by applicable law or agreed to in writing,
13+
# software distributed under the License is distributed on an
14+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
# KIND, either express or implied. See the License for the
16+
# specific language governing permissions and limitations
17+
# under the License.
18+
#
19+
20+
name: clean disk
21+
description: makes some more space available on the disk by removing files
22+
inputs:
23+
mode:
24+
description: "Use 'full' to clean as much as possible"
25+
required: false
26+
runs:
27+
using: composite
28+
steps:
29+
- run: |
30+
if [[ "$OSTYPE" == "linux-gnu"* ]]; then
31+
directories=(/usr/local/lib/android /opt/ghc)
32+
if [[ "${{ inputs.mode }}" == "full" ]]; then
33+
# remove these directories only when mode is 'full'
34+
directories+=(/usr/share/dotnet /opt/hostedtoolcache/CodeQL)
35+
fi
36+
emptydir=/tmp/empty$$/
37+
mkdir $emptydir
38+
echo "::group::Available diskspace"
39+
time df -BM / /mnt
40+
echo "::endgroup::"
41+
for directory in "${directories[@]}"; do
42+
echo "::group::Removing $directory"
43+
# fast way to delete a lot of files on linux
44+
time sudo eatmydata rsync -a --delete $emptydir ${directory}/
45+
time sudo eatmydata rm -rf ${directory}
46+
time df -BM / /mnt
47+
echo "::endgroup::"
48+
done
49+
echo "::group::Cleaning apt state"
50+
time sudo bash -c "apt-get clean; apt-get autoclean; apt-get -y --purge autoremove"
51+
time df -BM / /mnt
52+
echo "::endgroup::"
53+
fi
54+
echo "::group::Available diskspace"
55+
time df -BM / /mnt
56+
echo "::endgroup::"
57+
shell: bash

‎.github/actions/copy-test-reports/action.yml‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
#
2+
# Licensed to the Apache Software Foundation (ASF) under one
3+
# or more contributor license agreements. See the NOTICE file
4+
# distributed with this work for additional information
5+
# regarding copyright ownership. The ASF licenses this file
6+
# to you under the Apache License, Version 2.0 (the
7+
# "License"); you may not use this file except in compliance
8+
# with the License. You may obtain a copy of the License at
9+
#
10+
# http://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# Unless required by applicable law or agreed to in writing,
13+
# software distributed under the License is distributed on an
14+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
# KIND, either express or implied. See the License for the
16+
# specific language governing permissions and limitations
17+
# under the License.
18+
#
19+
20+
name: Copy test reports
21+
description: Aggregates all test reports to ./test-reports and ./surefire-reports directories
22+
runs:
23+
using: composite
24+
steps:
25+
- run: |
26+
$GITHUB_WORKSPACE/build/pulsar_ci_tool.sh move_test_reports
27+
shell: bash

‎.github/actions/ssh-access/action.yml‎

Lines changed: 161 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,161 @@
1+
#
2+
# Licensed to the Apache Software Foundation (ASF) under one
3+
# or more contributor license agreements. See the NOTICE file
4+
# distributed with this work for additional information
5+
# regarding copyright ownership. The ASF licenses this file
6+
# to you under the Apache License, Version 2.0 (the
7+
# "License"); you may not use this file except in compliance
8+
# with the License. You may obtain a copy of the License at
9+
#
10+
# http://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# Unless required by applicable law or agreed to in writing,
13+
# software distributed under the License is distributed on an
14+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
# KIND, either express or implied. See the License for the
16+
# specific language governing permissions and limitations
17+
# under the License.
18+
#
19+
20+
name: ssh access
21+
description: Sets up SSH access to build VM with upterm
22+
inputs:
23+
action:
24+
description: |
25+
Action to perform: options are "start" and "wait"
26+
"start" will install, configure and start upterm.
27+
"wait" will wait until a connection is established to upterm and will continue to wait until the session is closed.
28+
required: false
29+
default: 'start'
30+
limit-access-to-actor:
31+
description: 'If only the public SSH keys of the user triggering the workflow should be authorized'
32+
required: false
33+
default: 'false'
34+
limit-access-to-users:
35+
description: 'If only the public SSH keys of the listed GitHub users should be authorized. Comma separate list of GitHub user names.'
36+
required: false
37+
default: ''
38+
secure-access:
39+
description: |
40+
Set to false for allowing public access when limit-access-to-actor and limit-access-to-users are unset.
41+
required: false
42+
default: 'true'
43+
timeout:
44+
description: 'When action=wait, the timeout in seconds to wait for the user to connect'
45+
required: false
46+
default: '300'
47+
runs:
48+
using: composite
49+
steps:
50+
- run: |
51+
if [[ "${{ inputs.action }}" == "start" ]]; then
52+
echo "::group::Installing upterm & tmux"
53+
if [[ "$OSTYPE" == "linux-gnu"* ]]; then
54+
# install upterm
55+
curl -sL https://github.com/owenthereal/upterm/releases/download/v0.7.6/upterm_linux_amd64.tar.gz | tar zxvf - -C /tmp upterm && sudo install /tmp/upterm /usr/local/bin/ && rm -rf /tmp/upterm
56+
57+
# install tmux if it's not present
58+
if ! command -v tmux &>/dev/null; then
59+
sudo apt-get -y install tmux
60+
fi
61+
elif [[ "$OSTYPE" == "darwin"* ]]; then
62+
brew install owenthereal/upterm/upterm
63+
# install tmux if it's not present
64+
if ! command -v tmux &>/dev/null; then
65+
brew install tmux
66+
fi
67+
else
68+
echo "Unsupported $OSTYPE"
69+
exit 0
70+
fi
71+
echo '::endgroup::'
72+
echo "::group::Configuring ssh and ssh keys"
73+
# generate ssh key
74+
mkdir -p ~/.ssh
75+
chmod 0700 ~/.ssh
76+
if [ ! -f ~/.ssh/id_rsa ]; then
77+
ssh-keygen -q -t rsa -N "" -f ~/.ssh/id_rsa
78+
fi
79+
if [ ! -f ~/.ssh/id_ed25519 ]; then
80+
ssh-keygen -q -t ed25519 -N "" -f ~/.ssh/id_ed25519
81+
fi
82+
# configure ssh
83+
echo -e "Host *\nStrictHostKeyChecking no\nCheckHostIP no\nTCPKeepAlive yes\nServerAliveInterval 30\nServerAliveCountMax 180\nVerifyHostKeyDNS yes\nUpdateHostKeys yes\n" > ~/.ssh/config
84+
# Auto-generate ~/.ssh/known_hosts by attempting connection to uptermd.upterm.dev
85+
ssh -i ~/.ssh/id_ed25519 uptermd.upterm.dev || true
86+
# @cert-authority entry is a mandatory entry when connecting to upterm. generate the entry based on the known_hosts entry key
87+
cat <(cat ~/.ssh/known_hosts | awk '{ print "@cert-authority * " $2 " " $3 }') >> ~/.ssh/known_hosts
88+
authorizedKeysParameter=""
89+
authorizedKeysFile=${HOME}/.ssh/authorized_keys
90+
if [[ "${{ inputs.secure-access }}" != "false" ]]; then
91+
ssh-keygen -q -t ed25519 -N "$(echo $RANDOM | md5sum | awk '{ print $1 }')" -C "Prevent public access" -f /tmp/dummykey$$
92+
cat /tmp/dummykey$$.pub >> $authorizedKeysFile
93+
rm /tmp/dummykey$$ /tmp/dummykey$$.pub
94+
fi
95+
limit_access_to_actor="${{ inputs.limit-access-to-actor }}"
96+
if [[ "${limit_access_to_actor}" == "true" ]]; then
97+
echo "Adding ${GITHUB_ACTOR} to allowed users (identified by ssh key registered in GitHub)"
98+
curl -s https://github.com/${GITHUB_ACTOR}.keys >> $authorizedKeysFile
99+
fi
100+
limit_access_to_users="${{ inputs.limit-access-to-users }}"
101+
for github_user in ${limit_access_to_users//,/ }; do
102+
if [[ -n "${github_user}" ]]; then
103+
echo "Adding ${github_user} to allowed users (identified by ssh key registered in GitHub)"
104+
curl -s https://github.com/${github_user}.keys >> $authorizedKeysFile
105+
fi
106+
done
107+
if [ -f $authorizedKeysFile ]; then
108+
chmod 0600 $authorizedKeysFile
109+
authorizedKeysParameter="-a $authorizedKeysFile"
110+
echo -e "Using $authorizedKeysFile\nContent:\n---------------------------"
111+
cat $authorizedKeysFile
112+
echo "---------------------------"
113+
fi
114+
echo '::endgroup::'
115+
echo "::group::Starting terminal session and connecting to server"
116+
tmux new -d -s upterm-wrapper -x 132 -y 43 "upterm host ${authorizedKeysParameter} --force-command 'tmux attach -t upterm' -- tmux new -s upterm -x 132 -y 43"
117+
sleep 2
118+
tmux send-keys -t upterm-wrapper q C-m
119+
sleep 1
120+
tmux set -t upterm-wrapper window-size largest
121+
tmux set -t upterm window-size largest
122+
echo '::endgroup::'
123+
echo -e "\nSSH connection information"
124+
# wait up to 10 seconds for upterm admin socket to appear
125+
for i in {1..10}; do
126+
ADMIN_SOCKET=$(find $HOME/.upterm -name "*.sock")
127+
if [ ! -S "$ADMIN_SOCKET" ]; then
128+
echo "Waiting for upterm admin socket to appear in ~/.upterm/*.sock ..."
129+
sleep 1
130+
else
131+
echo "upterm admin socket available in $ADMIN_SOCKET"
132+
break
133+
fi
134+
done
135+
shopt -s nullglob
136+
upterm session current --admin-socket ~/.upterm/*.sock || {
137+
echo "Starting upterm failed."
138+
exit 0
139+
}
140+
elif [[ "${{ inputs.action }}" == "wait" ]]; then
141+
# only wait if upterm was installed
142+
if command -v upterm &>/dev/null; then
143+
shopt -s nullglob
144+
echo "SSH connection information"
145+
upterm session current --admin-socket ~/.upterm/*.sock || {
146+
echo "upterm isn't running. Not waiting any longer."
147+
exit 0
148+
}
149+
timeout=${{ inputs.timeout }}
150+
echo "Waiting $timeout seconds..."
151+
sleep $timeout
152+
echo "Keep waiting as long as there's a connected session"
153+
while upterm session current --admin-socket ~/.upterm/*.sock|grep Connected &>/dev/null; do
154+
sleep 30
155+
done
156+
echo "No session is connected. Not waiting any longer."
157+
else
158+
echo "upterm isn't installed"
159+
fi
160+
fi
161+
shell: bash

‎.github/actions/tune-runner-vm/action.yml‎

Lines changed: 102 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,102 @@
1+
#
2+
# Licensed to the Apache Software Foundation (ASF) under one
3+
# or more contributor license agreements. See the NOTICE file
4+
# distributed with this work for additional information
5+
# regarding copyright ownership. The ASF licenses this file
6+
# to you under the Apache License, Version 2.0 (the
7+
# "License"); you may not use this file except in compliance
8+
# with the License. You may obtain a copy of the License at
9+
#
10+
# http://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# Unless required by applicable law or agreed to in writing,
13+
# software distributed under the License is distributed on an
14+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
# KIND, either express or implied. See the License for the
16+
# specific language governing permissions and limitations
17+
# under the License.
18+
#
19+
20+
name: Tune Runner VM performance
21+
description: tunes the GitHub Runner VM operation system
22+
runs:
23+
using: composite
24+
steps:
25+
- run: |
26+
if [[ "$OSTYPE" == "linux-gnu"* ]]; then
27+
echo "::group::Configure and tune OS"
28+
# Ensure that reverse lookups for current hostname are handled properly
29+
# Add the current IP address, long hostname and short hostname record to /etc/hosts file
30+
echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts
31+
32+
# The default vm.swappiness setting is 60 which has a tendency to start swapping when memory
33+
# consumption is high.
34+
# Set vm.swappiness=1 to avoid swapping and allow high RAM usage
35+
echo 1 | sudo tee /proc/sys/vm/swappiness
36+
(
37+
shopt -s nullglob
38+
# Set swappiness to 1 for all cgroups and sub-groups
39+
for swappiness_file in /sys/fs/cgroup/memory/*/memory.swappiness /sys/fs/cgroup/memory/*/*/memory.swappiness; do
40+
echo 1 | sudo tee $swappiness_file > /dev/null
41+
done
42+
) || true
43+
44+
# use "madvise" Linux Transparent HugePages (THP) setting
45+
# https://www.kernel.org/doc/html/latest/admin-guide/mm/transhuge.html
46+
# "madvise" is generally a better option than the default "always" setting
47+
# Based on Azul instructions from https://docs.azul.com/prime/Enable-Huge-Pages#transparent-huge-pages-thp
48+
echo madvise | sudo tee /sys/kernel/mm/transparent_hugepage/enabled
49+
echo advise | sudo tee /sys/kernel/mm/transparent_hugepage/shmem_enabled
50+
echo defer+madvise | sudo tee /sys/kernel/mm/transparent_hugepage/defrag
51+
echo 1 | sudo tee /sys/kernel/mm/transparent_hugepage/khugepaged/defrag
52+
53+
# tune filesystem mount options, https://www.kernel.org/doc/Documentation/filesystems/ext4.txt
54+
# commit=999999, effectively disables automatic syncing to disk (default is every 5 seconds)
55+
# nobarrier/barrier=0, loosen data consistency on system crash (no negative impact to empheral CI nodes)
56+
sudo mount -o remount,nodiscard,commit=999999,barrier=0 /
57+
sudo mount -o remount,nodiscard,commit=999999,barrier=0 /mnt
58+
# disable discard/trim at device level since remount with nodiscard doesn't seem to be effective
59+
# https://www.spinics.net/lists/linux-ide/msg52562.html
60+
for i in /sys/block/sd*/queue/discard_max_bytes; do
61+
echo 0 | sudo tee $i
62+
done
63+
# disable any background jobs that run SSD discard/trim
64+
sudo systemctl disable fstrim.timer || true
65+
sudo systemctl stop fstrim.timer || true
66+
sudo systemctl disable fstrim.service || true
67+
sudo systemctl stop fstrim.service || true
68+
69+
# stop php-fpm
70+
sudo systemctl stop php8.0-fpm.service || true
71+
sudo systemctl stop php7.4-fpm.service || true
72+
# stop mono-xsp4
73+
sudo systemctl disable mono-xsp4.service || true
74+
sudo systemctl stop mono-xsp4.service || true
75+
sudo killall mono || true
76+
77+
# stop Azure Linux agent to save RAM
78+
sudo systemctl stop walinuxagent.service || true
79+
80+
# enable docker experimental mode which is
81+
# required for using "docker build --squash" / "-Ddocker.squash=true"
82+
daemon_json="$(sudo cat /etc/docker/daemon.json | jq '.experimental = true')"
83+
echo "$daemon_json" | sudo tee /etc/docker/daemon.json
84+
# restart docker daemon
85+
sudo systemctl restart docker
86+
echo '::endgroup::'
87+
88+
# show memory
89+
echo "::group::Available Memory"
90+
free -m
91+
echo '::endgroup::'
92+
# show disk
93+
echo "::group::Available diskspace"
94+
df -BM
95+
echo "::endgroup::"
96+
# show cggroup
97+
echo "::group::Cgroup settings for current cgroup $CURRENT_CGGROUP"
98+
CURRENT_CGGROUP=$(cat /proc/self/cgroup | grep '0::' | awk -F: '{ print $3 }')
99+
sudo cgget -a $CURRENT_CGGROUP || true
100+
echo '::endgroup::'
101+
fi
102+
shell: bash

0 commit comments

Comments
 (0)