Skip to content

Commit 7951385

Browse files
rylevrylev
authored
Merge pull request #231 from rylev/docs-rs-builder-ansible
Ansible for the docs-rs builder
2 parents 7b992ce + 9882ae1 commit 7951385

File tree

11 files changed

+281
-2
lines changed

11 files changed

+281
-2
lines changed

‎ansible/envs/staging/group_vars/docs-rs-builder.yml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
---
2+
3+
sha: "{{ lookup('aws_ssm', '/docs-rs/builder/sha') }}"
4+
vars_repository_sha: "{{ sha | ternary(sha, 'HEAD') }}"

‎ansible/playbooks/docs-rs-builder.yml‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
---
2+
3+
- hosts: docs-rs-builder
4+
become: yes
5+
become_user: root
6+
7+
roles:
8+
9+
- role: common
10+
papertrail_url: "{{ vars_papertrail_url }}"
11+
collect_metrics_from: "{{ global_collect_metrics_from }}"
12+
sudo_users: "{{ global_sudo_users }}"
13+
14+
- role: docs-rs-builder

‎ansible/requirements.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,2 @@
11
ansible==2.9.26
2-
boto3==1.13.16
2+
boto3==1.14.0

‎ansible/roles/common/tasks/metrics.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@
3333
state: absent
3434

3535
when: |
36-
node_exporter_capture.rc != 0 or not node_exporter_capture.stderr.startswith(
36+
node_exporter_capture.rc != 0 or not node_exporter_capture.stdout.startswith(
3737
"node_exporter, version " + node_exporter_version
3838
)
3939

‎ansible/roles/docs-rs-builder/defaults/main.yml‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
---
2+
3+
vars_mountpoint_file_path: /builder.fs
4+
vars_mountpoint_size: 53687091200
5+
vars_mountpoint_path: /mnt/builder
6+
7+
vars_user: builder
8+
vars_group: "{{ vars_user }}"
9+
vars_home_path: "/home/{{ vars_user }}"
10+
11+
vars_repository_url: https://github.com/rust-lang/docs.rs
12+
vars_repository_sha: HEAD
13+
14+
vars_checkout_path: "{{ vars_home_path }}/builder"
15+
vars_working_dir: "{{ vars_checkout_path }}"
16+
vars_executable_path: "{{ vars_working_dir }}/target/release/cratesfyi"
17+
18+
vars_toolchain: nightly
19+
20+
vars_database_url: "{{ lookup('aws_ssm', '/docs-rs/database-url') }}"
21+
vars_s3_bucket: docs-rs-storage-519825364412

‎ansible/roles/docs-rs-builder/handlers/main.yml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
---
2+
3+
- name: restart-docker
4+
systemd:
5+
name: docker
6+
state: restarted
7+
daemon_reload: true

‎ansible/roles/docs-rs-builder/tasks/main.yml‎

Lines changed: 152 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,152 @@
1+
---
2+
3+
- name: Create user
4+
user:
5+
name: "{{ vars_user }}"
6+
groups: docker
7+
append: true
8+
9+
# --------------------
10+
11+
# Install Docker
12+
13+
- name: Add Docker APT repository GPG key
14+
apt_key:
15+
state: present
16+
keyserver: "https://download.docker.com/linux/{{ ansible_distribution|lower }}/gpg"
17+
id: 7EA0A9C3F273FCD8
18+
19+
- name: Add Docker APT repository
20+
apt_repository:
21+
repo: "deb [arch=amd64] https://download.docker.com/linux/{{ ansible_distribution|lower }} {{ ansible_distribution_release }} stable"
22+
state: present
23+
update_cache: true
24+
25+
- name: Install Docker
26+
apt:
27+
name: docker-ce
28+
state: present
29+
notify: restart-docker
30+
31+
- name: Make sure /etc/docker dir exists
32+
file:
33+
path: /etc/docker
34+
state: directory
35+
36+
- name: Configure Docker daemon
37+
template:
38+
src: daemon.json
39+
dest: /etc/docker/daemon.json
40+
mode: 0600
41+
notify: restart-docker
42+
43+
# --------------------
44+
45+
# Set up a partition with limited space to avoid temporary
46+
# input/output files consuming all of the space on the primary
47+
# partition.
48+
49+
- name: Check for mountpoint file
50+
stat:
51+
path: "{{ vars_mountpoint_file_path }}"
52+
register: mountpoint_stat
53+
54+
- name: Create mountpoint file
55+
block:
56+
- name: Allocate file
57+
command: "fallocate -l {{ vars_mountpoint_size }} {{ vars_mountpoint_file_path }}"
58+
59+
- name: Locate mountpoint loopback device
60+
command: "losetup -f --show {{ vars_mountpoint_file_path }}"
61+
register: loopback
62+
63+
- name: Partition mountpoint
64+
command: "mkfs -t ext3 -m 1 -v {{ loopback.stdout }}"
65+
when: mountpoint_stat.stat.size is not defined or mountpoint_stat.stat.size != vars_mountpoint_size
66+
67+
- name: Create mountpoint
68+
file:
69+
path: "{{ vars_mountpoint_path }}"
70+
state: directory
71+
72+
- name: Mount mountpoint
73+
mount:
74+
src: "{{ vars_mountpoint_file_path }}"
75+
name: "{{ vars_mountpoint_path }}"
76+
fstype: ext3
77+
state: mounted
78+
79+
- name: Change kubeconfig file permission
80+
file:
81+
path: "{{ vars_mountpoint_path }}"
82+
owner: "{{ vars_user }}"
83+
group: docker
84+
85+
# --------------------
86+
87+
- name: Checkout repository
88+
become: true
89+
become_user: "{{ vars_user }}"
90+
git:
91+
repo: "{{ vars_repository_url }}"
92+
dest: "{{ vars_checkout_path }}"
93+
version: "{{ vars_repository_sha }}"
94+
95+
- name: Install awscli
96+
apt:
97+
name: awscli
98+
state: present
99+
100+
- name: Allow user to start and stop service
101+
template:
102+
src: sudoers
103+
dest: "/etc/sudoers.d/{{ vars_user }}"
104+
mode: 0440
105+
106+
# ---------
107+
108+
- name: install apt packages
109+
apt:
110+
name:
111+
- build-essential
112+
- cmake
113+
- libssl-dev
114+
- pkg-config
115+
state: present
116+
117+
- name: check if cargo is installed
118+
shell: |
119+
test -f ~/.cargo/env && . ~/.cargo/env && command -v cargo
120+
register: cargo_exists
121+
ignore_errors: yes
122+
123+
- name: Download Installer
124+
when: cargo_exists is failed
125+
get_url:
126+
url: https://sh.rustup.rs
127+
dest: /tmp/sh.rustup.rs
128+
mode: '0755'
129+
force: 'yes'
130+
131+
- name: install rust/cargo
132+
when: cargo_exists is failed
133+
shell: /tmp/sh.rustup.rs -y
134+
135+
- name: Build builder
136+
shell: |
137+
. ~/.cargo/env
138+
cargo build --release
139+
args:
140+
chdir: "{{ vars_checkout_path }}"
141+
142+
- name: Configure service
143+
template:
144+
src: builder.service
145+
dest: /etc/systemd/system/builder.service
146+
mode: 0644
147+
148+
- name: Start and enable service
149+
systemd:
150+
name: builder
151+
state: started
152+
enabled: true

‎ansible/roles/docs-rs-builder/templates/builder.service‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
#
2+
# {{ ansible_managed }}
3+
#
4+
5+
[Unit]
6+
Description=The docs.rs Builder
7+
8+
[Service]
9+
Environment=TMPDIR={{ vars_mountpoint_path }}
10+
Environment=DOCSRS_PREFIX={{ vars_mountpoint_path }}
11+
Environment=DOCSRS_RUSTWIDE_WORKSPACE={{ vars_mountpoint_path }}
12+
Environment=DOCSRS_LOG=debug
13+
Environment=DOCSRS_DATABASE_URL={{ vars_database_url }}
14+
Environment=DOCSRS_DOCKER_IMAGE=ghcr.io/rust-lang/crates-build-env/linux
15+
Environment=DOCSRS_STORAGE_BACKEND=s3
16+
Environment=S3_REGION=us-east-1
17+
Environment=DOCSRS_S3_BUCKET={{ vars_s3_bucket }}
18+
Environment=RUST_BACKTRACE=1
19+
Environment=DOCSRS_TOOLCHAIN={{ vars_toolchain }}
20+
21+
LimitNOFILE=20000 # Consider removing now that we upload zip files
22+
Restart=on-failure
23+
WorkingDirectory={{ vars_working_dir }}
24+
ExecStart={{ vars_executable_path }} start-build-server
25+
26+
[Install]
27+
WantedBy=multi-user.target

‎ansible/roles/docs-rs-builder/templates/daemon.json‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
{
2+
"storage-driver": "overlay2"
3+
}

‎ansible/roles/docs-rs-builder/templates/sudoers‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
#
2+
# {{ ansible_managed }}
3+
#
4+
5+
builder ALL= NOPASSWD: /bin/systemctl stop builder.service
6+
builder ALL= NOPASSWD: /bin/systemctl start builder.service
7+
builder ALL= NOPASSWD: /bin/systemctl restart builder.service

0 commit comments

Comments
 (0)