Add a Terraform configuration to deploy lnt.llvm.org

.github/workflows/deploy-lnt.llvm.org.yaml

@@ -0,0 +1,36 @@
+name: Deploy lnt.llvm.org
+
+on:
+  push:
+    branches: ['main']
+    paths:
+      - '.github/workflows/deploy-lnt.llvm.org.yaml'
+      - 'deployment/*'
+
+permissions:
+  contents: read
+
+jobs:
+  deploy:
+    runs-on: ubuntu-24.04
+
+    steps:
+    - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8
+
+    - name: Setup Terraform
+      uses: hashicorp/setup-terraform@v3
+
+    - name: Configure AWS Credentials
+      uses: aws-actions/configure-aws-credentials@v4
+      with:
+        aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+        aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+
+    - name: Initialize Terraform
+      run: terraform -chdir=deployment init
+
+    - name: Apply Terraform changes
+      run: terraform -chdir=deployment apply -auto-approve
+      env:
+        TF_VAR_lnt_db_password: ${{ secrets.LNT_DB_PASSWORD }}
+        TF_VAR_lnt_auth_token: ${{ secrets.LNT_AUTH_TOKEN }}

.gitignore

@@ -3,6 +3,7 @@
 .tox/
 /llvm_lnt.egg-info
 build
+deployment/.terraform
 dist
 docs/_build
 lnt/server/ui/static/docs

deployment/README.md

@@ -0,0 +1,23 @@
+This directory contains configuration files to deploy lnt.llvm.org.
+
+The https://lnt.llvm.org instance gets re-deployed automatically whenever changes
+are made to the configuration files under `deployment/` on the `main` branch via
+a Github Action. Manually deploying the instance is also possible by directly using
+Terraform:
+
+```bash
+aws configure # provide appropriate access keys
+terraform -chdir=deployment init
+terraform -chdir=deployment plan # to see what will be done
+terraform -chdir=deployment apply
+```
+
+At a high level, lnt.llvm.org is running in a Docker container on an EC2 instance.
+The database is stored in an independent EBS storage that gets attached and detached
+to/from the EC2 instance when it is created/destroyed, but the EBS storage has its own
+independent life cycle (because we want the data to outlive any specific EC2 instance).
+
+The state used by Terraform to track the current status of the instance, EBS storage, etc
+is located in a S3 bucket defined in the Terraform file. It is updated automatically when
+changes are performed via the `terraform` command-line. Terraform is able to access that
+data via the AWS credentials that are set up by `aws configure`.

deployment/compose.env.tpl

@@ -0,0 +1,4 @@
+LNT_DB_PASSWORD=${__db_password__}
+LNT_AUTH_TOKEN=${__auth_token__}
+LNT_IMAGE=${__lnt_image__}
+LNT_HOST_PORT=${__lnt_host_port__}

deployment/ec2-startup.sh

@@ -0,0 +1,49 @@
+#!/bin/bash
+
+#
+# This is the startup script that gets executed when the EC2 instance running lnt.llvm.org
+# is brought up. This script references some files under /etc/lnt that are put into place
+# by cloud-init, which is specified in the Terraform configuration file.
+#
+
+set -e
+
+echo "Installing docker"
+sudo yum update -y
+sudo yum install -y docker
+docker --version
+
+echo "Installing docker compose"
+sudo mkdir -p /usr/local/lib/docker/cli-plugins
+sudo curl -L https://github.com/docker/compose/releases/latest/download/docker-compose-linux-$(uname -m) \
+          -o /usr/local/lib/docker/cli-plugins/docker-compose
+sudo chmod +x /usr/local/lib/docker/cli-plugins/docker-compose
+docker compose version
+
+echo "Starting the Docker service"
+sudo service docker start
+sudo systemctl enable docker # also ensure the Docker service starts on reboot
+
+if ! lsblk --output FSTYPE -f /dev/sdh | grep --quiet ext4; then
+    echo "Formatting /dev/sdh -- this is a new EBS volume"
+    sudo mkfs -t ext4 /dev/sdh
+else
+    echo "/dev/sdh already contains a filesystem -- reusing previous EBS volume"
+fi
+
+echo "Mounting EBS volume with persistent information at /persistent-state"
+sudo mkdir /persistent-state
+sudo mount /dev/sdh /persistent-state
+
+echo "Creating folders to map volumes in the Docker container to locations on the EC2 instance"
+sudo mkdir -p /persistent-state/var/lib/lnt
+(cd /var/lib && ln -s /persistent-state/var/lib/lnt lnt)
+sudo mkdir -p /persistent-state/var/lib/postgresql
+(cd /var/lib && ln -s /persistent-state/var/lib/postgresql postgresql)
+sudo mkdir -p /var/log/lnt # logs are not persisted
+
+echo "Starting LNT service with Docker compose"
+sudo docker compose --file /etc/lnt/compose.yaml               \
+                    --file /etc/lnt/ec2-volume-mapping.yaml    \
+                    --env-file /etc/lnt/compose.env            \
+                    up --detach

deployment/ec2-volume-mapping.yaml

@@ -0,0 +1,27 @@
+#
+# This file maps volumes in the Docker container to actual locations on the EC2 instance.
+# We basically bind the volumes inside the Docker image to the same filesystem location
+# on the EC2 instance (e.g. /var/lib/lnt -> /var/lib/lnt) for ease of access.
+#
+
+volumes:
+  instance:
+    driver: local
+    driver_opts:
+      o: bind
+      type: none
+      device: /var/lib/lnt
+
+  logs:
+    driver: local
+    driver_opts:
+      o: bind
+      type: none
+      device: /var/log/lnt
+
+  database:
+    driver: local
+    driver_opts:
+      o: bind
+      type: none
+      device: /var/lib/postgresql

deployment/main.tf

@@ -0,0 +1,154 @@
+#
+# Terraform file for deploying lnt.llvm.org.
+#
+
+variable "lnt_db_password" {
+  type        = string
+  description = "The database password for the lnt.llvm.org database."
+  sensitive   = true
+}
+
+variable "lnt_auth_token" {
+  type        = string
+  description = "The authentication token to perform destructive operations on lnt.llvm.org."
+  sensitive   = true
+}
+
+locals {
+  # The Docker image to use for the webserver part of the LNT service
+  lnt_image     = "d9ffa5317a9a42a1d2fa337cba97ec51d931f391"
+
+  # The port on the EC2 instance used by the Docker webserver for communication
+  lnt_host_port = "80"
+}
+
+terraform {
+  backend "s3" {
+    bucket  = "lnt.llvm.org-test-bucket" # TODO: Adjust this for the real LLVM Foundation account
+    key     = "terraform.tfstate"
+    region  = "us-west-2"
+    encrypt = true
+  }
+}
+
+locals {
+  availability_zone = "us-west-2a"
+}
+
+provider "aws" {
+  region = "us-west-2"
+}
+
+#
+# Setup the EC2 instance
+#
+data "aws_ami" "amazon_linux_2023" {
+  most_recent = true
+  owners      = ["amazon"]
+
+  filter {
+    name   = "name"
+    values = ["al2023-ami-ecs-hvm-*-kernel-*-x86_64"]
+  }
+}
+
+data "cloudinit_config" "startup_scripts" {
+  base64_encode = true
+
+  part {
+    filename     = "ec2-startup.sh"
+    content_type = "text/x-shellscript"
+    content      = file("${path.module}/ec2-startup.sh")
+  }
+
+  part {
+    content_type = "text/cloud-config"
+    content = yamlencode({
+      write_files = [
+        {
+          path        = "/etc/lnt/compose.yaml"
+          permissions = "0400" # read-only for owner
+          content     = file("${path.module}/../docker/compose.yaml")
+        },
+        {
+          path        = "/etc/lnt/ec2-volume-mapping.yaml"
+          permissions = "0400" # read-only for owner
+          content     = file("${path.module}/ec2-volume-mapping.yaml")
+        },
+        {
+          path        = "/etc/lnt/compose.env"
+          permissions = "0400" # read-only for owner
+          content     = templatefile("${path.module}/compose.env.tpl", {
+            __db_password__   = var.lnt_db_password,
+            __auth_token__    = var.lnt_auth_token,
+            __lnt_image__     = local.lnt_image,
+            __lnt_host_port__ = local.lnt_host_port,
+          })
+        }
+      ]
+    })
+  }
+}
+
+resource "aws_security_group" "server" {
+  name        = "lnt.llvm.org/server-security-group"
+  description = "Allow SSH and HTTP traffic"
+
+  ingress {
+    description = "Allow incoming SSH traffic from anywhere"
+    from_port   = 22
+    to_port     = 22
+    protocol    = "tcp"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+
+  ingress {
+    description = "Allow incoming HTTP traffic from anywhere"
+    from_port   = 80
+    to_port     = 80
+    protocol    = "tcp"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+
+  egress {
+    description = "Allow outgoing traffic to anywhere"
+    from_port   = 0
+    to_port     = 0
+    protocol    = "-1"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+}
+
+resource "aws_instance" "server" {
+  ami                         = data.aws_ami.amazon_linux_2023.id
+  availability_zone           = local.availability_zone
+  instance_type               = "t2.micro" # TODO: Adjust the size of the real instance
+  associate_public_ip_address = true
+  security_groups             = [aws_security_group.server.name]
+  tags = {
+    Name = "lnt.llvm.org/server"
+  }
+
+  user_data_base64 = data.cloudinit_config.startup_scripts.rendered
+}
+
+#
+# Setup the EBS volume attached to the instance that stores the DB
+# and other instance-related configuration (e.g. the schema files,
+# profiles and anything else that should persist).
+#
+resource "aws_ebs_volume" "persistent_state" {
+  availability_zone = local.availability_zone
+  # TODO: Put a real size once we're ready to go to production
+  size              = 20 # GiB
+  type              = "gp2"
+  tags = {
+    Name = "lnt.llvm.org/persistent-state"
+  }
+}
+
+resource "aws_volume_attachment" "persistent_state_attachment" {
+  instance_id = aws_instance.server.id
+  volume_id   = aws_ebs_volume.persistent_state.id
+  device_name = "/dev/sdh"
+}