feat: starttls (#558)

Co-authored-by: Ee Durbin <ewdurbin@gmail.com>
Co-authored-by: Ee Durbin <ee@python.org>

Author: JacobCoffee

pillar/base/tls.sls

@@ -61,3 +61,80 @@ tls:
     svn.psf.io:
       roles:
         - hg
+
+  acme_cert_configs:
+
+    bugs.python.org:
+      validation: http
+      name: bugs.python.org
+      roles:
+        - loadbalancer
+        - bugs
+      aliases:
+        - bugs.jython.org
+        - issues.roundup-tracker.org
+
+    buildbot.python.org:
+      validation: http
+      roles:
+        - loadbalancer
+
+    console.python.org:
+      validation: http
+      roles:
+        - loadbalancer
+
+    jobs.pyfound.org:
+      validation: http
+      roles:
+        - loadbalancer
+
+    jython.org:
+      validation: http
+      roles:
+        - loadbalancer
+      aliases:
+        - jython.net
+        - www.jython.net
+        - jython.com
+        - www.jython.com
+
+    legacy.python.org:
+      validation: http
+      roles:
+        - loadbalancer
+      aliases:
+        - hg.python.org
+        - svn.python.org
+
+    planetpython.org:
+      validation: http
+      roles:
+        - loadbalancer
+      aliases:
+        - www.planetpython.org
+        - planet.python.org
+
+    pypa.io:
+      validation: http
+      roles:
+        - loadbalancer
+
+    salt-public.psf.io:
+      validation: http
+      roles:
+        - loadbalancer
+
+    speed.python.org:
+      validation: http
+      roles:
+        - loadbalancer
+      aliases:
+        - speed.pypy.org
+
+    wiki.python.org:
+      validation: http
+      roles:
+        - loadbalancer
+      aliases:
+        - wiki.jython.org

pillar/dev/top.sls

@@ -8,6 +8,7 @@ base:
     - tls
     - users.*
     - postgres.clusters
+    - pebble # needing to do this to have pebble rum in dev
     # - secrets.sentry # Uncomment and update sentry secrets if you want to work in dev
 
   'backup-server':

salt/_extensions/pillar/ca.py

@@ -295,46 +295,69 @@ def get_ca_signed_cert(cacert_path, ca_name, CN):
     return "\n".join([cert, key])
 
 
+def _read_cert_file(path: str) -> str:
+    """Helper to read certificate files, which might be symlinks"""
+    try:
+        with open(path, 'r') as f:
+            return f.read()
+    except (IOError, OSError):
+        return None
+
+
 def ext_pillar(minion_id, pillar, base="/etc/ssl", name="PSFCA", cert_opts=None):
     if cert_opts is None:
         cert_opts = {}
 
-    # Ensure we have a CA created.
+    # Create CA certificate
     opts = cert_opts.copy()
     opts["CN"] = name
     create_ca(base, name, **opts)
 
-    # Start our pillar with just the ca certificate.
     data = {
         "tls": {
             "ca": {
                 name: get_ca_cert(base, name),
             },
             "certs": {},
+            "acme_certs": {},
         },
     }
 
-    # Create all of the certificates required by this minion
+    minion_roles = []
+    minion_roles.extend(
+        role_name
+        for role_name, role_config in pillar.get("roles", {}).items()
+        if role_config.get("pattern")
+        and compound(role_config["pattern"], minion_id)
+    )
+
+    # Process CA-signed certificates (gen_certs)
     gen_certs = pillar.get("tls", {}).get("gen_certs", {})
     for certificate, config in gen_certs.items():
-        role_patterns = [
-            role.get("pattern")
-            for role in [
-                pillar.get("roles", {}).get(r) for r in config.get("roles", "")
-            ]
-            if role and role.get("pattern") is not None
-        ]
-        if any([compound(pat, minion_id) for pat in role_patterns]):
+        cert_roles = config.get("roles", [])
+        # Check if any of the minion's roles are in the certificate's required roles
+        if any(role in minion_roles for role in cert_roles):
             # Create the options
             opts = cert_opts.copy()
             opts["CN"] = certificate
             opts["days"] = config.get("days", 1)
 
-            # Create the signed certificates
             create_ca_signed_cert(base, name, **opts)
 
             # Add the signed certificates to the pillar data
             cert_data = get_ca_signed_cert(base, name, certificate)
             data["tls"]["certs"][certificate] = cert_data
 
-    return data
+    # Collect ACME certs (acme.cert) for this minion based on its roles
+    acme_cert_configs = pillar.get("tls", {}).get("acme_cert_configs", {})
+    for domain, domain_config in acme_cert_configs.items():
+        cert_roles = domain_config.get("roles", [])
+        if any(role in minion_roles for role in cert_roles):
+            cert_name = domain_config.get('name', domain)
+            full_cert_chain = _read_cert_file(f"/etc/letsencrypt/live/{cert_name}/fullchain.pem")
+            privkey = _read_cert_file(f"/etc/letsencrypt/live/{cert_name}/privkey.pem")
+
+            if full_cert_chain and privkey:
+                data["tls"]["acme_certs"][domain] = full_cert_chain + "\n" + privkey
+
+    return data

salt/bugs/config/postfix/main.cf

@@ -24,8 +24,8 @@ compatibility_level = 3.6
 
 
 # TLS parameters
-smtpd_tls_cert_file=ssl_certificate /etc/ssl/private/bugs.psf.io.pem;
-smtpd_tls_key_file=etc/ssl/private/bugs.psf.io.pem;
+smtpd_tls_cert_file=/etc/ssl/private/acme-bugs.python.org.pem
+smtpd_tls_key_file=/etc/ssl/private/acme-bugs.python.org.pem
 smtpd_tls_security_level=may
 
 smtp_tls_CApath=/etc/ssl/certs

salt/tls/init.sls

@@ -1,6 +1,12 @@
+include:
+  - .pebble
+  - .lego
+
 ssl-cert:
   pkg.installed
 
+certbot:
+  pkg.installed
 
 {% for name in salt["pillar.get"]("tls:ca", {}) %}  # " Syntax Hack
 /etc/ssl/certs/{{ name }}.pem:
@@ -11,8 +17,21 @@ ssl-cert:
     - mode: "0644"
     - require:
       - pkg: ssl-cert
+
+/usr/local/share/ca-certificates/{{ name }}.crt:
+  file.managed:
+    - contents_pillar: tls:ca:{{ name }}
+    - user: root
+    - group: ssl-cert
+    - mode: "0644"
+    - require:
+      - pkg: ssl-cert
 {% endfor %}
 
+/usr/sbin/update-ca-certificates:
+  cmd.run:
+    - onchanges:
+      - file: /usr/local/share/ca-certificates/*.crt
 
 {% for name in salt["pillar.get"]("tls:certs", {}) %}  # " Syntax Hack
 /etc/ssl/private/{{ name }}.pem:
@@ -25,3 +44,44 @@ ssl-cert:
     - require:
       - pkg: ssl-cert
 {% endfor %}
+
+# Install acme.cert certs prepended with acme-* to avoic conflicts
+{% for name in salt["pillar.get"]("tls:acme_certs", {}) %}
+/etc/ssl/private/acme-{{ name }}.pem:
+  file.managed:
+    - contents_pillar: tls:acme_certs:{{ name }}
+    - user: root
+    - group: ssl-cert
+    - mode: "0640"
+    - show_diff: False
+    - require:
+      - pkg: ssl-cert
+{% endfor %}
+
+{% if salt["match.compound"](pillar["roles"]["salt-master"]["pattern"]) %}
+# Process ACME certificates
+{% for domain, domain_config in salt["pillar.get"]("tls:acme_cert_configs", {}).items() %}
+{{ domain }}:
+  acme.cert:
+    - email: infrastructure-staff@python.org
+    - webroot: /etc/lego
+    - renew: 14
+    {% if domain_config.get('aliases') %}
+    - aliases:
+      {% for alias in domain_config.get('aliases', []) %}
+      - {{ alias }}
+      {% endfor %}
+    {% endif %}
+    {% if pillar["dc"] == "vagrant" %}
+    - server: https://salt-master.vagrant.psf.io:14000/dir
+    {% endif %}
+    {% if domain_config.get('validation') == "dns" %}
+    - dns_plugin: {{ domain_config.get('dns_plugin') }}
+    - dns_plugin_credentials: {{ domain_config.get('dns_plugin_credentials') }}
+    {% else %}
+    - require:
+      - sls: tls.lego
+      - pkg: certbot
+    {% endif %}
+{% endfor %}
+{% endif %}

salt/tls/pebble.sls

@@ -1,3 +1,4 @@
+{% if salt["match.compound"](pillar["roles"]["salt-master"]["pattern"]) %}
 {% if pillar.get('pebble', {'enabled': False}).enabled %}
 pebble-build-deps:
   pkg.installed:
@@ -60,3 +61,4 @@ pebble-service:
       - file: /etc/ssl/certs/PSF_CA.pem
       - file: /etc/ssl/private/salt-master.vagrant.psf.io.pem
 {% endif %}
+{% endif %}