workbench: make it new Nomad cluster compatible

fmaste · fmaste · commit 9e90ff5d11ad · 2023-08-10T02:32:48.000Z
diff --git a/nix/workbench/backend/nomad-job.nix b/nix/workbench/backend/nomad-job.nix
@@ -91,9 +91,10 @@ let
       # The SUPERVISORD_CONFIG variable must be set
       [ -z "''${SUPERVISORD_CONFIG:-}" ] && echo "SUPERVISORD_CONFIG env var must be set -- aborting" && exit 1
 
-      # Create symlink to 'supervisor' Nix folder so we can call it from 'ssh'
-      #  or 'nomad exec' without having to know the currently version running.
-      # First check if already exists to be able to restart containers.
+      # Create a symlink to 'supervisor' Nix Store folder so we can call it from
+      # 'ssh' or 'nomad exec' without having it in PATH or knowing the currently
+      # running version. But first check if it already exists to be able to
+      # restart containers without errors.
       if ! test -e "''${SUPERVISOR_NIX}"
       then
         ${coreutils}/bin/ln -s "${supervisor}" "''${SUPERVISOR_NIX}"
@@ -163,7 +164,7 @@ let
     # namespace can be specified either with the flag -namespace or read from
     # the NOMAD_NAMESPACE environment variable."
     # https://developer.hashicorp.com/nomad/tutorials/manage-clusters/namespaces
-    namespace = "perf";
+    namespace = "perf"; # Default to "perf" to avoid errors were possible.
 
     # The region in which to execute the job.
     region = "global"; # SRE: They are actually using global.
@@ -270,10 +271,12 @@ let
         constraint = {
           attribute = "\${node.class}";
           operator = "=";
-          # For cloud benchmarking dedicated static machines in the "perf"
-          # class are used. We replicate that for local/test runs.
-          # Class "qa" nodes are also available but must be limited to short
-          # test and avoid using "infra" node class as HA jobs runs there.
+          # For cloud benchmarking, dedicated static machines in the "perf"
+          # class are used. We mimic that for local/test runs.
+          # This default is just a precaution, like the top level namespace,
+          # because there are also available "qa" Class nodes but usage of these
+          # must be limited to short test and "infra" Class nodes are used for
+          # HA jobs and must be avoided entirely.
           value = "perf";
         };
 
@@ -389,8 +392,22 @@ let
 
           # Specifies environment variables that will be passed to the running
           # process.
-          # `null` because we are using a "template" (see below).
-          env = {};
+          env = {
+            # The "old Nomad" setup somehow included the CA certs inside the
+            # task namespace but apparently not "new Nomad". We are adding the
+            # necessary Nix package ("cacert") and making sure `wget` finds it.
+            #
+            # "All nix-docker images set environment variables which point to
+            # cacert.", see:
+            # - https://github.com/NixOS/nixpkgs/issues/48211#issuecomment-434102565
+            # - https://github.com/LnL7/nix-docker/blob/8dcfb3aff1f87cdafeecb0d27964b27c3fb8b1d2/default.nix#L70-L71
+            #
+            # Error when using `wget` to deploy the genesis tar file was:
+            # ERROR: cannot verify iog-cardano-perf.s3.eu-central-1.amazonaws.com's certificate, issued by 'CN=Amazon RSA 2048 M01,O=Amazon,C=US':
+            # Unable to locally verify the issuer's authority.
+            # To connect to iog-cardano-perf.s3.eu-central-1.amazonaws.com insecurely, use `--no-check-certificate'.
+            SSL_CERT_FILE = "${containerSpecs.containerPkgs.cacert.nix-store-path}/etc/ssl/certs/ca-bundle.crt";
+          };
 
           # Sensible defaults to run cloud version of "default", "ci-test" and
           # "ci-bench" in cardano-world qa class Nomad nodes.
@@ -434,7 +451,7 @@ let
               # When using Cardano World (nomad.world.dev.cardano.org) "perf"
               # class nodes we use public IPs/routing, all the other cloud runs
               # are behind a VPC/firewall. Local runs just use 12.0.0.1.
-              if lib.strings.hasPrefix "cw-perf" profileData.profileName
+              if lib.strings.hasInfix "cw-perf" profileData.profileName
               then "\${attr.unique.platform.aws.public-ipv4}"
               else ""
             ;
@@ -1161,17 +1178,19 @@ let
         # Port string from
         ''--port ${toString nodeSpec.port}''
       ]
-      # On cloud deployments with cardano world, that uses AWS, the hosts at the
-      # Linux level aren't aware of the EIP public address they have, so the
-      # private IP is the only network interface address to bind to.
-      # An alternative is to bind to 0.0.0.0 but I prefer being more specific.
+      # On cloud deployments to SRE-managed Nomad, that uses AWS, the hosts at
+      # Linux level may not be aware of the EIP public address they have so we
+      # can't bind to the public IP (that we can resolve to using templates).
+      # I prefer being more specific but the "all-weather" alternative is to
+      # bind to 0.0.0.0 instead of the private IP, just in case the Nomad Client
+      # was not started with the correct `-network-interface XX` parameter.
       [
         # Address string to
-        ''--host-addr {{ env "NOMAD_HOST_IP_${servicePortName}" }}''
+        ''--host-addr 0.0.0.0''
         # Alternatives (may not work):
+        #''--host-addr {{ env "NOMAD_HOST_IP_${servicePortName}" }}''
         #''--host-addr {{ env "NOMAD_IP_${servicePortName}" }}''
         #''--host-addr {{range nomadService "${servicePortName}"}}{{.Address}}{{end}}''
-        #''--host-addr 0.0.0.0''
 
         # Port string to
         ''--port {{ env "NOMAD_HOST_PORT_${servicePortName}" }}''
diff --git a/nix/workbench/backend/nomad.sh b/nix/workbench/backend/nomad.sh
@@ -223,11 +223,24 @@ backend_nomad() {
 
     allocate-run-nomad-job-patch-namespace )
       local usage="USAGE: wb backend $op RUN-DIR NAMESPACE"
-      local dir=${1:?$usage};       shift
-      local namespace=${1:?$usage}; shift
+      local dir=${1:?$usage}; shift
       local nomad_job_name=$(jq -r ". [\"job\"] | keys[0]" "${dir}"/nomad/nomad-job.json)
-      msg "Setting Nomad job namespace to \"${namespace}\""
-      jq ".job[\"${nomad_job_name}\"][\"namespace\"] = \"${namespace}\"" "${dir}"/nomad/nomad-job.json | sponge "${dir}"/nomad/nomad-job.json
+      if test $# -gt 0 && test -n "${1}"
+      then
+        local namespace
+        namespace="${1:?$usage}"; shift
+        msg "Setting Nomad job top level namespace to \"${namespace}\""
+          jq                                                                \
+            ".job[\"${nomad_job_name}\"][\"namespace\"] = \"${namespace}\"" \
+            "${dir}"/nomad/nomad-job.json                                   \
+        | sponge "${dir}"/nomad/nomad-job.json
+      else
+        msg "Setting Nomad job top level namespace to null"
+          jq                                                                \
+            ".job[\"${nomad_job_name}\"][\"namespace\"] = null"             \
+            "${dir}"/nomad/nomad-job.json                                   \
+        | sponge "${dir}"/nomad/nomad-job.json
+      fi
     ;;
 
     allocate-run-nomad-job-patch-nix )
diff --git a/nix/workbench/backend/nomad/cloud.sh b/nix/workbench/backend/nomad/cloud.sh
@@ -167,36 +167,51 @@ setenv-defaults-nomadcloud() {
   local profile_container_specs_file
   profile_container_specs_file="${backend_dir}"/container-specs.json
 
-  # If the most important `nomad` cli envars is present this is not a local
-  # test, I repeat, this is not a drill =)
-  if test -z "${NOMAD_ADDR:-}"
+  if test -z "${NOMAD_ADDR+set}"
   then
+    # The variable is not set, not set but empty, just not set!
     msg $(yellow "WARNING: Nomad address \"NOMAD_ADDR\" envar is not set")
+    # TODO: New Nomad cluster:export NOMAD_ADDR=http://10.200.0.1:4646
     export NOMAD_ADDR="https://nomad.world.dev.cardano.org"
     msg $(blue "INFO: Setting \"NOMAD_ADDR\" to the SRE provided address for \"Performance and Tracing\" (\"${NOMAD_ADDR}\")")
   else
+    # The variable is set and maybe empty!
+    msg $(blue "INFO: Nomad address \"NOMAD_ADDR\" envar is \"${NOMAD_ADDR}\"")
     if test "${NOMAD_ADDR}" != "https://nomad.world.dev.cardano.org"
     then
       msg $(yellow "WARNING: Nomad address \"NOMAD_ADDR\" envar is not \"https://nomad.world.dev.cardano.org\"")
     fi
   fi
   # The abscence of `NOMAD_NAMESPACE` or `NOMAD_TOKEN` needs confirmation
-  if test -z "${NOMAD_NAMESPACE:-}"
+  if test -z ${NOMAD_NAMESPACE+set}
   then
+    # The variable is not set, not set but empty, just not set!
     msg $(yellow "WARNING: Nomad namespace \"NOMAD_NAMESPACE\" envar is not set")
+    # TODO: New Nomad cluster: export NOMAD_NAMESPACE=""
     export NOMAD_NAMESPACE="perf"
     msg $(blue "INFO: Setting \"NOMAD_NAMESPACE\" to the SRE provided namespace for \"Performance and Tracing\" (\"${NOMAD_NAMESPACE}\")")
   else
+    # The variable is set and maybe empty!
+    msg $(blue "INFO: Nomad namespace \"NOMAD_NAMESPACE\" envar is \"${NOMAD_NAMESPACE}\"")
     if test "${NOMAD_NAMESPACE}" != "perf"
     then
       msg $(yellow "WARNING: Nomad namespace \"NOMAD_NAMESPACE\" envar is not \"perf\"")
     fi
   fi
-  if test -z "${NOMAD_TOKEN:-}"
+  if test -z "${NOMAD_TOKEN+set}"
   then
+    # The variable is not set, not set but empty, just not set!
     msg $(yellow "WARNING: Nomad token \"NOMAD_TOKEN\" envar is not set")
-    msg $(blue "INFO: Fetching a \"NOMAD_TOKEN\" from SRE provided Vault for \"Performance and Tracing\"")
-    export NOMAD_TOKEN="$(wb_nomad vault world nomad-token)"
+    msg $(yellow "If you need to fetch a NOMAD_TOKEN for world.dev.cardano.org provide an empty string")
+  else
+    # The variable is set and maybe empty!
+    if test -z "${NOMAD_TOKEN}"
+    then
+      msg $(blue "INFO: Fetching a \"NOMAD_TOKEN\" from SRE provided Vault for \"Performance and Tracing\"")
+      export NOMAD_TOKEN="$(wb_nomad vault world nomad-token)"
+    else
+      msg $(blue "INFO: Using provided Nomad token \"NOMAD_TOKEN\" envar")
+    fi
   fi
   # Check all the AWS S3 envars needed for the HTTP PUT request
   # Using same names as the AWS CLI
@@ -284,8 +299,16 @@ allocate-run-nomadcloud() {
     > "${dir}"/nomad/nomad-job.json
   fi
   # The job file is "slightly" modified (jq) to suit the running environment.
-  backend_nomad allocate-run-nomad-job-patch-namespace "${dir}" "${NOMAD_NAMESPACE}"
-  backend_nomad allocate-run-nomad-job-patch-nix       "${dir}"
+  if test -n "${NOMAD_NAMESPACE:-}"
+  then
+    # This sets only the global namespace, the job level namespace. Not groups!
+    backend_nomad allocate-run-nomad-job-patch-namespace "${dir}" "${NOMAD_NAMESPACE}"
+  else
+    # Empty the global namespace
+    backend_nomad allocate-run-nomad-job-patch-namespace "${dir}"
+  fi
+  # Will set the flake URIs from ".installable" in container-specs.json
+  backend_nomad allocate-run-nomad-job-patch-nix "${dir}"
 
   # Set the placement info and resources accordingly
   local nomad_job_name
@@ -348,12 +371,12 @@ allocate-run-nomadcloud() {
     ########################################################################
     local group_constraints_array
     # "perf" class nodes are the default unless the profile name contains
-    # "cw-qa", we try to limit the usage of Nomad nodes that are not
-    # dedicated Perf team nodes.
-    # But also, we have to be careful that "perf" runs do not overlap. We
-    # are making "perf" class nodes runs can't clash because service names
-    # and resources definitions currently won't allow that to happen but
-    # still a new "perf" run may mess up a previously running cluster.
+    # "cw-qa", we try to limit the usage of Nomad nodes that are not dedicated
+    # Perf team nodes.
+    # But also, we have to be careful that "perf" runs do not overlap. We are
+    # making sure "perf" class nodes runs can't clash because service names
+    # and resources definitions currently won't allow that to happen but a new
+    # "perf" run may still mess up a previously running cluster.
     if echo "${WB_SHELL_PROFILE}" | grep --quiet "cw-qa"
     then
       # Using "qa" class distinct nodes. Only "short" test allowed here.
@@ -366,29 +389,42 @@ allocate-run-nomadcloud() {
           }
         ]
       '
-    else
+    elif test -n "${NOMAD_NAMESPACE:-}"
+    then
       # Using Performance & Tracing exclusive "perf" class distinct nodes!
-      group_constraints_array='
-        [
-          {
-            "operator":  "="
-          , "attribute": "${node.class}"
-          , "value":     "perf"
-          }
-        ]
-      '
+      group_constraints_array="                   \
+        [                                         \
+          {                                       \
+            \"operator\":  \"=\"                  \
+          , \"attribute\": \"\${node.class}\"     \
+          , \"value\":     \"${NOMAD_NAMESPACE}\" \
+          }                                       \
+        ]                                         \
+      "
+    fi
+    # It there something to change related to group constraints ?
+    # Sets or deletes all groups level constraints.
+    if test -n "${group_constraints_array:-}"
+    then
+      # Adds it as a group level contraint to all groups.
+        jq \
+          --argjson group_constraints_array "${group_constraints_array}" \
+          ".[\"job\"][\"${nomad_job_name}\"][\"group\"] |= with_entries(.value.constraint = \$group_constraints_array)" \
+          "${dir}"/nomad/nomad-job.json \
+      | \
+        sponge "${dir}"/nomad/nomad-job.json
+    else
+      # Else, empties all group level constraints, like previous namespaces.
+        jq \
+          ".[\"job\"][\"${nomad_job_name}\"][\"group\"] |= with_entries(.value.constraint = null)" \
+          "${dir}"/nomad/nomad-job.json \
+      | \
+        sponge "${dir}"/nomad/nomad-job.json
     fi
-    # Adds it as a group level contraint.
-      jq \
-        --argjson group_constraints_array "${group_constraints_array}" \
-        ".[\"job\"][\"${nomad_job_name}\"][\"group\"] |= with_entries(.value.constraint = \$group_constraints_array)" \
-        "${dir}"/nomad/nomad-job.json \
-    | \
-      sponge "${dir}"/nomad/nomad-job.json
     ########################################################################
     # Memory/resources: ####################################################
     ########################################################################
-    # Set the resources, only for perf!
+    # Set the resources, only for perf exlusive cloud runs!
     # When not "perf", when "cw-qa", only "short" tests are allowed on
     # whatever resources we are given.
     if echo "${WB_SHELL_PROFILE}" | grep --quiet "cw-perf"
diff --git a/nix/workbench/backend/nomad/exec.sh b/nix/workbench/backend/nomad/exec.sh
@@ -162,8 +162,10 @@ allocate-run-nomadexec() {
     "${dir}"/container-specs.json              \
   > "${dir}"/nomad/nomad-job.json
   # The job file is "slightly" modified (jq) to suit the running environment.
-  backend_nomad allocate-run-nomad-job-patch-namespace "${dir}" "default"
-  backend_nomad allocate-run-nomad-job-patch-nix       "${dir}"
+  ## Empty the global namespace. Local runs ignore "${NOMAD_NAMESPACE:-}"
+  backend_nomad allocate-run-nomad-job-patch-namespace "${dir}"
+  # Will set the /nix/store paths from ".nix-store-path" in container-specs.json
+  backend_nomad allocate-run-nomad-job-patch-nix "${dir}"
 }
 
 deploy-genesis-nomadexec() {
diff --git a/nix/workbench/backend/nomad/podman.sh b/nix/workbench/backend/nomad/podman.sh
@@ -163,7 +163,8 @@ allocate-run-nomadpodman() {
     "${dir}"/container-specs.json              \
   > "${dir}"/nomad/nomad-job.json
   # The job file is "slightly" modified (jq) to suit the running environment.
-  backend_nomad allocate-run-nomad-job-patch-namespace "${dir}" "default"
+  ## Empty the global namespace. Local runs ignore "${NOMAD_NAMESPACE:-}"
+  backend_nomad allocate-run-nomad-job-patch-namespace "${dir}"
   podman_create_image "${dir}"
   # Make sure the "genesis-volume" dir is present when the Nomad job is
   # started because with the podman task driver (always local, not used for
