[reconfigurator] Enable execution and planning by default (#8980)

The change to enable blueprint execution will only affect newly-deployed
systems that go through RSS after this change lands. Existing systems
will need to have execution enabled via `omdb`.

The changes to the default chicken switches will:

* enable the planner 
* disable zone additions if a mupdate override is present

on any systems that have not had reconfigurator chicken switches
explicitly set (this is _probably_ all deployed systems we care about;
even the dogfood rack has not had any of these set explicitly).

Fixes #8960. Fixes #8961.

Author: jgallagher

Cargo.lock

@@ -1684,7 +1684,12 @@ EXECUTING COMMAND: omdb ["nexus", "chicken-switches", "show", "current"]
 termination: Exited(0)
 ---------------------------------------------
 stdout:
-No chicken switches enabled
+Reconfigurator chicken switches:
+    version: 1
+    modified time: <REDACTED_TIMESTAMP>
+    planner enabled: false
+    planner switches:
+        add zones with mupdate override:   false
 ---------------------------------------------
 stderr:
 note: using Nexus URL http://127.0.0.1:REDACTED_PORT/
@@ -1693,22 +1698,22 @@ EXECUTING COMMAND: omdb ["-w", "nexus", "chicken-switches", "set", "--planner-en
 termination: Exited(0)
 ---------------------------------------------
 stdout:
-chicken switches updated to version 1:
-    planner enabled: true
+chicken switches updated to version 2:
+*   planner enabled:   false -> true
     planner switches:
-        add zones with mupdate override:   true
+        add zones with mupdate override:   false (unchanged)
 ---------------------------------------------
 stderr:
 note: using Nexus URL http://127.0.0.1:REDACTED_PORT/
 =============================================
-EXECUTING COMMAND: omdb ["-w", "nexus", "chicken-switches", "set", "--add-zones-with-mupdate-override", "false"]
+EXECUTING COMMAND: omdb ["-w", "nexus", "chicken-switches", "set", "--add-zones-with-mupdate-override", "true"]
 termination: Exited(0)
 ---------------------------------------------
 stdout:
-chicken switches updated to version 2:
+chicken switches updated to version 3:
     planner enabled:   true (unchanged)
     planner switches:
-    *   add zones with mupdate override:   true -> false
+    *   add zones with mupdate override:   false -> true
 ---------------------------------------------
 stderr:
 note: using Nexus URL http://127.0.0.1:REDACTED_PORT/
@@ -1718,11 +1723,11 @@ termination: Exited(0)
 ---------------------------------------------
 stdout:
 Reconfigurator chicken switches:
-    version: 2
+    version: 3
     modified time: <REDACTED_TIMESTAMP>
     planner enabled: true
     planner switches:
-        add zones with mupdate override:   false
+        add zones with mupdate override:   true
 ---------------------------------------------
 stderr:
 note: using Nexus URL http://127.0.0.1:REDACTED_PORT/

dev-tools/omdb/tests/successes.out

@@ -248,6 +248,10 @@ async fn test_omdb_success_cases(cptestctx: &ControlPlaneTestContext) {
         &["nexus", "blueprints", "diff", &initial_blueprint_id],
         // chicken switches: show and set
         &["nexus", "chicken-switches", "show", "current"],
+        // NOTE: Enabling the planner here _may_ cause Nexus to start creating
+        // new blueprints; any commands whose output is only stable if the set
+        // of blueprints is stable must come before this command to avoid being
+        // racy.
         &[
             "-w",
             "nexus",
@@ -262,7 +266,7 @@ async fn test_omdb_success_cases(cptestctx: &ControlPlaneTestContext) {
             "chicken-switches",
             "set",
             "--add-zones-with-mupdate-override",
-            "false",
+            "true",
         ],
         // After the set commands above, we should see chicken switches
         // populated.

dev-tools/omdb/tests/test_all_output.rs

@@ -40,9 +40,6 @@ INFO skipping noop image source check for all sleds, reason: no target release i
 WARN cannot issue more MGS-driven updates (no current artifacts)
 generated blueprint 8da82a8e-bf97-4fbd-8ddd-9f6462732cf1 based on parent blueprint dbcbd3d6-41ff-48ae-ac0b-1becc9b2fd21
 planning report for blueprint 8da82a8e-bf97-4fbd-8ddd-9f6462732cf1:
-chicken switches:
-    add zones with mupdate override:   false
-
 * no zpools in service for NTP zones on sleds: 00320471-945d-413c-85e7-03e091a70b3c
 * discretionary zone placement waiting for NTP zones on sleds: 00320471-945d-413c-85e7-03e091a70b3c
 
@@ -279,9 +276,6 @@ parent:    dbcbd3d6-41ff-48ae-ac0b-1becc9b2fd21
  PENDING MGS-MANAGED UPDATES: 0
 
 planning report for blueprint 8da82a8e-bf97-4fbd-8ddd-9f6462732cf1:
-chicken switches:
-    add zones with mupdate override:   false
-
 * no zpools in service for NTP zones on sleds: 00320471-945d-413c-85e7-03e091a70b3c
 * discretionary zone placement waiting for NTP zones on sleds: 00320471-945d-413c-85e7-03e091a70b3c
 

dev-tools/reconfigurator-cli/tests/output/cmds-add-sled-no-disks-stdout

@@ -556,9 +556,6 @@ WARN cannot issue more MGS-driven updates (no current artifacts)
 INFO some zones not yet up-to-date, sled_id: 89d02b1b-478c-401a-8e28-7a26f74fa41b, zones_currently_updating: [ZoneCurrentlyUpdating { zone_id: b3c9c041-d2f0-4767-bdaf-0e52e9d7a013 (service), zone_kind: InternalNtp, reason: MissingInInventory { bp_image_source: InstallDataset } }]
 generated blueprint 86db3308-f817-4626-8838-4085949a6a41 based on parent blueprint ade5749d-bdf3-4fab-a8ae-00bea01b3a5a
 planning report for blueprint 86db3308-f817-4626-8838-4085949a6a41:
-chicken switches:
-    add zones with mupdate override:   false
-
 * discretionary zone placement waiting for NTP zones on sleds: 89d02b1b-478c-401a-8e28-7a26f74fa41b
 * missing NTP zone on sled 89d02b1b-478c-401a-8e28-7a26f74fa41b
 * only placed 0/1 desired clickhouse zones

dev-tools/reconfigurator-cli/tests/output/cmds-example-stdout

@@ -842,9 +842,6 @@ INFO skipping noop image source check for all sleds, reason: no target release i
 WARN cannot issue more MGS-driven updates (no current artifacts)
 generated blueprint 9c998c1d-1a7b-440a-ae0c-40f781dea6e2 based on parent blueprint 366b0b68-d80e-4bc1-abd3-dc69837847e0
 planning report for blueprint 9c998c1d-1a7b-440a-ae0c-40f781dea6e2:
-chicken switches:
-    add zones with mupdate override:   false
-
 * discretionary zones placed:
   * 1 zone on sled 711ac7f8-d19e-4572-bdb9-e9b50f6e362a: external_dns
 * zone updates waiting on discretionary zones
@@ -1342,9 +1339,6 @@ parent:    366b0b68-d80e-4bc1-abd3-dc69837847e0
  PENDING MGS-MANAGED UPDATES: 0
 
 planning report for blueprint 9c998c1d-1a7b-440a-ae0c-40f781dea6e2:
-chicken switches:
-    add zones with mupdate override:   false
-
 * discretionary zones placed:
   * 1 zone on sled 711ac7f8-d19e-4572-bdb9-e9b50f6e362a: external_dns
 * zone updates waiting on discretionary zones

dev-tools/reconfigurator-cli/tests/output/cmds-expunge-newly-added-external-dns-stdout

@@ -650,9 +650,6 @@ INFO skipping noop image source check for all sleds, reason: no target release i
 WARN cannot issue more MGS-driven updates (no current artifacts)
 generated blueprint af934083-59b5-4bf6-8966-6fb5292c29e1 based on parent blueprint 58d5e830-0884-47d8-a7cd-b2b3751adeb4
 planning report for blueprint af934083-59b5-4bf6-8966-6fb5292c29e1:
-chicken switches:
-    add zones with mupdate override:   false
-
 * discretionary zones placed:
   * 1 zone on sled 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c: internal_dns
 * zone updates waiting on discretionary zones

dev-tools/reconfigurator-cli/tests/output/cmds-expunge-newly-added-internal-dns-stdout

@@ -502,9 +502,6 @@ INFO skipped noop image source check on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0
 INFO skipped noop image source check on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, reason: remove_mupdate_override is set in the blueprint (203fa72c-85c1-466a-8ed3-338ee029530d)
 generated blueprint a5a8f242-ffa5-473c-8efd-2acf2dc0b736 based on parent blueprint d60afc57-f15d-476c-bd0f-b1071e2bb976
 planning report for blueprint a5a8f242-ffa5-473c-8efd-2acf2dc0b736:
-chicken switches:
-    add zones with mupdate override:   false
-
 * zone adds waiting on blockers
 * zone adds and updates are blocked:
   - current target release generation (2) is lower than minimum required by blueprint (3)
@@ -777,9 +774,6 @@ INFO skipped noop image source check on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0
 INFO skipped noop image source check on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, reason: remove_mupdate_override is set in the blueprint (203fa72c-85c1-466a-8ed3-338ee029530d)
 generated blueprint 626487fa-7139-45ec-8416-902271fc730b based on parent blueprint a5a8f242-ffa5-473c-8efd-2acf2dc0b736
 planning report for blueprint 626487fa-7139-45ec-8416-902271fc730b:
-chicken switches:
-    add zones with mupdate override:   false
-
 * zone adds waiting on blockers
 * zone adds and updates are blocked:
   - sleds have remove mupdate override set in blueprint: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, d81c6a84-79b8-4958-ae41-ea46c9b19763
@@ -902,9 +896,6 @@ INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noo
 INFO skipped noop image source check on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, reason: remove_mupdate_override is set in the blueprint (1c0ce176-6dc8-4a90-adea-d4a8000751da)
 generated blueprint c1a0d242-9160-40f4-96ae-61f8f40a0b1b based on parent blueprint 626487fa-7139-45ec-8416-902271fc730b
 planning report for blueprint c1a0d242-9160-40f4-96ae-61f8f40a0b1b:
-chicken switches:
-    add zones with mupdate override:   false
-
 * noop converting 6/6 install-dataset zones to artifact store on sled 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6
 * zone adds waiting on blockers
 * zone adds and updates are blocked:
@@ -1091,9 +1082,6 @@ INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noo
 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: b, expected_hash: 0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b
 generated blueprint afb09faf-a586-4483-9289-04d4f1d8ba23 based on parent blueprint c1a0d242-9160-40f4-96ae-61f8f40a0b1b
 planning report for blueprint afb09faf-a586-4483-9289-04d4f1d8ba23:
-chicken switches:
-    add zones with mupdate override:   false
-
 * skipping noop zone image source check on sled 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6: all 6 zones are already from artifacts
 * zone adds waiting on blockers
 * zone adds and updates are blocked:
@@ -1269,9 +1257,6 @@ parent:    c1a0d242-9160-40f4-96ae-61f8f40a0b1b
  PENDING MGS-MANAGED UPDATES: 0
 
 planning report for blueprint afb09faf-a586-4483-9289-04d4f1d8ba23:
-chicken switches:
-    add zones with mupdate override:   false
-
 * skipping noop zone image source check on sled 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6: all 6 zones are already from artifacts
 * zone adds waiting on blockers
 * zone adds and updates are blocked:
@@ -1410,9 +1395,6 @@ INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noo
 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: b, expected_hash: 0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b
 generated blueprint ce365dff-2cdb-4f35-a186-b15e20e1e700 based on parent blueprint afb09faf-a586-4483-9289-04d4f1d8ba23
 planning report for blueprint ce365dff-2cdb-4f35-a186-b15e20e1e700:
-chicken switches:
-    add zones with mupdate override:   false
-
 * skipping noop zone image source check on sled 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6: all 6 zones are already from artifacts
 * noop converting 6/6 install-dataset zones to artifact store on sled d81c6a84-79b8-4958-ae41-ea46c9b19763
 * zone adds waiting on blockers
@@ -1588,9 +1570,6 @@ parent:    afb09faf-a586-4483-9289-04d4f1d8ba23
  PENDING MGS-MANAGED UPDATES: 0
 
 planning report for blueprint ce365dff-2cdb-4f35-a186-b15e20e1e700:
-chicken switches:
-    add zones with mupdate override:   false
-
 * skipping noop zone image source check on sled 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6: all 6 zones are already from artifacts
 * noop converting 6/6 install-dataset zones to artifact store on sled d81c6a84-79b8-4958-ae41-ea46c9b19763
 * zone adds waiting on blockers
@@ -1708,9 +1687,6 @@ INFO configuring MGS-driven update, artifact_version: 2.0.0, artifact_hash: 8f89
 INFO reached maximum number of pending MGS-driven updates, max: 1
 generated blueprint 8f2d1f39-7c88-4701-aa43-56bf281b28c1 based on parent blueprint ce365dff-2cdb-4f35-a186-b15e20e1e700
 planning report for blueprint 8f2d1f39-7c88-4701-aa43-56bf281b28c1:
-chicken switches:
-    add zones with mupdate override:   false
-
 * skipping noop zone image source check on sled 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6: all 6 zones are already from artifacts
 * skipping noop zone image source check on sled d81c6a84-79b8-4958-ae41-ea46c9b19763: all 6 zones are already from artifacts
 * 1 pending MGS update:
@@ -1891,9 +1867,6 @@ parent:    ce365dff-2cdb-4f35-a186-b15e20e1e700
 
 
 planning report for blueprint 8f2d1f39-7c88-4701-aa43-56bf281b28c1:
-chicken switches:
-    add zones with mupdate override:   false
-
 * skipping noop zone image source check on sled 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6: all 6 zones are already from artifacts
 * skipping noop zone image source check on sled d81c6a84-79b8-4958-ae41-ea46c9b19763: all 6 zones are already from artifacts
 * 1 pending MGS update:
@@ -2018,9 +1991,6 @@ INFO ran out of boards for MGS-driven update
 INFO some zones not yet up-to-date, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, zones_currently_updating: [ZoneCurrentlyUpdating { zone_id: 0c71b3b2-6ceb-4e8f-b020-b08675e83038 (service), zone_kind: Nexus, reason: ImageSourceMismatch { bp_image_source: Artifact { version: Available { version: ArtifactVersion("1.0.0") }, hash: ArtifactHash("0e32b4a3e5d3668bb1d6a16fb06b74dc60b973fa479dcee0aae3adbb52bf1388") }, inv_image_source: InstallDataset } }, ZoneCurrentlyUpdating { zone_id: 427ec88f-f467-42fa-9bbb-66a91a36103c (service), zone_kind: InternalDns, reason: ImageSourceMismatch { bp_image_source: Artifact { version: Available { version: ArtifactVersion("1.0.0") }, hash: ArtifactHash("ffbf1373f7ee08dddd74c53ed2a94e7c4c572a982d3a9bc94000c6956b700c6a") }, inv_image_source: InstallDataset } }, ZoneCurrentlyUpdating { zone_id: 5199c033-4cf9-4ab6-8ae7-566bd7606363 (service), zone_kind: Crucible, reason: ImageSourceMismatch { bp_image_source: Artifact { version: Available { version: ArtifactVersion("1.0.0") }, hash: ArtifactHash("6f17cf65fb5a5bec5542dd07c03cd0acc01e59130f02c532c8d848ecae810047") }, inv_image_source: InstallDataset } }, ZoneCurrentlyUpdating { zone_id: 6444f8a5-6465-4f0b-a549-1993c113569c (service), zone_kind: InternalNtp, reason: ImageSourceMismatch { bp_image_source: Artifact { version: Available { version: ArtifactVersion("1.0.0") }, hash: ArtifactHash("67593d686ed04a1709f93972b71f4ebc148a9362120f65d239943e814a9a7439") }, inv_image_source: InstallDataset } }, ZoneCurrentlyUpdating { zone_id: 803bfb63-c246-41db-b0da-d3b87ddfc63d (service), zone_kind: ExternalDns, reason: ImageSourceMismatch { bp_image_source: Artifact { version: Available { version: ArtifactVersion("1.0.0") }, hash: ArtifactHash("ccca13ed19b8731f9adaf0d6203b02ea3b9ede4fa426b9fac0a07ce95440046d") }, inv_image_source: InstallDataset } }, ZoneCurrentlyUpdating { zone_id: ba4994a8-23f9-4b1a-a84f-a08d74591389 (service), zone_kind: CruciblePantry, reason: ImageSourceMismatch { bp_image_source: Artifact { version: Available { version: ArtifactVersion("1.0.0") }, hash: ArtifactHash("21f0ada306859c23917361f2e0b9235806c32607ec689c7e8cf16bb898bc5a02") }, inv_image_source: InstallDataset } }]
 generated blueprint 12d602a6-5ab4-487a-b94e-eb30cdf30300 based on parent blueprint 8f2d1f39-7c88-4701-aa43-56bf281b28c1
 planning report for blueprint 12d602a6-5ab4-487a-b94e-eb30cdf30300:
-chicken switches:
-    add zones with mupdate override:   false
-
 * skipping noop zone image source check on sled 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6: all 6 zones are already from artifacts
 * skipping noop zone image source check on sled d81c6a84-79b8-4958-ae41-ea46c9b19763: all 6 zones are already from artifacts
 * noop converting host phase 2 slot B to Artifact on sled 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c
@@ -2256,9 +2226,6 @@ INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-495
 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a
 generated blueprint 61a93ea3-c872-48e0-aace-e86b0c52b839 based on parent blueprint 12d602a6-5ab4-487a-b94e-eb30cdf30300
 planning report for blueprint 61a93ea3-c872-48e0-aace-e86b0c52b839:
-chicken switches:
-    add zones with mupdate override:   false
-
 * skipping noop zone image source check on sled c3bc4c6d-fdde-4fc4-8493-89d2a1e5ee6b: all 0 zones are already from artifacts
 * skipping noop zone image source check on sled d81c6a84-79b8-4958-ae41-ea46c9b19763: all 6 zones are already from artifacts
 * zone adds waiting on blockers
@@ -2384,6 +2351,9 @@ INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noo
 INFO altered physical disks, sled_id: c3bc4c6d-fdde-4fc4-8493-89d2a1e5ee6b, sled_edits: SledEditCounts { disks: EditCounts { added: 10, updated: 0, expunged: 0, removed: 0 }, datasets: EditCounts { added: 20, updated: 0, expunged: 0, removed: 0 }, zones: EditCounts { added: 0, updated: 0, expunged: 0, removed: 0 } }
 generated blueprint 27e755bc-dc10-4647-853c-f89bb3a15a2c based on parent blueprint 61a93ea3-c872-48e0-aace-e86b0c52b839
 planning report for blueprint 27e755bc-dc10-4647-853c-f89bb3a15a2c:
+chicken switches:
+    add zones with mupdate override:   true
+
 * skipping noop zone image source check on sled c3bc4c6d-fdde-4fc4-8493-89d2a1e5ee6b: all 0 zones are already from artifacts
 * skipping noop zone image source check on sled d81c6a84-79b8-4958-ae41-ea46c9b19763: all 6 zones are already from artifacts
 * zone adds and updates are blocked:

dev-tools/reconfigurator-cli/tests/output/cmds-mupdate-update-flow-stdout

@@ -178,9 +178,6 @@ INFO skipped noop image source check on sled, sled_id: d81c6a84-79b8-4958-ae41-e
 INFO skipped noop image source check on sled, sled_id: e96e226f-4ed9-4c01-91b9-69a9cd076c9e, reason: sled not found in inventory
 generated blueprint 58d5e830-0884-47d8-a7cd-b2b3751adeb4 based on parent blueprint 8da82a8e-bf97-4fbd-8ddd-9f6462732cf1
 planning report for blueprint 58d5e830-0884-47d8-a7cd-b2b3751adeb4:
-chicken switches:
-    add zones with mupdate override:   false
-
 * noop converting 6/6 install-dataset zones to artifact store on sled 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6
 * noop converting 5/6 install-dataset zones to artifact store on sled aff6c093-197d-42c5-ad80-9f10ba051a34
 * zone adds waiting on blockers
@@ -419,9 +416,6 @@ INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noo
 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: e96e226f-4ed9-4c01-91b9-69a9cd076c9e, slot: b, expected_hash: 0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b
 generated blueprint af934083-59b5-4bf6-8966-6fb5292c29e1 based on parent blueprint 58d5e830-0884-47d8-a7cd-b2b3751adeb4
 planning report for blueprint af934083-59b5-4bf6-8966-6fb5292c29e1:
-chicken switches:
-    add zones with mupdate override:   false
-
 * skipping noop zone image source check on sled 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6: all 6 zones are already from artifacts
 * noop converting 2/2 install-dataset zones to artifact store on sled e96e226f-4ed9-4c01-91b9-69a9cd076c9e
 * zone adds waiting on blockers