Move schema validation progress table and model to bootstrap_model (#40572)

Moves `_schema_validation_progress` table and model to `bootstrap_model` so that we can delete documents in `_schema_validation_progress` atomically in `SchemaModel::mark_failed` and `SchemaModel::mark_validated`.

GitOrigin-RevId: 954691071c2e01fa624166fa92d27823a1c06538

Author: emmaling27

crates/database/src/bootstrap_model/defaults.rs

@@ -38,6 +38,10 @@ use crate::{
         index_backfills::IndexBackfillTable,
         index_workers::IndexWorkerMetadataTable,
         schema::SchemasTable,
+        schema_validation_progress::{
+            SchemaValidationProgressTable,
+            SCHEMA_VALIDATION_PROGRESS_TABLE,
+        },
         table::TablesTable,
     },
     system_tables::ErasedSystemTable,
@@ -58,6 +62,7 @@ pub fn bootstrap_system_tables() -> Vec<&'static dyn ErasedSystemTable> {
         &IndexWorkerMetadataTable,
         &ComponentDefinitionsTable,
         &ComponentsTable,
+        &SchemaValidationProgressTable,
     ]
 }
 
@@ -72,6 +77,7 @@ pub static DEFAULT_BOOTSTRAP_TABLE_NUMBERS: LazyLock<BTreeMap<TableName, TableNu
             COMPONENT_DEFINITIONS_TABLE.clone() => tn(31),
             COMPONENTS_TABLE.clone() => tn(32),
             INDEX_BACKFILLS_TABLE.clone() => tn(36),
+            SCHEMA_VALIDATION_PROGRESS_TABLE.clone() => tn(37)
             // To add a bootstrap system table, first add to model/src/lib and then
             // replicate that table number to here.
         }

crates/database/src/bootstrap_model/mod.rs

@@ -9,6 +9,7 @@ pub mod index;
 pub mod index_backfills;
 pub mod index_workers;
 pub mod schema;
+pub mod schema_validation_progress;
 pub mod system_metadata;
 pub mod table;
 pub mod user_facing;

crates/model/src/schema_validation_progress/mod.rs renamed to crates/database/src/bootstrap_model/schema_validation_progress/mod.rs

@@ -3,15 +3,16 @@ pub mod types;
 use std::sync::LazyLock;
 
 use common::document::CREATION_TIME_FIELD_PATH;
-use database::system_tables::{
-    SystemIndex,
-    SystemTable,
-};
 use value::{
     FieldPath,
     TableName,
 };
 
+use crate::system_tables::{
+    SystemIndex,
+    SystemTable,
+};
+
 pub static SCHEMA_VALIDATION_PROGRESS_TABLE: LazyLock<TableName> = LazyLock::new(|| {
     "_schema_validation_progress"
         .parse()

crates/model/src/schema_validation_progress/types.rs renamed to crates/database/src/bootstrap_model/schema_validation_progress/types.rs

@@ -136,6 +136,12 @@ pub use self::{
             SCHEMAS_TABLE,
             SCHEMA_STATE_FIELD,
         },
+        schema_validation_progress::{
+            types::SchemaValidationProgressMetadata,
+            SchemaValidationProgressTable,
+            SCHEMA_VALIDATION_PROGRESS_BY_SCHEMA_ID,
+            SCHEMA_VALIDATION_PROGRESS_TABLE,
+        },
         system_metadata::SystemMetadataModel,
         table::{
             TableModel,

crates/database/src/lib.rs

@@ -28,6 +28,7 @@ use database::{
     IndexModel,
     SchemaDiff,
     SchemaModel,
+    SchemaValidationProgressTable,
     SchemasTable,
     SerializedSchemaDiff,
     SystemMetadataModel,
@@ -372,6 +373,13 @@ impl<'a, RT: Runtime> ComponentConfigModel<'a, RT> {
             &DEFAULT_TABLE_NUMBERS,
         )
         .await?;
+        initialize_application_system_table(
+            self.tx,
+            &SchemaValidationProgressTable,
+            component_id.into(),
+            &DEFAULT_TABLE_NUMBERS,
+        )
+        .await?;
         for table in component_system_tables() {
             initialize_application_system_table(
                 self.tx,

crates/model/src/components/config.rs

@@ -97,6 +97,7 @@ use database::{
     IndexModel,
     IndexTable,
     IndexWorkerMetadataTable,
+    SchemaValidationProgressTable,
     SchemasTable,
     TablesTable,
     Transaction,
@@ -110,6 +111,8 @@ use database::{
     NUM_RESERVED_LEGACY_TABLE_NUMBERS,
     SCHEMAS_STATE_INDEX,
     SCHEMAS_TABLE,
+    SCHEMA_VALIDATION_PROGRESS_BY_SCHEMA_ID,
+    SCHEMA_VALIDATION_PROGRESS_TABLE,
     TABLES_BY_NAME_INDEX,
 };
 use database_globals::{
@@ -197,11 +200,6 @@ use crate::{
     exports::ExportsTable,
     external_packages::EXTERNAL_PACKAGES_TABLE,
     log_sinks::LOG_SINKS_TABLE,
-    schema_validation_progress::{
-        SchemaValidationProgressTable,
-        SCHEMA_VALIDATION_PROGRESS_BY_SCHEMA_ID,
-        SCHEMA_VALIDATION_PROGRESS_TABLE,
-    },
 };
 
 pub mod airbyte_import;
@@ -225,7 +223,6 @@ mod metrics;
 pub mod migrations;
 pub mod modules;
 pub mod scheduled_jobs;
-pub mod schema_validation_progress;
 pub mod session_requests;
 pub mod snapshot_imports;
 pub mod source_packages;
@@ -551,8 +548,8 @@ pub fn app_system_tables() -> Vec<&'static dyn ErasedSystemTable> {
     system_tables
 }
 
-/// NOTE: Does not include _schemas because that's not an app system table,
-/// but it is created for each component.
+/// NOTE: Does not include _schemas or _schema_validation_progress because they
+/// are in bootstrapped system tables, but they are created for each component.
 pub fn component_system_tables() -> Vec<&'static dyn ErasedSystemTable> {
     vec![
         &FileStorageTable,
@@ -563,7 +560,6 @@ pub fn component_system_tables() -> Vec<&'static dyn ErasedSystemTable> {
         &ModulesTable,
         &UdfConfigTable,
         &SourcePackagesTable,
-        &SchemaValidationProgressTable,
     ]
 }