NoSQL: Realms handling (#3007)

Introduces handling for realms including realm-state management/transition.

The `RealmStore` implementation for NoSQL depends on CDI components, coming in a follo-up PR.

Author: snazy

bom/build.gradle.kts

@@ -48,6 +48,10 @@ dependencies {
     api(project(":polaris-nodes-impl"))
     api(project(":polaris-nodes-spi"))
 
+    api(project(":polaris-persistence-nosql-realms-api"))
+    api(project(":polaris-persistence-nosql-realms-impl"))
+    api(project(":polaris-persistence-nosql-realms-spi"))
+
     api(project(":polaris-persistence-nosql-api"))
     api(project(":polaris-persistence-nosql-impl"))
     api(project(":polaris-persistence-nosql-benchmark"))

gradle/projects.main.properties

@@ -62,6 +62,10 @@ polaris-idgen-spi=persistence/nosql/idgen/spi
 polaris-nodes-api=persistence/nosql/nodes/api
 polaris-nodes-impl=persistence/nosql/nodes/impl
 polaris-nodes-spi=persistence/nosql/nodes/spi
+# realms
+polaris-persistence-nosql-realms-api=persistence/nosql/realms/api
+polaris-persistence-nosql-realms-impl=persistence/nosql/realms/impl
+polaris-persistence-nosql-realms-spi=persistence/nosql/realms/spi
 # persistence / database agnostic
 polaris-persistence-nosql-api=persistence/nosql/persistence/api
 polaris-persistence-nosql-impl=persistence/nosql/persistence/impl

persistence/nosql/realms/README.md

@@ -0,0 +1,32 @@
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+ 
+   http://www.apache.org/licenses/LICENSE-2.0
+ 
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied.  See the License for the
+  specific language governing permissions and limitations
+  under the License.
+-->
+
+# Dynamic realm management
+
+Framework to manage realms.
+
+## Code structure
+
+The code is structured into multiple modules. Consuming code should almost always pull in only the API module.
+
+* `polaris-persistence-nosql-realms-api` provides the necessary Java interfaces and immutable types.
+* `polaris-persistence-nosql-realms-id` provides a type-safe holder for a realm ID.
+* `polaris-persistence-nosql-realms-impl` provides the storage agnostic implementation.
+* `polaris-persistence-nosql-realms-spi` provides the necessary interfaces to provide a storage specific implementation.
+* `polaris-persistence-nosql-realms-store-nosql` provides the storage implementation based on `polaris-persistence-nosql-api`.

persistence/nosql/realms/api/build.gradle.kts

@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+plugins {
+  id("org.kordamp.gradle.jandex")
+  id("polaris-server")
+}
+
+description = "Polaris realms API, no concrete implementations"
+
+dependencies {
+  implementation(project(":polaris-idgen-api"))
+  implementation(libs.guava)
+
+  compileOnly(project(":polaris-immutables"))
+  annotationProcessor(project(":polaris-immutables", configuration = "processor"))
+
+  implementation(platform(libs.jackson.bom))
+  implementation("com.fasterxml.jackson.core:jackson-databind")
+
+  compileOnly(libs.jakarta.annotation.api)
+  compileOnly(libs.jakarta.validation.api)
+  compileOnly(libs.jakarta.inject.api)
+  compileOnly(libs.jakarta.enterprise.cdi.api)
+}

persistence/nosql/realms/api/src/main/java/org/apache/polaris/persistence/nosql/realms/api/RealmAlreadyExistsException.java

@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.persistence.nosql.realms.api;
+
+public class RealmAlreadyExistsException extends RuntimeException {
+  public RealmAlreadyExistsException(String message) {
+    super(message);
+  }
+}

persistence/nosql/realms/api/src/main/java/org/apache/polaris/persistence/nosql/realms/api/RealmDefinition.java

@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.persistence.nosql.realms.api;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
+import com.fasterxml.jackson.databind.annotation.JsonSerialize;
+import java.time.Instant;
+import java.util.Map;
+import org.apache.polaris.immutables.PolarisImmutable;
+import org.immutables.value.Value;
+
+@PolarisImmutable
+@JsonSerialize(as = ImmutableRealmDefinition.class)
+@JsonDeserialize(as = ImmutableRealmDefinition.class)
+public interface RealmDefinition {
+  String id();
+
+  Instant created();
+
+  Instant updated();
+
+  RealmStatus status();
+
+  @JsonInclude(JsonInclude.Include.NON_EMPTY)
+  Map<String, String> properties();
+
+  static ImmutableRealmDefinition.Builder builder() {
+    return ImmutableRealmDefinition.builder();
+  }
+
+  @JsonIgnore
+  @Value.NonAttribute
+  default boolean needsBootstrap() {
+    return switch (status()) {
+      case CREATED, LOADING, INITIALIZING -> true;
+      default -> false;
+    };
+  }
+
+  /** Realms are assigned */
+  enum RealmStatus {
+    /**
+     * The initial state of a realm is "created", which means that the realm ID is reserved, but the
+     * realm is not yet usable. This state can transition to {@link #LOADING} or {@link
+     * #INITIALIZING} or the realm can be directly deleted.
+     */
+    CREATED,
+    /**
+     * State used to indicate that the realm data is being imported. This state can transition to
+     * {@link #ACTIVE} or {@link #INACTIVE} or {@link #PURGING}.
+     */
+    LOADING,
+    /**
+     * State used to indicate that the realm is being initialized. This state can transition to
+     * {@link #ACTIVE} or {@link #INACTIVE} or {@link #PURGING}.
+     */
+    INITIALIZING,
+    /**
+     * When a realm is fully set up, its state is "active". This state can only transition to {@link
+     * #INACTIVE}.
+     */
+    ACTIVE,
+    /**
+     * An {@link #ACTIVE} realm can be put into "inactive" state, which means that the realm cannot
+     * be used, but it can be put back into {@link #ACTIVE} state.
+     */
+    INACTIVE,
+    /**
+     * An {@link #INACTIVE} realm can be put into "purging" state, which means that the realm's data
+     * is being purged from the persistence database. This is next to the final and terminal state
+     * {@link #PURGED} of a realm. Once all data of the realm has been purged, it must at least be
+     * set into {@link #PURGED} status or be entirely removed.
+     */
+    PURGING,
+    /**
+     * "Purged" is the terminal state of every realm. A purged realm can be safely {@linkplain
+     * RealmManagement#delete(RealmDefinition) deleted}. The difference between a "purged" realm and
+     * a non-existing (deleted) realm is that the ID of a purged realm cannot be (re)used.
+     */
+    PURGED,
+  }
+}

persistence/nosql/realms/api/src/main/java/org/apache/polaris/persistence/nosql/realms/api/RealmExpectedStateMismatchException.java

@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.persistence.nosql.realms.api;
+
+public class RealmExpectedStateMismatchException extends RuntimeException {
+  public RealmExpectedStateMismatchException(String message) {
+    super(message);
+  }
+}

persistence/nosql/realms/api/src/main/java/org/apache/polaris/persistence/nosql/realms/api/RealmManagement.java

@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.persistence.nosql.realms.api;
+
+import com.google.errorprone.annotations.MustBeClosed;
+import jakarta.annotation.Nonnull;
+import java.util.Optional;
+import java.util.stream.Stream;
+
+/**
+ * Low-level realm management functionality.
+ *
+ * <p>Realm IDs must conform to the following constraints:
+ *
+ * <ul>
+ *   <li>Must not start or end with whitespaces.
+ *   <li>Must only consist of US-ASCII letters or digits or hyphens ({@code -}) or underscores
+ *       ({@code _}).
+ *   <li>Must not start with two consecutive colons ({@code ::}).
+ *   <li>Must not be empty.
+ *   <li>Must not be longer than 128 characters.
+ * </ul>
+ *
+ * <p>Note: In a CDI container {@link RealmManagement} can be directly injected.
+ */
+public interface RealmManagement {
+  /**
+   * Creates a new realm in {@linkplain RealmDefinition.RealmStatus#CREATED created status} with the
+   * given realm ID.
+   *
+   * @return the persisted state of the realm definition
+   * @throws RealmAlreadyExistsException if a realm with the given ID already exists
+   */
+  @Nonnull
+  RealmDefinition create(@Nonnull String realmId);
+
+  /** Returns a stream of all realm definitions. The returned stream must be closed. */
+  @Nonnull
+  @MustBeClosed
+  Stream<RealmDefinition> list();
+
+  /**
+   * Retrieve a realm definition by realm ID.
+   *
+   * @return the realm definition if it exists.
+   */
+  @Nonnull
+  Optional<RealmDefinition> get(@Nonnull String realmId);
+
+  /**
+   * Updates a realm definition to {@code update}, if the persisted state matches the {@code
+   * expected} state, and if the {@linkplain RealmDefinition#status() status} transition is valid.
+   *
+   * @param expected The expected persisted state of the realm definition. This must exactly
+   *     represent the persisted realm definition as returned by {@link #create(String)} or {@link
+   *     #get(String)} or a prior {@link #update(RealmDefinition, RealmDefinition)}.
+   * @param update the new state of the realm definition to be persisted, the {@link
+   *     RealmDefinition#created() created} and {@link RealmDefinition#updated() updated} attributes
+   *     are solely managed by the implementation.
+   * @return the persisted state of the realm definition
+   * @throws RealmNotFoundException if a realm with the given ID does not exist
+   * @throws RealmExpectedStateMismatchException if the expected state does not match
+   * @throws IllegalArgumentException if the transition is not valid.
+   */
+  @Nonnull
+  RealmDefinition update(@Nonnull RealmDefinition expected, @Nonnull RealmDefinition update);
+
+  /**
+   * Deletes the given realm.
+   *
+   * @param expected The expected persisted state of the realm definition. This must exactly
+   *     represent the persisted realm definition as returned by {@link #create(String)} or {@link
+   *     #get(String)} or {@link #update(RealmDefinition, RealmDefinition)}.
+   * @throws RealmNotFoundException if a realm with the given ID does not exist
+   * @throws RealmExpectedStateMismatchException if the expected state does not match
+   */
+  void delete(@Nonnull RealmDefinition expected);
+}

persistence/nosql/realms/api/src/main/java/org/apache/polaris/persistence/nosql/realms/api/RealmNotFoundException.java

@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.persistence.nosql.realms.api;
+
+public class RealmNotFoundException extends RuntimeException {
+  public RealmNotFoundException(String message) {
+    super(message);
+  }
+}