feat: Add queryables configuration support in pgstac bootstrap (#323)

Co-authored-by: xıʃǝɟ <felix@developmentseed.org>

Author: emmanuelmathot

CHANGELOG.md

@@ -8,6 +8,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ## [Unreleased]
 
 ### Added
+
+- Add queryables configuration support using pypgstac load-queryables [#323](https://github.com/developmentseed/eoapi-k8s/pull/323)
 - Added local testing with k3s and minikube
 - Base local development values file (`local-base-values.yaml`)
 - Unified local cluster management with `CLUSTER_TYPE` variable

charts/eoapi/Chart.yaml

@@ -39,7 +39,7 @@ annotations:
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: "0.7.12"
+version: "0.8.0"
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

charts/eoapi/initdb-data/queryables/test-queryables.json

@@ -0,0 +1,44 @@
+{
+  "$schema": "https://json-schema.org/draft/2019-09/schema",
+  "$id": "https://stac-extensions.github.io/item-search/v1.0.0/schema.json",
+  "title": "Test Queryables",
+  "description": "Test queryables for eoapi",
+  "type": "object",
+  "properties": {
+    "eo:cloud_cover": {
+      "description": "Estimate of cloud cover as a percentage (0-100) of the entire scene",
+      "type": "number",
+      "title": "Cloud Cover",
+      "minimum": 0,
+      "maximum": 100
+    },
+    "view:sun_azimuth": {
+      "description": "Sun azimuth angle in degrees",
+      "type": "number",
+      "title": "Sun Azimuth",
+      "minimum": 0,
+      "maximum": 360
+    },
+    "view:sun_elevation": {
+      "description": "Sun elevation angle in degrees",
+      "type": "number",
+      "title": "Sun Elevation",
+      "minimum": -90,
+      "maximum": 90
+    },
+    "platform": {
+      "description": "Platform or satellite name",
+      "type": "string",
+      "title": "Platform"
+    },
+    "instruments": {
+      "description": "Instrument(s) used",
+      "type": "array",
+      "title": "Instruments",
+      "items": {
+        "type": "string"
+      }
+    }
+  },
+  "additionalProperties": true
+}

charts/eoapi/local-base-values.yaml

@@ -14,6 +14,11 @@ database:
 pgstacBootstrap:
   enabled: true
   settings:
+    # Queryables configuration for testing
+    queryables:
+      - file: "initdb-data/queryables/test-queryables.json"
+        indexFields: ["platform", "instruments"]
+        deleteMissing: true
     resources:
       requests:
         cpu: "256m"

charts/eoapi/templates/pgstacbootstrap/configmap.yaml

@@ -46,6 +46,23 @@ data:
   {{- base $path | nindent 2 -}}: | {{- $.Files.Get $path | nindent 4 -}}
   {{- end }}
 {{- end }}
+---
+{{- if .Values.pgstacBootstrap.settings.queryables }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ $.Release.Name }}-pgstac-queryables-config
+  annotations:
+    helm.sh/hook: "post-install,post-upgrade"
+    helm.sh/hook-weight: "-7"
+    helm.sh/hook-delete-policy: "before-hook-creation,hook-succeeded"
+data:
+  {{- range $config := .Values.pgstacBootstrap.settings.queryables }}
+  {{- $filename := splitList "/" $config.file | last }}
+  {{ $filename }}: |
+    {{- $.Files.Get $config.file | nindent 4 }}
+  {{- end }}
+{{- end }}
 {{- end }}
 ---
 {{- if .Values.postgrescluster.enabled }}

charts/eoapi/templates/pgstacbootstrap/job.yaml

@@ -156,3 +156,86 @@ spec:
       {{- end }}
   backoffLimit: 3
 {{- end }}
+
+{{- if and .Values.pgstacBootstrap.enabled .Values.pgstacBootstrap.settings.queryables }}
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: {{ .Release.Name }}-pgstac-load-queryables
+  labels:
+    app: {{ .Release.Name }}-pgstac-load-queryables
+  annotations:
+    helm.sh/hook: "post-install,post-upgrade"
+    helm.sh/hook-weight: "-3"
+    helm.sh/hook-delete-policy: "before-hook-creation"
+spec:
+  template:
+    metadata:
+      labels:
+        app: {{ .Release.Name }}-pgstac-load-queryables
+    spec:
+      restartPolicy: Never
+      containers:
+        - name: pgstac-load-queryables
+          image: {{ .Values.pgstacBootstrap.image.name }}:{{ .Values.pgstacBootstrap.image.tag }}
+          command:
+            - "/bin/sh"
+            - "-c"
+          args:
+            - |
+              # Exit immediately if a command exits with a non-zero status
+              set -e
+
+              # Database connection configured through standard PG* environment variables
+              # Environment variables are already set by the container
+
+              # Wait for the database to be ready
+              echo "Waiting for database to be ready..."
+              pypgstac pgready
+
+              # Load queryables configurations
+              echo "Loading queryables configurations..."
+              {{- range $idx, $config := .Values.pgstacBootstrap.settings.queryables }}
+              {{- $filename := splitList "/" $config.file | last }}
+              echo "Processing queryables file: {{ $filename }}"
+              pypgstac load-queryables \
+              {{- if $config.deleteMissing }}
+                --delete-missing \
+              {{- end }}
+              {{- if $config.collections }}
+                # Complex escaping needed due to multiple interpretation layers:
+                # 1. Helm processes the template: \\\" becomes \" in the rendered YAML
+                # 2. YAML processes the string: \" becomes " in the shell script
+                # 3. Shell receives: --collection-ids '["collection1","collection2"]'
+                # 4. pypgstac gets a proper JSON array string as expected
+                --collection-ids "[\\\"{{ join "\\\",\\\"" $config.collections }}\\\"]" \
+              {{- end }}
+              {{- if $config.indexFields }}
+                --index-fields {{ join "," $config.indexFields }} \
+              {{- end }}
+                "/opt/queryables/{{ $filename }}"
+              {{- end }}
+
+              echo "Queryables loading complete"
+          resources:
+            {{- toYaml .Values.pgstacBootstrap.settings.resources | nindent 12 }}
+          volumeMounts:
+            - mountPath: /opt/queryables
+              name: {{ .Release.Name }}-queryables-volume
+          env:
+            {{- include "eoapi.postgresqlEnv" . | nindent 12 }}
+      volumes:
+        - name: {{ .Release.Name }}-queryables-volume
+          configMap:
+            name: {{ .Release.Name }}-pgstac-queryables-config
+      {{- with .Values.pgstacBootstrap.settings.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.pgstacBootstrap.settings.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+  backoffLimit: 3
+{{- end }}

charts/eoapi/values.yaml

@@ -163,6 +163,20 @@ pgstacBootstrap:
     # General configuration options
     loadSamples: true      # Set to false to disable sample data loading
 
+    # Queryables configuration
+    # List of queryables configurations to load using pypgstac load-queryables
+    # Each item specifies a file path and optional parameters
+    # Example:
+    # queryables:
+    #   - file: "initdb-data/queryables/common-queryables.json"
+    #     indexFields: ["platform", "instruments"]
+    #     deleteMissing: true
+    #   - file: "initdb-data/queryables/collection-specific.json"
+    #     collections: ["my-collection-1", "my-collection-2"]
+    #     indexFields: ["custom:field1", "custom:field2"]
+    #     deleteMissing: true
+    queryables: []
+
     # Wait configuration for init containers waiting for pgstac jobs
     # These parameters control how long services wait for pgstac migration jobs to complete
     waitConfig: