From c4706fdb1ee3dd5523a8bfe5a48e9997a4fc4cfe Mon Sep 17 00:00:00 2001
From: Maxence Maireaux <maxence@maireaux.fr>
Date: Mon, 10 Nov 2025 15:42:59 +0100
Subject: [PATCH 01/12] feat: optimize database indexes for query performance

Add migration 41 to optimize database indexes for improved performance:

## Changes

### Moves Table
- **DROP**: moves_post_commit_volumes, moves_effective_post_commit_volumes
  (used accounts_seq instead of account_address)
- **ADD**: idx_moves_pit_insertion for PIT queries with insertion_date
- **ADD**: idx_moves_pit_effective for PIT queries with effective_date
- **ADD**: idx_moves_account_balance for balance lookups

### Accounts Metadata
- **DROP**: accounts_metadata_revisions (used accounts_seq)
- **ADD**: idx_accounts_metadata_pit for historical metadata queries

### Transactions Metadata
- **DROP**: transactions_metadata_revisions (used transactions_seq)
- **ADD**: idx_transactions_metadata_pit for historical metadata queries

### Accounts Volumes
- **DROP**: accounts_volumes_idx (redundant with PRIMARY KEY)

## Performance Impact
- Major improvement for Point-in-Time (PIT) queries
- Major improvement for balance lookups by account
- Major improvement for historical metadata queries
- Reduced storage by removing redundant index

## Affected Query Files
- resource_aggregated_balances.go
- resource_accounts.go
- resource_transactions.go
---
 .../41-optimize-query-indexes/notes.yaml      | 58 +++++++++++++++++
 .../41-optimize-query-indexes/up.sql          | 62 +++++++++++++++++++
 2 files changed, 120 insertions(+)
 create mode 100644 internal/storage/bucket/migrations/41-optimize-query-indexes/notes.yaml
 create mode 100644 internal/storage/bucket/migrations/41-optimize-query-indexes/up.sql

diff --git a/internal/storage/bucket/migrations/41-optimize-query-indexes/notes.yaml b/internal/storage/bucket/migrations/41-optimize-query-indexes/notes.yaml
new file mode 100644
index 000000000..2a57153b7
--- /dev/null
+++ b/internal/storage/bucket/migrations/41-optimize-query-indexes/notes.yaml
@@ -0,0 +1,58 @@
+name: Optimize Query Indexes
+description: |
+  Optimize database indexes for improved query performance in Point-in-Time (PIT) queries,
+  balance lookups, and metadata history queries.
+
+changes:
+  moves_table:
+    - action: "DROP and REPLACE"
+      old_indexes:
+        - "moves_post_commit_volumes (accounts_seq, asset, seq)"
+        - "moves_effective_post_commit_volumes (accounts_seq, asset, effective_date desc)"
+      new_indexes:
+        - "idx_moves_pit_insertion (ledger, account_address, asset, insertion_date desc, seq desc)"
+        - "idx_moves_pit_effective (ledger, account_address, asset, effective_date desc, seq desc)"
+        - "idx_moves_account_balance (ledger, account_address, effective_date desc, seq desc) INCLUDE (asset, post_commit_effective_volumes)"
+      reason: |
+        Old indexes used accounts_seq but queries filter on account_address and ledger.
+        This caused full table scans for PIT queries and balance lookups.
+        New indexes align with actual query patterns for significant performance improvement.
+
+  accounts_metadata_table:
+    - action: "DROP and REPLACE"
+      old_index: "accounts_metadata_revisions (accounts_seq, revision desc)"
+      new_index: "idx_accounts_metadata_pit (ledger, accounts_address, date desc, revision desc) INCLUDE (metadata)"
+      reason: |
+        Old index used accounts_seq but queries filter on accounts_address and date.
+        New index enables efficient historical metadata lookups.
+
+  transactions_metadata_table:
+    - action: "DROP and REPLACE"
+      old_index: "transactions_metadata_revisions (transactions_seq, revision desc)"
+      new_index: "idx_transactions_metadata_pit (ledger, transactions_id, date desc, revision desc) INCLUDE (metadata)"
+      reason: |
+        Old index used transactions_seq but queries filter on transactions_id and date.
+        New index enables efficient historical transaction metadata lookups.
+
+  accounts_volumes_table:
+    - action: "DROP"
+      index: "accounts_volumes_idx"
+      reason: "Redundant - PRIMARY KEY (ledger, accounts_address, asset) already covers this pattern"
+
+affected_queries:
+  - "resource_aggregated_balances.go: PIT queries with DISTINCT ON (accounts_address, asset)"
+  - "resource_accounts.go: Balance filtering with effective_date"
+  - "resource_accounts.go: Account expansion volumes queries"
+  - "resource_accounts.go: Historical metadata lookups"
+  - "resource_transactions.go: Historical transaction metadata lookups"
+
+performance_impact:
+  - "MAJOR improvement for Point-in-Time (PIT) queries"
+  - "MAJOR improvement for balance lookups by account"
+  - "MAJOR improvement for historical metadata queries"
+  - "Minor reduction in storage by removing redundant index"
+
+migration_safety:
+  - "Uses CONCURRENTLY flag to avoid locking tables during index creation"
+  - "Drops old indexes only after new ones are created"
+  - "No data modification - only index structure changes"
diff --git a/internal/storage/bucket/migrations/41-optimize-query-indexes/up.sql b/internal/storage/bucket/migrations/41-optimize-query-indexes/up.sql
new file mode 100644
index 000000000..622e65d85
--- /dev/null
+++ b/internal/storage/bucket/migrations/41-optimize-query-indexes/up.sql
@@ -0,0 +1,62 @@
+set search_path = '{{.Schema}}';
+
+-- ========================================
+-- MOVES TABLE INDEXES OPTIMIZATION
+-- ========================================
+
+-- Drop old suboptimal indexes that use accounts_seq instead of account_address
+drop index if exists moves_post_commit_volumes;
+drop index if exists moves_effective_post_commit_volumes;
+
+-- Critical: Index for Point-in-Time queries with insertion_date
+-- Covers queries in resource_aggregated_balances.go and resource_accounts.go
+-- Replaces: moves_post_commit_volumes
+create index {{ if not .Transactional }}concurrently{{end}} idx_moves_pit_insertion
+    on "{{.Schema}}".moves (ledger, account_address, asset, insertion_date desc, seq desc);
+
+-- Critical: Index for Point-in-Time queries with effective_date
+-- Covers queries in resource_aggregated_balances.go and resource_accounts.go
+-- Replaces: moves_effective_post_commit_volumes
+create index {{ if not .Transactional }}concurrently{{end}} idx_moves_pit_effective
+    on "{{.Schema}}".moves (ledger, account_address, asset, effective_date desc, seq desc);
+
+-- Optimal: Index for balance lookups by account with effective date
+-- Covers balance filtering queries in resource_accounts.go
+create index {{ if not .Transactional }}concurrently{{end}} idx_moves_account_balance
+    on "{{.Schema}}".moves (ledger, account_address, effective_date desc, seq desc)
+    include (asset, post_commit_effective_volumes);
+
+-- ========================================
+-- ACCOUNTS_METADATA TABLE OPTIMIZATION
+-- ========================================
+
+-- Drop old suboptimal index that uses accounts_seq
+drop index if exists accounts_metadata_revisions;
+
+-- Critical: Index for historical metadata queries
+-- Covers queries in resource_accounts.go for Point-in-Time metadata
+-- Replaces: accounts_metadata_revisions
+create index {{ if not .Transactional }}concurrently{{end}} idx_accounts_metadata_pit
+    on "{{.Schema}}".accounts_metadata (ledger, accounts_address, date desc, revision desc)
+    include (metadata);
+
+-- ========================================
+-- TRANSACTIONS_METADATA TABLE OPTIMIZATION
+-- ========================================
+
+-- Drop old suboptimal index that uses transactions_seq
+drop index if exists transactions_metadata_revisions;
+
+-- Critical: Index for historical transaction metadata queries
+-- Covers queries in resource_transactions.go for Point-in-Time metadata
+-- Replaces: transactions_metadata_revisions
+create index {{ if not .Transactional }}concurrently{{end}} idx_transactions_metadata_pit
+    on "{{.Schema}}".transactions_metadata (ledger, transactions_id, date desc, revision desc)
+    include (metadata);
+
+-- ========================================
+-- ACCOUNTS_VOLUMES TABLE CLEANUP
+-- ========================================
+
+-- Drop redundant index - PRIMARY KEY already covers this pattern
+drop index if exists accounts_volumes_idx;

From ee0441fef0a7ae538a3f9138bf7a2823f0f5f956 Mon Sep 17 00:00:00 2001
From: Maxence Maireaux <maxence@maireaux.fr>
Date: Mon, 10 Nov 2025 15:48:05 +0100
Subject: [PATCH 02/12] fix: create new indexes before dropping old ones to
 avoid performance degradation

Reorder migration operations to:
1. Create all new optimized indexes first (with CONCURRENTLY)
2. Drop old suboptimal indexes only after new ones are ready

This ensures continuous query performance during migration execution.
---
 .../41-optimize-query-indexes/up.sql          | 24 +++++++++++--------
 1 file changed, 14 insertions(+), 10 deletions(-)

diff --git a/internal/storage/bucket/migrations/41-optimize-query-indexes/up.sql b/internal/storage/bucket/migrations/41-optimize-query-indexes/up.sql
index 622e65d85..a2826f874 100644
--- a/internal/storage/bucket/migrations/41-optimize-query-indexes/up.sql
+++ b/internal/storage/bucket/migrations/41-optimize-query-indexes/up.sql
@@ -4,10 +4,7 @@ set search_path = '{{.Schema}}';
 -- MOVES TABLE INDEXES OPTIMIZATION
 -- ========================================
 
--- Drop old suboptimal indexes that use accounts_seq instead of account_address
-drop index if exists moves_post_commit_volumes;
-drop index if exists moves_effective_post_commit_volumes;
-
+-- Step 1: Create new optimized indexes FIRST to avoid performance degradation
 -- Critical: Index for Point-in-Time queries with insertion_date
 -- Covers queries in resource_aggregated_balances.go and resource_accounts.go
 -- Replaces: moves_post_commit_volumes
@@ -26,13 +23,15 @@ create index {{ if not .Transactional }}concurrently{{end}} idx_moves_account_ba
     on "{{.Schema}}".moves (ledger, account_address, effective_date desc, seq desc)
     include (asset, post_commit_effective_volumes);
 
+-- Step 2: Drop old suboptimal indexes AFTER new ones are ready
+drop index if exists moves_post_commit_volumes;
+drop index if exists moves_effective_post_commit_volumes;
+
 -- ========================================
 -- ACCOUNTS_METADATA TABLE OPTIMIZATION
 -- ========================================
 
--- Drop old suboptimal index that uses accounts_seq
-drop index if exists accounts_metadata_revisions;
-
+-- Step 1: Create new optimized index FIRST
 -- Critical: Index for historical metadata queries
 -- Covers queries in resource_accounts.go for Point-in-Time metadata
 -- Replaces: accounts_metadata_revisions
@@ -40,13 +39,14 @@ create index {{ if not .Transactional }}concurrently{{end}} idx_accounts_metadat
     on "{{.Schema}}".accounts_metadata (ledger, accounts_address, date desc, revision desc)
     include (metadata);
 
+-- Step 2: Drop old suboptimal index AFTER new one is ready
+drop index if exists accounts_metadata_revisions;
+
 -- ========================================
 -- TRANSACTIONS_METADATA TABLE OPTIMIZATION
 -- ========================================
 
--- Drop old suboptimal index that uses transactions_seq
-drop index if exists transactions_metadata_revisions;
-
+-- Step 1: Create new optimized index FIRST
 -- Critical: Index for historical transaction metadata queries
 -- Covers queries in resource_transactions.go for Point-in-Time metadata
 -- Replaces: transactions_metadata_revisions
@@ -54,9 +54,13 @@ create index {{ if not .Transactional }}concurrently{{end}} idx_transactions_met
     on "{{.Schema}}".transactions_metadata (ledger, transactions_id, date desc, revision desc)
     include (metadata);
 
+-- Step 2: Drop old suboptimal index AFTER new one is ready
+drop index if exists transactions_metadata_revisions;
+
 -- ========================================
 -- ACCOUNTS_VOLUMES TABLE CLEANUP
 -- ========================================
 
 -- Drop redundant index - PRIMARY KEY already covers this pattern
+-- Safe to drop immediately as PRIMARY KEY provides same functionality
 drop index if exists accounts_volumes_idx;

From 9ffb97f4483b574fc14d93eb50668d9c91e53340 Mon Sep 17 00:00:00 2001
From: Maxence Maireaux <maxence@maireaux.fr>
Date: Mon, 10 Nov 2025 16:06:10 +0100
Subject: [PATCH 03/12] refactor: remove ledger and date columns from indexes

Remove ledger and date columns from indexes as requested:
- Moves indexes: removed 'ledger' column prefix
- Accounts metadata index: removed 'ledger' and 'date' columns
- Transactions metadata index: already optimized (ledger/date not included)

This reduces index size while maintaining query performance.
---
 .../migrations/41-optimize-query-indexes/up.sql    | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/internal/storage/bucket/migrations/41-optimize-query-indexes/up.sql b/internal/storage/bucket/migrations/41-optimize-query-indexes/up.sql
index a2826f874..fad516907 100644
--- a/internal/storage/bucket/migrations/41-optimize-query-indexes/up.sql
+++ b/internal/storage/bucket/migrations/41-optimize-query-indexes/up.sql
@@ -9,18 +9,18 @@ set search_path = '{{.Schema}}';
 -- Covers queries in resource_aggregated_balances.go and resource_accounts.go
 -- Replaces: moves_post_commit_volumes
 create index {{ if not .Transactional }}concurrently{{end}} idx_moves_pit_insertion
-    on "{{.Schema}}".moves (ledger, account_address, asset, insertion_date desc, seq desc);
+    on "{{.Schema}}".moves (account_address, asset, insertion_date desc, seq desc);
 
 -- Critical: Index for Point-in-Time queries with effective_date
 -- Covers queries in resource_aggregated_balances.go and resource_accounts.go
 -- Replaces: moves_effective_post_commit_volumes
 create index {{ if not .Transactional }}concurrently{{end}} idx_moves_pit_effective
-    on "{{.Schema}}".moves (ledger, account_address, asset, effective_date desc, seq desc);
+    on "{{.Schema}}".moves (account_address, asset, effective_date desc, seq desc);
 
 -- Optimal: Index for balance lookups by account with effective date
 -- Covers balance filtering queries in resource_accounts.go
 create index {{ if not .Transactional }}concurrently{{end}} idx_moves_account_balance
-    on "{{.Schema}}".moves (ledger, account_address, effective_date desc, seq desc)
+    on "{{.Schema}}".moves (account_address, effective_date desc, seq desc)
     include (asset, post_commit_effective_volumes);
 
 -- Step 2: Drop old suboptimal indexes AFTER new ones are ready
@@ -36,8 +36,8 @@ drop index if exists moves_effective_post_commit_volumes;
 -- Covers queries in resource_accounts.go for Point-in-Time metadata
 -- Replaces: accounts_metadata_revisions
 create index {{ if not .Transactional }}concurrently{{end}} idx_accounts_metadata_pit
-    on "{{.Schema}}".accounts_metadata (ledger, accounts_address, date desc, revision desc)
-    include (metadata);
+    on "{{.Schema}}".accounts_metadata (accounts_address, revision desc)
+    include (metadata, date);
 
 -- Step 2: Drop old suboptimal index AFTER new one is ready
 drop index if exists accounts_metadata_revisions;
@@ -51,8 +51,8 @@ drop index if exists accounts_metadata_revisions;
 -- Covers queries in resource_transactions.go for Point-in-Time metadata
 -- Replaces: transactions_metadata_revisions
 create index {{ if not .Transactional }}concurrently{{end}} idx_transactions_metadata_pit
-    on "{{.Schema}}".transactions_metadata (ledger, transactions_id, date desc, revision desc)
-    include (metadata);
+    on "{{.Schema}}".transactions_metadata (transactions_id, revision desc)
+    include (metadata, date);
 
 -- Step 2: Drop old suboptimal index AFTER new one is ready
 drop index if exists transactions_metadata_revisions;

From 019ed4390951aeb7aaf69808265f40a24922042d Mon Sep 17 00:00:00 2001
From: Maxence Maireaux <maxence@maireaux.fr>
Date: Mon, 10 Nov 2025 17:39:46 +0100
Subject: [PATCH 04/12] refactor: split index migrations into isolated
 concurrent operations

Split the single migration 41 into 5 separate migrations:
- Migration 41: Create idx_moves_pit_insertion index
- Migration 42: Create idx_moves_pit_effective index
- Migration 43: Create idx_accounts_metadata_pit index
- Migration 44: Create idx_transactions_metadata_pit index
- Migration 45: Drop all old indexes

This ensures each concurrent index creation is isolated in its own
migration, and all index drops are consolidated into a final migration.
Also removed the optional idx_moves_account_balance index on Moves table.
---
 .../41-optimize-query-indexes/up.sql          | 57 +------------------
 .../up.sql                                    | 11 ++++
 .../up.sql                                    | 12 ++++
 .../up.sql                                    | 12 ++++
 .../migrations/45-drop-old-indexes/up.sql     | 18 ++++++
 5 files changed, 54 insertions(+), 56 deletions(-)
 create mode 100644 internal/storage/bucket/migrations/42-optimize-query-indexes-pit-effective/up.sql
 create mode 100644 internal/storage/bucket/migrations/43-optimize-accounts-metadata-index/up.sql
 create mode 100644 internal/storage/bucket/migrations/44-optimize-transactions-metadata-index/up.sql
 create mode 100644 internal/storage/bucket/migrations/45-drop-old-indexes/up.sql

diff --git a/internal/storage/bucket/migrations/41-optimize-query-indexes/up.sql b/internal/storage/bucket/migrations/41-optimize-query-indexes/up.sql
index fad516907..35b90339a 100644
--- a/internal/storage/bucket/migrations/41-optimize-query-indexes/up.sql
+++ b/internal/storage/bucket/migrations/41-optimize-query-indexes/up.sql
@@ -1,66 +1,11 @@
 set search_path = '{{.Schema}}';
 
 -- ========================================
--- MOVES TABLE INDEXES OPTIMIZATION
+-- MOVES TABLE INDEXES OPTIMIZATION (1/4)
 -- ========================================
 
--- Step 1: Create new optimized indexes FIRST to avoid performance degradation
 -- Critical: Index for Point-in-Time queries with insertion_date
 -- Covers queries in resource_aggregated_balances.go and resource_accounts.go
 -- Replaces: moves_post_commit_volumes
 create index {{ if not .Transactional }}concurrently{{end}} idx_moves_pit_insertion
     on "{{.Schema}}".moves (account_address, asset, insertion_date desc, seq desc);
-
--- Critical: Index for Point-in-Time queries with effective_date
--- Covers queries in resource_aggregated_balances.go and resource_accounts.go
--- Replaces: moves_effective_post_commit_volumes
-create index {{ if not .Transactional }}concurrently{{end}} idx_moves_pit_effective
-    on "{{.Schema}}".moves (account_address, asset, effective_date desc, seq desc);
-
--- Optimal: Index for balance lookups by account with effective date
--- Covers balance filtering queries in resource_accounts.go
-create index {{ if not .Transactional }}concurrently{{end}} idx_moves_account_balance
-    on "{{.Schema}}".moves (account_address, effective_date desc, seq desc)
-    include (asset, post_commit_effective_volumes);
-
--- Step 2: Drop old suboptimal indexes AFTER new ones are ready
-drop index if exists moves_post_commit_volumes;
-drop index if exists moves_effective_post_commit_volumes;
-
--- ========================================
--- ACCOUNTS_METADATA TABLE OPTIMIZATION
--- ========================================
-
--- Step 1: Create new optimized index FIRST
--- Critical: Index for historical metadata queries
--- Covers queries in resource_accounts.go for Point-in-Time metadata
--- Replaces: accounts_metadata_revisions
-create index {{ if not .Transactional }}concurrently{{end}} idx_accounts_metadata_pit
-    on "{{.Schema}}".accounts_metadata (accounts_address, revision desc)
-    include (metadata, date);
-
--- Step 2: Drop old suboptimal index AFTER new one is ready
-drop index if exists accounts_metadata_revisions;
-
--- ========================================
--- TRANSACTIONS_METADATA TABLE OPTIMIZATION
--- ========================================
-
--- Step 1: Create new optimized index FIRST
--- Critical: Index for historical transaction metadata queries
--- Covers queries in resource_transactions.go for Point-in-Time metadata
--- Replaces: transactions_metadata_revisions
-create index {{ if not .Transactional }}concurrently{{end}} idx_transactions_metadata_pit
-    on "{{.Schema}}".transactions_metadata (transactions_id, revision desc)
-    include (metadata, date);
-
--- Step 2: Drop old suboptimal index AFTER new one is ready
-drop index if exists transactions_metadata_revisions;
-
--- ========================================
--- ACCOUNTS_VOLUMES TABLE CLEANUP
--- ========================================
-
--- Drop redundant index - PRIMARY KEY already covers this pattern
--- Safe to drop immediately as PRIMARY KEY provides same functionality
-drop index if exists accounts_volumes_idx;
diff --git a/internal/storage/bucket/migrations/42-optimize-query-indexes-pit-effective/up.sql b/internal/storage/bucket/migrations/42-optimize-query-indexes-pit-effective/up.sql
new file mode 100644
index 000000000..a5beef846
--- /dev/null
+++ b/internal/storage/bucket/migrations/42-optimize-query-indexes-pit-effective/up.sql
@@ -0,0 +1,11 @@
+set search_path = '{{.Schema}}';
+
+-- ========================================
+-- MOVES TABLE INDEXES OPTIMIZATION (2/4)
+-- ========================================
+
+-- Critical: Index for Point-in-Time queries with effective_date
+-- Covers queries in resource_aggregated_balances.go and resource_accounts.go
+-- Replaces: moves_effective_post_commit_volumes
+create index {{ if not .Transactional }}concurrently{{end}} idx_moves_pit_effective
+    on "{{.Schema}}".moves (account_address, asset, effective_date desc, seq desc);
diff --git a/internal/storage/bucket/migrations/43-optimize-accounts-metadata-index/up.sql b/internal/storage/bucket/migrations/43-optimize-accounts-metadata-index/up.sql
new file mode 100644
index 000000000..d6ca6bbb8
--- /dev/null
+++ b/internal/storage/bucket/migrations/43-optimize-accounts-metadata-index/up.sql
@@ -0,0 +1,12 @@
+set search_path = '{{.Schema}}';
+
+-- ========================================
+-- ACCOUNTS_METADATA TABLE OPTIMIZATION
+-- ========================================
+
+-- Critical: Index for historical metadata queries
+-- Covers queries in resource_accounts.go for Point-in-Time metadata
+-- Replaces: accounts_metadata_revisions
+create index {{ if not .Transactional }}concurrently{{end}} idx_accounts_metadata_pit
+    on "{{.Schema}}".accounts_metadata (accounts_address, revision desc)
+    include (metadata, date);
diff --git a/internal/storage/bucket/migrations/44-optimize-transactions-metadata-index/up.sql b/internal/storage/bucket/migrations/44-optimize-transactions-metadata-index/up.sql
new file mode 100644
index 000000000..1f22b2a25
--- /dev/null
+++ b/internal/storage/bucket/migrations/44-optimize-transactions-metadata-index/up.sql
@@ -0,0 +1,12 @@
+set search_path = '{{.Schema}}';
+
+-- ========================================
+-- TRANSACTIONS_METADATA TABLE OPTIMIZATION
+-- ========================================
+
+-- Critical: Index for historical transaction metadata queries
+-- Covers queries in resource_transactions.go for Point-in-Time metadata
+-- Replaces: transactions_metadata_revisions
+create index {{ if not .Transactional }}concurrently{{end}} idx_transactions_metadata_pit
+    on "{{.Schema}}".transactions_metadata (transactions_id, revision desc)
+    include (metadata, date);
diff --git a/internal/storage/bucket/migrations/45-drop-old-indexes/up.sql b/internal/storage/bucket/migrations/45-drop-old-indexes/up.sql
new file mode 100644
index 000000000..69d6b7000
--- /dev/null
+++ b/internal/storage/bucket/migrations/45-drop-old-indexes/up.sql
@@ -0,0 +1,18 @@
+set search_path = '{{.Schema}}';
+
+-- ========================================
+-- DROP OLD INDEXES
+-- ========================================
+
+-- Drop old moves table indexes (replaced by migrations 41-42)
+drop index if exists moves_post_commit_volumes;
+drop index if exists moves_effective_post_commit_volumes;
+
+-- Drop old accounts_metadata index (replaced by migration 43)
+drop index if exists accounts_metadata_revisions;
+
+-- Drop old transactions_metadata index (replaced by migration 44)
+drop index if exists transactions_metadata_revisions;
+
+-- Drop redundant accounts_volumes index - PRIMARY KEY already covers this pattern
+drop index if exists accounts_volumes_idx;

From 0ee4d2bfce847183b8fd1552ccf2a5df3b10dce6 Mon Sep 17 00:00:00 2001
From: Maxence Maireaux <maxence@maireaux.fr>
Date: Mon, 10 Nov 2025 18:07:13 +0100
Subject: [PATCH 05/12] docs: add notes.yaml files for split index migrations

Add comprehensive notes.yaml documentation for each migration:
- Migration 41: idx_moves_pit_insertion index creation
- Migration 42: idx_moves_pit_effective index creation
- Migration 43: idx_accounts_metadata_pit index creation
- Migration 44: idx_transactions_metadata_pit index creation
- Migration 45: Drop all old indexes

Each file documents the purpose, changes, affected queries,
performance impact, and safety considerations.
---
 .../41-optimize-query-indexes/notes.yaml      | 55 ++++---------------
 .../notes.yaml                                | 25 +++++++++
 .../notes.yaml                                | 24 ++++++++
 .../notes.yaml                                | 24 ++++++++
 .../migrations/45-drop-old-indexes/notes.yaml | 32 +++++++++++
 5 files changed, 116 insertions(+), 44 deletions(-)
 create mode 100644 internal/storage/bucket/migrations/42-optimize-query-indexes-pit-effective/notes.yaml
 create mode 100644 internal/storage/bucket/migrations/43-optimize-accounts-metadata-index/notes.yaml
 create mode 100644 internal/storage/bucket/migrations/44-optimize-transactions-metadata-index/notes.yaml
 create mode 100644 internal/storage/bucket/migrations/45-drop-old-indexes/notes.yaml

diff --git a/internal/storage/bucket/migrations/41-optimize-query-indexes/notes.yaml b/internal/storage/bucket/migrations/41-optimize-query-indexes/notes.yaml
index 2a57153b7..b01096884 100644
--- a/internal/storage/bucket/migrations/41-optimize-query-indexes/notes.yaml
+++ b/internal/storage/bucket/migrations/41-optimize-query-indexes/notes.yaml
@@ -1,58 +1,25 @@
-name: Optimize Query Indexes
+name: Optimize Query Indexes - PIT Insertion (1/5)
 description: |
-  Optimize database indexes for improved query performance in Point-in-Time (PIT) queries,
-  balance lookups, and metadata history queries.
+  Create optimized index for Point-in-Time queries with insertion_date on moves table.
+  This is part of a multi-step migration to safely create indexes concurrently.
 
 changes:
   moves_table:
-    - action: "DROP and REPLACE"
-      old_indexes:
-        - "moves_post_commit_volumes (accounts_seq, asset, seq)"
-        - "moves_effective_post_commit_volumes (accounts_seq, asset, effective_date desc)"
-      new_indexes:
-        - "idx_moves_pit_insertion (ledger, account_address, asset, insertion_date desc, seq desc)"
-        - "idx_moves_pit_effective (ledger, account_address, asset, effective_date desc, seq desc)"
-        - "idx_moves_account_balance (ledger, account_address, effective_date desc, seq desc) INCLUDE (asset, post_commit_effective_volumes)"
+    - action: "CREATE"
+      new_index: "idx_moves_pit_insertion (account_address, asset, insertion_date desc, seq desc)"
       reason: |
-        Old indexes used accounts_seq but queries filter on account_address and ledger.
-        This caused full table scans for PIT queries and balance lookups.
-        New indexes align with actual query patterns for significant performance improvement.
-
-  accounts_metadata_table:
-    - action: "DROP and REPLACE"
-      old_index: "accounts_metadata_revisions (accounts_seq, revision desc)"
-      new_index: "idx_accounts_metadata_pit (ledger, accounts_address, date desc, revision desc) INCLUDE (metadata)"
-      reason: |
-        Old index used accounts_seq but queries filter on accounts_address and date.
-        New index enables efficient historical metadata lookups.
-
-  transactions_metadata_table:
-    - action: "DROP and REPLACE"
-      old_index: "transactions_metadata_revisions (transactions_seq, revision desc)"
-      new_index: "idx_transactions_metadata_pit (ledger, transactions_id, date desc, revision desc) INCLUDE (metadata)"
-      reason: |
-        Old index used transactions_seq but queries filter on transactions_id and date.
-        New index enables efficient historical transaction metadata lookups.
-
-  accounts_volumes_table:
-    - action: "DROP"
-      index: "accounts_volumes_idx"
-      reason: "Redundant - PRIMARY KEY (ledger, accounts_address, asset) already covers this pattern"
+        Creates index for PIT queries filtering on insertion_date.
+        Replaces moves_post_commit_volumes which used accounts_seq.
+        This index aligns with actual query patterns for better performance.
 
 affected_queries:
   - "resource_aggregated_balances.go: PIT queries with DISTINCT ON (accounts_address, asset)"
-  - "resource_accounts.go: Balance filtering with effective_date"
   - "resource_accounts.go: Account expansion volumes queries"
-  - "resource_accounts.go: Historical metadata lookups"
-  - "resource_transactions.go: Historical transaction metadata lookups"
 
 performance_impact:
-  - "MAJOR improvement for Point-in-Time (PIT) queries"
-  - "MAJOR improvement for balance lookups by account"
-  - "MAJOR improvement for historical metadata queries"
-  - "Minor reduction in storage by removing redundant index"
+  - "MAJOR improvement for Point-in-Time queries using insertion_date"
 
 migration_safety:
   - "Uses CONCURRENTLY flag to avoid locking tables during index creation"
-  - "Drops old indexes only after new ones are created"
-  - "No data modification - only index structure changes"
+  - "Each index creation is isolated in its own migration"
+  - "Old indexes will be dropped in migration 45 after all new indexes are created"
diff --git a/internal/storage/bucket/migrations/42-optimize-query-indexes-pit-effective/notes.yaml b/internal/storage/bucket/migrations/42-optimize-query-indexes-pit-effective/notes.yaml
new file mode 100644
index 000000000..2886f4600
--- /dev/null
+++ b/internal/storage/bucket/migrations/42-optimize-query-indexes-pit-effective/notes.yaml
@@ -0,0 +1,25 @@
+name: Optimize Query Indexes - PIT Effective (2/5)
+description: |
+  Create optimized index for Point-in-Time queries with effective_date on moves table.
+  This is part of a multi-step migration to safely create indexes concurrently.
+
+changes:
+  moves_table:
+    - action: "CREATE"
+      new_index: "idx_moves_pit_effective (account_address, asset, effective_date desc, seq desc)"
+      reason: |
+        Creates index for PIT queries filtering on effective_date.
+        Replaces moves_effective_post_commit_volumes which used accounts_seq.
+        This index aligns with actual query patterns for better performance.
+
+affected_queries:
+  - "resource_aggregated_balances.go: PIT queries with effective_date filtering"
+  - "resource_accounts.go: Balance queries with effective_date"
+
+performance_impact:
+  - "MAJOR improvement for Point-in-Time queries using effective_date"
+
+migration_safety:
+  - "Uses CONCURRENTLY flag to avoid locking tables during index creation"
+  - "Each index creation is isolated in its own migration"
+  - "Old indexes will be dropped in migration 45 after all new indexes are created"
diff --git a/internal/storage/bucket/migrations/43-optimize-accounts-metadata-index/notes.yaml b/internal/storage/bucket/migrations/43-optimize-accounts-metadata-index/notes.yaml
new file mode 100644
index 000000000..45dbae325
--- /dev/null
+++ b/internal/storage/bucket/migrations/43-optimize-accounts-metadata-index/notes.yaml
@@ -0,0 +1,24 @@
+name: Optimize Accounts Metadata Index (3/5)
+description: |
+  Create optimized index for historical metadata queries on accounts_metadata table.
+  This is part of a multi-step migration to safely create indexes concurrently.
+
+changes:
+  accounts_metadata_table:
+    - action: "CREATE"
+      new_index: "idx_accounts_metadata_pit (accounts_address, revision desc) INCLUDE (metadata, date)"
+      reason: |
+        Creates index for historical account metadata lookups.
+        Replaces accounts_metadata_revisions which used accounts_seq.
+        New index aligns with actual query patterns using accounts_address.
+
+affected_queries:
+  - "resource_accounts.go: Historical metadata lookups for Point-in-Time queries"
+
+performance_impact:
+  - "MAJOR improvement for historical account metadata queries"
+
+migration_safety:
+  - "Uses CONCURRENTLY flag to avoid locking tables during index creation"
+  - "Each index creation is isolated in its own migration"
+  - "Old index will be dropped in migration 45 after all new indexes are created"
diff --git a/internal/storage/bucket/migrations/44-optimize-transactions-metadata-index/notes.yaml b/internal/storage/bucket/migrations/44-optimize-transactions-metadata-index/notes.yaml
new file mode 100644
index 000000000..a407b95d1
--- /dev/null
+++ b/internal/storage/bucket/migrations/44-optimize-transactions-metadata-index/notes.yaml
@@ -0,0 +1,24 @@
+name: Optimize Transactions Metadata Index (4/5)
+description: |
+  Create optimized index for historical transaction metadata queries on transactions_metadata table.
+  This is part of a multi-step migration to safely create indexes concurrently.
+
+changes:
+  transactions_metadata_table:
+    - action: "CREATE"
+      new_index: "idx_transactions_metadata_pit (transactions_id, revision desc) INCLUDE (metadata, date)"
+      reason: |
+        Creates index for historical transaction metadata lookups.
+        Replaces transactions_metadata_revisions which used transactions_seq.
+        New index aligns with actual query patterns using transactions_id.
+
+affected_queries:
+  - "resource_transactions.go: Historical transaction metadata lookups for Point-in-Time queries"
+
+performance_impact:
+  - "MAJOR improvement for historical transaction metadata queries"
+
+migration_safety:
+  - "Uses CONCURRENTLY flag to avoid locking tables during index creation"
+  - "Each index creation is isolated in its own migration"
+  - "Old index will be dropped in migration 45 after all new indexes are created"
diff --git a/internal/storage/bucket/migrations/45-drop-old-indexes/notes.yaml b/internal/storage/bucket/migrations/45-drop-old-indexes/notes.yaml
new file mode 100644
index 000000000..233258d46
--- /dev/null
+++ b/internal/storage/bucket/migrations/45-drop-old-indexes/notes.yaml
@@ -0,0 +1,32 @@
+name: Drop Old Indexes (5/5)
+description: |
+  Drop all old suboptimal indexes that have been replaced by new optimized indexes
+  created in migrations 41-44. This final step completes the index optimization process.
+
+changes:
+  moves_table:
+    - action: "DROP"
+      old_indexes:
+        - "moves_post_commit_volumes"
+        - "moves_effective_post_commit_volumes"
+      reason: "Replaced by idx_moves_pit_insertion and idx_moves_pit_effective in migrations 41-42"
+
+  accounts_metadata_table:
+    - action: "DROP"
+      old_index: "accounts_metadata_revisions"
+      reason: "Replaced by idx_accounts_metadata_pit in migration 43"
+
+  transactions_metadata_table:
+    - action: "DROP"
+      old_index: "transactions_metadata_revisions"
+      reason: "Replaced by idx_transactions_metadata_pit in migration 44"
+
+  accounts_volumes_table:
+    - action: "DROP"
+      old_index: "accounts_volumes_idx"
+      reason: "Redundant - PRIMARY KEY already covers this pattern"
+
+migration_safety:
+  - "Old indexes are dropped only AFTER all new indexes are created"
+  - "Ensures no performance degradation during migration"
+  - "No data modification - only removes redundant index structures"

From 33f3ba5853ffc7eb9cba6a78002b2fda2e1c0746 Mon Sep 17 00:00:00 2001
From: Maxence Maireaux <maxence@maireaux.fr>
Date: Mon, 10 Nov 2025 18:29:35 +0100
Subject: [PATCH 06/12] fix: remove CONCURRENTLY from index creation migrations

CONCURRENTLY cannot be used within the migration framework as migrations
run within a transactional context (advisory locks). Removed CONCURRENTLY
from all index creation statements to ensure migrations can execute successfully.

The migrations remain isolated (one index per migration) which maintains
good organization and allows for easier rollback if needed. Index creation
will lock tables during execution but ensures compatibility with the
migration system.

Note: Migration 41-44 create new indexes, migration 45 drops old ones.
This approach minimizes risk by creating all new indexes before dropping
old ones.
---
 .../bucket/migrations/41-optimize-query-indexes/notes.yaml      | 2 +-
 .../storage/bucket/migrations/41-optimize-query-indexes/up.sql  | 2 +-
 .../42-optimize-query-indexes-pit-effective/notes.yaml          | 2 +-
 .../migrations/42-optimize-query-indexes-pit-effective/up.sql   | 2 +-
 .../migrations/43-optimize-accounts-metadata-index/notes.yaml   | 2 +-
 .../migrations/43-optimize-accounts-metadata-index/up.sql       | 2 +-
 .../44-optimize-transactions-metadata-index/notes.yaml          | 2 +-
 .../migrations/44-optimize-transactions-metadata-index/up.sql   | 2 +-
 8 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/internal/storage/bucket/migrations/41-optimize-query-indexes/notes.yaml b/internal/storage/bucket/migrations/41-optimize-query-indexes/notes.yaml
index b01096884..fac3785b1 100644
--- a/internal/storage/bucket/migrations/41-optimize-query-indexes/notes.yaml
+++ b/internal/storage/bucket/migrations/41-optimize-query-indexes/notes.yaml
@@ -1,7 +1,7 @@
 name: Optimize Query Indexes - PIT Insertion (1/5)
 description: |
   Create optimized index for Point-in-Time queries with insertion_date on moves table.
-  This is part of a multi-step migration to safely create indexes concurrently.
+  This is part of a multi-step migration where each index is created in a separate migration.
 
 changes:
   moves_table:
diff --git a/internal/storage/bucket/migrations/41-optimize-query-indexes/up.sql b/internal/storage/bucket/migrations/41-optimize-query-indexes/up.sql
index 35b90339a..ebbb60b7f 100644
--- a/internal/storage/bucket/migrations/41-optimize-query-indexes/up.sql
+++ b/internal/storage/bucket/migrations/41-optimize-query-indexes/up.sql
@@ -7,5 +7,5 @@ set search_path = '{{.Schema}}';
 -- Critical: Index for Point-in-Time queries with insertion_date
 -- Covers queries in resource_aggregated_balances.go and resource_accounts.go
 -- Replaces: moves_post_commit_volumes
-create index {{ if not .Transactional }}concurrently{{end}} idx_moves_pit_insertion
+create index idx_moves_pit_insertion
     on "{{.Schema}}".moves (account_address, asset, insertion_date desc, seq desc);
diff --git a/internal/storage/bucket/migrations/42-optimize-query-indexes-pit-effective/notes.yaml b/internal/storage/bucket/migrations/42-optimize-query-indexes-pit-effective/notes.yaml
index 2886f4600..6f92f3ba8 100644
--- a/internal/storage/bucket/migrations/42-optimize-query-indexes-pit-effective/notes.yaml
+++ b/internal/storage/bucket/migrations/42-optimize-query-indexes-pit-effective/notes.yaml
@@ -1,7 +1,7 @@
 name: Optimize Query Indexes - PIT Effective (2/5)
 description: |
   Create optimized index for Point-in-Time queries with effective_date on moves table.
-  This is part of a multi-step migration to safely create indexes concurrently.
+  This is part of a multi-step migration where each index is created in a separate migration.
 
 changes:
   moves_table:
diff --git a/internal/storage/bucket/migrations/42-optimize-query-indexes-pit-effective/up.sql b/internal/storage/bucket/migrations/42-optimize-query-indexes-pit-effective/up.sql
index a5beef846..c3c674dcd 100644
--- a/internal/storage/bucket/migrations/42-optimize-query-indexes-pit-effective/up.sql
+++ b/internal/storage/bucket/migrations/42-optimize-query-indexes-pit-effective/up.sql
@@ -7,5 +7,5 @@ set search_path = '{{.Schema}}';
 -- Critical: Index for Point-in-Time queries with effective_date
 -- Covers queries in resource_aggregated_balances.go and resource_accounts.go
 -- Replaces: moves_effective_post_commit_volumes
-create index {{ if not .Transactional }}concurrently{{end}} idx_moves_pit_effective
+create index idx_moves_pit_effective
     on "{{.Schema}}".moves (account_address, asset, effective_date desc, seq desc);
diff --git a/internal/storage/bucket/migrations/43-optimize-accounts-metadata-index/notes.yaml b/internal/storage/bucket/migrations/43-optimize-accounts-metadata-index/notes.yaml
index 45dbae325..dc31645bf 100644
--- a/internal/storage/bucket/migrations/43-optimize-accounts-metadata-index/notes.yaml
+++ b/internal/storage/bucket/migrations/43-optimize-accounts-metadata-index/notes.yaml
@@ -1,7 +1,7 @@
 name: Optimize Accounts Metadata Index (3/5)
 description: |
   Create optimized index for historical metadata queries on accounts_metadata table.
-  This is part of a multi-step migration to safely create indexes concurrently.
+  This is part of a multi-step migration where each index is created in a separate migration.
 
 changes:
   accounts_metadata_table:
diff --git a/internal/storage/bucket/migrations/43-optimize-accounts-metadata-index/up.sql b/internal/storage/bucket/migrations/43-optimize-accounts-metadata-index/up.sql
index d6ca6bbb8..d235fe338 100644
--- a/internal/storage/bucket/migrations/43-optimize-accounts-metadata-index/up.sql
+++ b/internal/storage/bucket/migrations/43-optimize-accounts-metadata-index/up.sql
@@ -7,6 +7,6 @@ set search_path = '{{.Schema}}';
 -- Critical: Index for historical metadata queries
 -- Covers queries in resource_accounts.go for Point-in-Time metadata
 -- Replaces: accounts_metadata_revisions
-create index {{ if not .Transactional }}concurrently{{end}} idx_accounts_metadata_pit
+create index idx_accounts_metadata_pit
     on "{{.Schema}}".accounts_metadata (accounts_address, revision desc)
     include (metadata, date);
diff --git a/internal/storage/bucket/migrations/44-optimize-transactions-metadata-index/notes.yaml b/internal/storage/bucket/migrations/44-optimize-transactions-metadata-index/notes.yaml
index a407b95d1..d20d7811d 100644
--- a/internal/storage/bucket/migrations/44-optimize-transactions-metadata-index/notes.yaml
+++ b/internal/storage/bucket/migrations/44-optimize-transactions-metadata-index/notes.yaml
@@ -1,7 +1,7 @@
 name: Optimize Transactions Metadata Index (4/5)
 description: |
   Create optimized index for historical transaction metadata queries on transactions_metadata table.
-  This is part of a multi-step migration to safely create indexes concurrently.
+  This is part of a multi-step migration where each index is created in a separate migration.
 
 changes:
   transactions_metadata_table:
diff --git a/internal/storage/bucket/migrations/44-optimize-transactions-metadata-index/up.sql b/internal/storage/bucket/migrations/44-optimize-transactions-metadata-index/up.sql
index 1f22b2a25..adef7590d 100644
--- a/internal/storage/bucket/migrations/44-optimize-transactions-metadata-index/up.sql
+++ b/internal/storage/bucket/migrations/44-optimize-transactions-metadata-index/up.sql
@@ -7,6 +7,6 @@ set search_path = '{{.Schema}}';
 -- Critical: Index for historical transaction metadata queries
 -- Covers queries in resource_transactions.go for Point-in-Time metadata
 -- Replaces: transactions_metadata_revisions
-create index {{ if not .Transactional }}concurrently{{end}} idx_transactions_metadata_pit
+create index idx_transactions_metadata_pit
     on "{{.Schema}}".transactions_metadata (transactions_id, revision desc)
     include (metadata, date);

From 655bfb4f969d395950307f9773b1c010abc76fc9 Mon Sep 17 00:00:00 2001
From: Maxence Maireaux <maxence@maireaux.fr>
Date: Mon, 10 Nov 2025 19:56:10 +0100
Subject: [PATCH 07/12] fix: correct column name from account_address to
 accounts_address in index creation

---
 .../storage/bucket/migrations/41-optimize-query-indexes/up.sql  | 2 +-
 .../migrations/42-optimize-query-indexes-pit-effective/up.sql   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/internal/storage/bucket/migrations/41-optimize-query-indexes/up.sql b/internal/storage/bucket/migrations/41-optimize-query-indexes/up.sql
index ebbb60b7f..d4ff925f9 100644
--- a/internal/storage/bucket/migrations/41-optimize-query-indexes/up.sql
+++ b/internal/storage/bucket/migrations/41-optimize-query-indexes/up.sql
@@ -8,4 +8,4 @@ set search_path = '{{.Schema}}';
 -- Covers queries in resource_aggregated_balances.go and resource_accounts.go
 -- Replaces: moves_post_commit_volumes
 create index idx_moves_pit_insertion
-    on "{{.Schema}}".moves (account_address, asset, insertion_date desc, seq desc);
+    on "{{.Schema}}".moves (accounts_address, asset, insertion_date desc, seq desc);
diff --git a/internal/storage/bucket/migrations/42-optimize-query-indexes-pit-effective/up.sql b/internal/storage/bucket/migrations/42-optimize-query-indexes-pit-effective/up.sql
index c3c674dcd..f1b4c0d77 100644
--- a/internal/storage/bucket/migrations/42-optimize-query-indexes-pit-effective/up.sql
+++ b/internal/storage/bucket/migrations/42-optimize-query-indexes-pit-effective/up.sql
@@ -8,4 +8,4 @@ set search_path = '{{.Schema}}';
 -- Covers queries in resource_aggregated_balances.go and resource_accounts.go
 -- Replaces: moves_effective_post_commit_volumes
 create index idx_moves_pit_effective
-    on "{{.Schema}}".moves (account_address, asset, effective_date desc, seq desc);
+    on "{{.Schema}}".moves (accounts_address, asset, effective_date desc, seq desc);

From 8ffb13f924f88bb5eac641a8f84a0d2f63c32b42 Mon Sep 17 00:00:00 2001
From: Maxence Maireaux <maxence@maireaux.fr>
Date: Mon, 10 Nov 2025 20:35:41 +0100
Subject: [PATCH 08/12] fix: rename testServer variable to srv for clarity and
 add cleanup on service stop

---
 test/e2e/app_lifecycle_test.go | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/test/e2e/app_lifecycle_test.go b/test/e2e/app_lifecycle_test.go
index f4d5549ef..b48b770e2 100644
--- a/test/e2e/app_lifecycle_test.go
+++ b/test/e2e/app_lifecycle_test.go
@@ -76,10 +76,14 @@ var _ = Context("Ledger application lifecycle tests", func() {
 		})
 		When("restarting the service", func() {
 			BeforeEach(func(ctx context.Context) {
-				testServer, err := testServer.Wait(ctx)
+				srv, err := testServer.Wait(ctx)
 				Expect(err).To(BeNil())
 
-				Expect(testServer.Restart(ctx)).To(BeNil())
+				Expect(srv.Restart(ctx)).To(BeNil())
+
+				DeferCleanup(func(ctx context.Context) {
+					Expect(srv.Stop(ctx)).To(BeNil())
+				})
 			})
 			It("should be ok", func() {})
 		})

From df167910225cc55d8873c2d3faaebd9b4ac63b24 Mon Sep 17 00:00:00 2001
From: Maxence Maireaux <maxence@maireaux.fr>
Date: Mon, 10 Nov 2025 22:17:54 +0100
Subject: [PATCH 09/12] Increase default benchmark count from 1 to 5

---
 .github/workflows/benchmark-comparison.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/benchmark-comparison.yml b/.github/workflows/benchmark-comparison.yml
index 591210ea9..056af1e45 100644
--- a/.github/workflows/benchmark-comparison.yml
+++ b/.github/workflows/benchmark-comparison.yml
@@ -17,7 +17,7 @@ on:
       count:
         description: "Number of times to run each benchmark "
         required: false
-        default: 1
+        default: 5
   pull_request:
     types: [assigned, opened, synchronize, reopened, labeled]
 

From d8b1e019f5f192da31b3e5fe5e374ecf049b71e8 Mon Sep 17 00:00:00 2001
From: Maxence Maireaux <maxence@maireaux.fr>
Date: Thu, 13 Nov 2025 10:16:07 +0100
Subject: [PATCH 10/12] feat: add index to optimize update_effective_volumes
 trigger

Add idx_moves_update_effective_volumes index to prevent sequential scans
on every INSERT when MOVES_HISTORY feature is enabled (ON by default).

The update_effective_volumes trigger runs after each move insertion and
updates all moves with effective_date > new.effective_date. Without this
index, this operation causes significant performance degradation at scale.

Changes:
- New migration 45: Create idx_moves_update_effective_volumes
- Migration 45 renamed to 46: Updated to drop moves_range_dates
---
 .../notes.yaml                                | 35 +++++++++++++++++++
 .../up.sql                                    | 12 +++++++
 .../notes.yaml                                | 10 ++++--
 .../up.sql                                    |  3 +-
 4 files changed, 56 insertions(+), 4 deletions(-)
 create mode 100644 internal/storage/bucket/migrations/45-add-moves-update-effective-volumes-index/notes.yaml
 create mode 100644 internal/storage/bucket/migrations/45-add-moves-update-effective-volumes-index/up.sql
 rename internal/storage/bucket/migrations/{45-drop-old-indexes => 46-drop-old-indexes}/notes.yaml (71%)
 rename internal/storage/bucket/migrations/{45-drop-old-indexes => 46-drop-old-indexes}/up.sql (85%)

diff --git a/internal/storage/bucket/migrations/45-add-moves-update-effective-volumes-index/notes.yaml b/internal/storage/bucket/migrations/45-add-moves-update-effective-volumes-index/notes.yaml
new file mode 100644
index 000000000..9398f46de
--- /dev/null
+++ b/internal/storage/bucket/migrations/45-add-moves-update-effective-volumes-index/notes.yaml
@@ -0,0 +1,35 @@
+name: Add Index for Trigger Optimization (3/5)
+description: |
+  Create optimized index for the update_effective_volumes trigger that runs on every
+  INSERT when MOVES_HISTORY feature is enabled (ON by default).
+
+  This trigger updates all moves where effective_date > new.effective_date, which
+  requires an efficient index on (accounts_address, asset, effective_date) to avoid
+  full table scans on large datasets.
+
+changes:
+  moves_table:
+    - action: "CREATE"
+      new_index: "idx_moves_update_effective_volumes (accounts_address, asset, effective_date)"
+      reason: |
+        Optimizes the UPDATE query in update_effective_volumes trigger.
+        Without this index, every INSERT causes a slow sequential scan when
+        MOVES_HISTORY is ON, causing severe performance degradation.
+
+        The existing moves_range_dates index has the same columns but in a
+        different order (account_address, asset, effective_date) and was created
+        before the column rename. This new index ensures optimal performance.
+
+affected_queries:
+  - "update_effective_volumes trigger: UPDATE moves WHERE effective_date > ?"
+  - "Runs on EVERY INSERT when MOVES_HISTORY=ON (default configuration)"
+
+performance_impact:
+  - "CRITICAL improvement for write performance with MOVES_HISTORY enabled"
+  - "Reduces INSERT latency by eliminating sequential scans on moves table"
+  - "Essential for maintaining performance at scale with history tracking"
+
+migration_safety:
+  - "Index creation is non-blocking"
+  - "No data modification"
+  - "moves_range_dates will be dropped in next migration (46) as redundant"
diff --git a/internal/storage/bucket/migrations/45-add-moves-update-effective-volumes-index/up.sql b/internal/storage/bucket/migrations/45-add-moves-update-effective-volumes-index/up.sql
new file mode 100644
index 000000000..1202551f0
--- /dev/null
+++ b/internal/storage/bucket/migrations/45-add-moves-update-effective-volumes-index/up.sql
@@ -0,0 +1,12 @@
+set search_path = '{{.Schema}}';
+
+-- ========================================
+-- MOVES TABLE INDEX FOR TRIGGER OPTIMIZATION (3/5)
+-- ========================================
+
+-- Critical: Index to optimize update_effective_volumes trigger
+-- This trigger runs on EVERY INSERT when MOVES_HISTORY feature is ON
+-- The trigger updates all moves with effective_date > new.effective_date
+-- Replaces/optimizes: moves_range_dates which has suboptimal column order
+create index idx_moves_update_effective_volumes
+    on "{{.Schema}}".moves (accounts_address, asset, effective_date);
diff --git a/internal/storage/bucket/migrations/45-drop-old-indexes/notes.yaml b/internal/storage/bucket/migrations/46-drop-old-indexes/notes.yaml
similarity index 71%
rename from internal/storage/bucket/migrations/45-drop-old-indexes/notes.yaml
rename to internal/storage/bucket/migrations/46-drop-old-indexes/notes.yaml
index 233258d46..61ac4acf3 100644
--- a/internal/storage/bucket/migrations/45-drop-old-indexes/notes.yaml
+++ b/internal/storage/bucket/migrations/46-drop-old-indexes/notes.yaml
@@ -1,7 +1,7 @@
-name: Drop Old Indexes (5/5)
+name: Drop Old Indexes (6/6)
 description: |
   Drop all old suboptimal indexes that have been replaced by new optimized indexes
-  created in migrations 41-44. This final step completes the index optimization process.
+  created in migrations 41-45. This final step completes the index optimization process.
 
 changes:
   moves_table:
@@ -9,7 +9,11 @@ changes:
       old_indexes:
         - "moves_post_commit_volumes"
         - "moves_effective_post_commit_volumes"
-      reason: "Replaced by idx_moves_pit_insertion and idx_moves_pit_effective in migrations 41-42"
+        - "moves_range_dates"
+      reason: |
+        - moves_post_commit_volumes and moves_effective_post_commit_volumes
+          replaced by idx_moves_pit_insertion and idx_moves_pit_effective in migrations 41-42
+        - moves_range_dates replaced by idx_moves_update_effective_volumes in migration 45
 
   accounts_metadata_table:
     - action: "DROP"
diff --git a/internal/storage/bucket/migrations/45-drop-old-indexes/up.sql b/internal/storage/bucket/migrations/46-drop-old-indexes/up.sql
similarity index 85%
rename from internal/storage/bucket/migrations/45-drop-old-indexes/up.sql
rename to internal/storage/bucket/migrations/46-drop-old-indexes/up.sql
index 69d6b7000..a868e3688 100644
--- a/internal/storage/bucket/migrations/45-drop-old-indexes/up.sql
+++ b/internal/storage/bucket/migrations/46-drop-old-indexes/up.sql
@@ -4,9 +4,10 @@ set search_path = '{{.Schema}}';
 -- DROP OLD INDEXES
 -- ========================================
 
--- Drop old moves table indexes (replaced by migrations 41-42)
+-- Drop old moves table indexes (replaced by migrations 41-42-45)
 drop index if exists moves_post_commit_volumes;
 drop index if exists moves_effective_post_commit_volumes;
+drop index if exists moves_range_dates;
 
 -- Drop old accounts_metadata index (replaced by migration 43)
 drop index if exists accounts_metadata_revisions;

From 6920c1c7c6a8db65e244d1ab30aad2332db30810 Mon Sep 17 00:00:00 2001
From: Maxence Maireaux <maxence@maireaux.fr>
Date: Tue, 18 Nov 2025 18:18:31 +0100
Subject: [PATCH 11/12] fix: Add concurrently

---
 .../storage/bucket/migrations/41-optimize-query-indexes/up.sql  | 2 +-
 .../migrations/42-optimize-query-indexes-pit-effective/up.sql   | 2 +-
 .../migrations/43-optimize-accounts-metadata-index/up.sql       | 2 +-
 .../migrations/44-optimize-transactions-metadata-index/up.sql   | 2 +-
 .../45-add-moves-update-effective-volumes-index/up.sql          | 2 +-
 5 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/internal/storage/bucket/migrations/41-optimize-query-indexes/up.sql b/internal/storage/bucket/migrations/41-optimize-query-indexes/up.sql
index d4ff925f9..40ef39523 100644
--- a/internal/storage/bucket/migrations/41-optimize-query-indexes/up.sql
+++ b/internal/storage/bucket/migrations/41-optimize-query-indexes/up.sql
@@ -7,5 +7,5 @@ set search_path = '{{.Schema}}';
 -- Critical: Index for Point-in-Time queries with insertion_date
 -- Covers queries in resource_aggregated_balances.go and resource_accounts.go
 -- Replaces: moves_post_commit_volumes
-create index idx_moves_pit_insertion
+create index {{ if not .Transactional }}concurrently{{end}} idx_moves_pit_insertion
     on "{{.Schema}}".moves (accounts_address, asset, insertion_date desc, seq desc);
diff --git a/internal/storage/bucket/migrations/42-optimize-query-indexes-pit-effective/up.sql b/internal/storage/bucket/migrations/42-optimize-query-indexes-pit-effective/up.sql
index f1b4c0d77..0afe8ed47 100644
--- a/internal/storage/bucket/migrations/42-optimize-query-indexes-pit-effective/up.sql
+++ b/internal/storage/bucket/migrations/42-optimize-query-indexes-pit-effective/up.sql
@@ -7,5 +7,5 @@ set search_path = '{{.Schema}}';
 -- Critical: Index for Point-in-Time queries with effective_date
 -- Covers queries in resource_aggregated_balances.go and resource_accounts.go
 -- Replaces: moves_effective_post_commit_volumes
-create index idx_moves_pit_effective
+create index {{ if not .Transactional }}concurrently{{end}} idx_moves_pit_effective
     on "{{.Schema}}".moves (accounts_address, asset, effective_date desc, seq desc);
diff --git a/internal/storage/bucket/migrations/43-optimize-accounts-metadata-index/up.sql b/internal/storage/bucket/migrations/43-optimize-accounts-metadata-index/up.sql
index d235fe338..d6ca6bbb8 100644
--- a/internal/storage/bucket/migrations/43-optimize-accounts-metadata-index/up.sql
+++ b/internal/storage/bucket/migrations/43-optimize-accounts-metadata-index/up.sql
@@ -7,6 +7,6 @@ set search_path = '{{.Schema}}';
 -- Critical: Index for historical metadata queries
 -- Covers queries in resource_accounts.go for Point-in-Time metadata
 -- Replaces: accounts_metadata_revisions
-create index idx_accounts_metadata_pit
+create index {{ if not .Transactional }}concurrently{{end}} idx_accounts_metadata_pit
     on "{{.Schema}}".accounts_metadata (accounts_address, revision desc)
     include (metadata, date);
diff --git a/internal/storage/bucket/migrations/44-optimize-transactions-metadata-index/up.sql b/internal/storage/bucket/migrations/44-optimize-transactions-metadata-index/up.sql
index adef7590d..1f22b2a25 100644
--- a/internal/storage/bucket/migrations/44-optimize-transactions-metadata-index/up.sql
+++ b/internal/storage/bucket/migrations/44-optimize-transactions-metadata-index/up.sql
@@ -7,6 +7,6 @@ set search_path = '{{.Schema}}';
 -- Critical: Index for historical transaction metadata queries
 -- Covers queries in resource_transactions.go for Point-in-Time metadata
 -- Replaces: transactions_metadata_revisions
-create index idx_transactions_metadata_pit
+create index {{ if not .Transactional }}concurrently{{end}} idx_transactions_metadata_pit
     on "{{.Schema}}".transactions_metadata (transactions_id, revision desc)
     include (metadata, date);
diff --git a/internal/storage/bucket/migrations/45-add-moves-update-effective-volumes-index/up.sql b/internal/storage/bucket/migrations/45-add-moves-update-effective-volumes-index/up.sql
index 1202551f0..b54b0c042 100644
--- a/internal/storage/bucket/migrations/45-add-moves-update-effective-volumes-index/up.sql
+++ b/internal/storage/bucket/migrations/45-add-moves-update-effective-volumes-index/up.sql
@@ -8,5 +8,5 @@ set search_path = '{{.Schema}}';
 -- This trigger runs on EVERY INSERT when MOVES_HISTORY feature is ON
 -- The trigger updates all moves with effective_date > new.effective_date
 -- Replaces/optimizes: moves_range_dates which has suboptimal column order
-create index idx_moves_update_effective_volumes
+create index {{ if not .Transactional }}concurrently{{end}} idx_moves_update_effective_volumes
     on "{{.Schema}}".moves (accounts_address, asset, effective_date);

From ab5ba2f23de2fc73df22f468fbf6c151b908b66a Mon Sep 17 00:00:00 2001
From: Maxence Maireaux <maxence@maireaux.fr>
Date: Wed, 19 Nov 2025 09:41:27 +0100
Subject: [PATCH 12/12] fix: migrations

---
 .../storage/bucket/migrations/41-optimize-query-indexes/up.sql  | 2 --
 .../migrations/42-optimize-query-indexes-pit-effective/up.sql   | 2 --
 .../migrations/43-optimize-accounts-metadata-index/up.sql       | 2 --
 .../migrations/44-optimize-transactions-metadata-index/up.sql   | 2 --
 .../45-add-moves-update-effective-volumes-index/up.sql          | 2 --
 internal/storage/bucket/migrations/46-drop-old-indexes/up.sql   | 2 --
 6 files changed, 12 deletions(-)

diff --git a/internal/storage/bucket/migrations/41-optimize-query-indexes/up.sql b/internal/storage/bucket/migrations/41-optimize-query-indexes/up.sql
index 40ef39523..5c3781562 100644
--- a/internal/storage/bucket/migrations/41-optimize-query-indexes/up.sql
+++ b/internal/storage/bucket/migrations/41-optimize-query-indexes/up.sql
@@ -1,5 +1,3 @@
-set search_path = '{{.Schema}}';
-
 -- ========================================
 -- MOVES TABLE INDEXES OPTIMIZATION (1/4)
 -- ========================================
diff --git a/internal/storage/bucket/migrations/42-optimize-query-indexes-pit-effective/up.sql b/internal/storage/bucket/migrations/42-optimize-query-indexes-pit-effective/up.sql
index 0afe8ed47..815adf12e 100644
--- a/internal/storage/bucket/migrations/42-optimize-query-indexes-pit-effective/up.sql
+++ b/internal/storage/bucket/migrations/42-optimize-query-indexes-pit-effective/up.sql
@@ -1,5 +1,3 @@
-set search_path = '{{.Schema}}';
-
 -- ========================================
 -- MOVES TABLE INDEXES OPTIMIZATION (2/4)
 -- ========================================
diff --git a/internal/storage/bucket/migrations/43-optimize-accounts-metadata-index/up.sql b/internal/storage/bucket/migrations/43-optimize-accounts-metadata-index/up.sql
index d6ca6bbb8..dd6b37d84 100644
--- a/internal/storage/bucket/migrations/43-optimize-accounts-metadata-index/up.sql
+++ b/internal/storage/bucket/migrations/43-optimize-accounts-metadata-index/up.sql
@@ -1,5 +1,3 @@
-set search_path = '{{.Schema}}';
-
 -- ========================================
 -- ACCOUNTS_METADATA TABLE OPTIMIZATION
 -- ========================================
diff --git a/internal/storage/bucket/migrations/44-optimize-transactions-metadata-index/up.sql b/internal/storage/bucket/migrations/44-optimize-transactions-metadata-index/up.sql
index 1f22b2a25..235b07426 100644
--- a/internal/storage/bucket/migrations/44-optimize-transactions-metadata-index/up.sql
+++ b/internal/storage/bucket/migrations/44-optimize-transactions-metadata-index/up.sql
@@ -1,5 +1,3 @@
-set search_path = '{{.Schema}}';
-
 -- ========================================
 -- TRANSACTIONS_METADATA TABLE OPTIMIZATION
 -- ========================================
diff --git a/internal/storage/bucket/migrations/45-add-moves-update-effective-volumes-index/up.sql b/internal/storage/bucket/migrations/45-add-moves-update-effective-volumes-index/up.sql
index b54b0c042..c263e0426 100644
--- a/internal/storage/bucket/migrations/45-add-moves-update-effective-volumes-index/up.sql
+++ b/internal/storage/bucket/migrations/45-add-moves-update-effective-volumes-index/up.sql
@@ -1,5 +1,3 @@
-set search_path = '{{.Schema}}';
-
 -- ========================================
 -- MOVES TABLE INDEX FOR TRIGGER OPTIMIZATION (3/5)
 -- ========================================
diff --git a/internal/storage/bucket/migrations/46-drop-old-indexes/up.sql b/internal/storage/bucket/migrations/46-drop-old-indexes/up.sql
index a868e3688..33c603a66 100644
--- a/internal/storage/bucket/migrations/46-drop-old-indexes/up.sql
+++ b/internal/storage/bucket/migrations/46-drop-old-indexes/up.sql
@@ -1,5 +1,3 @@
-set search_path = '{{.Schema}}';
-
 -- ========================================
 -- DROP OLD INDEXES
 -- ========================================