perf: suboptimal tables query schema filter happens late (#980)

* wip: optimize queries

* wip: refactor queries for root filtering

* perf: add root level filtering to all queries

* feat: add functions args retrieval

* chore: fix functions

* fix(table): use or for table relationships

* test(functions): add tests for retrival argument based

Author: avallete

src/lib/PostgresMetaColumnPrivileges.ts

@@ -1,7 +1,7 @@
 import { ident, literal } from 'pg-format'
 import { DEFAULT_SYSTEM_SCHEMAS } from './constants.js'
-import { filterByList } from './helpers.js'
-import { columnPrivilegesSql } from './sql/index.js'
+import { filterByValue, filterByList } from './helpers.js'
+import { COLUMN_PRIVILEGES_SQL } from './sql/column_privileges.sql.js'
 import {
   PostgresMetaResult,
   PostgresColumnPrivileges,
@@ -29,25 +29,12 @@ export default class PostgresMetaColumnPrivileges {
     limit?: number
     offset?: number
   } = {}): Promise<PostgresMetaResult<PostgresColumnPrivileges[]>> {
-    let sql = `
-with column_privileges as (${columnPrivilegesSql})
-select *
-from column_privileges
-`
-    const filter = filterByList(
+    const schemaFilter = filterByList(
       includedSchemas,
       excludedSchemas,
       !includeSystemSchemas ? DEFAULT_SYSTEM_SCHEMAS : undefined
     )
-    if (filter) {
-      sql += ` where relation_schema ${filter}`
-    }
-    if (limit) {
-      sql += ` limit ${limit}`
-    }
-    if (offset) {
-      sql += ` offset ${offset}`
-    }
+    const sql = COLUMN_PRIVILEGES_SQL({ schemaFilter, limit, offset })
     return await this.query(sql)
   }
 
@@ -86,12 +73,8 @@ end $$;
 
     // Return the updated column privileges for modified columns.
     const columnIds = [...new Set(grants.map(({ column_id }) => column_id))]
-    sql = `
-with column_privileges as (${columnPrivilegesSql})
-select *
-from column_privileges
-where column_id in (${columnIds.map(literal).join(',')})
-`
+    const columnIdsFilter = filterByValue(columnIds)
+    sql = COLUMN_PRIVILEGES_SQL({ columnIdsFilter })
     return await this.query(sql)
   }
 
@@ -130,12 +113,8 @@ end $$;
 
     // Return the updated column privileges for modified columns.
     const columnIds = [...new Set(revokes.map(({ column_id }) => column_id))]
-    sql = `
-with column_privileges as (${columnPrivilegesSql})
-select *
-from column_privileges
-where column_id in (${columnIds.map(literal).join(',')})
-`
+    const columnIdsFilter = filterByValue(columnIds)
+    sql = COLUMN_PRIVILEGES_SQL({ columnIdsFilter })
     return await this.query(sql)
   }
 }

src/lib/PostgresMetaColumns.ts

@@ -1,9 +1,9 @@
 import { ident, literal } from 'pg-format'
 import PostgresMetaTables from './PostgresMetaTables.js'
 import { DEFAULT_SYSTEM_SCHEMAS } from './constants.js'
-import { columnsSql } from './sql/index.js'
 import { PostgresMetaResult, PostgresColumn } from './types.js'
-import { filterByList } from './helpers.js'
+import { filterByValue, filterByList } from './helpers.js'
+import { COLUMNS_SQL } from './sql/columns.sql.js'
 
 export default class PostgresMetaColumns {
   query: (sql: string) => Promise<PostgresMetaResult<any>>
@@ -29,32 +29,13 @@ export default class PostgresMetaColumns {
     limit?: number
     offset?: number
   } = {}): Promise<PostgresMetaResult<PostgresColumn[]>> {
-    let sql = `
-WITH
-  columns AS (${columnsSql})
-SELECT
-  *
-FROM
-  columns
-WHERE
-  true`
-    const filter = filterByList(
+    const schemaFilter = filterByList(
       includedSchemas,
       excludedSchemas,
       !includeSystemSchemas ? DEFAULT_SYSTEM_SCHEMAS : undefined
     )
-    if (filter) {
-      sql += ` AND schema ${filter}`
-    }
-    if (tableId !== undefined) {
-      sql += ` AND table_id = ${literal(tableId)}`
-    }
-    if (limit) {
-      sql += ` LIMIT ${limit}`
-    }
-    if (offset) {
-      sql += ` OFFSET ${offset}`
-    }
+    const tableIdFilter = tableId ? filterByValue([`${tableId}`]) : undefined
+    const sql = COLUMNS_SQL({ schemaFilter, tableIdFilter, limit, offset })
     return await this.query(sql)
   }
 
@@ -79,14 +60,16 @@ WHERE
     table?: string
     schema?: string
   }): Promise<PostgresMetaResult<PostgresColumn>> {
+    const schemaFilter = schema ? filterByList([schema], []) : undefined
     if (id) {
       const regexp = /^(\d+)\.(\d+)$/
       if (!regexp.test(id)) {
         return { data: null, error: { message: 'Invalid format for column ID' } }
       }
       const matches = id.match(regexp) as RegExpMatchArray
       const [tableId, ordinalPos] = matches.slice(1).map(Number)
-      const sql = `${columnsSql} AND c.oid = ${tableId} AND a.attnum = ${ordinalPos};`
+      const idsFilter = filterByValue([`${tableId}.${ordinalPos}`])
+      const sql = COLUMNS_SQL({ idsFilter })
       const { data, error } = await this.query(sql)
       if (error) {
         return { data, error }
@@ -96,9 +79,8 @@ WHERE
         return { data: data[0], error }
       }
     } else if (name && table) {
-      const sql = `${columnsSql} AND a.attname = ${literal(name)} AND c.relname = ${literal(
-        table
-      )} AND nc.nspname = ${literal(schema)};`
+      const columnNameFilter = filterByValue([`${table}.${name}`])
+      const sql = `${COLUMNS_SQL({ schemaFilter, columnNameFilter })};`
       const { data, error } = await this.query(sql)
       if (error) {
         return { data, error }

src/lib/PostgresMetaConfig.ts

@@ -1,4 +1,4 @@
-import { configSql } from './sql/index.js'
+import { CONFIG_SQL } from './sql/config.sql.js'
 import { PostgresMetaResult, PostgresConfig } from './types.js'
 
 export default class PostgresMetaConfig {
@@ -15,13 +15,7 @@ export default class PostgresMetaConfig {
     limit?: number
     offset?: number
   } = {}): Promise<PostgresMetaResult<PostgresConfig[]>> {
-    let sql = configSql
-    if (limit) {
-      sql = `${sql} LIMIT ${limit}`
-    }
-    if (offset) {
-      sql = `${sql} OFFSET ${offset}`
-    }
+    const sql = CONFIG_SQL({ limit, offset })
     return await this.query(sql)
   }
 }

src/lib/PostgresMetaExtensions.ts

@@ -1,6 +1,7 @@
 import { ident, literal } from 'pg-format'
-import { extensionsSql } from './sql/index.js'
 import { PostgresMetaResult, PostgresExtension } from './types.js'
+import { EXTENSIONS_SQL } from './sql/extensions.sql.js'
+import { filterByValue } from './helpers.js'
 
 export default class PostgresMetaExtensions {
   query: (sql: string) => Promise<PostgresMetaResult<any>>
@@ -16,18 +17,13 @@ export default class PostgresMetaExtensions {
     limit?: number
     offset?: number
   } = {}): Promise<PostgresMetaResult<PostgresExtension[]>> {
-    let sql = extensionsSql
-    if (limit) {
-      sql = `${sql} LIMIT ${limit}`
-    }
-    if (offset) {
-      sql = `${sql} OFFSET ${offset}`
-    }
+    const sql = EXTENSIONS_SQL({ limit, offset })
     return await this.query(sql)
   }
 
   async retrieve({ name }: { name: string }): Promise<PostgresMetaResult<PostgresExtension>> {
-    const sql = `${extensionsSql} WHERE name = ${literal(name)};`
+    const nameFilter = filterByValue([name])
+    const sql = EXTENSIONS_SQL({ nameFilter })
     const { data, error } = await this.query(sql)
     if (error) {
       return { data, error }

src/lib/PostgresMetaForeignTables.ts

@@ -1,7 +1,7 @@
-import { literal } from 'pg-format'
-import { coalesceRowsToArray, filterByList } from './helpers.js'
-import { columnsSql, foreignTablesSql } from './sql/index.js'
+import { coalesceRowsToArray, filterByList, filterByValue } from './helpers.js'
 import { PostgresMetaResult, PostgresForeignTable } from './types.js'
+import { FOREIGN_TABLES_SQL } from './sql/foreign_tables.sql.js'
+import { COLUMNS_SQL } from './sql/columns.sql.js'
 
 export default class PostgresMetaForeignTables {
   query: (sql: string) => Promise<PostgresMetaResult<any>>
@@ -37,17 +37,8 @@ export default class PostgresMetaForeignTables {
     offset?: number
     includeColumns?: boolean
   } = {}): Promise<PostgresMetaResult<PostgresForeignTable[]>> {
-    let sql = generateEnrichedForeignTablesSql({ includeColumns })
-    const filter = filterByList(includedSchemas, excludedSchemas)
-    if (filter) {
-      sql += ` where schema ${filter}`
-    }
-    if (limit) {
-      sql += ` limit ${limit}`
-    }
-    if (offset) {
-      sql += ` offset ${offset}`
-    }
+    const schemaFilter = filterByList(includedSchemas, excludedSchemas)
+    const sql = generateEnrichedForeignTablesSql({ includeColumns, schemaFilter, limit, offset })
     return await this.query(sql)
   }
 
@@ -69,9 +60,11 @@ export default class PostgresMetaForeignTables {
     schema?: string
   }): Promise<PostgresMetaResult<PostgresForeignTable>> {
     if (id) {
-      const sql = `${generateEnrichedForeignTablesSql({
+      const idsFilter = filterByValue([`${id}`])
+      const sql = generateEnrichedForeignTablesSql({
         includeColumns: true,
-      })} where foreign_tables.id = ${literal(id)};`
+        idsFilter,
+      })
       const { data, error } = await this.query(sql)
       if (error) {
         return { data, error }
@@ -81,11 +74,11 @@ export default class PostgresMetaForeignTables {
         return { data: data[0], error }
       }
     } else if (name) {
-      const sql = `${generateEnrichedForeignTablesSql({
+      const nameFilter = filterByValue([`${schema}.${name}`])
+      const sql = generateEnrichedForeignTablesSql({
         includeColumns: true,
-      })} where foreign_tables.name = ${literal(name)} and foreign_tables.schema = ${literal(
-        schema
-      )};`
+        tableIdentifierFilter: nameFilter,
+      })
       const { data, error } = await this.query(sql)
       if (error) {
         return { data, error }
@@ -103,9 +96,23 @@ export default class PostgresMetaForeignTables {
   }
 }
 
-const generateEnrichedForeignTablesSql = ({ includeColumns }: { includeColumns: boolean }) => `
-with foreign_tables as (${foreignTablesSql})
-  ${includeColumns ? `, columns as (${columnsSql})` : ''}
+const generateEnrichedForeignTablesSql = ({
+  includeColumns,
+  schemaFilter,
+  idsFilter,
+  tableIdentifierFilter,
+  limit,
+  offset,
+}: {
+  includeColumns: boolean
+  schemaFilter?: string
+  idsFilter?: string
+  tableIdentifierFilter?: string
+  limit?: number
+  offset?: number
+}) => `
+with foreign_tables as (${FOREIGN_TABLES_SQL({ schemaFilter, tableIdentifierFilter, limit, offset })})
+  ${includeColumns ? `, columns as (${COLUMNS_SQL({ schemaFilter, tableIdentifierFilter, tableIdFilter: idsFilter })})` : ''}
 select
   *
   ${