Merge branch 'v1.12.x' into develop

Author: pskrbasu

Makefile

@@ -7,10 +7,10 @@ PLATFORM=$(shell uname)
 GETTEXT_INCLUDE=$(shell dirname $(shell dirname $(shell readlink -f $(shell which gettext))))/include
 
 install: build
-	if test -d ~/.steampipe/db/14.2.0; then \
-		cp ./build-$(PLATFORM)/steampipe_postgres_fdw--1.0.sql $(STEAMPIPE_INSTALL_DIR)/db/14.2.0/postgres/share/postgresql/extension/; \
-		cp ./build-$(PLATFORM)/steampipe_postgres_fdw.control $(STEAMPIPE_INSTALL_DIR)/db/14.2.0/postgres/share/postgresql/extension/; \
-		cp ./build-$(PLATFORM)/steampipe_postgres_fdw.so $(STEAMPIPE_INSTALL_DIR)/db/14.2.0/postgres/lib/postgresql/; \
+	if test -d ~/.steampipe/db/14.17.0; then \
+		cp ./build-$(PLATFORM)/steampipe_postgres_fdw--1.0.sql $(STEAMPIPE_INSTALL_DIR)/db/14.17.0/postgres/share/postgresql/extension/; \
+		cp ./build-$(PLATFORM)/steampipe_postgres_fdw.control $(STEAMPIPE_INSTALL_DIR)/db/14.17.0/postgres/share/postgresql/extension/; \
+		cp ./build-$(PLATFORM)/steampipe_postgres_fdw.so $(STEAMPIPE_INSTALL_DIR)/db/14.17.0/postgres/lib/postgresql/; \
 	fi
 
 # build standalone 

fdw.go

@@ -18,6 +18,9 @@ import (
 	"fmt"
 	"io"
 	"log"
+	"net"
+	"net/http"
+	"os"
 	"time"
 	"unsafe"
 
@@ -71,6 +74,18 @@ func init() {
 	log.Printf("[INFO] Version:   v%s\n", version.FdwVersion.String())
 	log.Printf("[INFO] Log level: %s\n", level)
 
+	if _, found := os.LookupEnv("STEAMPIPE_FDW_PPROF"); found {
+		log.Printf("[INFO] PROFILING!!!!")
+		go func() {
+			listener, err := net.Listen("tcp", "localhost:0")
+			if err != nil {
+				log.Println(err)
+				return
+			}
+			log.Printf("[INFO] Check http://localhost:%d/debug/pprof/", listener.Addr().(*net.TCPAddr).Port)
+			log.Println(http.Serve(listener, nil))
+		}()
+	}
 }
 
 // Given a list of FdwDeparsedSortGroup and a FdwPlanState,

go.mod

@@ -57,64 +57,99 @@ func keyColumnArrayContainsColumn(keyColumns []*proto.KeyColumn, c string) bool
 }
 
 // keyColumnsToColumnPath returns a list of all the column sets to use in path keys
-func keyColumnsToColumnPath(keyColumns []*proto.KeyColumn) (columnPaths [][]string, baseCost Cost) {
+func keyColumnsToColumnPath(keyColumns []*proto.KeyColumn) ([][]string, Cost) {
+	var columnPaths [][]string
+	var baseCost Cost
+
 	if len(keyColumns) == 0 {
-		return
+		return columnPaths, baseCost
 	}
 
-	// collect required and optional columns - we build a single path for all of them
-	var requiredKeys, optionalKeys []string
-	// an array of paths for any of keys - each path will have a single element (the any-of key)
-	var anyOfKeys [][]string
+	// collect required and optional and any of keys
+	var requiredKeys, optionalKeys, anyOfKeys []string
 	for _, c := range keyColumns {
 		if c.Require == plugin.Required {
 			requiredKeys = append(requiredKeys, c.Name)
 		} else if c.Require == plugin.Optional {
 			optionalKeys = append(optionalKeys, c.Name)
 		} else if c.Require == plugin.AnyOf {
-			anyOfKeys = append(anyOfKeys, []string{c.Name})
+			anyOfKeys = append(anyOfKeys, c.Name)
 		}
 	}
 
-	if len(requiredKeys) > 0 {
-		// add required keys as a single path
-		columnPaths = append(columnPaths, requiredKeys)
-	}
+	var requiredColumnPaths [][]string
+	// if we have any-of keys, for each any-of key we will create a separate path containing that key and all required keys
 	if len(anyOfKeys) > 0 {
-		// add a separate path for _each_ any-of key
-		columnPaths = append(columnPaths, anyOfKeys...)
+		for _, a := range anyOfKeys {
+			columnPath := append([]string{a}, requiredKeys...)
+			requiredColumnPaths = append(requiredColumnPaths, columnPath)
+		}
+	} else if len(requiredKeys) > 0 {
+		// add required keys as a single path
+		requiredColumnPaths = append(requiredColumnPaths, requiredKeys)
 	}
+
 	// if we have any column paths (meaning we have aither required or any-of columns),
 	// we have required keys so make the base cost CHEAP
-	if len(columnPaths) > 0 {
+	if len(requiredColumnPaths) > 0 {
 		baseCost = requiredKeyColumnBaseCost
 	} else {
 		baseCost = optionalKeyColumnBaseCost
 	}
-	// if there are optional keys, add:
-	//	- a path with required keys and each optional key
-	//	- a path with each any-of key and each optional key
+
+	// create output column paths - init to the required column paths - we will add permutations with optional keys
+	columnPaths = requiredColumnPaths
+	// now create additional paths based on the required paths, with each of the optional keys
 	for _, optional := range optionalKeys {
+		col := optional
 		// NOTE: append onto optional, NOT requiredKeys - otherwise we end up reusing the underlying array
 		// and mutating values in columnPaths
 
 		// if there are any required keys, build path from optional AND required
-		if len(requiredKeys) > 0 {
-			p := append([]string{optional}, requiredKeys...)
-			columnPaths = append(columnPaths, p)
-		}
-		// if there are any anyOf keys, build path from optional AND required
-		for _, a := range anyOfKeys {
-			p := append([]string{optional}, a...)
-			columnPaths = append(columnPaths, p)
+		if len(requiredColumnPaths) > 0 {
+			for _, requiredPath := range requiredColumnPaths {
+				p := append(append([]string{}, requiredPath...), col)
+				columnPaths = append(columnPaths, p)
+			}
+		} else {
+			// if there are no required keys or anyof keys, just create a path from the optional
+			columnPaths = append(columnPaths, []string{col})
 		}
-		// if there are no required keys or anyof keys, just create a path from the optional
-		if baseCost == optionalKeyColumnBaseCost {
-			columnPaths = append(columnPaths, []string{optional})
+	}
+
+	return columnPaths, baseCost
+}
+
+func columnPathsToPathKeys(columnPaths [][]string, allColumns []string, baseCost Cost) []PathKey {
+
+	var res []PathKey
+
+	// generate path keys each column set
+	for _, s := range columnPaths {
+		// create a path for just the column path
+		res = append(res, PathKey{
+			ColumnNames: s,
+			// make this cheap so the planner prefers to give us the qual
+			Rows: baseCost * keyColumnOnlyCostMultiplier,
+		})
+		// also create paths for the columns path WITH each other column
+		for _, c := range allColumns {
+			if !slices.Contains(s, c) {
+				// NOTE: create a new slice rather than appending onto s - to avoid clash between loop iterations
+				columnNames := append([]string{c}, s...)
+
+				res = append(res, PathKey{
+					ColumnNames: columnNames,
+					// make this even cheaper - prefer to include all quals which were provided
+					Rows: baseCost,
+				})
+			}
 		}
 	}
 
-	return
+	log.Printf("[TRACE] columnPathsToPathKeys %d column paths %d all columns, %d pathkeys", len(columnPaths), len(allColumns), len(res))
+
+	return res
 }
 
 func LegacyKeyColumnsToPathKeys(requiredColumns, optionalColumns *proto.KeyColumnsSet, allColumns []string) []PathKey {
@@ -155,34 +190,3 @@ func LegacyKeyColumnsToColumnPaths(k *proto.KeyColumnsSet) [][]string {
 	}
 	return res
 }
-func columnPathsToPathKeys(columnPaths [][]string, allColumns []string, baseCost Cost) []PathKey {
-
-	var res []PathKey
-
-	// generate path keys each column set
-	for _, s := range columnPaths {
-		// create a path for just the column path
-		res = append(res, PathKey{
-			ColumnNames: s,
-			// make this cheap so the planner prefers to give us the qual
-			Rows: baseCost * keyColumnOnlyCostMultiplier,
-		})
-		// also create paths for the columns path WITH each other column
-		for _, c := range allColumns {
-			if !slices.Contains(s, c) {
-				// NOTE: create a new slice rather than appending onto s - to avoid clash between loop iterations
-				columnNames := append([]string{c}, s...)
-
-				res = append(res, PathKey{
-					ColumnNames: columnNames,
-					// make this even cheaper - prefer to include all quals which were provided
-					Rows: baseCost,
-				})
-			}
-		}
-	}
-
-	log.Printf("[TRACE] columnPathsToPathKeys %d column paths %d all columns, %d pathkeys", len(columnPaths), len(allColumns), len(res))
-
-	return res
-}