Merge branch 'check-dle-for-all-postrges-versions' into 'master'

chore: run DLE tests for all supported PostgreSQL versions

See merge request postgres-ai/database-lab!350

Author: agneum

.gitlab-ci.yml

@@ -22,7 +22,6 @@ lint:
   script:
     - make lint
 
-
 .only_var_template: &only_tag_release
   only:
     variables:
@@ -320,7 +319,8 @@ build-image-swagger-latest:
     - export LATEST_TAG=$(echo ${CLEAN_TAG%.*}-latest)
     - export TAGS="${DOCKER_NAME}:${LATEST_TAG}"
 
-bash-test:
+
+.bash-test: &bash_test
   <<: *only_feature
   stage: integration-test
   variables:
@@ -333,9 +333,40 @@ bash-test:
   tags:
     - dle-test
 
+bash-test-9-6:
+  <<: *bash_test
+  variables:
+    POSTGRES_VERSION: "9.6"
+
+bash-test-10:
+  <<: *bash_test
+  variables:
+    POSTGRES_VERSION: 10
+
+bash-test-11:
+  <<: *bash_test
+  variables:
+    POSTGRES_VERSION: 11
+
+bash-test-12:
+  <<: *bash_test
+  variables:
+    POSTGRES_VERSION: 12
+
+bash-test-13:
+  <<: *bash_test
+  variables:
+    POSTGRES_VERSION: 13
+
+bash-test-14:
+  <<: *bash_test
+  variables:
+    POSTGRES_VERSION: 14
+
 integration-test:
   services:
-    - docker:dind
+    - name: docker:dind
+      command: ["--tls=false"]
   <<: *only_feature
   stage: integration-test
   variables:

Makefile

@@ -59,9 +59,6 @@ test:
 test-ci-integration:
 	GO111MODULE=on go test -race  -tags=integration ./...
 
-test-local-integration:
-	sudo GO111MODULE=on /usr/local/go/bin/go test -race  -tags=integration ./...
-
 fmt:
 	go fmt $$(go list ./... | grep -v /vendor/)
 

configs/standard/postgres/default/14/postgresql.conf renamed to configs/standard/postgres/default/14/postgresql.dblab.postgresql.conf

@@ -11,7 +11,6 @@ import (
 	"context"
 	"fmt"
 	"io"
-	"os"
 	"path"
 	"strings"
 	"sync"
@@ -652,14 +651,18 @@ func (p *PhysicalInitial) getDSAFromWAL(ctx context.Context, pgVersion float64,
 
 	walDirectory := walDir(cloneDir, pgVersion)
 
-	infos, err := os.ReadDir(walDirectory)
+	output, err := tools.ExecCommandWithOutput(ctx, p.dockerClient, containerID, types.ExecConfig{
+		Cmd: []string{"ls", "-t", walDirectory},
+	})
 	if err != nil {
-		return "", errors.Wrap(err, "failed to read the pg_wal dir")
+		return "", errors.Wrap(err, "failed to read the wal directory")
 	}
 
+	walFileList := strings.Fields(output)
+
 	// Walk in the reverse order.
-	for i := len(infos) - 1; i >= 0; i-- {
-		fileName := infos[i].Name()
+	for i := len(walFileList) - 1; i >= 0; i-- {
+		fileName := walFileList[i]
 		walFilePath := path.Join(walDirectory, fileName)
 
 		log.Dbg("Look up into file: ", walFilePath)

pkg/retrieval/engine/postgres/snapshot/physical.go

@@ -11,10 +11,8 @@ import (
 	"fmt"
 	"os"
 	"testing"
-	"time"
 
 	"github.com/docker/docker/client"
-	"github.com/docker/go-connections/nat"
 	_ "github.com/lib/pq"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -83,6 +81,7 @@ const (
 	dbname       = "postgres"
 	user         = "postgres"
 	testPassword = "password"
+	pgdata       = "/var/lib/postgresql/data/"
 )
 
 func TestParsingWAL96(t *testing.T) {
@@ -100,7 +99,7 @@ func TestParsingWAL(t *testing.T) {
 		t.Fatal("Failed to create a Docker client:", err)
 	}
 
-	postgresVersions := []float64{10, 11, 12, 13}
+	postgresVersions := []float64{10, 11, 12, 13, 14}
 
 	for _, pgVersion := range postgresVersions {
 		testWALParsing(t, dockerCLI, pgVersion, initialScript)
@@ -122,26 +121,17 @@ func testWALParsing(t *testing.T, dockerCLI *client.Client, pgVersion float64, i
 	logStrategyForAcceptingConnections := wait.NewLogStrategy("database system is ready to accept connections")
 	logStrategyForAcceptingConnections.Occurrence = 2
 
-	dbURL := func(port nat.Port) string {
-		return fmt.Sprintf("host=%s port=%s user=%s password=%s dbname=%s sslmode=disable",
-			"localhost", port.Port(), user, testPassword, dbname)
-	}
-
 	req := testcontainers.ContainerRequest{
 		Name:         "pg_test_" + pgVersionString,
 		Image:        "postgres:" + pgVersionString,
 		ExposedPorts: []string{port},
 		WaitingFor: wait.ForAll(
 			logStrategyForAcceptingConnections,
 			wait.ForLog("PostgreSQL init process complete; ready for start up."),
-			wait.ForSQL(nat.Port(port), "postgres", dbURL).Timeout(10*time.Second),
 		),
-		BindMounts: map[string]string{
-			dir:    "/var/lib/postgresql/data",
-			"/tmp": "/tmp",
-		},
 		Env: map[string]string{
 			"POSTGRES_PASSWORD": testPassword,
+			"PGDATA":            pgdata,
 		},
 	}
 
@@ -155,7 +145,6 @@ func testWALParsing(t *testing.T, dockerCLI *client.Client, pgVersion float64, i
 
 	// Prepare test data.
 	code, err := postgresContainer.Exec(ctx, []string{"psql", "-U", user, "-d", dbname, "-XAtc", initialSQL})
-
 	require.Nil(t, err)
 	assert.Equal(t, 0, code)
 
@@ -164,7 +153,7 @@ func testWALParsing(t *testing.T, dockerCLI *client.Client, pgVersion float64, i
 	}
 
 	// Check WAL parsing.
-	dsa, err := p.getDSAFromWAL(ctx, pgVersion, postgresContainer.GetContainerID(), dir)
+	dsa, err := p.getDSAFromWAL(ctx, pgVersion, postgresContainer.GetContainerID(), pgdata)
 	assert.Nil(t, err)
 	assert.NotEmpty(t, dsa)
 

pkg/retrieval/engine/postgres/snapshot/physical_integration_test.go

@@ -1,7 +1,7 @@
 #!/bin/bash
 set -euxo pipefail
 
-TAG="${TAG:-"master"}"
+TAG=${TAG:-${CI_COMMIT_REF_SLUG}}
 IMAGE2TEST="registry.gitlab.com/postgres-ai/database-lab/dblab-server:${TAG}"
 POSTGRES_VERSION="${POSTGRES_VERSION:-13}"
 
@@ -25,16 +25,24 @@ sudo docker run \
   postgres:"${POSTGRES_VERSION}"-alpine
 
 for i in {1..300}; do
-  sudo docker exec -it dblab_pg_initdb psql -U postgres -c 'select' > /dev/null 2>&1  && break || echo "test database is not ready yet"
+  sudo docker exec dblab_pg_initdb psql -U postgres -c 'select' > /dev/null 2>&1  && break || echo "test database is not ready yet"
+  sleep 1
+done
+
+# Restart container explicitly after initdb to make sure that the server will not receive a shutdown request and queries will not be interrupted.
+sudo docker restart dblab_pg_initdb
+
+for i in {1..300}; do
+  sudo docker exec dblab_pg_initdb psql -U postgres -c 'select' > /dev/null 2>&1  && break || echo "test database is not ready yet"
   sleep 1
 done
 
 # Create the test database
-sudo docker exec -it dblab_pg_initdb psql -U postgres -c 'create database test'
+sudo docker exec dblab_pg_initdb psql -U postgres -c 'create database test'
 
 # Generate data in the test database using pgbench
 # 1,000,000 accounts, ~0.14 GiB of data.
-sudo docker exec -it dblab_pg_initdb pgbench -U postgres -i -s 10 test
+sudo docker exec dblab_pg_initdb pgbench -U postgres -i -s 10 test
 
 # Stop and remove the container
 sudo docker stop dblab_pg_initdb
@@ -51,11 +59,18 @@ curl https://gitlab.com/postgres-ai/database-lab/-/raw/"${TAG}"/configs/config.e
 
 # Edit the following options
 sed -ri 's/^(\s*)(debug:.*$)/\1debug: true/' "${configDir}/server.yml"
+sed -ri '/^ *telemetry:/,/^ *[^:]*:/s/enabled: true/enabled: false/' "${configDir}/server.yml"
 sed -ri 's/^(\s*)(- logicalDump$)/\1/' "${configDir}/server.yml"
 sed -ri 's/^(\s*)(- logicalRestore$)/\1/' "${configDir}/server.yml"
 # replace postgres version
 sed -ri "s/:13/:${POSTGRES_VERSION}/g"  "${configDir}/server.yml"
 
+
+# logerrors is not supported in PostgreSQL 9.6
+if [ "${POSTGRES_VERSION}" = "9.6" ]; then
+  sed -ri 's/^(\s*)(shared_preload_libraries:.*$)/\1shared_preload_libraries: "pg_stat_statements, auto_explain"/' "${configDir}/server.yml"
+fi
+
 ## Launch Database Lab server
 sudo docker run \
   --name dblab_server \

test/1.synthetic.sh

@@ -60,6 +60,7 @@ curl https://gitlab.com/postgres-ai/database-lab/-/raw/"${TAG}"/configs/config.e
 
 # Edit the following options
 sed -ri "s/^(\s*)(debug:.*$)/\1debug: true/" "${configDir}/server.yml"
+sed -ri '/^ *telemetry:/,/^ *[^:]*:/s/enabled: true/enabled: false/' "${configDir}/server.yml"
 sed -ri "s/^(\s*)(dbname:.*$)/\1dbname: ${SOURCE_DBNAME}/" "${configDir}/server.yml"
 sed -ri "s/^(\s*)(host: 34.56.78.90$)/\1host: ${SOURCE_HOST}/" "${configDir}/server.yml"
 sed -ri "s/^(\s*)(port: 5432$)/\1port: ${SOURCE_PORT}/" "${configDir}/server.yml"

test/2.logical_generic.sh

@@ -38,6 +38,7 @@ curl https://gitlab.com/postgres-ai/database-lab/-/raw/"${TAG}"/configs/config.e
 
 # Edit the following options
 sed -ri "s/^(\s*)(debug:.*$)/\1debug: true/" "${configDir}/server.yml"
+sed -ri '/^ *telemetry:/,/^ *[^:]*:/s/enabled: true/enabled: false/' "${configDir}/server.yml"
 # set WAL-G envs
 sed -ri "s/^(\s*)(backupName:.*$)/\1backupName: ${WALG_BACKUP_NAME}/" "${configDir}/server.yml"
 set +euxo pipefail # ---- do not display secrets

test/3.physical_walg.sh

@@ -64,6 +64,7 @@ curl https://gitlab.com/postgres-ai/database-lab/-/raw/"${TAG}"/configs/config.e
 
 # Edit the following options
 sed -ri "s/^(\s*)(debug:.*$)/\1debug: true/" "${configDir}/server.yml"
+sed -ri '/^ *telemetry:/,/^ *[^:]*:/s/enabled: true/enabled: false/' "${configDir}/server.yml"
 sed -ri "s/^(\s*)(PGUSER:.*$)/\1PGUSER: ${SOURCE_USERNAME}/" "${configDir}/server.yml"
 sed -ri "s/^(\s*)(PGPASSWORD:.*$)/\1PGPASSWORD: ${SOURCE_PASSWORD}/" "${configDir}/server.yml"
 sed -ri "s/^(\s*)(PGHOST:.*$)/\1PGHOST: ${SOURCE_HOST}/" "${configDir}/server.yml"

test/4.physical_basebackup.sh

@@ -33,6 +33,7 @@ curl https://gitlab.com/postgres-ai/database-lab/-/raw/"${TAG}"/configs/config.e
 
 # Edit the following options
 sed -ri "s/^(\s*)(debug:.*$)/\1debug: true/" "${configDir}/server.yml"
+sed -ri '/^ *telemetry:/,/^ *[^:]*:/s/enabled: true/enabled: false/' "${configDir}/server.yml"
 sed -ri "s/^(\s*)(dbname:.*$)/\1dbname: ${SOURCE_DBNAME}/" "${configDir}/server.yml"
 sed -ri "s/^(\s*)(username: test_user.*$)/\1username: \"${SOURCE_USERNAME}\"/" "${configDir}/server.yml"
 sed -ri "s/^(\s*)(awsRegion:.*$)/\1awsRegion: \"${AWS_REGION}\"/" "${configDir}/server.yml"