Fix CI

Author: mkgrgis

.github/workflows/CI.yml

@@ -29,6 +29,7 @@ jobs:
       POSTGIS_VERSION : "3.4.2"
       HTTP_PROXY: ""
       HTTPS_PROXY: ""
+      SQLITE_FOR_TESTING_DIR: "/opt/CI_sqlite"
     strategy:
       fail-fast: false
       matrix:
@@ -46,32 +47,38 @@ jobs:
       - name: set_proxy
         run: bash GitHubActions/env.sh
 
+      - name: download PostGIS, ${{ matrix.config }} mode
+        run: |
+          if [[ "${{ matrix.config }}" == "postgis" ]]; then
+            bash GitHubActions/download_postgis.sh ${{ env.POSTGIS_VERSION }}
+          fi
+
       - name: install locales
         run: bash GitHubActions/install_locales.sh
 
-      - name: build PostgreSQL ${{ matrix.pg }} with ${{ matrix.config }}
+      - name: build PostgreSQL ${{ matrix.pg }}
         run: bash GitHubActions/build_postgres.sh ${{ matrix.pg }}
 
-      - name: build PostGIS ${{ env.POSTGIS_VERSION }} for PostgreSQL ${{ matrix.pg }}, ${{ matrix.config }} mode
-        run: |
-          if [[ "${{ matrix.config }}" == "postgis" ]]; then
-            bash GitHubActions/build_postgis.sh ${{ matrix.pg }} ${{ env.POSTGIS_VERSION }}
-          fi
-
       - name: install SQLite, ${{ matrix.config }} mode
         run: |
           if [[ "${{ matrix.config }}" == "default" ]]; then
-            bash GitHubActions/install_sqlite.sh ${{ env.SQLITE_VERSION }} ${{ env.SQLITE_YEAR }} "${{ matrix.config }}"
+            bash GitHubActions/install_sqlite.sh ${{ env.SQLITE_VERSION }} ${{ env.SQLITE_YEAR }} ${{ matrix.config }} ${{ env.SQLITE_FOR_TESTING_DIR }}
           elif [[ "${{ matrix.config }}" == "postgis" ]]; then
-            bash GitHubActions/install_sqlite.sh ${{ env.SQLITE_VERSION }} ${{ env.SQLITE_YEAR }} "${{ matrix.config }}" --enable-rtree
+            bash GitHubActions/install_sqlite.sh ${{ env.SQLITE_VERSION }} ${{ env.SQLITE_YEAR }} ${{ matrix.config }} ${{ env.SQLITE_FOR_TESTING_DIR }} --enable-rtree
+          fi
+
+      - name: build PostGIS ${{ env.POSTGIS_VERSION }} for PostgreSQL ${{ matrix.pg }}, ${{ matrix.config }} mode
+        run: |
+          if [[ "${{ matrix.config }}" == "postgis" ]]; then
+            bash GitHubActions/build_postgis.sh ${{ matrix.pg }} ${{ env.POSTGIS_VERSION }}
           fi
 
       - name: build sqlite_fdw, ${{ matrix.config }} mode
         run: |
-          bash GitHubActions/build_sqlite_fdw.sh ${{ matrix.pg }} ${{ matrix.config }}
+          bash GitHubActions/build_sqlite_fdw.sh ${{ matrix.pg }} ${{ matrix.config }} ${{ env.SQLITE_FOR_TESTING_DIR }}
 
       - name: execute sqlite_fdw test
-        run: bash GitHubActions/execute_test.sh ${{ matrix.pg }} ${{ matrix.config }}
+        run: bash GitHubActions/execute_test.sh ${{ matrix.pg }} ${{ matrix.config }} ${{ env.SQLITE_FOR_TESTING_DIR }}
 
       - name: download output files (regression.diffs)
         if: failure()
@@ -83,3 +90,4 @@ jobs:
             workdir/postgresql-${{ matrix.pg }}/contrib/sqlite_fdw/regression.out
             workdir/postgresql-${{ matrix.pg }}/contrib/sqlite_fdw/results
           retention-days: 7
+

GitHubActions/build_postgis.sh

@@ -2,7 +2,7 @@
 
 ################################################################################
 #
-# This script downloads PostgreSQL from the official web site into ./workdir
+# This script downloads PostGIS from the official web site into ./workdir
 # then builds it.
 #
 # Usage: ./build_postgis.sh pg_version postgis_version
@@ -17,23 +17,18 @@
 POSTGRESQL_VERSION=$1
 POSTGIS_VERSION=$2
 
-cd ./workdir
-cd postgresql-${POSTGRESQL_VERSION}
-
 # Install necessary dependencies
 sudo apt update
 sudo apt install -y build-essential libxml2-dev libgeos-dev libproj-dev libgdal-dev libjson-c-dev libprotobuf-c-dev protobuf-c-compiler
 
-GEOS_CONFIG_PATH=$(which geos-config)
-
+cd ./workdir
 # Download and compile PostGIS
-cd contrib
-wget http://download.osgeo.org/postgis/source/postgis-${POSTGIS_VERSION}.tar.gz
-tar -xzf postgis-${POSTGIS_VERSION}.tar.gz
-mv postgis-${POSTGIS_VERSION} postgis -v
-cd postgis
+cp -vr postgis postgresql-${POSTGRESQL_VERSION}/contrib
+cd postgresql-${POSTGRESQL_VERSION}/contrib/postgis
 echo " - PostGIS directory"
+GEOS_CONFIG_PATH=$(which geos-config)
 export LD_LIBRARY_PATH=/usr/local/lib/:$LD_LIBRARY_PATH
 ./configure --with-pgconfig=/usr/local/pgsql/bin/pg_config --with-geosconfig=$GEOS_CONFIG_PATH
 make
 sudo make install
+

GitHubActions/build_sqlite_fdw.sh

@@ -4,9 +4,10 @@
 #
 # This script builds sqlite_fdw in PostgreSQL source tree.
 #
-# Usage: ./build_sqlite_fdw.sh pg_version mode
+# Usage: ./build_sqlite_fdw.sh pg_version mode sqlite_for_testing_dir
 #     pg_version is a PostgreSQL version like 17.0 to be built in.
 #     mode is flag for sqlite_fdw compiler.
+#     sqlite_for_testing_dir: path to install directory of SQLite version for testing
 #
 # Requirements
 # - the source code of sqlite_fdw is available by git clone.
@@ -18,14 +19,16 @@ VERSION="$1"
 MODE="$2"
 
 mkdir -p ./workdir/postgresql-${VERSION}/contrib/sqlite_fdw
-tar zxvf ./sqlite_fdw.tar.gz -C ./workdir/postgresql-${VERSION}/contrib/sqlite_fdw/
+tar zxf ./sqlite_fdw.tar.gz -C ./workdir/postgresql-${VERSION}/contrib/sqlite_fdw/
 cd ./workdir/postgresql-${VERSION}/contrib/sqlite_fdw
-export LD_LIBRARY_PATH=/usr/local/lib/:$LD_LIBRARY_PATH
+
+# show locally compiled sqlite library
+ls -la /usr/local/lib
 
 if [ "$MODE" == "postgis" ]; then
-  make ENABLE_GIS=1
+  make ENABLE_GIS=1 SQLITE_FOR_TESTING_DIR="$3"
 else
-  make
+  make SQLITE_FOR_TESTING_DIR="$3"
 fi
 
 sudo make install

GitHubActions/download_postgis.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+
+################################################################################
+#
+# This script downloads PostGIS from the official web site into ./workdir
+# then builds it.
+#
+# Usage: ./download_postgis.sh postgis_version
+#     postgis_version is a PostGIS version to be installed.
+#
+# Requirements
+# - be able to connect to the PostGIS official web site by wget.
+#
+################################################################################
+
+POSTGIS_VERSION=$1
+
+mkdir -p ./workdir
+cd ./workdir
+pgisfile="postgis-${POSTGIS_VERSION}.tar.gz"
+if [ ! -f "$pgisfile" ]; then
+  wget -nv "http://download.osgeo.org/postgis/source/$pgisfile"
+  tar -xzf "$pgisfile"
+  mv postgis-${POSTGIS_VERSION} postgis -v
+  echo "PostGIS source code directory " $(dirname $(readlink -f postgis))
+else
+  echo "PostGIS downloaded"
+fi

GitHubActions/execute_test.sh

@@ -6,9 +6,10 @@
 # sqlite_fdw. If all tests are passed, this script will exit successfully.
 # Otherwise, it will exit with failure.
 
-# Usage: ./execute_test.sh pg_version mode
+# Usage: ./execute_test.sh pg_version mode sqlite_for_testing_dir
 #     pg_version is a PostgreSQL version to be tested like 17.0.
 #	  mode is flag for sqlite_fdw compiler.
+#     sqlite_for_testing_dir: path to install directory of SQLite version for testing
 #
 # Requiremets
 # - the source code of PostgreSQL is located in ./workdir/postgresql-{pg_version}.
@@ -22,6 +23,7 @@
 
 VERSION=$1
 MODE="$2"
+SQLITE_FOR_TESTING_DIR="$3"
 
 cd ./workdir/postgresql-${VERSION}/contrib/sqlite_fdw
 

GitHubActions/install_locales.sh

@@ -17,5 +17,6 @@ sudo apt-get install locales language-pack-ja
 sudo locale-gen ja_JP.EUC-JP
 sudo apt-get install language-pack-ko-base language-pack-ko
 sudo locale-gen ko_KR.EUC-KR
-sudo apt -get install language-pack-bg-base language-pack-bg
+sudo apt-get install language-pack-bg-base language-pack-bg
 sudo locale-gen bg_BG
+sudo apt-get install libreadline8 libreadline-dev

GitHubActions/install_sqlite.sh

@@ -5,13 +5,14 @@
 # This sript downloads SQLite source code from the official web site into
 # ./workdir then builds and installs it.
 #
-# Usage: ./install_sqlite.sh version year testing_mode [configure_options]
+# Usage: ./install_sqlite.sh version year testing_mode sqlite_for_testing_dir [configure_options]
 #     version: SQLite version to be installed.
 #     year: A year of SQLite released. It is used for determining a download URL.
 #     testing_mode:	'default' or 'postgis' value.
+#     sqlite_for_testing_dir: path to install directory of the specified SQLite version
 #     configure_options: are a list of option for sqlite server.
 #
-#     Ex) ./install_sqlite.sh 3420000 2023 postgis --enable-rtree
+#     Ex) ./install_sqlite.sh 3420000 2023 postgis /opt/sqlite_for_testing --enable-rtree
 #
 # Requirements
 # - be able to connect to the SQLite official web site by curl.
@@ -22,27 +23,36 @@
 VERSION="$1"
 YEAR="$2"
 TESTING_MODE="$3"
+SQLITE_FOR_TESTING_DIR="$4"
 
 CONFIGURE_OPTIONS=""
 
 while (( "$#" )); do
-  CONFIGURE_OPTIONS="$CONFIGURE_OPTIONS $4"
+  CONFIGURE_OPTIONS="$CONFIGURE_OPTIONS $5"
   shift
 done
 
+echo "SQLite ver $VERSION ($YEAR), mode $TESTING_MODE, install to $SQLITE_FOR_TESTING_DIR with options $CONFIGURE_OPTIONS";
+
 mkdir -p ./workdir
 cd ./workdir
-curl -O https://www.sqlite.org/${YEAR}/sqlite-src-${VERSION}.zip
-unzip sqlite-src-${VERSION}.zip > /dev/null
-cd sqlite-src-${VERSION}
-
-if [ -z "$CONFIGURE_OPTIONS" ]; then
-  ./configure --enable-fts5
-else
-  ./configure --enable-fts5 $CONFIGURE_OPTIONS
+vsrc="sqlite-src-${VERSION}"
+adr="https://www.sqlite.org/${YEAR}/$vsrc.zip"
+echo "SQLite source code archive: $adr"
+wget "$adr" -O "$vsrc.zip"
+unzip "$vsrc.zip" > /dev/null
+cd "$vsrc"
+
+export CFLAGS=-DSQLITE_ENABLE_COLUMN_METADATA
+confcom="./configure --enable-fts5 --prefix=$SQLITE_FOR_TESTING_DIR"
+if [ ! -z "$CONFIGURE_OPTIONS" ]; then
+  confcom+="$CONFIGURE_OPTIONS"  
 fi
+echo "SQLite configure call: $confcom"
+$confcom
 
 make
+echo "----- SQLITE INSTALL directory $SQLITE_FOR_TESTING_DIR -----"
 sudo make install
 
 if [ "$TESTING_MODE" == "postgis" ]; then

Makefile

@@ -33,10 +33,10 @@ GIS_DEP_TESTS = $(GIS_DEP_TESTS_DIR)/type $(GIS_DEP_TESTS_DIR)/auto_import $(GIS
 
 ifndef REGRESS
 # System tests, full default sequence
-REGRESS = extra/sqlite_fdw_post $(DATA_TYPE_TESTS) extra/join extra/limit extra/aggregates extra/prepare extra/select_having extra/select extra/insert extra/update extra/encodings sqlite_fdw aggregate selectfunc $(GIS_DEP_TESTS)
+REGRESS = libsqlite extra/sqlite_fdw_post $(DATA_TYPE_TESTS) extra/join extra/limit extra/aggregates extra/prepare extra/select_having extra/select extra/insert extra/update extra/encodings sqlite_fdw aggregate selectfunc $(GIS_DEP_TESTS)
 endif
 
-# Other encodings also are tested. Client encoding should be UTF-8.
+# Other encodings also are tested. Client encoding should be UTF-8-
 REGRESS_OPTS = --encoding=utf8
 
 UNAME = uname
@@ -47,7 +47,12 @@ else
 DLSUFFIX = .so
 endif
 
+ifdef SQLITE_FOR_TESTING_DIR
+SHLIB_LINK := -L$(SQLITE_FOR_TESTING_DIR)/lib -lsqlite3
+PG_CFLAGS += -I$(SQLITE_FOR_TESTING_DIR)/include -Wl,-rpath,$(SQLITE_FOR_TESTING_DIR)/lib
+else
 SHLIB_LINK := -lsqlite3
+endif
 
 ifdef ENABLE_GIS
 override SHLIB_LINK += -lspatialite
@@ -81,9 +86,17 @@ $(shell mkdir -p results/$(REGRESS_PREFIX_SUB)/extra)
 $(shell mkdir -p results/$(REGRESS_PREFIX_SUB)/types)
 $(shell mkdir -p results/$(REGRESS_PREFIX_SUB)/$(GIS_DEP_TESTS_DIR))
 
+# $(info    ENABLE_GIS      is $(ENABLE_GIS))
+ $(info    SHLIB_LINK      is $(SHLIB_LINK))
+# $(info    LD_LIBRARY_PATH is $(LD_LIBRARY_PATH))
+ $(info    PG_CFLAGS       is $(PG_CFLAGS))
+# $(info    PG_CPPFLAGS     is $(PG_CPPFLAGS))
+# $(info    REGRESS         is $(REGRESS))
+# $(info    DLSUFFIX        is $(DLSUFFIX))
 
 ifdef ENABLE_GIS
 check: temp-install
 temp-install: EXTRA_INSTALL+=contrib/postgis
 checkprep: EXTRA_INSTALL+=contrib/postgis
 endif
+

README.md

@@ -45,15 +45,15 @@ Features
 - Support mixed SQLite [data affinity](https://www.sqlite.org/datatype3.html) input and filtering (`SELECT`/`WHERE` usage) for such data types as
 	- `timestamp`: `text` and `int`,
 	- `uuid`: `text`(32..39) and `blob`(16),
- 	- `bool`: `text`(1..5) and `int`,
- 	- `double precision`, `float` and `numeric`: `real` values and special values with `text` affinity (`+Infinity`, `-Infinity`, `NaN`),
- 	- `macaddr`: `text`(12..17) or `blob`(6) or `integer`,
- 	- `macaddr8`: `text`(16..23) or `blob`(8) or `integer`.
+	- `bool`: `text`(1..5) and `int`,
+	- `double precision`, `float` and `numeric`: `real` values and special values with `text` affinity (`+Infinity`, `-Infinity`, `NaN`),
+	- `macaddr`: `text`(12..17) or `blob`(6) or `integer`,
+	- `macaddr8`: `text`(16..23) or `blob`(8) or `integer`.
 - Support mixed SQLite [data affinity](https://www.sqlite.org/datatype3.html) output (`INSERT`/`UPDATE`) for such data types as
 	- `timestamp`: `text`(default) or `int`,
- 	- `uuid`: `text`(36) or `blob`(16)(default),
- 	- `macaddr`: `text`(17) or `blob`(6) or `integer`(default),
- 	- `macaddr8`: `text`(23) or `blob`(8) or `integer`(default).
+	- `uuid`: `text`(36) or `blob`(16)(default),
+	- `macaddr`: `text`(17) or `blob`(6) or `integer`(default),
+	- `macaddr8`: `text`(23) or `blob`(8) or `integer`(default).
 - Full support for `+Infinity` (means ∞) and `-Infinity` (means -∞) special values for IEEE 754-2008 numbers in `double precision`, `float` and `numeric` columns including such conditions as ` n < '+Infinity'` or ` m > '-Infinity'`.
 - Bidirectional data transformation for `geometry` and `geography` data types for SpatiaLite ↔ PostGIS. [EWKB](https://libgeos.org/specifications/wkb/#extended-wkb) data transport is used. See [GIS support description](GIS.md).
 
@@ -500,7 +500,7 @@ Once for a foreign datasource you need, as PostgreSQL superuser. Please specify
 	CREATE SERVER sqlite_server
 	FOREIGN DATA WRAPPER sqlite_fdw
 	OPTIONS (
-          database '/path/to/database'
+			  database '/path/to/database'
 	);
 ```
 

expected/13.15/libsqlite.out

@@ -0,0 +1,26 @@
+-- Test for SQLite library code source and defaults
+--Testcase 1:
+CREATE EXTENSION sqlite_fdw;
+--Testcase 2:
+SELECT sqlite_fdw_sqlite_version();
+ sqlite_fdw_sqlite_version 
+---------------------------
+                   3046000
+(1 row)
+
+--Testcase 3:
+SELECT length(sqlite_fdw_sqlite_code_source());
+ length 
+--------
+     84
+(1 row)
+
+--Testcase 4:
+SELECT sqlite_fdw_sqlite_code_source();
+                            sqlite_fdw_sqlite_code_source                             
+--------------------------------------------------------------------------------------
+ 2024-05-23 13:25:27 96c92aba00c8375bc32fafcdf12429c58bd8aabfcadab6683e35bbb9cdebf19e
+(1 row)
+
+--Testcase 7:
+DROP EXTENSION sqlite_fdw CASCADE;