fix: fixed replication lag on slave

- added query based on recovery control functions instead of timestamp evaluation;
- added new bootstrap optional parameter '-c/--config': bootstrap uses some plugin parameters set by user for functions generating;

Author: cuprumtan

github-actions-tests/tools/bootstrap.sh

@@ -52,9 +52,6 @@ function check_db_objects() {
   echo "MAMONSU_USER = $MAMONSU_USER, MAMONSU_DB = $MAMONSU_DB"
 
   $PSQL $DB -c "\dt mamonsu.config" | grep "mamonsu.*config.*table.*$MAMONSU_USER"  || exit 11
-  MAMONSU_TIMESTAMP_NAME=`echo $MAMONSU_VERSION | sed 's/\./\_/g'`
-  $PSQL $DB -c "\dt mamonsu.timestamp_master_$MAMONSU_TIMESTAMP_NAME" | grep  "mamonsu.*timestamp_master_$MAMONSU_TIMESTAMP_NAME.*table.*$MAMONSU_USER"  || exit 11
-  $PSQL $DB -c "\df mamonsu.timestamp_master_update" | grep  "mamonsu.*timestamp_master_update"  || exit 11
   $PSQL $DB -c "\df mamonsu.timestamp_get" | grep  "mamonsu.*timestamp_get"  || exit 11
   $PSQL $DB -c "\df mamonsu.count_autovacuum" | grep  "mamonsu.*count_autovacuum"  || exit 11
   $PSQL $DB -c "\df mamonsu.get_oldest_xid" | grep  "mamonsu.*get_oldest_xid"  || exit 11

mamonsu/plugins/pgsql/driver/pool.py

@@ -7,12 +7,11 @@ class Pool(object):
 
     SQL = {
         # query type: ( 'if_not_installed', 'if_installed' )
-        'replication_lag_master_query': (
-            'select 1 as replication_lag_master_query',
-            'select mamonsu.timestamp_master_update()'
-        ),
         'replication_lag_slave_query': (
-            'select extract(epoch from now()-pg_last_xact_replay_timestamp())',
+            "SELECT"
+            "    CASE WHEN pg_last_{1}() = pg_last_{2}() THEN 0"
+            "   ELSE extract (epoch FROM now() - coalesce(pg_last_xact_replay_timestamp(), now() - INTERVAL '{0} seconds'))"
+            "   END",
             'select mamonsu.timestamp_get()'
         ),
         'count_xlog_files': (

mamonsu/plugins/pgsql/xlog.py

@@ -54,12 +54,17 @@ class Xlog(Plugin):
     def run(self, zbx):
 
         if Pooler.in_recovery():
-            # replication lag
-            lag = Pooler.run_sql_type('replication_lag_slave_query')
+            if Pooler.server_version_greater('10.0'):
+                lag = Pooler.run_sql_type('replication_lag_slave_query'.format(self.plugin_config('interval'),
+                                                                               'wal_receive_lsn',
+                                                                               'wal_replay_lsn'))
+            else:
+                lag = Pooler.run_sql_type('replication_lag_slave_query'.format(self.plugin_config('interval'),
+                                                                               'xlog_receive_location',
+                                                                               'xlog_replay_location'))
             if lag[0][0] is not None:
                 zbx.send('pgsql.replication_lag[sec]', float(lag[0][0]))
         else:
-            Pooler.run_sql_type('replication_lag_master_query')
             if Pooler.server_version_greater('10.0'):
                 result = Pooler.query(self.query_wal_lsn_diff)
                 result_lags = Pooler.run_sql_type('wal_lag_lsn')

mamonsu/tools/bootstrap/sql.py

@@ -29,31 +29,12 @@
 
 INSERT INTO mamonsu.config(version) VALUES('{0}');
 
-DROP TABLE IF EXISTS mamonsu.timestamp_master_{1};
-
-CREATE TABLE mamonsu.timestamp_master_{1}(
-    id int primary key,
-    ts double precision,
-    lsn pg_lsn
-);
-
-INSERT INTO mamonsu.timestamp_master_{1} (id) values (1);
-
-CREATE OR REPLACE FUNCTION mamonsu.timestamp_master_update()
-RETURNS void AS $$
-  UPDATE mamonsu.timestamp_master_{1} SET
-    ts = extract(epoch from now() at time zone 'utc')::double precision,
-    lsn = pg_catalog.pg_current_{4}()
-  WHERE
-    id = 1;
-$$ LANGUAGE SQL SECURITY DEFINER;
-
 CREATE OR REPLACE FUNCTION mamonsu.timestamp_get()
 RETURNS double precision AS $$
   SELECT
-    (extract(epoch from now() at time zone 'utc') - ts)::double precision
-  FROM mamonsu.timestamp_master_{1}
-  WHERE id = 1 LIMIT 1;
+      CASE WHEN pg_last_{11}() = pg_last_{12}() THEN 0
+      ELSE extract (epoch FROM now() - coalesce(pg_last_xact_replay_timestamp(), now() - INTERVAL '{13} seconds'))
+      END
 $$ LANGUAGE SQL SECURITY DEFINER;
 
 CREATE OR REPLACE FUNCTION mamonsu.count_autovacuum()
@@ -178,10 +159,6 @@
 GrantsOnDefaultSchemaSQL = """
 ALTER TABLE mamonsu.config OWNER TO {1};
 
-ALTER TABLE mamonsu.timestamp_master_{0} OWNER TO {1};
-
-GRANT EXECUTE ON FUNCTION mamonsu.timestamp_master_update() TO {1};
-
 GRANT EXECUTE ON FUNCTION mamonsu.timestamp_get() TO {1};
 
 GRANT EXECUTE ON FUNCTION mamonsu.count_autovacuum() TO {1};

mamonsu/tools/bootstrap/start.py

@@ -4,14 +4,15 @@
 import optparse
 import sys
 
+from mamonsu.lib.config import Config
 import mamonsu.lib.platform as platform
 from mamonsu.lib.parser import MissOptsParser
 from mamonsu.plugins.pgsql.driver.checks import is_conn_to_db
 from mamonsu import __version__ as mamonsu_version
 from mamonsu.lib.default_config import DefaultConfig
 from mamonsu.plugins.pgsql.pool import Pooler
 from mamonsu.tools.bootstrap.sql import CreateMamonsuUserSQL, CreateSchemaExtensionSQL, \
-CreateSchemaDefaultSQL,GrantsOnDefaultSchemaSQL, GrantsOnExtensionSchemaSQL, QuerySplit
+    CreateSchemaDefaultSQL, GrantsOnDefaultSchemaSQL, GrantsOnExtensionSchemaSQL, QuerySplit
 
 
 class Args(DefaultConfig):
@@ -67,6 +68,11 @@ def __init__(self):
             action='store_true',
             dest='create_extensions',
             help='create pg_buffercache extension in mamonsu schema')
+        bootstrap_group.add_option(
+            '-c', '--config',
+            dest='config',
+            default=DefaultConfig.default_config_path(),
+            help='Mamonsu config file')
         parser.add_option_group(group)
         parser.add_option_group(bootstrap_group)
 
@@ -153,8 +159,9 @@ def test_db(self, host_pre):
         return True
 
 
-def fill_query_params(queries):
+def fill_query_params(queries, args):
     formatted_queries = ""
+    cfg = Config(args.args.config)
     for sql in queries.format(
             mamonsu_version,
             mamonsu_version.replace('.', '_'),
@@ -168,7 +175,10 @@ def fill_query_params(queries):
             'flush_lag INTERVAL, replay_lag INTERVAL, write_lag INTERVAL,' if Pooler.server_version_greater('10.0')
             else '',
             'lsn' if Pooler.server_version_greater('10.0') else 'location',
-            'walfile' if Pooler.server_version_greater('10.0') else 'xlogfile'
+            'walfile' if Pooler.server_version_greater('10.0') else 'xlogfile',
+            'wal_receive_lsn' if Pooler.server_version_greater('10.0') else 'xlog_receive_location',
+            'wal_replay_lsn' if Pooler.server_version_greater('10.0') else 'xlog_replay_location',
+            cfg.fetch('xlog', 'interval')
     ).split(QuerySplit):
         formatted_queries += sql
     return formatted_queries
@@ -215,15 +225,15 @@ def run_deploy():
         sys.exit(2)
 
     try:
-        bootstrap_queries = fill_query_params(CreateSchemaDefaultSQL)
+        bootstrap_queries = fill_query_params(CreateSchemaDefaultSQL, args)
         Pooler.query(bootstrap_queries)
     except Exception as e:
         sys.stderr.write("Bootstrap execution have exited with an error: {0}\n".format(e))
         sys.exit(2)
 
     if args.args.create_extensions:
         try:
-            bootstrap_extension_queries = fill_query_params(CreateSchemaExtensionSQL)
+            bootstrap_extension_queries = fill_query_params(CreateSchemaExtensionSQL, args)
             Pooler.query(bootstrap_extension_queries)
         except Exception as e:
             sys.stderr.write(