show N last log lines in exceptions (impl file_tail())

Author: funbringer

testgres/config.py

@@ -4,19 +4,20 @@
 class TestgresConfig:
     """
     Global config (override default settings).
-    """
 
-    # shall we cache pg_config results?
-    cache_pg_config = True
+    Attributes:
+        cache_initdb:       shall we use cached initdb instance?
+        cache_pg_config:    shall we cache pg_config results?
+        cached_initdb_dir:  shall we create a temp dir for cached initdb?
+        node_cleanup_full:  shall we remove EVERYTHING (including logs)?
+        error_log_lines:    N of log lines to be included into exception (0=inf).
+    """
 
-    # shall we use cached initdb instance?
     cache_initdb = True
-
-    # shall we create a temp dir for cached initdb?
+    cache_pg_config = True
     cached_initdb_dir = None
-
-    # shall we remove EVERYTHING (including logs)?
     node_cleanup_full = True
+    error_log_lines = 20
 
 
 def configure_testgres(**options):

testgres/node.py

@@ -37,6 +37,7 @@
 
 from .utils import \
     get_bin_path, \
+    file_tail as _file_tail, \
     pg_version_ge as _pg_version_ge, \
     reserve_port as _reserve_port, \
     release_port as _release_port, \
@@ -168,12 +169,12 @@ def _maybe_stop_logger(self):
             self._logger.stop()
 
     def _format_verbose_error(self, message=None):
-        # list of important files
+        # list of important files + N of last lines
         files = [
-            os.path.join(self.data_dir, "postgresql.conf"),
-            os.path.join(self.data_dir, "recovery.conf"),
-            os.path.join(self.data_dir, "pg_hba.conf"),
-            self.pg_log_name  # main log file
+            (os.path.join(self.data_dir, "postgresql.conf"), 0),
+            (os.path.join(self.data_dir, "recovery.conf"), 0),
+            (os.path.join(self.data_dir, "pg_hba.conf"), 0),
+            (self.pg_log_name, TestgresConfig.error_log_lines)
         ]
 
         error_text = ""
@@ -183,14 +184,20 @@ def _format_verbose_error(self, message=None):
             error_text += message
             error_text += '\n' * 2
 
-        for f in files:
+        for f, num_lines in files:
             # skip missing files
             if not os.path.exists(f):
                 continue
 
-            # append contents
-            with io.open(f, "r") as _f:
-                lines = _f.read()
+            with io.open(f, "rb") as _f:
+                if num_lines > 0:
+                    # take last N lines of file
+                    lines = b''.join(_file_tail(_f, num_lines)).decode('utf-8')
+                else:
+                    # read whole file
+                    lines = _f.read().decode('utf-8')
+
+                # append contents
                 error_text += u"{}:\n----\n{}\n".format(f, lines)
 
         return error_text

testgres/utils.py

@@ -1,5 +1,7 @@
 # coding: utf-8
 
+from __future__ import division
+
 import io
 import os
 import port_for
@@ -203,3 +205,28 @@ def pg_version_ge(version):
     min_ver = LooseVersion(version)
 
     return cur_ver >= min_ver
+
+
+def file_tail(f, num_lines):
+    """
+    Get last N lines of a file.
+    """
+
+    assert(num_lines > 0)
+
+    bufsize = 8192
+    buffers = 1
+
+    end_pos = f.seek(0, os.SEEK_END)
+
+    while True:
+        offset = max(0, end_pos - bufsize * buffers)
+        pos = f.seek(offset, os.SEEK_SET)
+
+        lines = f.readlines()
+        cur_lines = len(lines)
+
+        if cur_lines > num_lines or pos == 0:
+            return lines[-num_lines:]
+
+        buffers = int(buffers * max(2, num_lines / max(cur_lines, 1)))