Merge pull request #3266 from vkarak/feat/add-custom-session-metadata

[feat] Support session annotations with a new `--session-extras` option

Author: vkarak

docs/manpage.rst

@@ -1129,6 +1129,15 @@ Miscellaneous options
    .. versionadded:: 3.9.3
 
 
+.. option:: --session-extras KV_DATA
+
+   Annotate the current session with custom key/value metadata.
+
+   The key/value data is specified as a comma-separated list of `key=value` pairs.
+   When listing stored sessions with the :option:`--list-stored-sessions` option, any associated custom metadata will be presented by default.
+
+   .. versionadded:: 4.7
+
 .. option:: --system=NAME
 
    Load the configuration for system ``NAME``.

reframe/frontend/cli.py

@@ -616,6 +616,10 @@ def main():
         help=('Print a report for performance tests '
               '(default: "now:now/last:+job_nodelist/+result")')
     )
+    reporting_options.add_argument(
+        '--session-extras', action='store', metavar='KV_DATA',
+        help='Annotate session with custom key/value data'
+    )
 
     # Miscellaneous options
     misc_options.add_argument(
@@ -1590,6 +1594,15 @@ def module_unuse(*paths):
             if options.restore_session is not None:
                 report.update_restored_cases(restored_cases, restored_session)
 
+            if options.session_extras:
+                # Update report's extras
+                extras = {}
+                for arg in options.session_extras.split(','):
+                    k, v = arg.split('=', maxsplit=1)
+                    extras[k] = v
+
+                report.update_extras(extras)
+
             # Print a retry report if we did any retries
             if options.max_retries and runner.stats.failed(run=0):
                 printer.retry_report(report)

reframe/frontend/printer.py

@@ -283,7 +283,14 @@ def table(self, data, **kwargs):
             colidx = [i for i, col in enumerate(data[0])
                       if col not in hide_columns]
 
-            tab_data = [[rec[col] for col in colidx] for rec in data]
+            def _access(seq, i, default=None):
+                # Safe access of i-th element of a sequence
+                try:
+                    return seq[i]
+                except IndexError:
+                    return default
+
+            tab_data = [[_access(rec, col) for col in colidx] for rec in data]
         else:
             tab_data = data
 

reframe/frontend/reporting/__init__.py

@@ -25,7 +25,7 @@
 from reframe.core.logging import getlogger, _format_time_rfc3339, time_function
 from reframe.core.runtime import runtime
 from reframe.core.warnings import suppress_deprecations
-from reframe.utility import nodelist_abbrev
+from reframe.utility import nodelist_abbrev, OrderedSet
 from .storage import StorageBackend
 from .utility import Aggregator, parse_cmp_spec, parse_time_period, is_uuid
 
@@ -269,6 +269,14 @@ def update_timestamps(self, ts_start, ts_end):
             'time_elapsed': ts_end - ts_start
         })
 
+    def update_extras(self, extras):
+        '''Attach user-specific metadata to the session'''
+
+        # We prepend a special character to the user extras in order to avoid
+        # possible conflicts with existing keys
+        for k, v in extras.items():
+            self.__report['session_info'][f'${k}'] = v
+
     def update_run_stats(self, stats):
         session_uuid = self.__report['session_info']['uuid']
         for runidx, tasks in stats.runs():
@@ -645,17 +653,38 @@ def session_data(time_period):
     '''Retrieve all sessions'''
 
     data = [['UUID', 'Start time', 'End time', 'Num runs', 'Num cases']]
+    extra_cols = OrderedSet()
     for sess_data in StorageBackend.default().fetch_sessions_time_period(
         *parse_time_period(time_period) if time_period else (None, None)
     ):
         session_info = sess_data['session_info']
-        data.append(
-            [session_info['uuid'],
-             session_info['time_start'],
-             session_info['time_end'],
-             len(sess_data['runs']),
-             len(sess_data['runs'][0]['testcases'])]
-        )
+        record = [session_info['uuid'],
+                  session_info['time_start'],
+                  session_info['time_end'],
+                  len(sess_data['runs']),
+                  len(sess_data['runs'][0]['testcases'])]
+
+        # Expand output with any user metadata
+        for k in session_info:
+            if k.startswith('$'):
+                extra_cols.add(k[1:])
+
+        # Add any extras recorded so far
+        for key in extra_cols:
+            record.append(session_info.get(f'${key}', ''))
+
+        data.append(record)
+
+    # Do a final grooming pass of the data to expand short records
+    if extra_cols:
+        data[0] += extra_cols
+
+    for rec in data:
+        diff = len(extra_cols) - len(rec)
+        if diff == 0:
+            break
+
+        rec += ['n/a' for _ in range(diff)]
 
     return data
 

unittests/test_cli.py

@@ -1262,13 +1262,14 @@ def table_format(request):
     return request.param
 
 
-def test_storage_options(run_reframe, tmp_path, table_format):
-    def assert_no_crash(returncode, stdout, stderr, exitcode=0):
-        assert returncode == exitcode
-        assert 'Traceback' not in stdout
-        assert 'Traceback' not in stderr
-        return returncode, stdout, stderr
+def assert_no_crash(returncode, stdout, stderr, exitcode=0):
+    assert returncode == exitcode
+    assert 'Traceback' not in stdout
+    assert 'Traceback' not in stderr
+    return returncode, stdout, stderr
+
 
+def test_storage_options(run_reframe, tmp_path, table_format):
     run_reframe2 = functools.partial(
         run_reframe,
         checkpath=['unittests/resources/checks/frontend_checks.py'],
@@ -1328,6 +1329,19 @@ def assert_no_crash(returncode, stdout, stderr, exitcode=0):
     assert_no_crash(*run_reframe2(action=f'--delete-stored-session={uuid}'))
 
 
+def test_session_annotations(run_reframe):
+    assert_no_crash(*run_reframe(
+        checkpath=['unittests/resources/checks/frontend_checks.py'],
+        action='-r',
+        more_options=['--session-extras', 'key1=val1,key2=val2',
+                      '-n', '^PerformanceFailureCheck']
+    ), exitcode=1)
+
+    stdout = assert_no_crash(*run_reframe(action='--list-stored-sessions'))[1]
+    for text in ['key1', 'key2', 'val1', 'val2']:
+        assert text in stdout
+
+
 def test_performance_compare(run_reframe, table_format):
     def assert_no_crash(returncode, stdout, stderr, exitcode=0):
         assert returncode == exitcode