Miscellaneous Test Suite And Userland Core Support Changes (#324)

Contains the following changes:

1] New Crash Dump Directory Hierarchy

Instead of unarchiving one reference dump at a time and testing it it is
now possible to have multiple unpacked crash dumps at the same time and
run the regression tests against both. Each dump is now automatically
unpacked under `test/integration/data/dumps/<name>`.

It is also possible now to generate regression output for multiple dumps
at the same time.

2] Better support for automatically loading userland shared libs

The `-s`/`load_debug_info()` functionality would only look for `.ko`
files (e.g. kernel modules) given a directory. Now it can also
automatically detect debug links and shared objects.

3] Self-Descriptive Regression Output

All regression output files now contain the following appendix:
```
@#$ EXIT CODE $#@
<exit code number>
```

This makes it easier for us to not have to declare in the test suite
code whether a test case is negative or positive. Simplifying our test
code.

4] Test Infrastructure Refactor & Userland Support

The test suite infrastructure code has been refactored to handle
userland core dumps together with kernel crash dumps. Unfortuantely due
to a bug in drgn we can't use a ztest core dump for regressions (even
though sdb/drgn can handle a userland core dump just fine in the host
system). I'll come up with a smaller userland core dump soon so we can
test the userland version of stacks and work on the drgn bug when my
free time allows.

Author: sdimitro

.github/scripts/clear-dump.sh

@@ -7,17 +7,7 @@ if [ ! -d $DATA_DIR ]; then
 	exit 1
 fi
 
-echo "removing current crash dump if any ..."
-rm -f $DATA_DIR/dump.*
-
-echo "removing any extracted vmlinux ..."
-rm -f $DATA_DIR/vmlinux*
-
-echo "removing any extracted modules ..."
-rm -rf $DATA_DIR/mods
-rm -rf $DATA_DIR/usr
-
-echo "removing any savedump scripts ..."
-rm -rf $DATA_DIR/run-*.sh
+echo "removing all crash/core dumps ..."
+rm -rf $DATA_DIR/dumps
 
 echo "Done"

.github/scripts/download-dump-from-s3.sh

@@ -41,31 +41,33 @@ else
 	[ $? -eq 0 ] || exit 1
 fi
 
-if [[ $1 == *.lzma ]]; then
+if [[ $1 == *.tar.lzma ]]; then
 	# Profile A
+	dump_name=${1%.tar.lzma}
 
 	echo "decompressing dump ..."
 	tar -x --lzma -f $1
 
-	echo "moving contents to tests/integration/data ..."
-	mv dump-data/* $DATA_DIR
+	echo "moving contents to tests/integration/data/dumps/${dump_name} ..."
+	mkdir -p $DATA_DIR/dumps/${dump_name}
+	mv dump-data/* $DATA_DIR/dumps/${dump_name}
 	[ $? -eq 0 ] || exit 1
 
 	rmdir dump-data
 	[ $? -eq 0 ] || exit 1
 elif [[ $1 == *.tar.gz ]]; then
 	# Profile B
+	dump_name=${1%.tar.gz}
 
 	echo "decompressing dump ..."
 	tar xzf $1
 
-	decompressed_dir=${1%.tar.gz}
-
-	echo "moving contents to tests/integration/data ..."
-	mv *$decompressed_dir/* $DATA_DIR
+	echo "moving contents to tests/integration/data/dumps/${dump_name} ..."
+	mkdir -p $DATA_DIR/dumps/${dump_name}
+	mv *${dump_name}/* $DATA_DIR/dumps/${dump_name}
 	[ $? -eq 0 ] || exit 1
 
-	rmdir *$decompressed_dir
+	rmdir *${dump_name}
 	[ $? -eq 0 ] || exit 1
 else
 	echo "unknown dump profile"

.gitignore

@@ -114,3 +114,6 @@ tests/integration/data/mods/
 # zipped folders - usually crash dumps
 *.lzma
 *.tar.gz
+
+# crash dump directory
+tests/integration/data/dumps/

README.md

@@ -20,6 +20,7 @@
 
 import argparse
 import os
+import re
 import sys
 
 from typing import List
@@ -126,7 +127,8 @@ def parse_arguments() -> argparse.Namespace:
     return args
 
 
-def load_debug_info(prog: drgn.Program, dpaths: List[str]) -> None:
+def load_debug_info(prog: drgn.Program, dpaths: List[str], quiet: bool,
+                    no_filter: bool) -> None:
     """
     Iterates over all the paths provided (`dpaths`) and attempts
     to load any debug information it finds. If the path provided
@@ -140,9 +142,28 @@ def load_debug_info(prog: drgn.Program, dpaths: List[str]) -> None:
             kos = []
             for (ppath, __, files) in os.walk(path):
                 for i in files:
-                    if i.endswith(".ko"):
+                    if i.endswith(".ko") or i.endswith(".debug") or re.match(
+                            r".+\.so(\.\d)?", i) or no_filter:
+                        # matches:
+                        #     kernel modules - .ko suffix
+                        #     userland debug files - .debug suffix
+                        #     userland shared objects - .so suffix
                         kos.append(os.sep.join([ppath, i]))
-            prog.load_debug_info(kos)
+            try:
+                prog.load_debug_info(kos)
+            except drgn.MissingDebugInfoError as debug_info_err:
+                #
+                # If we encounter such an error it means that we can't
+                # find the debug info for one or more kernel modules.
+                # That's fine because the user may not need those, so
+                # print a warning and proceed.
+                #
+                # Again because of the aforementioned short-coming of drgn
+                # we quiet any errors when loading the *default debug info*
+                # if we are looking at a crash/core dump.
+                #
+                if not quiet:
+                    print("sdb: " + str(debug_info_err), file=sys.stderr)
         else:
             print("sdb: " + path + " is not a regular file or directory")
 
@@ -191,7 +212,7 @@ def setup_target(args: argparse.Namespace) -> drgn.Program:
 
     if args.symbol_search:
         try:
-            load_debug_info(prog, args.symbol_search)
+            load_debug_info(prog, args.symbol_search, args.quiet, False)
         except (
                 drgn.MissingDebugInfoError,
                 OSError,

sdb/internal/cli.py

@@ -1 +1,3 @@
 sdb: addr: symbol not found: bogus
+@#$ EXIT CODE $#@
+1

tests/integration/data/regression_output/dump.201912060006/core/addr bogus

@@ -1 +1,3 @@
 sdb: deref: cannot dereference function pointer
+@#$ EXIT CODE $#@
+1

tests/integration/data/regression_output/dump.201912060006/core/addr fget | deref

@@ -1 +1,3 @@
 (volatile unsigned long)4294968498
+@#$ EXIT CODE $#@
+0

tests/integration/data/regression_output/dump.201912060006/core/addr jiffies | deref

@@ -1 +1,3 @@
 sdb: deref: 'volatile unsigned long' is not a valid pointer type
+@#$ EXIT CODE $#@
+1

tests/integration/data/regression_output/dump.201912060006/core/addr jiffies | deref | deref

@@ -1 +1,3 @@
 (avl_tree_t *)spa_namespace_avl+0x0 = 0xffffffffc07d0fe0
+@#$ EXIT CODE $#@
+0