[RFC] GDB sanity test

[Keith Seitz]

This RFC is meant to start a discussion about testing. As bunsen
starts to mature and more people rely on it, it is imperative that
contributions don't randomly break other projects.

In that vein, this is a basic sanity test that I've written for GDB
which imports a single gdb.{log,sum} from a test fixture (.tar.xz).
It double-checks all results and test counts.

To do this, I've written a custom gdb test script that outputs every
valid test outcome and run this on a fedora 33 VM.

I'm only including the actual test file, not the glue code
(tests/core.py) necessary to make it work.

Does this look like an acceptable approach?

Keith
---
 tests/bunsen/test_init.py                     |  11 +++
 tests/core.py                                 |  43 +++++++++
 .../test_gdb_sanity/gdb-sanity.tar.xz         | Bin 0 -> 2120 bytes
 .../fixtures/test_gdb_sanity/test_outcomes.c  |  24 +++++
 .../test_gdb_sanity/test_outcomes.exp         |  50 +++++++++++
 tests/gdb/test_gdb_sanity.py                  |  82 ++++++++++++++++++
 6 files changed, 210 insertions(+)
 create mode 100644 tests/bunsen/test_init.py
 create mode 100644 tests/core.py
 create mode 100644 tests/gdb/fixtures/test_gdb_sanity/gdb-sanity.tar.xz
 create mode 100644 tests/gdb/fixtures/test_gdb_sanity/test_outcomes.c
 create mode 100644 tests/gdb/fixtures/test_gdb_sanity/test_outcomes.exp
 create mode 100644 tests/gdb/test_gdb_sanity.py

diff --git a/tests/gdb/test_gdb_sanity.py b/tests/gdb/test_gdb_sanity.py
new file mode 100644
index 0000000..1ce65be
--- /dev/null
+++ b/tests/gdb/test_gdb_sanity.py
@@ -0,0 +1,82 @@
+# A quick sanity test that the ensures that bunsen can import a really
+# basic log/sum test run which contains all test outcomes.
+
+import os.path
+from tests.core import setup_bunsen_repo, import_results
+
+TEST_FIXTURE = 'tests/gdb/fixtures/test_gdb_sanity/gdb-sanity.tar.xz'
+PATH_STRING = '/home/fedora33/work/gdb/fsf/virgin/linux/gdb/testsuite'
+
+def assert_test_name(test, subtest):
+        assert test['name'] == 'gdb.gdb/test_outcomes.exp'
+        assert test['subtest'] == subtest
+
+def print_test(test):
+    print(f'{test["outcome"]}: {test["name"]}: {test["subtest"]}')
+
+def test_gdb_sanity(tmp_path):
+    # Initialize a new bunsen instance.
+    instance = setup_bunsen_repo(tmp_path)
+    assert instance is not None
+
+    # Make sure the fixture for this test exists.
+    fixture = os.path.join(os.getcwd(), TEST_FIXTURE)
+    assert os.path.exists(fixture)
+
+    # Import the results.
+    commits = import_results(fixture, instance)
+    assert commits is not None
+    assert len(commits) == 1
+
+    # Print out the commit info.
+    canonical, commit = commits[0]
+    print(f'successfully imported test fixture, commit {commit}')
+    print(f'(canonically {canonical}')
+
+    # Get all test results and info strings.
+    testrun = instance.testrun(commit)
+    all_tests = testrun.testcases
+    info = testrun.get_info_strings()
+
+    # Verify metadata.
+    assert info["branch"] == 'f33'
+    assert info["architecture"] == 'x86_64'
+    assert info['target_board'] == 'native-gdbserver/-m32'
+    assert testrun.source_commit == '1234567890123456789012345678901234567890'
+    assert info['version'] == '13.0.50.20220412-git'
+
+    # Verify the tests.  First the easy ones with one result each.
+    for outcome in {'FAIL', 'XPASS', 'XFAIL', 'KPASS', 'KFAIL',
+                    'UNTESTED', 'UNRESOLVED', 'UNSUPPORTED'}:
+        test = [t for t in all_tests if t['outcome'] == outcome]
+        assert len(test) == 1
+        test = test[0]
+        print_test(test)
+        assert_test_name(test, 'expect ' + outcome)
+
+    # PATH should also have one result.
+    test = [t for t in all_tests if t['outcome'] == 'PATH']
+    assert len(test) == 1
+    print_test(test[0])
+    #assert_test_name(test[0], PATH_STRING)
+
+    # PASS should have 3 results.  One of these will be a duplicate, one will
+    # contain only a path name (which elicits GDB's 'PATH' outcome).
+    passes = [t for t in all_tests if t['outcome'] == 'PASS']
+    assert len(passes) == 3
+
+    # Verify the PASS result resulting in a PATH outcome.
+    path = [t for t in passes if t['subtest'].startswith('expect PATH')]
+    assert len(path) == 1
+    path = path[0]
+    print_test(path)
+    assert_test_name(path, 'expect PATH ' + PATH_STRING)
+
+    # Now verify the remaining two PASS results, which have duplicate "subtest"
+    # names.  The parser will append " <<2>>" for the duplicate to keep subtest
+    # names unique.
+    passes.remove(path)
+    print_test(passes[0])
+    assert_test_name(passes[0], 'expect PASS')
+    print_test(passes[1])
+    assert_test_name(passes[1], 'expect PASS <<2>>')
\ No newline at end of file
-- 
2.35.1