Merge pull request #259 from compnerd/lit
tests: convert the test suite to use more of the LLVM test utilities
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 58f5b79..8c539d0 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -334,6 +334,20 @@
if (IS_DIRECTORY "${PROJECT_SOURCE_DIR}/tests")
enable_testing()
+ find_package(Python COMPONENTS Interpreter)
+ if(Python_FOUND)
+ set(BLOATY_SRC_DIR ${PROJECT_SOURCE_DIR})
+ set(BLOATY_OBJ_DIR ${PROJECT_BINARY_DIR})
+ configure_file(tests/lit.site.cfg.in tests/lit.site.cfg @ONLY)
+
+ # TODO(abdulras) polish this further and convert this to `add_test` so that
+ # the default test suite run executes the tests.
+ add_custom_target(check-bloaty
+ COMMAND ${Python_EXECUTABLE} ${LIT_EXECUTABLE} -sv ${PROJECT_BINARY_DIR}/tests
+ COMMENT "Running bloaty tests..."
+ USES_TERMINAL)
+ endif()
+
if(BUILD_TESTING)
option(INSTALL_GTEST "" OFF)
add_subdirectory(third_party/googletest)
diff --git a/README.md b/README.md
index 096d7a8..fa5f66b 100644
--- a/README.md
+++ b/README.md
@@ -76,6 +76,32 @@
$ make -j6
```
+### Running the new extended test suite
+
+> NOTE: these tests are still experimental and not covered under CI yet. Please
+> report any issues in running them.
+
+When configuring the project via CMake, a few additional parameters must be
+specified currently:
+- `-DLIT_EXECUTABLE=<PATH>`: specifies where to find the lit tool
+- `-DFILECHECK_EXECUTABLE=<PATH>`: specifies where to find the FileCheck tool
+- `-DYAML2OBJ_EXECUTABLE=<PATH>`: specifies where to find the yaml2obj tool
+
+You can install lit via pip:
+```sh
+pip install --user lit
+```
+
+The FileCheck utility and yaml2obj currently need to be provided by the user.
+These are part of the LLVM toolchain and require a very recent build
+(development release from the main branch) to run the tests.
+
+```sh
+cmake -B build -G Ninja -S . -DLIT_EXECUTABLE=${HOME}/Library/Python/3.8/bin/lit -DFILECHECK_EXECUTABLE=${HOME}/BinaryCache/llvm.org/bin/FileCheck -DYAML2OBJ_EXECUTABLE=${HOME}/BinaryCache/llvm.org/bin/yaml2obj
+cmake --build build --config Debug
+cmake --build build --target check-bloaty
+```
+
## Running Bloaty
Run it directly on a binary target. For example, run it on itself.
diff --git a/tests/dwarf/range_lists/base-addr-selection.test b/tests/dwarf/range_lists/base-addr-selection.test
index 8efc530..ad3ea82 100644
--- a/tests/dwarf/range_lists/base-addr-selection.test
+++ b/tests/dwarf/range_lists/base-addr-selection.test
@@ -3,6 +3,10 @@
# a special kind of entry where the end address signifies the new base
# address.
+# RUN: %yaml2obj %s --docnum=1 -o %t.obj
+# RUN: %yaml2obj %s --docnum=2 -o %t.dwo
+# RUN: %bloaty %t.obj --debug-file %t.dwo -d compileunits --raw-map --domain=vm | %FileCheck %s
+
--- !ELF
FileHeader:
Class: ELFCLASS64
@@ -86,12 +90,8 @@
- AbbrCode: 0x0
...
-$ bloaty 1 --debug-file=2 -d compileunits --raw-map --domain=vm
-Maps for 1:
-
-VM MAP:
-0000-1000 4096 [-- Nothing mapped --]
-1000-1042 66 [section .text]
-1042-1047 5 test.c
-1047-1151 266 [section .text]
-
+# CHECK: VM MAP:
+# CHECK: 0000-1000 4096 [-- Nothing mapped --]
+# CHECK: 1000-1042 66 [section .text]
+# CHECK: 1042-1047 5 test.c
+# CHECK: 1047-1151 266 [section .text]
diff --git a/tests/elf/sections/empty-bin-64.test b/tests/elf/sections/empty-bin-64.test
index 9df7eb3..2873b4e 100644
--- a/tests/elf/sections/empty-bin-64.test
+++ b/tests/elf/sections/empty-bin-64.test
@@ -1,3 +1,6 @@
+# RUN: %yaml2obj %s -o %t.obj
+# RUN: %bloaty --raw-map %t.obj | %FileCheck %s
+
--- !ELF
FileHeader:
Class: ELFCLASS64
@@ -47,24 +50,20 @@
Section: .comment
...
-$ bloaty --raw-map 1
-Maps for 1:
+# CHECK: FILE MAP:
+# CHECK: 000-040 64 [ELF Headers]
+# CHECK: 040-070 48 .comment
+# CHECK: 070-118 168 .symtab
+# CHECK: 118-148 48 .strtab
+# CHECK: 148-190 72 .shstrtab
+# CHECK: 190-1d0 64 [ELF Headers]
+# CHECK: 1d0-210 64 .text
+# CHECK: 210-250 64 .data
+# CHECK: 250-290 64 .bss
+# CHECK: 290-2d0 64 .comment
+# CHECK: 2d0-310 64 .note.GNU-stack
+# CHECK: 310-350 64 .symtab
+# CHECK: 350-390 64 .strtab
+# CHECK: 390-3d0 64 .shstrtab
-FILE MAP:
-000-040 64 [ELF Headers]
-040-070 48 .comment
-070-118 168 .symtab
-118-148 48 .strtab
-148-190 72 .shstrtab
-190-1d0 64 [ELF Headers]
-1d0-210 64 .text
-210-250 64 .data
-250-290 64 .bss
-290-2d0 64 .comment
-2d0-310 64 .note.GNU-stack
-310-350 64 .symtab
-350-390 64 .strtab
-390-3d0 64 .shstrtab
-
-VM MAP:
-
+# CHECK: VM MAP:
diff --git a/tests/elf/sections/empty-obj-64.test b/tests/elf/sections/empty-obj-64.test
index fa94d1e..1645417 100644
--- a/tests/elf/sections/empty-obj-64.test
+++ b/tests/elf/sections/empty-obj-64.test
@@ -1,3 +1,6 @@
+# RUN: %yaml2obj %s -o %t.obj
+# RUN: %bloaty --raw-map %t.obj | %FileCheck %s
+
--- !ELF
FileHeader:
Class: ELFCLASS64
@@ -47,24 +50,20 @@
Section: .comment
...
-$ bloaty --raw-map 1
-Maps for 1:
+# CHECK: FILE MAP:
+# CHECK: 000-040 64 [ELF Headers]
+# CHECK: 040-070 48 .comment
+# CHECK: 070-118 168 .symtab
+# CHECK: 118-148 48 .strtab
+# CHECK: 148-190 72 .shstrtab
+# CHECK: 190-1d0 64 [ELF Headers]
+# CHECK: 1d0-210 64 .text
+# CHECK: 210-250 64 .data
+# CHECK: 250-290 64 .bss
+# CHECK: 290-2d0 64 .comment
+# CHECK: 2d0-310 64 .note.GNU-stack
+# CHECK: 310-350 64 .symtab
+# CHECK: 350-390 64 .strtab
+# CHECK: 390-3d0 64 .shstrtab
-FILE MAP:
-000-040 64 [ELF Headers]
-040-070 48 .comment
-070-118 168 .symtab
-118-148 48 .strtab
-148-190 72 .shstrtab
-190-1d0 64 [ELF Headers]
-1d0-210 64 .text
-210-250 64 .data
-250-290 64 .bss
-290-2d0 64 .comment
-2d0-310 64 .note.GNU-stack
-310-350 64 .symtab
-350-390 64 .strtab
-390-3d0 64 .shstrtab
-
-VM MAP:
-
+# CHECK: VM MAP:
diff --git a/tests/elf/sections/musl-static-bin.test b/tests/elf/sections/musl-static-bin.test
index 06c46ce..9a3c38f 100644
--- a/tests/elf/sections/musl-static-bin.test
+++ b/tests/elf/sections/musl-static-bin.test
@@ -18,6 +18,9 @@
# This binary does not successfully run due to bugs in obj2yaml
# (or possibly yaml2obj). The binary did not fully round-trip successfully.
+# RUN: %yaml2obj %s -o %t.obj
+# RUN: %bloaty --raw-map %t.obj | %FileCheck %s
+
--- !ELF
FileHeader:
Class: ELFCLASS64
@@ -530,62 +533,58 @@
Size: 0x0000000000000008
...
-$ bloaty --raw-map 1
-Maps for 1:
+# CHECK: FILE MAP:
+# CHECK: 0000-0040 64 [ELF Headers]
+# CHECK: 0040-0078 56 [LOAD #0 [R]]
+# CHECK: 0078-00b0 56 [LOAD #1 [RX]]
+# CHECK: 00b0-00e8 56 [LOAD #2 [R]]
+# CHECK: 00e8-0120 56 [LOAD #3 [RW]]
+# CHECK: 0120-0158 56 [LOAD #4 [RW]]
+# CHECK: 0158-0190 56 [LOAD #5 [R]]
+# CHECK: 0190-01a0 16 .init
+# CHECK: 01a0-09f9 2137 .text
+# CHECK: 09f9-0a00 7 .fini
+# CHECK: 0a00-0d20 800 .rodata
+# CHECK: 0d20-0d50 48 .eh_frame
+# CHECK: 0d50-0d58 8 .init_array
+# CHECK: 0d58-0d60 8 .fini_array
+# CHECK: 0d60-0d68 8 .got
+# CHECK: 0d68-0d80 24 .got.plt
+# CHECK: 0d80-0f78 504 .data
+# CHECK: 0f78-0f80 8 .data
+# CHECK: 0f80-0fd8 88 .comment
+# CHECK: 0fd8-1710 1848 .symtab
+# CHECK: 1710-1a43 819 .strtab
+# CHECK: 1a43-1ac0 125 .shstrtab
+# CHECK: 1ac0-1b00 64 [ELF Headers]
+# CHECK: 1b00-1b40 64 .init
+# CHECK: 1b40-1b80 64 .text
+# CHECK: 1b80-1bc0 64 .fini
+# CHECK: 1bc0-1c00 64 .rodata
+# CHECK: 1c00-1c40 64 .eh_frame
+# CHECK: 1c40-1c80 64 .init_array
+# CHECK: 1c80-1cc0 64 .fini_array
+# CHECK: 1cc0-1d00 64 .got
+# CHECK: 1d00-1d40 64 .got.plt
+# CHECK: 1d40-1d80 64 .data
+# CHECK: 1d80-1dc0 64 .bss
+# CHECK: 1dc0-1e00 64 .comment
+# CHECK: 1e00-1e40 64 .symtab
+# CHECK: 1e40-1e80 64 .strtab
+# CHECK: 1e80-1ec0 64 .shstrtab
-FILE MAP:
-0000-0040 64 [ELF Headers]
-0040-0078 56 [LOAD #0 [R]]
-0078-00b0 56 [LOAD #1 [RX]]
-00b0-00e8 56 [LOAD #2 [R]]
-00e8-0120 56 [LOAD #3 [RW]]
-0120-0158 56 [LOAD #4 [RW]]
-0158-0190 56 [LOAD #5 [R]]
-0190-01a0 16 .init
-01a0-09f9 2137 .text
-09f9-0a00 7 .fini
-0a00-0d20 800 .rodata
-0d20-0d50 48 .eh_frame
-0d50-0d58 8 .init_array
-0d58-0d60 8 .fini_array
-0d60-0d68 8 .got
-0d68-0d80 24 .got.plt
-0d80-0f78 504 .data
-0f78-0f80 8 .data
-0f80-0fd8 88 .comment
-0fd8-1710 1848 .symtab
-1710-1a43 819 .strtab
-1a43-1ac0 125 .shstrtab
-1ac0-1b00 64 [ELF Headers]
-1b00-1b40 64 .init
-1b40-1b80 64 .text
-1b80-1bc0 64 .fini
-1bc0-1c00 64 .rodata
-1c00-1c40 64 .eh_frame
-1c40-1c80 64 .init_array
-1c80-1cc0 64 .fini_array
-1cc0-1d00 64 .got
-1d00-1d40 64 .got.plt
-1d40-1d80 64 .data
-1d80-1dc0 64 .bss
-1dc0-1e00 64 .comment
-1e00-1e40 64 .symtab
-1e40-1e80 64 .strtab
-1e80-1ec0 64 .shstrtab
-
-VM MAP:
-000000-401000 4198400 [-- Nothing mapped --]
-401000-401010 16 .init
-401010-401869 2137 .text
-401869-40186c 3 .fini
-40186c-402000 1940 [-- Nothing mapped --]
-402000-402320 800 .rodata
-402320-402350 48 .eh_frame
-402350-403fe8 7320 [-- Nothing mapped --]
-403fe8-403ff0 8 .init_array
-403ff0-403ff8 8 .fini_array
-403ff8-404000 8 .got
-404000-404020 32 .got.plt
-404020-404220 512 .data
-404220-4044f8 728 .bss
-
+# CHECK: VM MAP:
+# CHECK: 000000-401000 4198400 [-- Nothing mapped --]
+# CHECK: 401000-401010 16 .init
+# CHECK: 401010-401869 2137 .text
+# CHECK: 401869-40186c 3 .fini
+# CHECK: 40186c-402000 1940 [-- Nothing mapped --]
+# CHECK: 402000-402320 800 .rodata
+# CHECK: 402320-402350 48 .eh_frame
+# CHECK: 402350-403fe8 7320 [-- Nothing mapped --]
+# CHECK: 403fe8-403ff0 8 .init_array
+# CHECK: 403ff0-403ff8 8 .fini_array
+# CHECK: 403ff8-404000 8 .got
+# CHECK: 404000-404020 32 .got.plt
+# CHECK: 404020-404220 512 .data
+# CHECK: 404220-4044f8 728 .bss
diff --git a/tests/elf/sections/normal-obj.test b/tests/elf/sections/normal-obj.test
index 1473507..f481b1a 100644
--- a/tests/elf/sections/normal-obj.test
+++ b/tests/elf/sections/normal-obj.test
@@ -1,3 +1,6 @@
+# RUN: %yaml2obj %s -o %t.obj
+# RUN: %bloaty --raw-map %t.obj | %FileCheck %s
+
--- !ELF
FileHeader:
Class: ELFCLASS64
@@ -83,43 +86,39 @@
Size: 0x0000000000000315
...
-$ bloaty --raw-map 1
-Maps for 1:
+# CHECK: FILE MAP:
+# CHECK: 000-040 64 [ELF Headers]
+# CHECK: 040-050 16 .text
+# CHECK: 050-220 464 .data
+# CHECK: 220-535 789 .rodata
+# CHECK: 535-558 35 .comment
+# CHECK: 558-590 56 .eh_frame
+# CHECK: 590-5a8 24 .rela.eh_frame
+# CHECK: 5a8-650 168 .symtab
+# CHECK: 650-66b 27 .strtab
+# CHECK: 66b-6d8 109 .shstrtab
+# CHECK: 6d8-718 64 [ELF Headers]
+# CHECK: 718-758 64 .text
+# CHECK: 758-798 64 .bss
+# CHECK: 798-7d8 64 .data
+# CHECK: 7d8-818 64 .rodata
+# CHECK: 818-858 64 .comment
+# CHECK: 858-898 64 .note.GNU-stack
+# CHECK: 898-8d8 64 .eh_frame
+# CHECK: 8d8-918 64 .rela.eh_frame
+# CHECK: 918-958 64 .llvm_addrsig
+# CHECK: 958-998 64 .symtab
+# CHECK: 998-9d8 64 .strtab
+# CHECK: 9d8-a18 64 .shstrtab
-FILE MAP:
-000-040 64 [ELF Headers]
-040-050 16 .text
-050-220 464 .data
-220-535 789 .rodata
-535-558 35 .comment
-558-590 56 .eh_frame
-590-5a8 24 .rela.eh_frame
-5a8-650 168 .symtab
-650-66b 27 .strtab
-66b-6d8 109 .shstrtab
-6d8-718 64 [ELF Headers]
-718-758 64 .text
-758-798 64 .bss
-798-7d8 64 .data
-7d8-818 64 .rodata
-818-858 64 .comment
-858-898 64 .note.GNU-stack
-898-8d8 64 .eh_frame
-8d8-918 64 .rela.eh_frame
-918-958 64 .llvm_addrsig
-958-998 64 .symtab
-998-9d8 64 .strtab
-9d8-a18 64 .shstrtab
-
-VM MAP:
-00000000000-10000000000 1099511627776 [-- Nothing mapped --]
-10000000000-1000000000b 11 .text
-1000000000b-20000000000 1099511627765 [-- Nothing mapped --]
-20000000000-2000000007b 123 .bss
-2000000007b-30000000000 1099511627653 [-- Nothing mapped --]
-30000000000-300000001c8 456 .data
-300000001c8-40000000000 1099511627320 [-- Nothing mapped --]
-40000000000-40000000315 789 .rodata
-40000000315-70000000000 3298534882539 [-- Nothing mapped --]
-70000000000-70000000038 56 .eh_frame
-
+# CHECK: VM MAP:
+# CHECK: 00000000000-10000000000 1099511627776 [-- Nothing mapped --]
+# CHECK: 10000000000-1000000000b 11 .text
+# CHECK: 1000000000b-20000000000 1099511627765 [-- Nothing mapped --]
+# CHECK: 20000000000-2000000007b 123 .bss
+# CHECK: 2000000007b-30000000000 1099511627653 [-- Nothing mapped --]
+# CHECK: 30000000000-300000001c8 456 .data
+# CHECK: 300000001c8-40000000000 1099511627320 [-- Nothing mapped --]
+# CHECK: 40000000000-40000000315 789 .rodata
+# CHECK: 40000000315-70000000000 3298534882539 [-- Nothing mapped --]
+# CHECK: 70000000000-70000000038 56 .eh_frame
diff --git a/tests/elf/sections/shn-xindex.test b/tests/elf/sections/shn-xindex.test
index ed068f3..396f33d 100644
--- a/tests/elf/sections/shn-xindex.test
+++ b/tests/elf/sections/shn-xindex.test
@@ -7,6 +7,9 @@
# These special cases are documented here:
# https://docs.oracle.com/cd/E19683-01/817-3677/chapter6-94076/index.html
+# RUN: %yaml2obj %s -o %t.obj
+# RUN: %bloaty --raw-map %t.obj | %FileCheck %s
+
--- !ELF
FileHeader:
Class: ELFCLASS64
@@ -21,16 +24,12 @@
Size: 0x3
...
-$ bloaty --raw-map 1
-Maps for 1:
+# CHECK: FILE MAP:
+# CHECK: 000-040 64 [ELF Headers]
+# CHECK: 040-041 1 .strtab
+# CHECK: 041-058 23 .shstrtab
+# CHECK: 058-098 64 [ELF Headers]
+# CHECK: 098-0d8 64 .strtab
+# CHECK: 0d8-118 64 .shstrtab
-FILE MAP:
-000-040 64 [ELF Headers]
-040-041 1 .strtab
-041-058 23 .shstrtab
-058-098 64 [ELF Headers]
-098-0d8 64 .strtab
-0d8-118 64 .shstrtab
-
-VM MAP:
-
+# CHECK: VM MAP:
diff --git a/tests/lit.cfg b/tests/lit.cfg
new file mode 100644
index 0000000..7d0e68e
--- /dev/null
+++ b/tests/lit.cfg
@@ -0,0 +1,46 @@
+# -*- Python -*-
+
+import os
+import platform
+import subprocess
+
+import lit
+import lit.formats
+import lit.util
+
+### Helper Functions
+
+def make_path(*args):
+ return os.path.normpath(os.path.join(*args))
+
+kIsWindows = platform.system() == 'Windows'
+
+###
+
+config.name = 'bloaty'
+
+bloaty_src_root = getattr(config, 'bloaty_src_root', None)
+if not bloaty_src_root:
+ lit_config.fatal("missing 'bloaty_src_root' key in the site specific config")
+bloaty_src_root = os.path.normpath(bloaty_src_root)
+
+bloaty_obj_root = getattr(config, 'bloaty_obj_root', None)
+if not bloaty_obj_root:
+ lit_config.fatal("missing 'bloaty_obj_root' key in the site specific config")
+bloaty_obj_root = os.path.normpath(bloaty_obj_root)
+
+use_lit_shell = os.environ.get('LIT_USE_INTERNAL_SHELL', kIsWindows)
+if not use_lit_shell:
+ config.available_features.add('shell')
+
+config.test_format = lit.formats.ShTest(execute_external = False)
+
+config.suffixes = ['.test']
+config.excludes = ['testdata']
+
+config.test_source_root = os.path.join(bloaty_src_root, 'tests')
+config.test_exec_root = os.path.join(bloaty_obj_root, 'tests')
+
+config.substitutions.append(('%FileCheck', config.filecheck_path))
+config.substitutions.append(('%yaml2obj', config.yaml2obj_path))
+config.substitutions.append(('%bloaty', make_path(config.bloaty_obj_root, 'bloaty')))
diff --git a/tests/lit.site.cfg.in b/tests/lit.site.cfg.in
new file mode 100644
index 0000000..6459ad1
--- /dev/null
+++ b/tests/lit.site.cfg.in
@@ -0,0 +1,13 @@
+# -*- Python -*-
+
+import os
+
+config.bloaty_src_root = "@BLOATY_SRC_DIR@"
+config.bloaty_obj_root = "@BLOATY_OBJ_DIR@"
+
+config.filecheck_path = "@FILECHECK_EXECUTABLE@"
+config.yaml2obj_path = "@YAML2OBJ_EXECUTABLE@"
+
+if not config.test_exec_root:
+ config.test_exec_root = os.path.dirname(os.path.realpath(__file__))
+lit_config.load_config(config, "@BLOATY_SRC_DIR@/tests/lit.cfg")
diff --git a/tests/tester.py b/tests/tester.py
deleted file mode 100755
index 556d579..0000000
--- a/tests/tester.py
+++ /dev/null
@@ -1,72 +0,0 @@
-#!/usr/bin/python3
-
-import os
-import glob
-import re
-import sys
-import subprocess
-import shutil
-import tempfile
-
-successes = 0
-failures = []
-
-def TestFile(filename):
- cwd = os.getcwd()
- abspath = os.path.abspath(filename)
- global successes
- global failures
- with open(abspath) as f:
- print(filename)
- tmpdir = tempfile.mkdtemp(prefix="/tmp/bloaty-")
- contents = f.read()
- file_count = len(re.findall(r'^---', contents, flags=re.MULTILINE))
- for i in range(1, file_count + 1):
- subprocess.check_call('yaml2obj {0} --docnum={1} > {1}'.format(abspath, str(i)), shell=True, cwd=tmpdir)
- lines = contents.splitlines()
- while len(lines[0]) == 0 or lines[0][0] != '$':
- lines.pop(0)
- bloaty_invocation = os.path.join(cwd, lines[0][2:]) + ' > actual'
- lines.pop(0)
- expected_output = "\n".join(lines) + "\n"
- failure = None
- if subprocess.call(bloaty_invocation, shell=True, cwd=tmpdir) != 0:
- failure = "CRASHED"
- else:
- with open(os.path.join(tmpdir, 'expected'), 'w') as expected:
- expected.write(expected_output)
- if subprocess.call('diff -u expected actual', shell=True, cwd=tmpdir) != 0:
- failure = "FAILED"
-
- if failure:
- print("{}: {}".format(failure, filename))
- print("{}: output in {}".format(failure, tmpdir))
- failures.append((filename, tmpdir))
- else:
- successes += 1
- shutil.rmtree(tmpdir)
-
-def TestArg(arg):
- if os.path.isdir(arg):
- files = sorted(glob.glob(os.path.join(arg, "**/*.test"), recursive=True))
- for file in files:
- TestFile(file)
- else:
- TestFile(arg)
-
-args = sys.argv[1:]
-
-if not args:
- print("Usage: tester.py <FILE-OR-DIR> ...")
- sys.exit(1)
-
-for arg in sys.argv[1:]:
- TestArg(arg)
-
-if not failures:
- print("SUCCESS: {} test(s) passed".format(successes))
-else:
- print("\nFAILURE: {} tests passed, {} tests failed:".format(successes, len(failures)))
- for failure in failures:
- print(" - {} (output in {})".format(*failure))
- sys.exit(1)
