Symbolize backtraces
This change also removes a sleep statement that was a workaround for an
issue that has been fixed.
Change-Id: Id0f8b67c6d7a20d7a58fbf941d5b68e23c4df43f
diff --git a/recipes/fuchsia.expected/backtrace.json b/recipes/fuchsia.expected/backtrace.json
new file mode 100644
index 0000000..93d2ff8
--- /dev/null
+++ b/recipes/fuchsia.expected/backtrace.json
@@ -0,0 +1,406 @@
+[
+ {
+ "cmd": [],
+ "name": "ensure_jiri"
+ },
+ {
+ "cmd": [
+ "cipd",
+ "ensure",
+ "--root",
+ "[START_DIR]/cipd/jiri",
+ "--list",
+ "fuchsia/tools/jiri/linux-amd64 latest",
+ "--json-output",
+ "/path/to/tmp/json"
+ ],
+ "infra_step": true,
+ "name": "ensure_jiri.ensure_installed",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@",
+ "@@@STEP_LOG_LINE@json.output@{@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"result\": [@@@",
+ "@@@STEP_LOG_LINE@json.output@ {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"resolved-instance_id-of-latest----------\", @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"package\": \"fuchsia/tools/jiri/linux-amd64\"@@@",
+ "@@@STEP_LOG_LINE@json.output@ }@@@",
+ "@@@STEP_LOG_LINE@json.output@ ]@@@",
+ "@@@STEP_LOG_LINE@json.output@}@@@",
+ "@@@STEP_LOG_END@json.output@@@"
+ ]
+ },
+ {
+ "cmd": [],
+ "name": "ensure_gsutil"
+ },
+ {
+ "cmd": [
+ "cipd",
+ "ensure",
+ "--root",
+ "[START_DIR]/cipd/gsutil",
+ "--list",
+ "infra/tools/gsutil latest",
+ "--json-output",
+ "/path/to/tmp/json"
+ ],
+ "infra_step": true,
+ "name": "ensure_gsutil.ensure_installed",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@",
+ "@@@STEP_LOG_LINE@json.output@{@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"result\": [@@@",
+ "@@@STEP_LOG_LINE@json.output@ {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"resolved-instance_id-of-latest----------\", @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"package\": \"infra/tools/gsutil\"@@@",
+ "@@@STEP_LOG_LINE@json.output@ }@@@",
+ "@@@STEP_LOG_LINE@json.output@ ]@@@",
+ "@@@STEP_LOG_LINE@json.output@}@@@",
+ "@@@STEP_LOG_END@json.output@@@"
+ ]
+ },
+ {
+ "cmd": [],
+ "name": "ensure_goma"
+ },
+ {
+ "cmd": [
+ "cipd",
+ "ensure",
+ "--root",
+ "[START_DIR]/cipd/goma",
+ "--list",
+ "infra_internal/goma/client/linux-amd64 release",
+ "--json-output",
+ "/path/to/tmp/json",
+ "--service-account-json",
+ "/creds/service_accounts/service-account-goma-client.json"
+ ],
+ "infra_step": true,
+ "name": "ensure_goma.ensure_installed",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@",
+ "@@@STEP_LOG_LINE@json.output@{@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"result\": [@@@",
+ "@@@STEP_LOG_LINE@json.output@ {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"resolved-instance_id-of-release---------\", @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"package\": \"infra_internal/goma/client/linux-amd64\"@@@",
+ "@@@STEP_LOG_LINE@json.output@ }@@@",
+ "@@@STEP_LOG_LINE@json.output@ ]@@@",
+ "@@@STEP_LOG_LINE@json.output@}@@@",
+ "@@@STEP_LOG_END@json.output@@@"
+ ]
+ },
+ {
+ "cmd": [],
+ "name": "ensure_qemu"
+ },
+ {
+ "cmd": [
+ "cipd",
+ "ensure",
+ "--root",
+ "[START_DIR]/cipd/qemu",
+ "--list",
+ "fuchsia/tools/qemu/linux-amd64 latest",
+ "--json-output",
+ "/path/to/tmp/json",
+ "--service-account-json",
+ "/creds/service_accounts/service-account-goma-client.json"
+ ],
+ "infra_step": true,
+ "name": "ensure_qemu.ensure_installed",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@",
+ "@@@STEP_LOG_LINE@json.output@{@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"result\": [@@@",
+ "@@@STEP_LOG_LINE@json.output@ {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"resolved-instance_id-of-latest----------\", @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"package\": \"fuchsia/tools/qemu/linux-amd64\"@@@",
+ "@@@STEP_LOG_LINE@json.output@ }@@@",
+ "@@@STEP_LOG_LINE@json.output@ ]@@@",
+ "@@@STEP_LOG_LINE@json.output@}@@@",
+ "@@@STEP_LOG_END@json.output@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "[START_DIR]/cipd/jiri/jiri",
+ "init",
+ "-cache",
+ "[CACHE]/git",
+ "-shared"
+ ],
+ "infra_step": true,
+ "name": "jiri init"
+ },
+ {
+ "cmd": [
+ "[START_DIR]/cipd/jiri/jiri",
+ "import",
+ "-overwrite=true",
+ "fuchsia",
+ "https://fuchsia.googlesource.com/manifest"
+ ],
+ "infra_step": true,
+ "name": "jiri import"
+ },
+ {
+ "cmd": [
+ "[START_DIR]/cipd/jiri/jiri",
+ "project",
+ "-clean-all"
+ ],
+ "infra_step": true,
+ "name": "jiri project clean"
+ },
+ {
+ "cmd": [
+ "[START_DIR]/cipd/jiri/jiri",
+ "update",
+ "-autoupdate=false",
+ "-gc=true"
+ ],
+ "infra_step": true,
+ "name": "jiri update"
+ },
+ {
+ "cmd": [
+ "[START_DIR]/cipd/jiri/jiri",
+ "snapshot",
+ "[TMP_BASE]/jiri.snapshot"
+ ],
+ "infra_step": true,
+ "name": "jiri snapshot"
+ },
+ {
+ "cmd": [
+ "python",
+ "-u",
+ "[START_DIR]/cipd/gsutil/gsutil",
+ "-o",
+ "GSUtil:software_update_check_period=0",
+ "cp",
+ "[TMP_BASE]/jiri.snapshot",
+ "gs://fuchsia/jiri/snapshots/8ac5404b688b34f2d34d1c8a648413aca30b7a97"
+ ],
+ "env": {
+ "BOTO_CONFIG": "[HOME]/.boto"
+ },
+ "infra_step": true,
+ "name": "upload jiri.snapshot",
+ "~followup_annotations": [
+ "@@@STEP_LINK@jiri.snapshot@https://storage.googleapis.com/fuchsia/jiri/snapshots/8ac5404b688b34f2d34d1c8a648413aca30b7a97@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "[START_DIR]/scripts/build-magenta.sh",
+ "-c",
+ "-t",
+ "x86_64"
+ ],
+ "name": "build magenta"
+ },
+ {
+ "cmd": [],
+ "name": "build fuchsia"
+ },
+ {
+ "cmd": [
+ "python",
+ "-u",
+ "[START_DIR]/cipd/goma/goma_ctl.py",
+ "restart"
+ ],
+ "env": {
+ "GOMA_CACHE_DIR": "[CACHE]/goma",
+ "GOMA_DEPS_CACHE_FILE": "goma_deps_cache",
+ "GOMA_SERVICE_ACCOUNT_JSON_FILE": "/creds/service_accounts/service-account-goma-client.json"
+ },
+ "infra_step": true,
+ "name": "build fuchsia.start_goma",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "[START_DIR]/packages/gn/gen.py",
+ "--target_cpu=x86-64",
+ "--modules=default,boot_test_runner",
+ "--with-dart-analysis",
+ "--goma=[START_DIR]/cipd/goma"
+ ],
+ "name": "build fuchsia.gen",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "python",
+ "-u",
+ "\nimport multiprocessing\nimport sys\n\njob_limit = 200\nif sys.platform.startswith('linux'):\n # Use 80 for linux not to load goma backend.\n job_limit = 80\n\ntry:\n jobs = min(job_limit, multiprocessing.cpu_count() * 10)\nexcept NotImplementedError:\n jobs = 50\n\nprint jobs\n"
+ ],
+ "name": "build fuchsia.calculate the number of recommended jobs",
+ "stdout": "/path/to/tmp/",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@",
+ "@@@STEP_LOG_LINE@python.inline@@@@",
+ "@@@STEP_LOG_LINE@python.inline@import multiprocessing@@@",
+ "@@@STEP_LOG_LINE@python.inline@import sys@@@",
+ "@@@STEP_LOG_LINE@python.inline@@@@",
+ "@@@STEP_LOG_LINE@python.inline@job_limit = 200@@@",
+ "@@@STEP_LOG_LINE@python.inline@if sys.platform.startswith('linux'):@@@",
+ "@@@STEP_LOG_LINE@python.inline@ # Use 80 for linux not to load goma backend.@@@",
+ "@@@STEP_LOG_LINE@python.inline@ job_limit = 80@@@",
+ "@@@STEP_LOG_LINE@python.inline@@@@",
+ "@@@STEP_LOG_LINE@python.inline@try:@@@",
+ "@@@STEP_LOG_LINE@python.inline@ jobs = min(job_limit, multiprocessing.cpu_count() * 10)@@@",
+ "@@@STEP_LOG_LINE@python.inline@except NotImplementedError:@@@",
+ "@@@STEP_LOG_LINE@python.inline@ jobs = 50@@@",
+ "@@@STEP_LOG_LINE@python.inline@@@@",
+ "@@@STEP_LOG_LINE@python.inline@print jobs@@@",
+ "@@@STEP_LOG_END@python.inline@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "[START_DIR]/buildtools/ninja",
+ "-C",
+ "[START_DIR]/out/debug-x86-64",
+ "-j",
+ "50"
+ ],
+ "name": "build fuchsia.ninja",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "python",
+ "-u",
+ "[START_DIR]/cipd/goma/goma_ctl.py",
+ "jsonstatus",
+ "[START_DIR]/cipd/goma/jsonstatus"
+ ],
+ "env": {
+ "GOMA_CACHE_DIR": "[CACHE]/goma",
+ "GOMA_DEPS_CACHE_FILE": "goma_deps_cache",
+ "GOMA_SERVICE_ACCOUNT_JSON_FILE": "/creds/service_accounts/service-account-goma-client.json"
+ },
+ "name": "build fuchsia.goma_jsonstatus",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "python",
+ "-u",
+ "[START_DIR]/cipd/goma/goma_ctl.py",
+ "stat"
+ ],
+ "env": {
+ "GOMA_CACHE_DIR": "[CACHE]/goma",
+ "GOMA_DEPS_CACHE_FILE": "goma_deps_cache",
+ "GOMA_SERVICE_ACCOUNT_JSON_FILE": "/creds/service_accounts/service-account-goma-client.json"
+ },
+ "name": "build fuchsia.goma_stat",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "python",
+ "-u",
+ "[START_DIR]/cipd/goma/goma_ctl.py",
+ "stop"
+ ],
+ "env": {
+ "GOMA_CACHE_DIR": "[CACHE]/goma",
+ "GOMA_DEPS_CACHE_FILE": "goma_deps_cache",
+ "GOMA_SERVICE_ACCOUNT_JSON_FILE": "/creds/service_accounts/service-account-goma-client.json"
+ },
+ "name": "build fuchsia.stop_goma",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "RECIPE_MODULE[infra::qemu]/resources/qemu.py",
+ "start",
+ "--executable",
+ "[START_DIR]/cipd/qemu/bin/qemu-system-x86_64",
+ "--memory",
+ "4096",
+ "--smp",
+ "4",
+ "--arch",
+ "x86_64",
+ "--initrd",
+ "[START_DIR]/out/debug-x86-64/user.bootfs",
+ "--netdev",
+ "user,id=net0,hostfwd=tcp::8342-:8342",
+ "--device",
+ "e1000,netdev=net0",
+ "[START_DIR]/out/build-magenta/build-magenta-pc-x86-64/magenta.bin"
+ ],
+ "name": "start qemu"
+ },
+ {
+ "cmd": [
+ "[START_DIR]/apps/test_runner/src/run_test",
+ "--test_file",
+ "[START_DIR]/tests.json",
+ "--server",
+ "127.0.0.1",
+ "--port",
+ "8342",
+ "--no-loglistener"
+ ],
+ "env": {
+ "FUCHSIA_BUILD_DIR": "[START_DIR]/out/debug-x86-64",
+ "FUCHSIA_OUT_DIR": "[START_DIR]/out"
+ },
+ "name": "run tests",
+ "~followup_annotations": [
+ "step returned non-zero exit code: 1",
+ "@@@STEP_FAILURE@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "RECIPE_MODULE[infra::qemu]/resources/qemu.py",
+ "stop",
+ "--log"
+ ],
+ "name": "stop qemu and read log"
+ },
+ {
+ "cmd": [
+ "[START_DIR]/magenta/scripts/symbolize",
+ "--no-echo",
+ "--file",
+ "qemu.stdout"
+ ],
+ "name": "symbolize",
+ "stdout": "/path/to/tmp/",
+ "~followup_annotations": [
+ "@@@STEP_LOG_LINE@symbolized backtraces@bt1@@@",
+ "@@@STEP_LOG_LINE@symbolized backtraces@bt2@@@",
+ "@@@STEP_LOG_END@symbolized backtraces@@@",
+ "@@@STEP_FAILURE@@@"
+ ]
+ },
+ {
+ "name": "$result",
+ "reason": "Step('run tests') failed with return_code 1",
+ "recipe_result": null,
+ "status_code": 1
+ }
+]
\ No newline at end of file
diff --git a/recipes/fuchsia.expected/boot_module.json b/recipes/fuchsia.expected/boot_module.json
index a01c544..1b8f5c3 100644
--- a/recipes/fuchsia.expected/boot_module.json
+++ b/recipes/fuchsia.expected/boot_module.json
@@ -371,17 +371,6 @@
},
{
"cmd": [
- "sleep",
- "3"
- ],
- "env": {
- "FUCHSIA_BUILD_DIR": "[START_DIR]/out/debug-x86-64",
- "FUCHSIA_OUT_DIR": "[START_DIR]/out"
- },
- "name": "sleep"
- },
- {
- "cmd": [
"RECIPE_MODULE[infra::qemu]/resources/qemu.py",
"stop",
"--log"
@@ -389,6 +378,16 @@
"name": "stop qemu and read log"
},
{
+ "cmd": [
+ "[START_DIR]/magenta/scripts/symbolize",
+ "--no-echo",
+ "--file",
+ "qemu.stdout"
+ ],
+ "name": "symbolize",
+ "stdout": "/path/to/tmp/"
+ },
+ {
"name": "$result",
"recipe_result": null,
"status_code": 0
diff --git a/recipes/fuchsia.expected/failed_tests.json b/recipes/fuchsia.expected/failed_tests.json
index bf04c00..bb2e954c 100644
--- a/recipes/fuchsia.expected/failed_tests.json
+++ b/recipes/fuchsia.expected/failed_tests.json
@@ -375,17 +375,6 @@
},
{
"cmd": [
- "sleep",
- "3"
- ],
- "env": {
- "FUCHSIA_BUILD_DIR": "[START_DIR]/out/debug-x86-64",
- "FUCHSIA_OUT_DIR": "[START_DIR]/out"
- },
- "name": "sleep"
- },
- {
- "cmd": [
"RECIPE_MODULE[infra::qemu]/resources/qemu.py",
"stop",
"--log"
@@ -393,6 +382,16 @@
"name": "stop qemu and read log"
},
{
+ "cmd": [
+ "[START_DIR]/magenta/scripts/symbolize",
+ "--no-echo",
+ "--file",
+ "qemu.stdout"
+ ],
+ "name": "symbolize",
+ "stdout": "/path/to/tmp/"
+ },
+ {
"name": "$result",
"reason": "Step('run tests') failed with return_code 1",
"recipe_result": null,
diff --git a/recipes/fuchsia.expected/tests.json b/recipes/fuchsia.expected/tests.json
index 24f21f0..ce2a5b4 100644
--- a/recipes/fuchsia.expected/tests.json
+++ b/recipes/fuchsia.expected/tests.json
@@ -371,17 +371,6 @@
},
{
"cmd": [
- "sleep",
- "3"
- ],
- "env": {
- "FUCHSIA_BUILD_DIR": "[START_DIR]/out/debug-x86-64",
- "FUCHSIA_OUT_DIR": "[START_DIR]/out"
- },
- "name": "sleep"
- },
- {
- "cmd": [
"RECIPE_MODULE[infra::qemu]/resources/qemu.py",
"stop",
"--log"
@@ -389,6 +378,16 @@
"name": "stop qemu and read log"
},
{
+ "cmd": [
+ "[START_DIR]/magenta/scripts/symbolize",
+ "--no-echo",
+ "--file",
+ "qemu.stdout"
+ ],
+ "name": "symbolize",
+ "stdout": "/path/to/tmp/"
+ },
+ {
"name": "$result",
"recipe_result": null,
"status_code": 0
diff --git a/recipes/fuchsia.py b/recipes/fuchsia.py
index 823b5df..0ee7385 100644
--- a/recipes/fuchsia.py
+++ b/recipes/fuchsia.py
@@ -165,29 +165,43 @@
netdev=netdev,
devices=['e1000,netdev=net0'])
- with qemu:
- run_tests_cmd = [
- start_dir.join('apps/test_runner/src/run_test'),
- '--test_file', start_dir.join(tests),
- '--server', '127.0.0.1',
- '--port', str(TEST_RUNNER_PORT),
- '--no-loglistener',
- ]
+ try:
+ with qemu:
+ run_tests_cmd = [
+ start_dir.join('apps/test_runner/src/run_test'),
+ '--test_file', start_dir.join(tests),
+ '--server', '127.0.0.1',
+ '--port', str(TEST_RUNNER_PORT),
+ '--no-loglistener',
+ ]
- env = {
- 'FUCHSIA_OUT_DIR': fuchsia_out_dir,
- 'FUCHSIA_BUILD_DIR': fuchsia_build_dir,
- }
+ env = {
+ 'FUCHSIA_OUT_DIR': fuchsia_out_dir,
+ 'FUCHSIA_BUILD_DIR': fuchsia_build_dir,
+ }
- # TODO(bgoldman): Update run_test so that it retries the TCP connection at
- # startup, to deal with a possible race condition where QEMU is not
- # forwarding the port by the time run_tests tries to connect.
- with api.context(env=env):
- try:
+ # TODO(bgoldman): Update run_test so that it retries the TCP connection at
+ # startup, to deal with a possible race condition where QEMU is not
+ # forwarding the port by the time run_test tries to connect.
+ with api.context(env=env):
api.step('run tests', run_tests_cmd)
- finally:
- # Workaround for: https://fuchsia.atlassian.net/browse/TO-273
- api.step('sleep', ['sleep', '3'])
+ finally:
+ symbolize_cmd = [
+ start_dir.join('magenta', 'scripts', 'symbolize'),
+ '--no-echo',
+ '--file',
+ 'qemu.stdout',
+ ]
+ step_result = api.step('symbolize', symbolize_cmd,
+ stdout=api.raw_io.output(),
+ step_test_data=lambda: api.raw_io.test_api.stream_output(''))
+ lines = step_result.stdout.splitlines()
+ if lines:
+ # If symbolize found any backtraces in qemu.stdout, mark the symbolize
+ # step as failed to indicate that it should be looked at.
+ step_result.presentation.logs['symbolized backtraces'] = lines
+ step_result.presentation.status = api.step.FAILURE
+
def RunSteps(api, category, patch_gerrit_url, patch_project, patch_ref,
patch_storage, patch_repository_url, manifest, remote, target,
@@ -249,6 +263,13 @@
target='x86-64',
tests='tests.json',
) + api.step_data('run tests', retcode=1)
+ yield api.test('backtrace') + api.properties(
+ manifest='fuchsia',
+ remote='https://fuchsia.googlesource.com/manifest',
+ target='x86-64',
+ tests='tests.json',
+ ) + api.step_data('run tests', retcode=1,
+ ) + api.step_data('symbolize', api.raw_io.stream_output('bt1\nbt2\n'))
yield api.test('no_goma') + api.properties(
manifest='fuchsia',
remote='https://fuchsia.googlesource.com/manifest',
