| // This schema manages the namespace of test parameters. By requiring all test parameters to |
| // be declared here, we manage test parameter namespace, provide a single source for |
| // test parameter descriptions, and supply the type information that test tools need to |
| // validate test configurations. |
| // |
| // To add a previously-unknown test parameter to a test configuration, add it here as a property |
| // of the top-level object. Please provide a thorough description as this file is considered the |
| // source of truth for identifying test parameters and describing their use. |
| // |
| // The complete configuration of a test run consists of a collection of test parameter |
| // name/value pairs. In order for such a configuration to be considered valid, its JSON |
| // representation must conform to this schema. The schema's top-level type is "object", and |
| // the properties of that top-level object comprise the set of allowed test parameters. |
| // |
| // Note that this schema is used to validate only *complete* test configurations, which must |
| // include the required properties. JSON inputs to test-pilot need not conform to the schema |
| // in that required properties may be missing. |
| { |
| description: "Schema for complete test configurations", |
| type: "object", |
| properties: { |
| // The following names are disallowed, because they are reserved for use by test-pilot |
| // as options that influence its behavior. |
| // schema |
| // debug |
| // strict |
| // include |
| // require |
| // prohibit |
| output_directory: { |
| description: "Directory in which output of the current test run is to be deposited", |
| type: "string", |
| }, |
| host_test_binary: { |
| description: "Host binary implementing the current test", |
| type: "string", |
| }, |
| host_test_args: { |
| description: "Arguments for the binary implementing the current test", |
| type: "array", |
| items: { |
| type: "string", |
| }, |
| }, |
| output_processors: { |
| description: "Processors to be run on the test output after test completion", |
| type: "array", |
| items: { |
| type: "object", |
| properties: { |
| binary: { |
| description: "Host binary implementing the output processor", |
| type: "string", |
| }, |
| args: { |
| description: "Arguments for the binary implementing the output processor", |
| type: "array", |
| items: { |
| type: "string", |
| }, |
| }, |
| use_on_success: { |
| description: "Whether the output processor should be used for successful test runs", |
| type: "boolean", |
| }, |
| use_on_failure: { |
| description: "Whether the output processor should be used for successful test runs", |
| type: "boolean", |
| }, |
| use_if_defined: { |
| description: "Parameters that must be defined for the output processor to be used", |
| type: "array", |
| items: { |
| type: "string", |
| }, |
| }, |
| }, |
| required: [ |
| "binary", |
| ], |
| additionalProperties: false, |
| }, |
| }, |
| resultdb_output_summary_file: { |
| description: "Path at which to deposit output summary file in resultdb format", |
| type: "string", |
| }, |
| ssh_private_key_path: { |
| description: "Path to ssh private key file", |
| type: "string", |
| }, |
| ffx_config: { |
| description: "ffx config settings in any of the formats supported by ffx --config", |
| type: "string", |
| }, |
| target: { |
| description: "IP address of the target device", |
| type: "string", |
| }, |
| max_severity_logs: { |
| description: "Max severity of the log messages that the test is supposed to produce", |
| type: "string", |
| enum: [ |
| "TRACE", |
| "DEBUG", |
| "INFO", |
| "WARN", |
| "ERROR", |
| "FATAL", |
| ], |
| }, |
| min_severity_logs: { |
| description: "Severity below which log messages will not be printed", |
| type: "string", |
| enum: [ |
| "TRACE", |
| "DEBUG", |
| "INFO", |
| "WARN", |
| "ERROR", |
| "FATAL", |
| ], |
| }, |
| tags: { |
| description: "Arbitrary tags to identify and categorize the tests", |
| type: "array", |
| items: { |
| type: "object", |
| properties: { |
| key: { |
| type: "string", |
| }, |
| value: { |
| type: "string", |
| }, |
| }, |
| }, |
| }, |
| target_test_url: { |
| description: "URL of the test component", |
| type: "string", |
| }, |
| target_test_args: { |
| description: "Arguments to be passed to the test component", |
| type: "array", |
| items: { |
| type: "string", |
| }, |
| }, |
| realm: { |
| description: "Realm in which to execute the test in the form <realm path>:<collection>", |
| type: "string", |
| }, |
| test_case_filters: { |
| description: "Glob patterns specifying which test cases to execute. A case is selected \ |
| if it matches any of the specified patterns", |
| type: "array", |
| items: { |
| type: "string", |
| }, |
| }, |
| run_disabled_cases: { |
| description: "Whether to run otherwise disabled cases. By default, disabled cases are \ |
| not run", |
| type: "boolean", |
| }, |
| break_on_failure: { |
| description: "Whether the first failure should cause test execution to stop with zxdb \ |
| attached to the failing process", |
| type: "boolean", |
| }, |
| no_exception_channel: { |
| description: "Whether the test manager should refrain from creating exception channels, \ |
| because the test creates conflicting ones", |
| type: "boolean", |
| }, |
| max_concurrent_test_case_runs: { |
| description: "Indicates the number of test cases that may be run in parallel", |
| type: "number", |
| }, |
| timeout: { |
| description: "Maximum number of seconds the test may run before being terminated \ |
| due to a timeout", |
| type: "number", |
| minimum: 1, |
| }, |
| show_full_moniker_in_logs: { |
| description: "Whether the full moniker should be shown in unstructured logs", |
| type: "boolean", |
| }, |
| }, |
| |
| // Only properties that must appear in *every* complete test configuration should be required. |
| required: [ |
| "output_directory", |
| "host_test_binary", |
| ], |
| |
| // Additional properties must *not* be allowed. test-pilot will reject additional properties |
| // in any case. |
| additionalProperties: false, |
| } |
