| { |
| select: { |
| backstop: "INSPECT:timekeeper.cmx:root:backstop", |
| current_mono: "INSPECT:timekeeper.cmx:root/current:monotonic", |
| current_utc: "INSPECT:timekeeper.cmx:root/current:clock_utc", |
| kernel_utc: "INSPECT:timekeeper.cmx:root/current:kernel_utc", |
| network_available_monotonic: "INSPECT:timekeeper.cmx:root:network_available_monotonic", |
| primary_source_status: "INSPECT:timekeeper.cmx:root/primary_time_source:status", |
| primary_estimate_0_counter: "INSPECT:timekeeper.cmx:root/primary_track/estimate_0:counter", |
| rtc_initialization: "INSPECT:timekeeper.cmx:root/real_time_clock:initialization", |
| rtc_write_failure_count: "INSPECT:timekeeper.cmx:root/real_time_clock:write_failures", |
| last_update_mono: "INSPECT:timekeeper.cmx:root/primary_track/last_update:retrieval_monotonic", |
| last_update_error: "INSPECT:timekeeper.cmx:root/primary_track/last_update:error_bounds", |
| }, |
| eval: { |
| network_unavailable: "Missing(network_available_monotonic)", |
| primary_source_unhealthy: "And(primary_source_status != \"Launched\", primary_source_status != \"Ok\")", |
| primary_estimate_ok: "primary_estimate_0_counter > 0", |
| clock_started: "current_utc > backstop", |
| clock_difference: "current_utc - kernel_utc", |
| rtc_present: "And(rtc_initialization != \"NoDevices\", rtc_initialization != \"InvalidBeforeBackstop\")", |
| rtc_read_ok: "rtc_initialization == \"Succeeded\"", |
| // Error bound is set to UINT64_MAX when the error is unknown but triage |
| // tests fail to parse a value this large, use INT64_MAX instead. |
| error_available: "And(Not(Missing(last_update_error)), last_update_error < 9223372036854775807)", |
| }, |
| act: { |
| not_started_due_to_network: { |
| type: "Warning", |
| trigger: "And(Not(clock_started), network_unavailable)", |
| print: "Time not yet available, network was not declared reachable", |
| }, |
| not_started_other_reason: { |
| type: "Warning", |
| trigger: "And(Not(clock_started), Not(network_unavailable))", |
| print: "Time not yet available even though network was declared reachable", |
| }, |
| rtc_time_only: { |
| type: "Warning", |
| trigger: "And(clock_started, Not(primary_estimate_ok))", |
| print: "Time is available from RTC but not network, network may have never been connected", |
| }, |
| started_but_source_unhealthy: { |
| type: "Warning", |
| trigger: "And(clock_started, primary_source_unhealthy)", |
| print: "Time is available but time source is reporting unhealthy. See timekeeper.cmx/primary_time_source", |
| }, |
| clock_inconsistency: { |
| type: "Warning", |
| trigger: "And(clock_started, Or(clock_difference > Seconds(2), clock_difference < Seconds(-2)))", |
| print: "Kernel and Userspace UTC clocks differ by more than 2 seconds, file bugs in the Time component", |
| }, |
| rtc_could_not_be_read: { |
| type: "Warning", |
| trigger: "And(rtc_present, Not(rtc_read_ok))", |
| print: "Real time clock present but could not be read. See timekeeper.cmx:root/real_time_clock. File bugs in the Time component", |
| }, |
| rtc_write_failures: { |
| type: "Warning", |
| trigger: "And(rtc_present, rtc_write_failure_count > 0)", |
| print: "Failures while writing to real time clock. See timekeeper.cmx:root/real_time_clock. File bugs in the Time component", |
| }, |
| last_update_stale: { |
| type: "Warning", |
| trigger: "And(Not(Missing(last_update_mono)), (current_mono - last_update_mono) > Hours(1))", |
| print: "Most recent update to the UTC clock was over a hour ago. File bugs in the Time component", |
| }, |
| error_too_high: { |
| type: "Warning", |
| trigger: "And(error_available, last_update_error > Seconds(5))", |
| print: "UTC error bound was over 5 seconds, may indicate network problems. See timekeeper.cmx:primary_track/last_update", |
| }, |
| utc_error_bound_ms: { |
| type: "Gauge", |
| value: "Option(last_update_error // Millis(1), \"unknown\")", |
| } |
| }, |
| test: { |
| status_ok: { |
| yes: [], |
| no: [ |
| "not_started_due_to_network", |
| "not_started_other_reason", |
| "started_but_source_unhealthy", |
| "rtc_time_only", |
| ], |
| values: { |
| backstop: 33333, |
| current_utc: 123456789, |
| network_available_monotonic: 4444444, |
| primary_source_status: "Ok", |
| primary_estimate_0_counter: 1, |
| }, |
| }, |
| no_time_network_problem: { |
| yes: [ |
| "not_started_due_to_network", |
| ], |
| no: [ |
| "not_started_other_reason", |
| "started_but_source_unhealthy", |
| "rtc_time_only", |
| ], |
| values: { |
| backstop: 33333, |
| current_utc: 33333, |
| primary_estimate_0_counter: 0, |
| }, |
| }, |
| no_time_other_problem: { |
| yes: [ |
| "not_started_other_reason", |
| ], |
| no: [ |
| "not_started_due_to_network", |
| "started_but_source_unhealthy", |
| "rtc_time_only", |
| ], |
| values: { |
| backstop: 33333, |
| current_utc: 33333, |
| network_available_monotonic: 4444444, |
| primary_source_status: "Ok", |
| primary_estimate_0_counter: 0, |
| }, |
| }, |
| rtc_time_only: { |
| yes: [ |
| "rtc_time_only", |
| ], |
| no: [ |
| "not_started_due_to_network", |
| "not_started_other_reason", |
| "started_but_source_unhealthy", |
| ], |
| values: { |
| backstop: 33333, |
| current_utc: 12345789, |
| primary_source_status: "Ok", |
| primary_estimate_0_counter: 0, |
| }, |
| }, |
| clocks_consistent: { |
| yes: [], |
| no: [ |
| "clock_inconsistency", |
| ], |
| values: { |
| backstop: 3333, |
| current_utc: 4000000000, |
| kernel_utc: 5000000000, |
| }, |
| }, |
| kernel_clock_exceeds_userspace_clock: { |
| yes: [ |
| "clock_inconsistency", |
| ], |
| no: [], |
| values: { |
| backstop: 3333, |
| current_utc: 4000000000, |
| kernel_utc: 7000000000, |
| }, |
| }, |
| userspace_clock_exceeds_kernel_clock: { |
| yes: [ |
| "clock_inconsistency", |
| ], |
| no: [], |
| values: { |
| backstop: 3333, |
| current_utc: 9000000000, |
| kernel_utc: 6000000000, |
| }, |
| }, |
| rtc_ok: { |
| yes: [], |
| no: [ |
| "rtc_could_not_be_read", |
| "rtc_write_failures", |
| ], |
| values: { |
| rtc_initialization: "InvalidBeforeBackstop", |
| rtc_write_failure_count: 0, |
| }, |
| }, |
| rtc_read_failure: { |
| yes: [ |
| "rtc_could_not_be_read", |
| ], |
| no: [ |
| "rtc_write_failures", |
| ], |
| values: { |
| rtc_initialization: "ConnectionFailed", |
| rtc_write_failure_count: 0, |
| }, |
| }, |
| rtc_write_failure: { |
| yes: [ |
| "rtc_write_failures", |
| ], |
| no: [ |
| "rtc_could_not_be_read", |
| ], |
| values: { |
| rtc_initialization: "Succeeded", |
| rtc_write_failure_count: 2, |
| }, |
| }, |
| last_update_missing: { |
| // Clock content warnings should not fire due to lack of time sync. |
| yes: [ ], |
| no: [ |
| "last_update_stale", |
| "error_too_high", |
| ], |
| values: { |
| current_mono: 40000000000, |
| }, |
| }, |
| last_update_valid: { |
| yes: [ ], |
| no: [ |
| "last_update_stale", |
| "error_too_high", |
| ], |
| values: { |
| current_mono: 40000000000, |
| last_update_mono: 2000000000, |
| last_update_error: 99000000, |
| }, |
| }, |
| last_update_valid_but_error_unknown: { |
| yes: [ ], |
| no: [ |
| "last_update_stale", |
| "error_too_high", |
| ], |
| values: { |
| current_mono: 40000000000, |
| last_update_mono: 2000000000, |
| last_update_error: 9223372036854775807, |
| }, |
| }, |
| last_update_error_too_high: { |
| yes: [ |
| "error_too_high", |
| ], |
| no: [ |
| "last_update_stale", |
| ], |
| values: { |
| current_mono: 40000000000, |
| last_update_mono: 2000000000, |
| last_update_error: 999999999999999, |
| }, |
| }, |
| last_update_stale: { |
| yes: [ |
| "last_update_stale", |
| ], |
| no: [ |
| "error_too_high", |
| ], |
| values: { |
| current_mono: 72000000000000, |
| last_update_mono: 2000000000, |
| last_update_error: 99000000, |
| }, |
| }, |
| }, |
| } |
