sdk/testing/sl4f/client/test/performance_test.dart

// Copyright 2019 The Fuchsia Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

import 'dart:convert';
import 'dart:io' show File, Directory;

import 'package:mockito/mockito.dart';
import 'package:path/path.dart' as path;
import 'package:sl4f/trace_processing.dart';
import 'package:test/test.dart';

import 'package:sl4f/sl4f.dart';

class MockSl4f extends Mock implements Sl4f {}

class MockDump extends Mock implements Dump {}

class RunProcessObserver {
  void runProcess(String executablePath, List<String> args) {}
}

class MockRunProcessObserver extends Mock implements RunProcessObserver {}

class FakePerformanceTools extends Performance {
  final RunProcessObserver _observer;
  FakePerformanceTools(Sl4f _sl4f, Dump _dump, RunProcessObserver observer)
      : _observer = observer,
        super(_sl4f, _dump);

  @override
  Future<bool> runProcess(String executablePath, List<String> args) async {
    _observer.runProcess(executablePath, args);
    return true;
  }
}

const String sl4fTraceRequestMethod = 'traceutil_facade.GetTraceFile';

void main(List<String> args) {
  Dump mockDump;
  Sl4f mockSl4f;
  setUp(() {
    mockDump = MockDump();
    mockSl4f = MockSl4f();
  });

  Directory createTempDir() {
    final tempDir = Directory.systemTemp.createTempSync();
    addTearDown(() {
      tempDir.deleteSync(recursive: true);
    });
    return tempDir;
  }

  // TODO(fxbug.dev/59861): Update this test to the new fuchsia perf json
  // format when the migration is done.
  test('process trace', () async {
    final metricsSpecs = [
      MetricsSpec(name: 'test_metric_1'),
      MetricsSpec(name: 'test_metric_2'),
    ];
    final metricsSpecSet =
        MetricsSpecSet(testName: 'test_name', metricsSpecs: metricsSpecs);
    final testMetricsRegistry = {
      'test_metric_1': (Model model, Map<String, dynamic> extraArgs) => [
            TestCaseResults('label_1', Unit.nanoseconds, [1.0])
          ],
      'test_metric_2': (Model model, Map<String, dynamic> extraArgs) => [
            TestCaseResults('label_2', Unit.milliseconds, [2.0])
          ],
    };

    final traceFile = File(path.join(createTempDir().path, 'sample-trace.json'))
      ..createSync()
      ..writeAsStringSync('''
{
  "traceEvents": [],
  "systemTraceEvents": {
    "type": "fuchsia",
    "events": []
  }
}
''');

    final performance = Performance(mockSl4f, mockDump);
    final resultsFile = await performance
        .processTrace(metricsSpecSet, traceFile, registry: testMetricsRegistry);

    final resultsFileContents = await resultsFile.readAsString();
    final resultsObject = json.decode(resultsFileContents);

    expect(resultsObject[0]['label'], equals('label_1'));
    expect(resultsObject[0]['test_suite'], equals('test_name'));
    expect(resultsObject[0]['unit'], equals('nanoseconds'));
    expect(resultsObject[0]['values'], equals([1.0]));

    expect(resultsObject[1]['label'], equals('label_2'));
    expect(resultsObject[1]['test_suite'], equals('test_name'));
    expect(resultsObject[1]['unit'], equals('milliseconds'));
    expect(resultsObject[1]['values'], equals([2.0]));
  });

  test('process trace with testSuite set but not testName', () async {
    final metricsSpecs = [
      MetricsSpec(name: 'test_metric_1'),
      MetricsSpec(name: 'test_metric_2'),
    ];
    final metricsSpecSet =
        MetricsSpecSet(testSuite: 'test_suite', metricsSpecs: metricsSpecs);
    final testMetricsRegistry = {
      'test_metric_1': (Model model, Map<String, dynamic> extraArgs) => [
            TestCaseResults('label_1', Unit.nanoseconds, [1.0])
          ],
      'test_metric_2': (Model model, Map<String, dynamic> extraArgs) => [
            TestCaseResults('label_2', Unit.milliseconds, [2.0])
          ],
    };

    final traceFile = File(path.join(createTempDir().path, 'sample-trace.json'))
      ..createSync()
      ..writeAsStringSync('''
{
  "traceEvents": [],
  "systemTraceEvents": {
    "type": "fuchsia",
    "events": []
  }
}
''');

    final performance = Performance(mockSl4f, mockDump);
    final resultsFile = await performance
        .processTrace(metricsSpecSet, traceFile, registry: testMetricsRegistry);

    final resultsFileContents = await resultsFile.readAsString();
    final resultsObject = json.decode(resultsFileContents);

    expect(resultsObject[0]['label'], equals('label_1'));
    expect(resultsObject[0]['test_suite'], equals('test_suite'));
    expect(resultsObject[0]['unit'], equals('nanoseconds'));
    expect(resultsObject[0]['values'], equals([1.0]));

    expect(resultsObject[1]['label'], equals('label_2'));
    expect(resultsObject[1]['test_suite'], equals('test_suite'));
    expect(resultsObject[1]['unit'], equals('milliseconds'));
    expect(resultsObject[1]['values'], equals([2.0]));
  });

  test('convert results', () async {
    final mockRunProcessObserver = MockRunProcessObserver();
    final performance =
        FakePerformanceTools(mockSl4f, mockDump, mockRunProcessObserver);

    // With no buildbucket id env variable, it should do a local run.
    await performance.convertResults('/bin/catapult_converter',
        File('test1-benchmark.fuchsiaperf.json'), {});
    var verifyMockRunProcessObserver = verify(mockRunProcessObserver.runProcess(
        argThat(endsWith('catapult_converter')), captureAny))
      ..called(1);
    var capturedArgs = verifyMockRunProcessObserver.captured.single;
    expect(capturedArgs[0], '--input');
    expect(capturedArgs[1], endsWith('test1-benchmark.fuchsiaperf.json'));
    expect(capturedArgs[2], '--output');
    expect(capturedArgs[3], endsWith('test1-benchmark.catapult_json_disabled'));
    expect(capturedArgs[4], '--execution-timestamp-ms');
    expect(int.parse(capturedArgs[5]) != null, true);
    expect(capturedArgs[6], '--masters');
    expect(capturedArgs[7], 'local-master');
    expect(capturedArgs[8], '--log-url');
    expect(capturedArgs[9], 'http://ci.example.com/build/300');
    expect(capturedArgs[10], '--bots');
    expect(capturedArgs[11], 'local-bot');

    // Otherwise, it should do a bot run.
    const environment = {
      'CATAPULT_DASHBOARD_MASTER': 'example.fuchsia.global.ci',
      'CATAPULT_DASHBOARD_BOT': 'example-fuchsia-x64-nuc',
      'BUILDBUCKET_ID': '8abc123',
      'BUILD_CREATE_TIME': '1561234567890',
    };

    await performance.convertResults('/bin/catapult_converter',
        File('test2-benchmark.fuchsiaperf.json'), environment);
    verifyMockRunProcessObserver = verify(mockRunProcessObserver.runProcess(
        argThat(endsWith('catapult_converter')), captureAny))
      ..called(1);
    capturedArgs = verifyMockRunProcessObserver.captured.single;
    expect(capturedArgs[0], '--input');
    expect(capturedArgs[1], endsWith('test2-benchmark.fuchsiaperf.json'));
    expect(capturedArgs[2], '--output');
    expect(capturedArgs[3], endsWith('test2-benchmark.catapult_json'));
    expect(capturedArgs[4], '--execution-timestamp-ms');
    expect(capturedArgs[5], '1561234567890');
    expect(capturedArgs[6], '--masters');
    expect(capturedArgs[7], 'example.fuchsia.global.ci');
    expect(capturedArgs[8], '--log-url');
    expect(capturedArgs[9], 'https://ci.chromium.org/b/8abc123');
    expect(capturedArgs[10], '--bots');
    expect(capturedArgs[11], 'example-fuchsia-x64-nuc');

    // If it is a bot run with release version, should have the product-versions arg.
    const environmentWithVersion = {
      'CATAPULT_DASHBOARD_MASTER': 'example.fuchsia.global.ci',
      'CATAPULT_DASHBOARD_BOT': 'example-fuchsia-x64-nuc',
      'BUILDBUCKET_ID': '8abc123',
      'BUILD_CREATE_TIME': '1561234567890',
      'RELEASE_VERSION': '0.001.20.3',
    };

    await performance.convertResults('/bin/catapult_converter',
        File('test3-benchmark.fuchsiaperf.json'), environmentWithVersion);
    verifyMockRunProcessObserver = verify(mockRunProcessObserver.runProcess(
        argThat(endsWith('catapult_converter')), captureAny))
      ..called(1);
    capturedArgs = verifyMockRunProcessObserver.captured.single;
    expect(capturedArgs[0], '--input');
    expect(capturedArgs[1], endsWith('test3-benchmark.fuchsiaperf.json'));
    expect(capturedArgs[2], '--output');
    expect(capturedArgs[3], endsWith('test3-benchmark.catapult_json'));
    expect(capturedArgs[4], '--execution-timestamp-ms');
    expect(capturedArgs[5], '1561234567890');
    expect(capturedArgs[6], '--masters');
    expect(capturedArgs[7], 'example.fuchsia.global.ci');
    expect(capturedArgs[8], '--log-url');
    expect(capturedArgs[9], 'https://ci.chromium.org/b/8abc123');
    expect(capturedArgs[10], '--bots');
    expect(capturedArgs[11], 'example-fuchsia-x64-nuc');
    expect(capturedArgs[12], '--product-versions');
    expect(capturedArgs[13], '0.001.20.3');
  });

  // convertResults() should raise an error if a non-empty subset of the
  // infra env vars are set.
  test('convertResults check env vars', () async {
    final mockRunProcessObserver = MockRunProcessObserver();
    final performance =
        FakePerformanceTools(mockSl4f, mockDump, mockRunProcessObserver);

    final environment = {
      'CATAPULT_DASHBOARD_MASTER': 'example.fuchsia.global.ci',
    };
    expect(
        performance.convertResults('/bin/catapult_converter',
            File('test1-benchmark.fuchsiaperf.json'), environment),
        throwsA(TypeMatcher<ArgumentError>()));
  });

  // convertResults() should raise an error when given a filename without the
  // proper filename extension.
  test('convertResults check filename extension', () async {
    final mockRunProcessObserver = MockRunProcessObserver();
    final performance =
        FakePerformanceTools(mockSl4f, mockDump, mockRunProcessObserver);

    expect(
        performance.convertResults(
            '/bin/catapult_converter', File('results-fuchsiaperf.json'), {}),
        throwsA(TypeMatcher<ArgumentError>()));
  });

  test('convert trace', () async {
    final mockRunProcessObserver = MockRunProcessObserver();
    final performance =
        FakePerformanceTools(mockSl4f, mockDump, mockRunProcessObserver);
    final convertedTraces = [
      await performance.convertTraceFileToJson(
          '/path/to/trace2json', File('/path/to/trace.fxt')),
      await performance.convertTraceFileToJson(
          '/path/to/trace2json', File('/path/to/trace.fxt.gz')),
      await performance.convertTraceFileToJson(
          '/path/to/trace2json', File('/path/to/trace.fxt'),
          compressedOutput: true),
      await performance.convertTraceFileToJson(
          '/path/to/trace2json', File('/path/to/trace.fxt.gz'),
          compressedOutput: true),
      // These compressedInput values intentionally do not match the file
      // extensions to test that the code isn't ignoring the compressedInput
      // argument.
      await performance.convertTraceFileToJson(
          '/path/to/trace2json', File('/path/to/trace.fxt'),
          compressedInput: true, compressedOutput: true),
      await performance.convertTraceFileToJson(
          '/path/to/trace2json', File('/path/to/trace.fxt.gz'),
          compressedInput: false, compressedOutput: true),
    ];
    expect(convertedTraces[0].path, '/path/to/trace.json');
    expect(convertedTraces[1].path, '/path/to/trace.json');
    expect(convertedTraces[2].path, '/path/to/trace.json.gz');
    expect(convertedTraces[3].path, '/path/to/trace.json.gz');
    expect(convertedTraces[4].path, '/path/to/trace.json.gz');
    expect(convertedTraces[5].path, '/path/to/trace.json.gz');

    final verifyMockRunProcessObserver = verify(mockRunProcessObserver
        .runProcess(argThat(endsWith('trace2json')), captureAny))
      ..called(6);
    final capturedArgs = verifyMockRunProcessObserver.captured;
    expect(capturedArgs[0], [
      '--input-file=/path/to/trace.fxt',
      '--output-file=/path/to/trace.json'
    ]);
    expect(capturedArgs[1], [
      '--input-file=/path/to/trace.fxt.gz',
      '--output-file=/path/to/trace.json',
      '--compressed-input'
    ]);
    expect(capturedArgs[2], [
      '--input-file=/path/to/trace.fxt',
      '--output-file=/path/to/trace.json.gz',
      '--compressed-output'
    ]);
    expect(capturedArgs[3], [
      '--input-file=/path/to/trace.fxt.gz',
      '--output-file=/path/to/trace.json.gz',
      '--compressed-input',
      '--compressed-output'
    ]);
    expect(capturedArgs[4], [
      '--input-file=/path/to/trace.fxt',
      '--output-file=/path/to/trace.json.gz',
      '--compressed-input',
      '--compressed-output'
    ]);
    expect(capturedArgs[5], [
      '--input-file=/path/to/trace.fxt.gz',
      '--output-file=/path/to/trace.json.gz',
      '--compressed-output'
    ]);
  });
}