[json] add tool and action to merge json
CP-80 #comment
CP-85 #comment
TEST: new unit tests, but not sure how to put them on CI
Change-Id: I58fdcae0c1224f7be15ab5b7058bad35805bb62b
diff --git a/packages/json_merge b/packages/json_merge
new file mode 100644
index 0000000..11243e3
--- /dev/null
+++ b/packages/json_merge
@@ -0,0 +1,8 @@
+{
+ "labels": [
+ "//build/tools/json_merge:install"
+ ],
+ "host_tests": [
+ "//build/tools/json_merge:json_merge_test"
+ ]
+}
diff --git a/tools/README.md b/tools/README.md
index 0e56e0b..75338f0 100644
--- a/tools/README.md
+++ b/tools/README.md
@@ -1,3 +1,4 @@
This directory contains a collection of tools used in the GN build:
+- [JSON merge](json_merge/): merges JSON files.
- [JSON validator](json_validator/): verifies that JSON files match a given
schema.
diff --git a/tools/json_merge/BUILD.gn b/tools/json_merge/BUILD.gn
new file mode 100644
index 0000000..0f017be
--- /dev/null
+++ b/tools/json_merge/BUILD.gn
@@ -0,0 +1,48 @@
+# Copyright 2018 The Fuchsia Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import("//build/host.gni")
+import("//build/test.gni")
+
+source_set("srcs") {
+ sources = [
+ "json_merge.cc",
+ "json_merge.h",
+ ]
+
+ public_deps = [
+ "//third_party/rapidjson",
+ ]
+}
+
+executable("json_merge") {
+ sources = [
+ "main.cc",
+ ]
+
+ deps = [
+ ":srcs",
+ ]
+}
+
+test("json_merge_test") {
+ sources = [
+ "test.cc",
+ ]
+
+ deps = [
+ ":srcs",
+ "//third_party/googletest:gtest_main",
+ ]
+}
+
+install_host_tools("install") {
+ deps = [
+ ":json_merge",
+ ]
+
+ outputs = [
+ "json_merge",
+ ]
+}
diff --git a/tools/json_merge/example/BUILD.gn b/tools/json_merge/example/BUILD.gn
new file mode 100644
index 0000000..9d2a291
--- /dev/null
+++ b/tools/json_merge/example/BUILD.gn
@@ -0,0 +1,14 @@
+# Copyright 2018 The Fuchsia Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import("//build/tools/json_merge/json_merge.gni")
+
+json_merge("merged.json") {
+ sources = [
+ "black_white.json",
+ "rgb.json",
+ ]
+
+ minify = true
+}
diff --git a/tools/json_merge/example/black_white.json b/tools/json_merge/example/black_white.json
new file mode 100644
index 0000000..db5d907
--- /dev/null
+++ b/tools/json_merge/example/black_white.json
@@ -0,0 +1,14 @@
+{
+ "black": {
+ "code": {
+ "rgba": [0,0,0,1],
+ "hex": "#000"
+ }
+ },
+ "white": {
+ "code": {
+ "rgba": [0,0,0,1],
+ "hex": "#FFF"
+ }
+ }
+}
diff --git a/tools/json_merge/example/rgb.json b/tools/json_merge/example/rgb.json
new file mode 100644
index 0000000..5949376
--- /dev/null
+++ b/tools/json_merge/example/rgb.json
@@ -0,0 +1,20 @@
+{
+ "red": {
+ "code": {
+ "rgba": [255,0,0,1],
+ "hex": "#F00"
+ }
+ },
+ "green": {
+ "code": {
+ "rgba": [0,255,0,1],
+ "hex": "#0F0"
+ }
+ },
+ "blue": {
+ "code": {
+ "rgba": [0,0,255,1],
+ "hex": "#00F"
+ }
+ }
+}
diff --git a/tools/json_merge/json_merge.cc b/tools/json_merge/json_merge.cc
new file mode 100644
index 0000000..e523486
--- /dev/null
+++ b/tools/json_merge/json_merge.cc
@@ -0,0 +1,59 @@
+// Copyright 2018 The Fuchsia Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "build/tools/json_merge/json_merge.h"
+
+#include "third_party/rapidjson/rapidjson/document.h"
+#include "third_party/rapidjson/rapidjson/error/en.h"
+#include "third_party/rapidjson/rapidjson/filewritestream.h"
+#include "third_party/rapidjson/rapidjson/istreamwrapper.h"
+#include "third_party/rapidjson/rapidjson/ostreamwrapper.h"
+#include "third_party/rapidjson/rapidjson/prettywriter.h"
+#include "third_party/rapidjson/rapidjson/writer.h"
+
+int JSONMerge(const std::vector<struct input_file>& inputs,
+ std::ostream& output, std::ostream& errors, bool minify) {
+ rapidjson::Document merged;
+ merged.SetObject();
+ auto& allocator = merged.GetAllocator();
+
+ for (auto input_it = inputs.begin(); input_it != inputs.end(); ++input_it) {
+ rapidjson::IStreamWrapper isw(*input_it->contents.get());
+ rapidjson::Document input_doc;
+ rapidjson::ParseResult parse_result = input_doc.ParseStream(isw);
+ if (!parse_result) {
+ errors << "Failed to parse " << input_it->name << "!\n";
+ errors << rapidjson::GetParseError_En(parse_result.Code()) << " (offset "
+ << parse_result.Offset() << ")\n";
+ return 1;
+ }
+ if (!input_doc.IsObject()) {
+ errors << input_it->name << " is not a JSON object!\n";
+ return 1;
+ }
+
+ for (auto value_it = input_doc.MemberBegin();
+ value_it != input_doc.MemberEnd(); ++value_it) {
+ if (merged.HasMember(value_it->name)) {
+ errors << input_it->name << " has a conflicting value for key \""
+ << value_it->name.GetString() << "\"!\n";
+ return 1;
+ }
+ merged.AddMember(value_it->name,
+ rapidjson::Value(value_it->value, allocator).Move(),
+ allocator);
+ }
+ }
+
+ rapidjson::OStreamWrapper osw(output);
+ if (minify) {
+ rapidjson::Writer<rapidjson::OStreamWrapper> writer(osw);
+ merged.Accept(writer);
+ } else {
+ rapidjson::PrettyWriter<rapidjson::OStreamWrapper> writer(osw);
+ merged.Accept(writer);
+ }
+
+ return 0;
+}
diff --git a/tools/json_merge/json_merge.gni b/tools/json_merge/json_merge.gni
new file mode 100644
index 0000000..763d919
--- /dev/null
+++ b/tools/json_merge/json_merge.gni
@@ -0,0 +1,67 @@
+# Copyright 2018 The Fuchsia Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import("//build/compiled_action.gni")
+
+# Merge one or more json files.
+#
+# If any input is not a valid JSON, the merge operation will fail. Consequently
+# you can "merge" one JSON file to perform validation.
+# If any two inputs overlap in key space, the merge operation will fail.
+# Optionally the merged output can be minified. Consequently you can "merge"
+# one JSON file to perform minification.
+#
+# Parameters
+#
+# sources (required)
+# [files list] One or more JSON files to merge.
+#
+# minify (optional)
+# [boolean] Whether to minify the result.
+# Minified JSON is functionally equivalent but less readable to humans.
+#
+# testonly (optional)
+# visibility (optional)
+# Standard GN meaning.
+#
+# Example of usage:
+#
+# json_merge("merged.json") {
+# sources = [ "one.json", "two.json" ]
+# minify = true
+# }
+template("json_merge") {
+ compiled_action(target_name) {
+ forward_variables_from(invoker,
+ [
+ "deps",
+ "sources",
+ "testonly",
+ "visibility",
+ "minify",
+ ])
+
+ tool = "//build/tools/json_merge"
+
+ merged_output = "$target_out_dir/$target_name"
+ outputs = [
+ merged_output,
+ ]
+
+ args = []
+ foreach(source, sources) {
+ args += [
+ "--input",
+ rebase_path(source, root_build_dir),
+ ]
+ }
+ args += [
+ "--output",
+ rebase_path(merged_output, root_build_dir),
+ ]
+ if (minify) {
+ args += [ "--minify" ]
+ }
+ }
+}
diff --git a/tools/json_merge/json_merge.h b/tools/json_merge/json_merge.h
new file mode 100644
index 0000000..f42d14e
--- /dev/null
+++ b/tools/json_merge/json_merge.h
@@ -0,0 +1,24 @@
+// Copyright 2018 The Fuchsia Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef BUILD_TOOLS_JSON_MERGE_JSON_MERGE_H_
+#define BUILD_TOOLS_JSON_MERGE_JSON_MERGE_H_
+
+#include <iostream>
+#include <memory>
+#include <vector>
+
+struct input_file {
+ std::string name;
+ std::unique_ptr<std::istream> contents;
+};
+
+// Merge one or more JSON documents.
+// Returns zero if successful.
+// On non-zero return value, writes human-readable errors to |errors|.
+// If |minify| then output merged JSON will be minified.
+int JSONMerge(const std::vector<struct input_file>& inputs,
+ std::ostream& output, std::ostream& errors, bool minify);
+
+#endif // BUILD_TOOLS_JSON_MERGE_JSON_MERGE_H_
diff --git a/tools/json_merge/main.cc b/tools/json_merge/main.cc
new file mode 100644
index 0000000..d163271
--- /dev/null
+++ b/tools/json_merge/main.cc
@@ -0,0 +1,83 @@
+// Copyright 2018 The Fuchsia Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include <getopt.h>
+#include <cstdio>
+#include <fstream>
+#include <iostream>
+#include <memory>
+#include <utility>
+#include <vector>
+
+#include "build/tools/json_merge/json_merge.h"
+
+static void usage(const char* exe_name) {
+ fprintf(
+ stderr,
+ "Usage: %s --input [infile] [--output outfile] [--minify]\n"
+ "\n"
+ "Merge one or more JSON files to a single JSON file.\n"
+ "If any input is not a valid JSON, the merge operation will fail.\n"
+ "Consequently you can \"merge\" one JSON file to perform validation.\n"
+ "If any two inputs overlap in the top-level key space, the merge "
+ "operation will fail.\n"
+ "Optionally the merged output can be minified.\n"
+ "Consequently you can \"merge\" one JSON file to perform "
+ "minification.\n"
+ "\n"
+ "Example usages:\n"
+ "%s --input in1.json --input in2.json # merges to STDOUT\n"
+ "%s --input in1.json --minify --output out.json # minifies to out.json\n"
+ "%s --help # prints this message\n",
+ exe_name, exe_name, exe_name, exe_name);
+}
+
+int main(int argc, char** argv) {
+ std::vector<input_file_t> inputs;
+ std::ofstream output_file;
+ auto output_buf = std::cout.rdbuf();
+ bool minify = false;
+
+ static struct option long_options[] = {
+ {"input", required_argument, 0, 'i'},
+ {"output", required_argument, 0, 'o'},
+ {"minify", no_argument, 0, 'm'},
+ {"help", no_argument, 0, 'h'},
+ };
+
+ int opt;
+ while ((opt = getopt_long(argc, argv, "iomh", long_options, nullptr)) != -1) {
+ switch (opt) {
+ case 'i': {
+ auto input = std::make_unique<std::ifstream>(optarg);
+ if (!input->is_open()) {
+ printf(stderr, "Could not read from input file %s\n", optarg);
+ return 1;
+ }
+ inputs.push_back({.name = optarg, .contents = std::move(input)});
+ break;
+ }
+
+ case 'o':
+ output_file.open(optarg);
+ if (!output_file.is_open()) {
+ printf(stderr, "Could not write to output file %s\n", optarg);
+ return 1;
+ }
+ output_buf = output_file.rdbuf();
+ break;
+
+ case 'm':
+ minify = true;
+ break;
+
+ case 'h':
+ usage(argv[0]);
+ return 0;
+ }
+ }
+
+ std::ostream output(output_buf);
+ return JSONMerge(inputs, output, std::cerr, minify);
+}
diff --git a/tools/json_merge/test.cc b/tools/json_merge/test.cc
new file mode 100644
index 0000000..5c412a2
--- /dev/null
+++ b/tools/json_merge/test.cc
@@ -0,0 +1,113 @@
+// Copyright 2018 The Fuchsia Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include <sstream>
+
+#include "build/tools/json_merge/json_merge.h"
+#include "gtest/gtest.h"
+
+namespace {
+
+class JsonMerge : public ::testing::Test {
+ protected:
+ void AddInput(const std::string& filename, const std::string& input) {
+ inputs.push_back({.name = filename,
+ .contents = std::make_unique<std::istringstream>(input)});
+ }
+
+ int Merge(bool minify) { return JSONMerge(inputs, output, errors, minify); }
+
+ std::string Output() { return output.str(); }
+
+ std::string Errors() { return errors.str(); }
+
+ void ExpectNoErrors() { EXPECT_TRUE(Errors().empty()); }
+
+ void ExpectError(const std::string& expected_error) {
+ EXPECT_EQ(Errors(), expected_error);
+ }
+
+ private:
+ std::vector<input_file> inputs;
+ std::ostringstream output;
+ std::ostringstream errors;
+};
+
+TEST_F(JsonMerge, MergeOne) {
+ const std::string input = R"JSON({
+ "key1": {
+ "key2": [
+ "value1",
+ "value2",
+ "value3"
+ ],
+ "key3": "value4"
+ }
+})JSON";
+ AddInput("file1.json", input);
+
+ EXPECT_EQ(Merge(false), 0);
+ EXPECT_EQ(Output(), input);
+ ExpectNoErrors();
+}
+
+TEST_F(JsonMerge, MergeOneAndMinify) {
+ const std::string input = R"JSON({
+ "key1": {
+ "key2": [
+ "value1",
+ "value2",
+ "value3"
+ ],
+ "key3": "value4"
+ }
+})JSON";
+ AddInput("file1.json", input);
+
+ EXPECT_EQ(Merge(true), 0);
+ const std::string output =
+ R"JSON({"key1":{"key2":["value1","value2","value3"],"key3":"value4"}})JSON";
+ EXPECT_EQ(Output(), output);
+ ExpectNoErrors();
+}
+
+TEST_F(JsonMerge, MergeThree) {
+ const std::string input1 = R"JSON({
+ "key1": "value1"
+})JSON";
+ AddInput("file1.json", input1);
+ const std::string input2 = R"JSON({
+ "key2": "value2"
+})JSON";
+ AddInput("file2.json", input2);
+ const std::string input3 = R"JSON({
+ "key3": "value3"
+})JSON";
+ AddInput("file3.json", input3);
+
+ EXPECT_EQ(Merge(false), 0);
+ const std::string output = R"JSON({
+ "key1": "value1",
+ "key2": "value2",
+ "key3": "value3"
+})JSON";
+ EXPECT_EQ(Output(), output);
+ ExpectNoErrors();
+}
+
+TEST_F(JsonMerge, MergeConflict) {
+ const std::string input1 = R"JSON({
+ "key1": "value1"
+})JSON";
+ AddInput("file1.json", input1);
+ const std::string input2 = R"JSON({
+ "key1": "value2"
+})JSON";
+ AddInput("file2.json", input2);
+
+ EXPECT_NE(Merge(false), 0);
+ ExpectError("file2.json has a conflicting value for key \"key1\"!\n");
+}
+
+} // namespace
