| //===-- SwiftTools.cpp ----------------------------------------------------===// |
| // |
| // This source file is part of the Swift.org open source project |
| // |
| // Copyright (c) 2014 - 2015 Apple Inc. and the Swift project authors |
| // Licensed under Apache License v2.0 with Runtime Library Exception |
| // |
| // See http://swift.org/LICENSE.txt for license information |
| // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors |
| // |
| //===----------------------------------------------------------------------===// |
| |
| #include "llbuild/BuildSystem/CommandResult.h" |
| #include "llbuild/BuildSystem/SwiftTools.h" |
| |
| #include "llbuild/Basic/FileSystem.h" |
| #include "llbuild/Basic/Hashing.h" |
| #include "llbuild/Basic/LLVM.h" |
| #include "llbuild/Basic/PlatformUtility.h" |
| #include "llbuild/BuildSystem/BuildExecutionQueue.h" |
| #include "llbuild/BuildSystem/BuildFile.h" |
| #include "llbuild/BuildSystem/BuildKey.h" |
| #include "llbuild/BuildSystem/BuildValue.h" |
| #include "llbuild/BuildSystem/BuildSystemCommandInterface.h" |
| #include "llbuild/BuildSystem/ExternalCommand.h" |
| #include "llbuild/Core/MakefileDepsParser.h" |
| |
| #include "llvm/ADT/ArrayRef.h" |
| #include "llvm/ADT/Hashing.h" |
| #include "llvm/ADT/STLExtras.h" |
| #include "llvm/ADT/SmallVector.h" |
| #include "llvm/ADT/Twine.h" |
| #include "llvm/Support/FileSystem.h" |
| #include "llvm/Support/MemoryBuffer.h" |
| #include "llvm/Support/Path.h" |
| #include "llvm/Support/raw_ostream.h" |
| |
| using namespace llbuild; |
| using namespace llbuild::core; |
| using namespace llbuild::buildsystem; |
| |
| namespace { |
| |
| class SwiftGetVersionCommand : public Command { |
| std::string executable; |
| |
| public: |
| SwiftGetVersionCommand(const BuildKey& key) |
| : Command("swift-get-version"), executable(key.getCustomTaskData()) { |
| } |
| |
| // FIXME: Should create a CustomCommand class, to avoid all the boilerplate |
| // required implementations. |
| |
| virtual void getShortDescription(SmallVectorImpl<char> &result) override { |
| llvm::raw_svector_ostream(result) << "Checking Swift Compiler Version"; |
| } |
| |
| virtual void getVerboseDescription(SmallVectorImpl<char> &result) override { |
| llvm::raw_svector_ostream(result) << '"' << executable << '"' |
| << " --version"; |
| } |
| |
| virtual void configureDescription(const ConfigureContext&, |
| StringRef value) override { } |
| virtual void configureInputs(const ConfigureContext&, |
| const std::vector<Node*>& value) override { } |
| virtual void configureOutputs(const ConfigureContext&, |
| const std::vector<Node*>& value) override { } |
| virtual bool configureAttribute(const ConfigureContext& ctx, StringRef name, |
| StringRef value) override { |
| // No supported attributes. |
| ctx.error("unexpected attribute: '" + name + "'"); |
| return false; |
| } |
| virtual bool configureAttribute(const ConfigureContext& ctx, StringRef name, |
| ArrayRef<StringRef> values) override { |
| // No supported attributes. |
| ctx.error("unexpected attribute: '" + name + "'"); |
| return false; |
| } |
| virtual bool configureAttribute( |
| const ConfigureContext& ctx, StringRef name, |
| ArrayRef<std::pair<StringRef, StringRef>> values) override { |
| // No supported attributes. |
| ctx.error("unexpected attribute: '" + name + "'"); |
| return false; |
| } |
| virtual BuildValue getResultForOutput(Node* node, |
| const BuildValue& value) override { |
| // This method should never be called on a custom command. |
| llvm_unreachable("unexpected"); |
| return BuildValue::makeInvalid(); |
| } |
| |
| virtual bool isResultValid(BuildSystem&, const BuildValue& value) override { |
| // Always rebuild this task. |
| return false; |
| } |
| |
| virtual void start(BuildSystemCommandInterface& bsci, |
| core::Task* task) override { } |
| virtual void providePriorValue(BuildSystemCommandInterface&, core::Task*, |
| const BuildValue&) override { } |
| virtual void provideValue(BuildSystemCommandInterface& bsci, core::Task*, |
| uintptr_t inputID, |
| const BuildValue& value) override { } |
| |
| virtual BuildValue execute(BuildSystemCommandInterface& bsci, |
| core::Task* task, |
| QueueJobContext* context) override { |
| // Construct the command line used to query the swift compiler version. |
| // |
| // FIXME: Need a decent subprocess interface. |
| SmallString<256> command; |
| llvm::raw_svector_ostream commandOS(command); |
| commandOS << executable; |
| commandOS << " " << "--version"; |
| |
| // Read the result. |
| FILE *fp = basic::sys::popen(commandOS.str().str().c_str(), "r"); |
| SmallString<4096> result; |
| if (fp) { |
| char buf[4096]; |
| for (;;) { |
| ssize_t numRead = fread(buf, 1, sizeof(buf), fp); |
| if (numRead == 0) { |
| // FIXME: Error handling. |
| break; |
| } |
| result.append(StringRef(buf, numRead)); |
| } |
| basic::sys::pclose(fp); |
| } |
| |
| // For now, we can get away with just encoding this as a successful |
| // command and relying on the signature to detect changes. |
| // |
| // FIXME: We should support BuildValues with arbitrary payloads. |
| return BuildValue::makeSuccessfulCommand( |
| basic::FileInfo{}, basic::hashString(result)); |
| } |
| }; |
| |
| class SwiftCompilerShellCommand : public ExternalCommand { |
| /// The compiler command to invoke. |
| std::string executable = "swiftc"; |
| |
| /// The name of the module. |
| std::string moduleName; |
| |
| /// The path of the output module. |
| std::string moduleOutputPath; |
| |
| /// The list of sources (combined). |
| std::vector<std::string> sourcesList; |
| |
| /// The list of objects (combined). |
| std::vector<std::string> objectsList; |
| |
| /// The list of import paths (combined). |
| std::vector<std::string> importPaths; |
| |
| /// The directory in which to store temporary files. |
| std::string tempsPath; |
| |
| /// Additional arguments, as a string. |
| std::vector<std::string> otherArgs; |
| |
| /// Whether the sources are part of a library or not. |
| bool isLibrary = false; |
| |
| /// Whether to enable -whole-module-optimization. |
| bool enableWholeModuleOptimization = false; |
| |
| /// Enables multi-threading with the thread count if > 0. |
| std::string numThreads = "0"; |
| |
| virtual uint64_t getSignature() override { |
| // FIXME: Use a more appropriate hashing infrastructure. |
| using llvm::hash_combine; |
| llvm::hash_code code = ExternalCommand::getSignature(); |
| code = hash_combine(code, executable); |
| code = hash_combine(code, moduleName); |
| code = hash_combine(code, moduleOutputPath); |
| for (const auto& item: sourcesList) { |
| code = hash_combine(code, item); |
| } |
| for (const auto& item: objectsList) { |
| code = hash_combine(code, item); |
| } |
| for (const auto& item: importPaths) { |
| code = hash_combine(code, item); |
| } |
| code = hash_combine(code, tempsPath); |
| for (const auto& item: otherArgs) { |
| code = hash_combine(code, item); |
| } |
| code = hash_combine(code, isLibrary); |
| return size_t(code); |
| } |
| |
| /// Get the path to use for the output file map. |
| void getOutputFileMapPath(SmallVectorImpl<char>& result) const { |
| llvm::sys::path::append(result, tempsPath, "output-file-map.json"); |
| } |
| |
| /// Compute the complete set of command line arguments to invoke swift with. |
| void constructCommandLineArgs(StringRef outputFileMapPath, |
| std::vector<StringRef>& result) const { |
| result.push_back(executable); |
| result.push_back("-module-name"); |
| result.push_back(moduleName); |
| result.push_back("-incremental"); |
| result.push_back("-emit-dependencies"); |
| if (!moduleOutputPath.empty()) { |
| result.push_back("-emit-module"); |
| result.push_back("-emit-module-path"); |
| result.push_back(moduleOutputPath); |
| } |
| result.push_back("-output-file-map"); |
| result.push_back(outputFileMapPath); |
| if (isLibrary) { |
| result.push_back("-parse-as-library"); |
| } |
| if (enableWholeModuleOptimization) { |
| result.push_back("-whole-module-optimization"); |
| } |
| result.push_back("-num-threads"); |
| result.push_back(numThreads); |
| result.push_back("-c"); |
| for (const auto& source: sourcesList) { |
| result.push_back(source); |
| } |
| for (const auto& import: importPaths) { |
| result.push_back("-I"); |
| result.push_back(import); |
| } |
| for (const auto& arg: otherArgs) { |
| result.push_back(arg); |
| } |
| } |
| |
| public: |
| using ExternalCommand::ExternalCommand; |
| |
| virtual void getShortDescription(SmallVectorImpl<char> &result) override { |
| llvm::raw_svector_ostream(result) |
| << "Compile Swift Module '" << moduleName |
| << "' (" << sourcesList.size() << " sources)"; |
| } |
| |
| virtual void getVerboseDescription(SmallVectorImpl<char> &result) override { |
| SmallString<64> outputFileMapPath; |
| getOutputFileMapPath(outputFileMapPath); |
| |
| std::vector<StringRef> commandLine; |
| constructCommandLineArgs(outputFileMapPath, commandLine); |
| |
| llvm::raw_svector_ostream os(result); |
| bool first = true; |
| for (const auto& arg: commandLine) { |
| if (!first) os << " "; |
| first = false; |
| // FIXME: This isn't correct, we need utilities for doing shell quoting. |
| if (arg.find(' ') != StringRef::npos) { |
| os << '"' << arg << '"'; |
| } else { |
| os << arg; |
| } |
| } |
| } |
| |
| virtual bool configureAttribute(const ConfigureContext& ctx, StringRef name, |
| StringRef value) override { |
| if (name == "executable") { |
| executable = value; |
| } else if (name == "module-name") { |
| moduleName = value; |
| } else if (name == "module-output-path") { |
| moduleOutputPath = value; |
| } else if (name == "sources") { |
| SmallVector<StringRef, 32> sources; |
| StringRef(value).split(sources, " ", /*MaxSplit=*/-1, |
| /*KeepEmpty=*/false); |
| sourcesList = std::vector<std::string>(sources.begin(), sources.end()); |
| } else if (name == "objects") { |
| SmallVector<StringRef, 32> objects; |
| StringRef(value).split(objects, " ", /*MaxSplit=*/-1, |
| /*KeepEmpty=*/false); |
| objectsList = std::vector<std::string>(objects.begin(), objects.end()); |
| } else if (name == "import-paths") { |
| SmallVector<StringRef, 32> imports; |
| StringRef(value).split(imports, " ", /*MaxSplit=*/-1, |
| /*KeepEmpty=*/false); |
| importPaths = std::vector<std::string>(imports.begin(), imports.end()); |
| } else if (name == "temps-path") { |
| tempsPath = value; |
| } else if (name == "is-library") { |
| if (!configureBool(ctx, isLibrary, name, value)) |
| return false; |
| } else if (name == "enable-whole-module-optimization") { |
| if (!configureBool(ctx, enableWholeModuleOptimization, name, value)) |
| return false; |
| } else if (name == "num-threads") { |
| int numThreadsInt = 0; |
| if (value.getAsInteger(10, numThreadsInt)) { |
| ctx.error("'" + name + "' should be an int."); |
| return false; |
| } |
| if (numThreadsInt < 0) { |
| ctx.error("'" + name + "' should be greater than or equal to zero."); |
| return false; |
| } |
| numThreads = value; |
| } else if (name == "other-args") { |
| SmallVector<StringRef, 32> args; |
| StringRef(value).split(args, " ", /*MaxSplit=*/-1, |
| /*KeepEmpty=*/false); |
| otherArgs = std::vector<std::string>(args.begin(), args.end()); |
| } else { |
| return ExternalCommand::configureAttribute(ctx, name, value); |
| } |
| |
| return true; |
| } |
| |
| // Extracts and stores the bool value of an attribute inside "to" variable. |
| // Returns true on success and false on error. |
| bool configureBool(const ConfigureContext& ctx, bool& to, StringRef name, StringRef value) { |
| if (value != "true" && value != "false") { |
| ctx.error("invalid value: '" + value + "' for attribute '" + |
| name + "'"); |
| return false; |
| } |
| to = value == "true"; |
| return true; |
| } |
| |
| virtual bool configureAttribute(const ConfigureContext& ctx, StringRef name, |
| ArrayRef<StringRef> values) override { |
| if (name == "sources") { |
| sourcesList = std::vector<std::string>(values.begin(), values.end()); |
| } else if (name == "objects") { |
| objectsList = std::vector<std::string>(values.begin(), values.end()); |
| } else if (name == "import-paths") { |
| importPaths = std::vector<std::string>(values.begin(), values.end()); |
| } else if (name == "other-args") { |
| otherArgs = std::vector<std::string>(values.begin(), values.end()); |
| } else { |
| return ExternalCommand::configureAttribute(ctx, name, values); |
| } |
| |
| return true; |
| } |
| |
| virtual bool configureAttribute( |
| const ConfigureContext& ctx, StringRef name, |
| ArrayRef<std::pair<StringRef, StringRef>> values) override { |
| return ExternalCommand::configureAttribute(ctx, name, values); |
| } |
| |
| bool writeOutputFileMap(BuildSystemCommandInterface& bsci, |
| StringRef outputFileMapPath, |
| std::vector<std::string>& depsFiles_out) const { |
| // FIXME: We need to properly escape everything we write here. |
| assert(sourcesList.size() == objectsList.size()); |
| |
| SmallString<16> data; |
| std::error_code ec; |
| llvm::raw_fd_ostream os(outputFileMapPath, ec, |
| llvm::sys::fs::OpenFlags::F_Text); |
| if (ec) { |
| bsci.getDelegate().error( |
| "", {}, |
| "unable to create output file map: '" + outputFileMapPath + "'"); |
| return false; |
| } |
| |
| os << "{\n"; |
| |
| // Write the master file dependencies entry. |
| SmallString<16> masterDepsPath; |
| llvm::sys::path::append(masterDepsPath, tempsPath, "master.swiftdeps"); |
| os << " \"\": {\n"; |
| if (enableWholeModuleOptimization) { |
| SmallString<16> depsPath; |
| llvm::sys::path::append(depsPath, tempsPath, moduleName + ".d"); |
| depsFiles_out.push_back(depsPath.str()); |
| SmallString<16> object; |
| llvm::sys::path::append(object, tempsPath, moduleName + ".o"); |
| os << " \"dependencies\": \"" << depsPath << "\",\n"; |
| os << " \"object\": \"" << object << "\",\n"; |
| } |
| os << " \"swift-dependencies\": \"" << masterDepsPath << "\"\n"; |
| os << " },\n"; |
| |
| // Write out the entries for each source file. |
| for (unsigned i = 0; i != sourcesList.size(); ++i) { |
| auto source = sourcesList[i]; |
| auto object = objectsList[i]; |
| auto sourceStem = llvm::sys::path::stem(source); |
| SmallString<16> partialModulePath; |
| llvm::sys::path::append(partialModulePath, tempsPath, |
| sourceStem + "~partial.swiftmodule"); |
| SmallString<16> swiftDepsPath; |
| llvm::sys::path::append(swiftDepsPath, tempsPath, |
| sourceStem + ".swiftdeps"); |
| |
| os << " \"" << source << "\": {\n"; |
| if (!enableWholeModuleOptimization) { |
| SmallString<16> depsPath; |
| llvm::sys::path::append(depsPath, tempsPath, sourceStem + ".d"); |
| os << " \"dependencies\": \"" << depsPath << "\",\n"; |
| depsFiles_out.push_back(depsPath.str()); |
| } |
| os << " \"object\": \"" << object << "\",\n"; |
| os << " \"swiftmodule\": \"" << partialModulePath << "\",\n"; |
| os << " \"swift-dependencies\": \"" << swiftDepsPath << "\"\n"; |
| os << " }" << ((i + 1) < sourcesList.size() ? "," : "") << "\n"; |
| } |
| |
| os << "}\n"; |
| |
| os.close(); |
| |
| return true; |
| } |
| |
| bool processDiscoveredDependencies(BuildSystemCommandInterface& bsci, |
| core::Task* task, StringRef depsPath) { |
| // Read the dependencies file. |
| auto input = bsci.getDelegate().getFileSystem().getFileContents(depsPath); |
| if (!input) { |
| bsci.getDelegate().error( |
| "", {}, |
| "unable to open dependencies file '" + depsPath + "'"); |
| return false; |
| } |
| |
| // Parse the output. |
| // |
| // We just ignore the rule, and add any dependency that we encounter in the |
| // file. |
| struct DepsActions : public core::MakefileDepsParser::ParseActions { |
| BuildSystemCommandInterface& bsci; |
| core::Task* task; |
| StringRef depsPath; |
| unsigned numErrors{0}; |
| unsigned ruleNumber{0}; |
| |
| DepsActions(BuildSystemCommandInterface& bsci, core::Task* task, |
| StringRef depsPath) |
| : bsci(bsci), task(task), depsPath(depsPath) {} |
| |
| virtual void error(const char* message, uint64_t position) override { |
| bsci.getDelegate().error( |
| "", {}, |
| "error reading dependency file: '" + depsPath + "' (" + |
| std::string(message) + ")"); |
| ++numErrors; |
| } |
| |
| virtual void actOnRuleDependency(const char* dependency, |
| uint64_t length, |
| const StringRef unescapedWord) override { |
| // Only process dependencies for the first rule (the output file), the |
| // rest are identical. |
| if (ruleNumber == 0) { |
| bsci.taskDiscoveredDependency( |
| task, BuildKey::makeNode(unescapedWord)); |
| } |
| } |
| |
| virtual void actOnRuleStart(const char* name, uint64_t length, |
| const StringRef unescapedWord) override {} |
| |
| virtual void actOnRuleEnd() override { |
| ++ruleNumber; |
| } |
| }; |
| |
| DepsActions actions(bsci, task, depsPath); |
| core::MakefileDepsParser(input->getBufferStart(), input->getBufferSize(), |
| actions).parse(); |
| return actions.numErrors == 0; |
| } |
| |
| /// Overridden start to also introduce a dependency on the Swift compiler |
| /// version. |
| virtual void start(BuildSystemCommandInterface& bsci, |
| core::Task* task) override { |
| ExternalCommand::start(bsci, task); |
| |
| // The Swift compiler version is also an input. |
| // |
| // FIXME: We need to fix the input ID situation, this is not extensible. We |
| // either have to build a registration of the custom tasks so they can divy |
| // up the input ID namespace, or we should just use the keys. Probably move |
| // to just using the keys, unless there is a place where that is really not |
| // cheap. |
| auto getVersionKey = BuildKey::makeCustomTask( |
| "swift-get-version", executable); |
| bsci.taskNeedsInput(task, getVersionKey, |
| core::BuildEngine::kMaximumInputID - 1); |
| } |
| |
| /// Overridden to access the Swift compiler version. |
| virtual void provideValue(BuildSystemCommandInterface& bsci, |
| core::Task* task, |
| uintptr_t inputID, |
| const BuildValue& value) override { |
| // We can ignore the 'swift-get-version' input, it is just used to detect |
| // that we need to rebuild. |
| if (inputID == core::BuildEngine::kMaximumInputID - 1) { |
| return; |
| } |
| |
| ExternalCommand::provideValue(bsci, task, inputID, value); |
| } |
| |
| virtual CommandResult executeExternalCommand(BuildSystemCommandInterface& bsci, |
| core::Task* task, |
| QueueJobContext* context) override { |
| // FIXME: Need to add support for required parameters. |
| if (sourcesList.empty()) { |
| bsci.getDelegate().error("", {}, "no configured 'sources'"); |
| return CommandResult::Failed; |
| } |
| if (objectsList.empty()) { |
| bsci.getDelegate().error("", {}, "no configured 'objects'"); |
| return CommandResult::Failed; |
| } |
| if (moduleName.empty()) { |
| bsci.getDelegate().error("", {}, "no configured 'module-name'"); |
| return CommandResult::Failed; |
| } |
| if (tempsPath.empty()) { |
| bsci.getDelegate().error("", {}, "no configured 'temps-path'"); |
| return CommandResult::Failed; |
| } |
| |
| if (sourcesList.size() != objectsList.size()) { |
| bsci.getDelegate().error( |
| "", {}, "'sources' and 'objects' are not the same size"); |
| return CommandResult::Failed; |
| } |
| |
| // Ensure the temporary directory exists. |
| // |
| // We ignore failures here, and just let things that depend on this fail. |
| // |
| // FIXME: This should really be done using an additional implicit input, so |
| // it only happens once per build. |
| (void) bsci.getDelegate().getFileSystem().createDirectories(tempsPath); |
| |
| SmallString<64> outputFileMapPath; |
| getOutputFileMapPath(outputFileMapPath); |
| |
| // Form the complete command. |
| std::vector<StringRef> commandLine; |
| constructCommandLineArgs(outputFileMapPath, commandLine); |
| |
| // Write the output file map. |
| std::vector<std::string> depsFiles; |
| if (!writeOutputFileMap(bsci, outputFileMapPath, depsFiles)) |
| return CommandResult::Failed; |
| |
| // Execute the command. |
| auto result = bsci.getExecutionQueue().executeProcess(context, commandLine); |
| |
| if (result != CommandResult::Succeeded) { |
| // If the command failed, there is no need to gather dependencies. |
| return result; |
| } |
| |
| // Load all of the discovered dependencies. |
| for (const auto& depsPath: depsFiles) { |
| if (!processDiscoveredDependencies(bsci, task, depsPath)) |
| return CommandResult::Failed; |
| } |
| |
| return result; |
| } |
| }; |
| |
| class SwiftCompilerTool : public Tool { |
| public: |
| SwiftCompilerTool(StringRef name) : Tool(name) {} |
| |
| virtual bool configureAttribute(const ConfigureContext& ctx, StringRef name, |
| StringRef value) override { |
| // No supported attributes. |
| ctx.error("unexpected attribute: '" + name + "'"); |
| return false; |
| } |
| virtual bool configureAttribute(const ConfigureContext& ctx, StringRef name, |
| ArrayRef<StringRef> values) override { |
| // No supported attributes. |
| ctx.error("unexpected attribute: '" + name + "'"); |
| return false; |
| } |
| virtual bool configureAttribute( |
| const ConfigureContext&ctx, StringRef name, |
| ArrayRef<std::pair<StringRef, StringRef>> values) override { |
| // No supported attributes. |
| ctx.error("unexpected attribute: '" + name + "'"); |
| return false; |
| } |
| |
| virtual std::unique_ptr<Command> createCommand(StringRef name) override { |
| return llvm::make_unique<SwiftCompilerShellCommand>(name); |
| } |
| |
| virtual std::unique_ptr<Command> |
| createCustomCommand(const BuildKey& key) override { |
| if (key.getCustomTaskName() == "swift-get-version" ) { |
| return llvm::make_unique<SwiftGetVersionCommand>(key); |
| } |
| |
| return nullptr; |
| } |
| }; |
| } |
| |
| std::unique_ptr<Tool> buildsystem::createSwiftCompilerTool(StringRef name) { |
| return llvm::make_unique<SwiftCompilerTool>(name); |
| } |