| //===-- ExternalCommand.cpp -----------------------------------------------===// |
| // |
| // This source file is part of the Swift.org open source project |
| // |
| // Copyright (c) 2014 - 2015 Apple Inc. and the Swift project authors |
| // Licensed under Apache License v2.0 with Runtime Library Exception |
| // |
| // See http://swift.org/LICENSE.txt for license information |
| // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors |
| // |
| //===----------------------------------------------------------------------===// |
| |
| #include "llbuild/BuildSystem/ExternalCommand.h" |
| |
| #include "llbuild/Basic/Hashing.h" |
| #include "llbuild/Basic/FileSystem.h" |
| #include "llbuild/BuildSystem/BuildExecutionQueue.h" |
| #include "llbuild/BuildSystem/BuildFile.h" |
| #include "llbuild/BuildSystem/BuildKey.h" |
| #include "llbuild/BuildSystem/BuildNode.h" |
| #include "llbuild/BuildSystem/BuildSystemCommandInterface.h" |
| #include "llbuild/BuildSystem/BuildValue.h" |
| |
| #include "llbuild/Basic/FileInfo.h" |
| #include "llbuild/Basic/LLVM.h" |
| |
| #include "llvm/ADT/Hashing.h" |
| #include "llvm/ADT/Twine.h" |
| #include "llvm/Support/FileSystem.h" |
| #include "llvm/Support/Path.h" |
| |
| using namespace llbuild; |
| using namespace llbuild::basic; |
| using namespace llbuild::buildsystem; |
| |
| uint64_t ExternalCommand::getSignature() { |
| // FIXME: Use a more appropriate hashing infrastructure. |
| using llvm::hash_combine; |
| llvm::hash_code code = hash_value(getName()); |
| for (const auto* input: inputs) { |
| code = hash_combine(code, input->getName()); |
| } |
| for (const auto* output: outputs) { |
| code = hash_combine(code, output->getName()); |
| } |
| code = hash_combine(code, allowMissingInputs); |
| code = hash_combine(code, allowModifiedOutputs); |
| code = hash_combine(code, alwaysOutOfDate); |
| return size_t(code); |
| } |
| |
| void ExternalCommand::configureDescription(const ConfigureContext&, |
| StringRef value) { |
| description = value; |
| } |
| |
| void ExternalCommand:: |
| configureInputs(const ConfigureContext&, |
| const std::vector<Node*>& value) { |
| inputs.reserve(value.size()); |
| for (auto* node: value) { |
| inputs.emplace_back(static_cast<BuildNode*>(node)); |
| } |
| } |
| |
| void ExternalCommand:: |
| configureOutputs(const ConfigureContext&, const std::vector<Node*>& value) { |
| outputs.reserve(value.size()); |
| for (auto* node: value) { |
| outputs.emplace_back(static_cast<BuildNode*>(node)); |
| } |
| } |
| |
| bool ExternalCommand:: |
| configureAttribute(const ConfigureContext& ctx, StringRef name, |
| StringRef value) { |
| if (name == "allow-missing-inputs") { |
| if (value != "true" && value != "false") { |
| ctx.error("invalid value: '" + value + "' for attribute '" + |
| name + "'"); |
| return false; |
| } |
| allowMissingInputs = value == "true"; |
| return true; |
| } else if (name == "allow-modified-outputs") { |
| if (value != "true" && value != "false") { |
| ctx.error("invalid value: '" + value + "' for attribute '" + |
| name + "'"); |
| return false; |
| } |
| allowModifiedOutputs = value == "true"; |
| return true; |
| } else if (name == "always-out-of-date") { |
| if (value != "true" && value != "false") { |
| ctx.error("invalid value: '" + value + "' for attribute '" + |
| name + "'"); |
| return false; |
| } |
| alwaysOutOfDate = value == "true"; |
| return true; |
| } else { |
| ctx.error("unexpected attribute: '" + name + "'"); |
| return false; |
| } |
| } |
| bool ExternalCommand:: |
| configureAttribute(const ConfigureContext& ctx, StringRef name, |
| ArrayRef<StringRef> values) { |
| ctx.error("unexpected attribute: '" + name + "'"); |
| return false; |
| } |
| bool ExternalCommand::configureAttribute( |
| const ConfigureContext& ctx, StringRef name, |
| ArrayRef<std::pair<StringRef, StringRef>> values) { |
| ctx.error("unexpected attribute: '" + name + "'"); |
| return false; |
| } |
| |
| BuildValue ExternalCommand:: |
| getResultForOutput(Node* node, const BuildValue& value) { |
| // If the value was a failed or cancelled command, propagate the failure. |
| if (value.isFailedCommand() || value.isPropagatedFailureCommand() || |
| value.isCancelledCommand()) |
| return BuildValue::makeFailedInput(); |
| if (value.isSkippedCommand()) |
| return BuildValue::makeSkippedCommand(); |
| |
| // Otherwise, we should have a successful command -- return the actual |
| // result for the output. |
| assert(value.isSuccessfulCommand()); |
| |
| // If the node is virtual, the output is always a virtual input value. |
| // |
| // FIXME: Eliminate this, and make the build value array contain an array of |
| // build values. |
| auto buildNode = static_cast<BuildNode*>(node); |
| if (buildNode->isVirtual() && !buildNode->isCommandTimestamp()) { |
| return BuildValue::makeVirtualInput(); |
| } |
| |
| // Find the index of the output node. |
| // |
| // FIXME: This is O(N). We don't expect N to be large in practice, but it |
| // could be. |
| auto it = std::find(outputs.begin(), outputs.end(), node); |
| assert(it != outputs.end()); |
| |
| auto idx = it - outputs.begin(); |
| assert(idx < value.getNumOutputs()); |
| |
| auto& info = value.getNthOutputInfo(idx); |
| if (info.isMissing()) |
| return BuildValue::makeMissingOutput(); |
| |
| return BuildValue::makeExistingInput(info); |
| } |
| |
| bool ExternalCommand::isResultValid(BuildSystem& system, |
| const BuildValue& value) { |
| // Treat the command as always out-of-date, if requested. |
| if (alwaysOutOfDate) |
| return false; |
| |
| // If the prior value wasn't for a successful command, recompute. |
| if (!value.isSuccessfulCommand()) |
| return false; |
| |
| // If the command's signature has changed since it was built, rebuild. |
| if (value.getCommandSignature() != getSignature()) |
| return false; |
| |
| // Check the timestamps on each of the outputs. |
| for (unsigned i = 0, e = outputs.size(); i != e; ++i) { |
| auto* node = outputs[i]; |
| |
| // Ignore virtual outputs. |
| if (node->isVirtual()) |
| continue; |
| |
| // Rebuild if the output information has changed. |
| // |
| // We intentionally allow missing outputs here, as long as they haven't |
| // changed. This is under the assumption that the commands themselves are |
| // behaving correctly when they exit successfully, and that downstream |
| // commands would diagnose required missing inputs. |
| // |
| // FIXME: CONSISTENCY: One consistency issue in this model currently is that |
| // if the output was missing, then appears, nothing will remove it; that |
| // results in an inconsistent build. What would be nice if we supported |
| // per-edge annotations on whether an output was optional -- in that case we |
| // could enforce and error on the missing output if not annotated, and we |
| // could enable behavior to remove such output files if annotated prior to |
| // running the command. |
| auto info = node->getFileInfo(system.getDelegate().getFileSystem()); |
| |
| // If this output is mutated by the build, we can't rely on equivalence, |
| // only existence. |
| if (node->isMutated()) { |
| if (value.getNthOutputInfo(i).isMissing() != info.isMissing()) |
| return false; |
| continue; |
| } |
| |
| if (value.getNthOutputInfo(i) != info) |
| return false; |
| } |
| |
| // Otherwise, the result is ok. |
| return true; |
| } |
| |
| void ExternalCommand::start(BuildSystemCommandInterface& bsci, |
| core::Task* task) { |
| // Initialize the build state. |
| skipValue = llvm::None; |
| hasMissingInput = false; |
| |
| // Request all of the inputs. |
| unsigned id = 0; |
| for (auto it = inputs.begin(), ie = inputs.end(); it != ie; ++it, ++id) { |
| bsci.taskNeedsInput(task, BuildKey::makeNode(*it), id); |
| } |
| } |
| |
| void ExternalCommand::providePriorValue(BuildSystemCommandInterface&, |
| core::Task*, |
| const BuildValue& value) { |
| if (value.isSuccessfulCommand()) { |
| hasPriorResult = true; |
| priorResultCommandSignature = value.getCommandSignature(); |
| } |
| } |
| |
| void ExternalCommand::provideValue(BuildSystemCommandInterface& bsci, |
| core::Task*, |
| uintptr_t inputID, |
| const BuildValue& value) { |
| // Process the input value to see if we should skip this command. |
| |
| // All direct inputs should be individual node values. |
| assert(!value.hasMultipleOutputs()); |
| assert(value.isExistingInput() || value.isMissingInput() || |
| value.isMissingOutput() || value.isFailedInput() || |
| value.isVirtualInput() || value.isSkippedCommand() || |
| value.isDirectoryTreeSignature() || value.isStaleFileRemoval()); |
| |
| // If the input should cause this command to skip, how should it skip? |
| auto getSkipValueForInput = [&]() -> llvm::Optional<BuildValue> { |
| // If the value is an signature, existing, or virtual input, we are always |
| // good. |
| if (value.isDirectoryTreeSignature() | value.isExistingInput() || |
| value.isVirtualInput() || value.isStaleFileRemoval()) |
| return llvm::None; |
| |
| // We explicitly allow running the command against a missing output, under |
| // the expectation that responsibility for reporting this situation falls to |
| // the command. |
| // |
| // FIXME: Eventually, it might be nice to harden the format so that we know |
| // when an output was actually required versus optional. |
| if (value.isMissingOutput()) |
| return llvm::None; |
| |
| // If the value is a missing input, but those are allowed, it is ok. |
| if (value.isMissingInput()) { |
| if (allowMissingInputs) |
| return llvm::None; |
| else |
| return BuildValue::makePropagatedFailureCommand(); |
| } |
| |
| // Propagate failure. |
| if (value.isFailedInput()) |
| return BuildValue::makePropagatedFailureCommand(); |
| |
| // A skipped dependency doesn't cause this command to skip. |
| if (value.isSkippedCommand()) |
| return llvm::None; |
| |
| llvm_unreachable("unexpected input"); |
| }; |
| |
| // Check if we need to skip the command because of this input. |
| auto skipValueForInput = getSkipValueForInput(); |
| if (skipValueForInput.hasValue()) { |
| skipValue = std::move(skipValueForInput); |
| if (value.isMissingInput()) { |
| hasMissingInput = true; |
| |
| // FIXME: Design the logging and status output APIs. |
| bsci.getDelegate().error( |
| "", {}, (Twine("missing input '") + inputs[inputID]->getName() + |
| "' and no rule to build it")); |
| } |
| } else { |
| // If there is a missing input file (from a successful command), we always |
| // need to run the command. |
| if (value.isMissingOutput()) |
| canUpdateIfNewer = false; |
| } |
| } |
| |
| bool ExternalCommand::canUpdateIfNewerWithResult(const BuildValue& result) { |
| // Unless `allowModifiedOutputs` is specified, we always need to update if |
| // ran. |
| if (!allowModifiedOutputs) |
| return false; |
| |
| // If it was specified, then we can update if all of our outputs simply exist. |
| for (unsigned i = 0, e = result.getNumOutputs(); i != e; ++i) { |
| const FileInfo& outputInfo = result.getNthOutputInfo(i); |
| |
| // If the output is missing, we need to rebuild. |
| if (outputInfo.isMissing()) |
| return false; |
| } |
| return true; |
| } |
| |
| BuildValue |
| ExternalCommand::computeCommandResult(BuildSystemCommandInterface& bsci) { |
| // Capture the file information for each of the output nodes. |
| // |
| // FIXME: We need to delegate to the node here. |
| SmallVector<FileInfo, 8> outputInfos; |
| for (auto* node: outputs) { |
| if (node->isCommandTimestamp()) { |
| // FIXME: We currently have to shoehorn the timestamp into a fake file |
| // info, but need to refactor the command result to just store the node |
| // subvalues instead. |
| FileInfo info{}; |
| info.size = bsci.getBuildEngine().getCurrentTimestamp(); |
| outputInfos.push_back(info); |
| } else if (node->isVirtual()) { |
| outputInfos.push_back(FileInfo{}); |
| } else { |
| outputInfos.push_back(node->getFileInfo( |
| bsci.getDelegate().getFileSystem())); |
| } |
| } |
| return BuildValue::makeSuccessfulCommand(outputInfos, getSignature()); |
| } |
| |
| BuildValue ExternalCommand::execute(BuildSystemCommandInterface& bsci, |
| core::Task* task, |
| QueueJobContext* context) { |
| // If this command should be skipped, do nothing. |
| if (skipValue.hasValue()) { |
| // If this command had a failed input, treat it as having failed. |
| if (hasMissingInput) { |
| // FIXME: Design the logging and status output APIs. |
| bsci.getDelegate().error( |
| "", {}, (Twine("cannot build '") + outputs[0]->getName() + |
| "' due to missing input")); |
| |
| // Report the command failure. |
| bsci.getDelegate().hadCommandFailure(); |
| } |
| |
| return std::move(skipValue.getValue()); |
| } |
| assert(!hasMissingInput); |
| |
| // If it is legal to simply update the command, then see if we can do so. |
| if (canUpdateIfNewer && |
| hasPriorResult && priorResultCommandSignature == getSignature()) { |
| BuildValue result = computeCommandResult(bsci); |
| if (canUpdateIfNewerWithResult(result)) { |
| return result; |
| } |
| } |
| |
| // Create the directories for the directories containing file outputs. |
| // |
| // FIXME: Implement a shared cache for this, to reduce the number of |
| // syscalls required to make this happen. |
| for (auto* node: outputs) { |
| if (!node->isVirtual()) { |
| // Attempt to create the directory; we ignore errors here under the |
| // assumption the command will diagnose the situation if necessary. |
| // |
| // FIXME: Need to use the filesystem interfaces. |
| auto parent = llvm::sys::path::parent_path(node->getName()); |
| if (!parent.empty()) { |
| (void) bsci.getDelegate().getFileSystem().createDirectories(parent); |
| } |
| } |
| } |
| |
| // Invoke the external command. |
| bsci.getDelegate().commandStarted(this); |
| auto result = executeExternalCommand(bsci, task, context); |
| bsci.getDelegate().commandFinished(this, result); |
| |
| // Process the result. |
| switch (result) { |
| case CommandResult::Failed: |
| return BuildValue::makeFailedCommand(); |
| case CommandResult::Cancelled: |
| return BuildValue::makeCancelledCommand(); |
| case CommandResult::Succeeded: |
| return computeCommandResult(bsci); |
| case CommandResult::Skipped: |
| // It is illegal to get skipped result at this point. |
| break; |
| } |
| llvm::report_fatal_error("unknown result"); |
| } |