blob: 07237e220c310cc1ab84403936bb00f81bdcdde4 [file] [log] [blame]
//===- tools/dsymutil/DwarfLinkerForBinary.cpp ----------------------------===//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
#include "DwarfLinkerForBinary.h"
#include "BinaryHolder.h"
#include "DebugMap.h"
#include "MachOUtils.h"
#include "dsymutil.h"
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/BitVector.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/DenseMapInfo.h"
#include "llvm/ADT/DenseSet.h"
#include "llvm/ADT/FoldingSet.h"
#include "llvm/ADT/Hashing.h"
#include "llvm/ADT/IntervalMap.h"
#include "llvm/ADT/None.h"
#include "llvm/ADT/Optional.h"
#include "llvm/ADT/PointerIntPair.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/SmallString.h"
#include "llvm/ADT/StringMap.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/ADT/Triple.h"
#include "llvm/ADT/Twine.h"
#include "llvm/BinaryFormat/Dwarf.h"
#include "llvm/BinaryFormat/MachO.h"
#include "llvm/CodeGen/AccelTable.h"
#include "llvm/CodeGen/AsmPrinter.h"
#include "llvm/CodeGen/DIE.h"
#include "llvm/CodeGen/NonRelocatableStringpool.h"
#include "llvm/Config/config.h"
#include "llvm/DWARFLinker/DWARFLinkerDeclContext.h"
#include "llvm/DebugInfo/DIContext.h"
#include "llvm/DebugInfo/DWARF/DWARFAbbreviationDeclaration.h"
#include "llvm/DebugInfo/DWARF/DWARFContext.h"
#include "llvm/DebugInfo/DWARF/DWARFDataExtractor.h"
#include "llvm/DebugInfo/DWARF/DWARFDebugLine.h"
#include "llvm/DebugInfo/DWARF/DWARFDebugRangeList.h"
#include "llvm/DebugInfo/DWARF/DWARFDie.h"
#include "llvm/DebugInfo/DWARF/DWARFFormValue.h"
#include "llvm/DebugInfo/DWARF/DWARFSection.h"
#include "llvm/DebugInfo/DWARF/DWARFUnit.h"
#include "llvm/MC/MCAsmBackend.h"
#include "llvm/MC/MCAsmInfo.h"
#include "llvm/MC/MCCodeEmitter.h"
#include "llvm/MC/MCContext.h"
#include "llvm/MC/MCDwarf.h"
#include "llvm/MC/MCInstrInfo.h"
#include "llvm/MC/MCObjectFileInfo.h"
#include "llvm/MC/MCObjectWriter.h"
#include "llvm/MC/MCRegisterInfo.h"
#include "llvm/MC/MCSection.h"
#include "llvm/MC/MCStreamer.h"
#include "llvm/MC/MCSubtargetInfo.h"
#include "llvm/MC/MCTargetOptions.h"
#include "llvm/Object/MachO.h"
#include "llvm/Object/ObjectFile.h"
#include "llvm/Object/SymbolicFile.h"
#include "llvm/Support/Allocator.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/Compiler.h"
#include "llvm/Support/DJB.h"
#include "llvm/Support/DataExtractor.h"
#include "llvm/Support/Error.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/ErrorOr.h"
#include "llvm/Support/FileSystem.h"
#include "llvm/Support/Format.h"
#include "llvm/Support/LEB128.h"
#include "llvm/Support/MathExtras.h"
#include "llvm/Support/MemoryBuffer.h"
#include "llvm/Support/Path.h"
#include "llvm/Support/TargetRegistry.h"
#include "llvm/Support/ThreadPool.h"
#include "llvm/Support/ToolOutputFile.h"
#include "llvm/Support/WithColor.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Target/TargetMachine.h"
#include "llvm/Target/TargetOptions.h"
#include "llvm/MC/MCTargetOptionsCommandFlags.h"
#include <algorithm>
#include <cassert>
#include <cinttypes>
#include <climits>
#include <cstdint>
#include <cstdlib>
#include <cstring>
#include <limits>
#include <map>
#include <memory>
#include <string>
#include <system_error>
#include <tuple>
#include <utility>
#include <vector>
namespace llvm {
static mc::RegisterMCTargetOptionsFlags MOF;
namespace dsymutil {
static Error copySwiftInterfaces(
const std::map<std::string, std::string> &ParseableSwiftInterfaces,
StringRef Architecture, const LinkOptions &Options) {
std::error_code EC;
SmallString<128> InputPath;
SmallString<128> Path;
sys::path::append(Path, *Options.ResourceDir, "Swift", Architecture);
if ((EC = sys::fs::create_directories(Path.str(), true,
return make_error<StringError>(
"cannot create directory: " + toString(errorCodeToError(EC)), EC);
unsigned BaseLength = Path.size();
for (auto &I : ParseableSwiftInterfaces) {
StringRef ModuleName = I.first;
StringRef InterfaceFile = I.second;
if (!Options.PrependPath.empty()) {
sys::path::append(InputPath, Options.PrependPath, InterfaceFile);
InterfaceFile = InputPath;
sys::path::append(Path, ModuleName);
if (Options.Verbose)
outs() << "copy parseable Swift interface " << InterfaceFile << " -> "
<< Path.str() << '\n';
// copy_file attempts an APFS clone first, so this should be cheap.
if ((EC = sys::fs::copy_file(InterfaceFile, Path.str())))
warn(Twine("cannot copy parseable Swift interface ") + InterfaceFile +
": " + toString(errorCodeToError(EC)));
return Error::success();
/// Report a warning to the user, optionally including information about a
/// specific \p DIE related to the warning.
void DwarfLinkerForBinary::reportWarning(const Twine &Warning,
StringRef Context,
const DWARFDie *DIE) const {
warn(Warning, Context);
if (!Options.Verbose || !DIE)
DIDumpOptions DumpOpts;
DumpOpts.ChildRecurseDepth = 0;
DumpOpts.Verbose = Options.Verbose;
WithColor::note() << " in DIE:\n";
DIE->dump(errs(), 6 /* Indent */, DumpOpts);
bool DwarfLinkerForBinary::createStreamer(const Triple &TheTriple,
raw_fd_ostream &OutFile) {
if (Options.NoOutput)
return true;
Streamer = std::make_unique<DwarfStreamer>(
Options.FileType, OutFile, Options.Translator, Options.Minimize,
[&](const Twine &Error, StringRef Context, const DWARFDie *) {
error(Error, Context);
[&](const Twine &Warning, StringRef Context, const DWARFDie *) {
warn(Warning, Context);
return Streamer->init(TheTriple);
ErrorOr<const object::ObjectFile &>
DwarfLinkerForBinary::loadObject(const DebugMapObject &Obj,
const Triple &Triple) {
auto ObjectEntry =
BinHolder.getObjectEntry(Obj.getObjectFilename(), Obj.getTimestamp());
if (!ObjectEntry) {
auto Err = ObjectEntry.takeError();
reportWarning(Twine(Obj.getObjectFilename()) + ": " +
return errorToErrorCode(std::move(Err));
auto Object = ObjectEntry->getObject(Triple);
if (!Object) {
auto Err = Object.takeError();
reportWarning(Twine(Obj.getObjectFilename()) + ": " +
return errorToErrorCode(std::move(Err));
return *Object;
static Error remarksErrorHandler(const DebugMapObject &DMO,
DwarfLinkerForBinary &Linker,
std::unique_ptr<FileError> FE) {
bool IsArchive = DMO.getObjectFilename().endswith(")");
// Don't report errors for missing remark files from static
// archives.
if (!IsArchive)
return Error(std::move(FE));
std::string Message = FE->message();
Error E = FE->takeError();
Error NewE = handleErrors(std::move(E), [&](std::unique_ptr<ECError> EC) {
if (EC->convertToErrorCode() != std::errc::no_such_file_or_directory)
return Error(std::move(EC));
Linker.reportWarning(Message, DMO.getObjectFilename());
return Error(Error::success());
if (!NewE)
return Error::success();
return createFileError(FE->getFileName(), std::move(NewE));
static Error emitRemarks(const LinkOptions &Options, StringRef BinaryPath,
StringRef ArchName, const remarks::RemarkLinker &RL) {
// Make sure we don't create the directories and the file if there is nothing
// to serialize.
if (RL.empty())
return Error::success();
SmallString<128> InputPath;
SmallString<128> Path;
// Create the "Remarks" directory in the "Resources" directory.
sys::path::append(Path, *Options.ResourceDir, "Remarks");
if (std::error_code EC = sys::fs::create_directories(Path.str(), true,
return errorCodeToError(EC);
// Append the file name.
// For fat binaries, also append a dash and the architecture name.
sys::path::append(Path, sys::path::filename(BinaryPath));
if (Options.NumDebugMaps > 1) {
// More than one debug map means we have a fat binary.
Path += '-';
Path += ArchName;
std::error_code EC;
raw_fd_ostream OS(Options.NoOutput ? "-" : Path.str(), EC, sys::fs::OF_None);
if (EC)
return errorCodeToError(EC);
if (Error E = RL.serialize(OS, Options.RemarksFormat))
return E;
return Error::success();
ErrorOr<DwarfFile &>
DwarfLinkerForBinary::loadObject(const DebugMapObject &Obj,
const DebugMap &DebugMap,
remarks::RemarkLinker &RL) {
auto ErrorOrObj = loadObject(Obj, DebugMap.getTriple());
if (ErrorOrObj) {
std::make_unique<AddressManager>(*this, *ErrorOrObj, Obj));
Obj.getObjectFilename(), ContextForLinking.back().get(),
Obj.empty() ? Obj.getWarnings() : EmptyWarnings));
Error E =*ErrorOrObj);
if (Error NewE = handleErrors(
std::move(E), [&](std::unique_ptr<FileError> EC) -> Error {
return remarksErrorHandler(Obj, *this, std::move(EC));
return errorToErrorCode(std::move(NewE));
return *ObjectsForLinking.back();
return ErrorOrObj.getError();
bool DwarfLinkerForBinary::link(const DebugMap &Map) {
if (!createStreamer(Map.getTriple(), OutFile))
return false;
DebugMap DebugMap(Map.getTriple(), Map.getBinaryPath());
DWARFLinker GeneralLinker(Streamer.get(), DwarfLinkerClient::Dsymutil);
remarks::RemarkLinker RL;
if (!Options.RemarksPrependPath.empty())
std::function<StringRef(StringRef)> TranslationLambda = [&](StringRef Input) {
return Options.Translator(Input);
if (Options.Translator)
[&](const Twine &Warning, StringRef Context, const DWARFDie *DIE) {
reportWarning(Warning, Context, DIE);
[&](const Twine &Error, StringRef Context, const DWARFDie *) {
error(Error, Context);
[&DebugMap, &RL, this](StringRef ContainerName,
StringRef Path) -> ErrorOr<DwarfFile &> {
auto &Obj = DebugMap.addDebugMapObject(
Path, sys::TimePoint<std::chrono::seconds>(), MachO::N_OSO);
if (auto ErrorOrObj = loadObject(Obj, DebugMap, RL)) {
return *ErrorOrObj;
} else {
// Try and emit more helpful warnings by applying some heuristics.
StringRef ObjFile = ContainerName;
bool IsClangModule = sys::path::extension(Path).equals(".pcm");
bool IsArchive = ObjFile.endswith(")");
if (IsClangModule) {
StringRef ModuleCacheDir = sys::path::parent_path(Path);
if (sys::fs::exists(ModuleCacheDir)) {
// If the module's parent directory exists, we assume that the
// module cache has expired and was pruned by clang. A more
// adventurous dsymutil would invoke clang to rebuild the module
// now.
if (!ModuleCacheHintDisplayed) {
<< "The clang module cache may have expired since "
"this object file was built. Rebuilding the "
"object file will rebuild the module cache.\n";
ModuleCacheHintDisplayed = true;
} else if (IsArchive) {
// If the module cache directory doesn't exist at all and the
// object file is inside a static library, we assume that the
// static library was built on a different machine. We don't want
// to discourage module debugging for convenience libraries within
// a project though.
if (!ArchiveHintDisplayed) {
<< "Linking a static library that was built with "
"-gmodules, but the module cache was not found. "
"Redistributable static libraries should never be "
"built with module debugging enabled. The debug "
"experience will be degraded due to incomplete "
"debug information.\n";
ArchiveHintDisplayed = true;
return ErrorOrObj.getError();
llvm_unreachable("Unhandled DebugMap object");
for (const auto &Obj : Map.objects()) {
// N_AST objects (swiftmodule files) should get dumped directly into the
// appropriate DWARF section.
if (Obj->getType() == MachO::N_AST) {
if (Options.Verbose)
outs() << "DEBUG MAP OBJECT: " << Obj->getObjectFilename() << "\n";
StringRef File = Obj->getObjectFilename();
auto ErrorOrMem = MemoryBuffer::getFile(File);
if (!ErrorOrMem) {
warn("Could not open '" + File + "'\n");
sys::fs::file_status Stat;
if (auto Err = sys::fs::status(File, Stat)) {
if (!Options.NoTimestamp) {
// The modification can have sub-second precision so we need to cast
// away the extra precision that's not present in the debug map.
auto ModificationTime =
if (ModificationTime != Obj->getTimestamp()) {
// Not using the helper here as we can easily stream TimePoint<>.
WithColor::warning() << "Timestamp mismatch for " << File << ": "
<< Stat.getLastModificationTime() << " and "
<< sys::TimePoint<>(Obj->getTimestamp()) << "\n";
// Copy the module into the .swift_ast section.
if (!Options.NoOutput)
if (auto ErrorOrObj = loadObject(*Obj, Map, RL))
else {
Obj->getObjectFilename(), nullptr, nullptr,
Obj->empty() ? Obj->getWarnings() : EmptyWarnings));
// link debug info for loaded object files.;
StringRef ArchName = Map.getTriple().getArchName();
if (Error E = emitRemarks(Options, Map.getBinaryPath(), ArchName, RL))
return error(toString(std::move(E)));
if (Options.NoOutput)
return true;
if (Options.ResourceDir && !ParseableSwiftInterfaces.empty()) {
StringRef ArchName = Triple::getArchTypeName(Map.getTriple().getArch());
if (auto E =
copySwiftInterfaces(ParseableSwiftInterfaces, ArchName, Options))
return error(toString(std::move(E)));
if (Map.getTriple().isOSDarwin() && !Map.getBinaryPath().empty() &&
Options.FileType == OutputFileType::Object)
return MachOUtils::generateDsymCompanion(
Options.VFS, Map, Options.Translator,
*Streamer->getAsmPrinter().OutStreamer, OutFile);
return true;
static bool isMachOPairedReloc(uint64_t RelocType, uint64_t Arch) {
switch (Arch) {
case Triple::x86:
return RelocType == MachO::GENERIC_RELOC_SECTDIFF ||
case Triple::x86_64:
return RelocType == MachO::X86_64_RELOC_SUBTRACTOR;
case Triple::arm:
case Triple::thumb:
return RelocType == MachO::ARM_RELOC_SECTDIFF ||
RelocType == MachO::ARM_RELOC_HALF ||
case Triple::aarch64:
return RelocType == MachO::ARM64_RELOC_SUBTRACTOR;
return false;
/// Iterate over the relocations of the given \p Section and
/// store the ones that correspond to debug map entries into the
/// ValidRelocs array.
void DwarfLinkerForBinary::AddressManager::findValidRelocsMachO(
const object::SectionRef &Section, const object::MachOObjectFile &Obj,
const DebugMapObject &DMO) {
Expected<StringRef> ContentsOrErr = Section.getContents();
if (!ContentsOrErr) {
Linker.reportWarning("error reading section", DMO.getObjectFilename());
DataExtractor Data(*ContentsOrErr, Obj.isLittleEndian(), 0);
bool SkipNext = false;
for (const object::RelocationRef &Reloc : Section.relocations()) {
if (SkipNext) {
SkipNext = false;
object::DataRefImpl RelocDataRef = Reloc.getRawDataRefImpl();
MachO::any_relocation_info MachOReloc = Obj.getRelocation(RelocDataRef);
if (isMachOPairedReloc(Obj.getAnyRelocationType(MachOReloc),
Obj.getArch())) {
SkipNext = true;
Linker.reportWarning("unsupported relocation in debug_info section.",
unsigned RelocSize = 1 << Obj.getAnyRelocationLength(MachOReloc);
uint64_t Offset64 = Reloc.getOffset();
if ((RelocSize != 4 && RelocSize != 8)) {
Linker.reportWarning("unsupported relocation in debug_info section.",
uint64_t OffsetCopy = Offset64;
// Mach-o uses REL relocations, the addend is at the relocation offset.
uint64_t Addend = Data.getUnsigned(&OffsetCopy, RelocSize);
uint64_t SymAddress;
int64_t SymOffset;
if (Obj.isRelocationScattered(MachOReloc)) {
// The address of the base symbol for scattered relocations is
// stored in the reloc itself. The actual addend will store the
// base address plus the offset.
SymAddress = Obj.getScatteredRelocationValue(MachOReloc);
SymOffset = int64_t(Addend) - SymAddress;
} else {
SymAddress = Addend;
SymOffset = 0;
auto Sym = Reloc.getSymbol();
if (Sym != Obj.symbol_end()) {
Expected<StringRef> SymbolName = Sym->getName();
if (!SymbolName) {
Linker.reportWarning("error getting relocation symbol name.",
if (const auto *Mapping = DMO.lookupSymbol(*SymbolName))
ValidRelocs.emplace_back(Offset64, RelocSize, Addend, Mapping);
} else if (const auto *Mapping = DMO.lookupObjectAddress(SymAddress)) {
// Do not store the addend. The addend was the address of the symbol in
// the object file, the address in the binary that is stored in the debug
// map doesn't need to be offset.
ValidRelocs.emplace_back(Offset64, RelocSize, SymOffset, Mapping);
/// Dispatch the valid relocation finding logic to the
/// appropriate handler depending on the object file format.
bool DwarfLinkerForBinary::AddressManager::findValidRelocs(
const object::SectionRef &Section, const object::ObjectFile &Obj,
const DebugMapObject &DMO) {
// Dispatch to the right handler depending on the file type.
if (auto *MachOObj = dyn_cast<object::MachOObjectFile>(&Obj))
findValidRelocsMachO(Section, *MachOObj, DMO);
Linker.reportWarning(Twine("unsupported object file type: ") +
if (ValidRelocs.empty())
return false;
// Sort the relocations by offset. We will walk the DIEs linearly in
// the file, this allows us to just keep an index in the relocation
// array that we advance during our walk, rather than resorting to
// some associative container. See DwarfLinkerForBinary::NextValidReloc.
return true;
/// Look for relocations in the debug_info section that match
/// entries in the debug map. These relocations will drive the Dwarf
/// link by indicating which DIEs refer to symbols present in the
/// linked binary.
/// \returns whether there are any valid relocations in the debug info.
bool DwarfLinkerForBinary::AddressManager::findValidRelocsInDebugInfo(
const object::ObjectFile &Obj, const DebugMapObject &DMO) {
// Find the debug_info section.
for (const object::SectionRef &Section : Obj.sections()) {
StringRef SectionName;
if (Expected<StringRef> NameOrErr = Section.getName())
SectionName = *NameOrErr;
SectionName = SectionName.substr(SectionName.find_first_not_of("._"));
if (SectionName != "debug_info")
return findValidRelocs(Section, Obj, DMO);
return false;
/// Checks that there is a relocation against an actual debug
/// map entry between \p StartOffset and \p NextOffset.
/// This function must be called with offsets in strictly ascending
/// order because it never looks back at relocations it already 'went past'.
/// \returns true and sets Info.InDebugMap if it is the case.
bool DwarfLinkerForBinary::AddressManager::hasValidRelocationAt(
uint64_t StartOffset, uint64_t EndOffset, CompileUnit::DIEInfo &Info) {
assert(NextValidReloc == 0 ||
StartOffset > ValidRelocs[NextValidReloc - 1].Offset);
if (NextValidReloc >= ValidRelocs.size())
return false;
uint64_t RelocOffset = ValidRelocs[NextValidReloc].Offset;
// We might need to skip some relocs that we didn't consider. For
// example the high_pc of a discarded DIE might contain a reloc that
// is in the list because it actually corresponds to the start of a
// function that is in the debug map.
while (RelocOffset < StartOffset && NextValidReloc < ValidRelocs.size() - 1)
RelocOffset = ValidRelocs[++NextValidReloc].Offset;
if (RelocOffset < StartOffset || RelocOffset >= EndOffset)
return false;
const auto &ValidReloc = ValidRelocs[NextValidReloc++];
const auto &Mapping = ValidReloc.Mapping->getValue();
const uint64_t BinaryAddress = Mapping.BinaryAddress;
const uint64_t ObjectAddress = Mapping.ObjectAddress
? uint64_t(*Mapping.ObjectAddress)
: std::numeric_limits<uint64_t>::max();
if (Linker.Options.Verbose)
outs() << "Found valid debug map entry: " << ValidReloc.Mapping->getKey()
<< "\t"
<< format("0x%016" PRIx64 " => 0x%016" PRIx64 "\n", ObjectAddress,
Info.AddrAdjust = BinaryAddress + ValidReloc.Addend;
if (Mapping.ObjectAddress)
Info.AddrAdjust -= ObjectAddress;
Info.InDebugMap = true;
return true;
/// Apply the valid relocations found by findValidRelocs() to
/// the buffer \p Data, taking into account that Data is at \p BaseOffset
/// in the debug_info section.
/// Like for findValidRelocs(), this function must be called with
/// monotonic \p BaseOffset values.
/// \returns whether any reloc has been applied.
bool DwarfLinkerForBinary::AddressManager::applyValidRelocs(
MutableArrayRef<char> Data, uint64_t BaseOffset, bool IsLittleEndian) {
assert((NextValidReloc == 0 ||
BaseOffset > ValidRelocs[NextValidReloc - 1].Offset) &&
"BaseOffset should only be increasing.");
if (NextValidReloc >= ValidRelocs.size())
return false;
// Skip relocs that haven't been applied.
while (NextValidReloc < ValidRelocs.size() &&
ValidRelocs[NextValidReloc].Offset < BaseOffset)
bool Applied = false;
uint64_t EndOffset = BaseOffset + Data.size();
while (NextValidReloc < ValidRelocs.size() &&
ValidRelocs[NextValidReloc].Offset >= BaseOffset &&
ValidRelocs[NextValidReloc].Offset < EndOffset) {
const auto &ValidReloc = ValidRelocs[NextValidReloc++];
assert(ValidReloc.Offset - BaseOffset < Data.size());
assert(ValidReloc.Offset - BaseOffset + ValidReloc.Size <= Data.size());
char Buf[8];
uint64_t Value = ValidReloc.Mapping->getValue().BinaryAddress;
Value += ValidReloc.Addend;
for (unsigned I = 0; I != ValidReloc.Size; ++I) {
unsigned Index = IsLittleEndian ? I : (ValidReloc.Size - I - 1);
Buf[I] = uint8_t(Value >> (Index * 8));
assert(ValidReloc.Size <= sizeof(Buf));
memcpy(&Data[ValidReloc.Offset - BaseOffset], Buf, ValidReloc.Size);
Applied = true;
return Applied;
bool linkDwarf(raw_fd_ostream &OutFile, BinaryHolder &BinHolder,
const DebugMap &DM, LinkOptions Options) {
DwarfLinkerForBinary Linker(OutFile, BinHolder, std::move(Options));
} // namespace dsymutil
} // namespace llvm