| //===--- IndexRecordWriter.cpp - Index record serialization ---------------===// |
| // |
| // The LLVM Compiler Infrastructure |
| // |
| // This file is distributed under the University of Illinois Open Source |
| // License. See LICENSE.TXT for details. |
| // |
| //===----------------------------------------------------------------------===// |
| |
| #include "clang/Index/IndexRecordWriter.h" |
| #include "IndexDataStoreUtils.h" |
| #include "indexstore/indexstore.h" |
| #include "clang/Index/IndexDataStoreSymbolUtils.h" |
| #include "llvm/ADT/APInt.h" |
| #include "llvm/ADT/DenseMap.h" |
| #include "llvm/ADT/StringSet.h" |
| #include "llvm/Bitcode/BitstreamWriter.h" |
| #include "llvm/Support/Errc.h" |
| #include "llvm/Support/FileSystem.h" |
| #include "llvm/Support/Path.h" |
| #include "llvm/Support/raw_ostream.h" |
| |
| using namespace clang; |
| using namespace clang::index; |
| using namespace clang::index::store; |
| using namespace llvm; |
| |
| using writer::OpaqueDecl; |
| |
| namespace { |
| struct DeclInfo { |
| OpaqueDecl D; |
| SymbolRoleSet Roles; |
| SymbolRoleSet RelatedRoles; |
| }; |
| |
| struct OccurrenceInfo { |
| unsigned DeclID; |
| OpaqueDecl D; |
| SymbolRoleSet Roles; |
| unsigned Line; |
| unsigned Column; |
| SmallVector<std::pair<writer::SymbolRelation, unsigned>, 4> Related; |
| }; |
| |
| struct RecordState { |
| std::string RecordPath; |
| SmallString<512> Buffer; |
| BitstreamWriter Stream; |
| |
| DenseMap<OpaqueDecl, unsigned> IndexForDecl; |
| std::vector<DeclInfo> Decls; |
| std::vector<OccurrenceInfo> Occurrences; |
| |
| RecordState(std::string &&RecordPath) |
| : RecordPath(std::move(RecordPath)), Stream(Buffer) {} |
| }; |
| } // end anonymous namespace |
| |
| static void writeBlockInfo(BitstreamWriter &Stream) { |
| RecordData Record; |
| |
| Stream.EnterBlockInfoBlock(); |
| #define BLOCK(X) emitBlockID(X ## _ID, #X, Stream, Record) |
| #define RECORD(X) emitRecordID(X, #X, Stream, Record) |
| |
| BLOCK(REC_VERSION_BLOCK); |
| RECORD(REC_VERSION); |
| |
| BLOCK(REC_DECLS_BLOCK); |
| RECORD(REC_DECLINFO); |
| |
| BLOCK(REC_DECLOFFSETS_BLOCK); |
| RECORD(REC_DECLOFFSETS); |
| |
| BLOCK(REC_DECLOCCURRENCES_BLOCK); |
| RECORD(REC_DECLOCCURRENCE); |
| |
| #undef RECORD |
| #undef BLOCK |
| Stream.ExitBlock(); |
| } |
| |
| static void writeVersionInfo(BitstreamWriter &Stream) { |
| using namespace llvm::sys; |
| |
| Stream.EnterSubblock(REC_VERSION_BLOCK_ID, 3); |
| |
| auto Abbrev = std::make_shared<BitCodeAbbrev>(); |
| Abbrev->Add(BitCodeAbbrevOp(REC_VERSION)); |
| Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Store format version |
| unsigned AbbrevCode = Stream.EmitAbbrev(std::move(Abbrev)); |
| |
| RecordData Record; |
| Record.push_back(REC_VERSION); |
| Record.push_back(STORE_FORMAT_VERSION); |
| Stream.EmitRecordWithAbbrev(AbbrevCode, Record); |
| |
| Stream.ExitBlock(); |
| } |
| |
| template <typename T, typename Allocator> |
| static StringRef data(const std::vector<T, Allocator> &v) { |
| if (v.empty()) |
| return StringRef(); |
| return StringRef(reinterpret_cast<const char *>(&v[0]), sizeof(T) * v.size()); |
| } |
| |
| template <typename T> static StringRef data(const SmallVectorImpl<T> &v) { |
| return StringRef(reinterpret_cast<const char *>(v.data()), |
| sizeof(T) * v.size()); |
| } |
| |
| static void writeDecls(BitstreamWriter &Stream, ArrayRef<DeclInfo> Decls, |
| ArrayRef<OccurrenceInfo> Occurrences, |
| writer::SymbolWriterCallback GetSymbolForDecl) { |
| SmallVector<uint32_t, 32> DeclOffsets; |
| DeclOffsets.reserve(Decls.size()); |
| |
| //===--------------------------------------------------------------------===// |
| // DECLS_BLOCK_ID |
| //===--------------------------------------------------------------------===// |
| |
| Stream.EnterSubblock(REC_DECLS_BLOCK_ID, 3); |
| |
| auto Abbrev = std::make_shared<BitCodeAbbrev>(); |
| Abbrev->Add(BitCodeAbbrevOp(REC_DECLINFO)); |
| Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 5)); // Kind |
| Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 5)); // SubKind |
| Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 5)); // Language |
| Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, SymbolPropertyBitNum)); // Properties |
| Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, SymbolRoleBitNum)); // Roles |
| Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, SymbolRoleBitNum)); // Related Roles |
| Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Length of name in block |
| Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Length of USR in block |
| Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob)); // Name + USR + CodeGen symbol name |
| unsigned AbbrevCode = Stream.EmitAbbrev(std::move(Abbrev)); |
| |
| #ifndef NDEBUG |
| StringSet<> USRSet; |
| #endif |
| |
| RecordData Record; |
| llvm::SmallString<256> Blob; |
| llvm::SmallString<256> Scratch; |
| for (auto &Info : Decls) { |
| DeclOffsets.push_back(Stream.GetCurrentBitNo()); |
| Blob.clear(); |
| Scratch.clear(); |
| |
| writer::Symbol SymInfo = GetSymbolForDecl(Info.D, Scratch); |
| assert(SymInfo.SymInfo.Kind != SymbolKind::Unknown); |
| assert(!SymInfo.USR.empty() && "Recorded decl without USR!"); |
| |
| Blob += SymInfo.Name; |
| Blob += SymInfo.USR; |
| Blob += SymInfo.CodeGenName; |
| |
| #ifndef NDEBUG |
| bool IsNew = USRSet.insert(SymInfo.USR).second; |
| if (!IsNew) { |
| llvm::errs() << "Index: Duplicate USR! " << SymInfo.USR << "\n"; |
| // FIXME: print more information so it's easier to find the declaration. |
| } |
| #endif |
| |
| Record.clear(); |
| Record.push_back(REC_DECLINFO); |
| Record.push_back(getIndexStoreKind(SymInfo.SymInfo.Kind)); |
| Record.push_back(getIndexStoreSubKind(SymInfo.SymInfo.SubKind)); |
| Record.push_back(getIndexStoreLang(SymInfo.SymInfo.Lang)); |
| Record.push_back(getIndexStoreProperties(SymInfo.SymInfo.Properties)); |
| Record.push_back(getIndexStoreRoles(Info.Roles)); |
| Record.push_back(getIndexStoreRoles(Info.RelatedRoles)); |
| Record.push_back(SymInfo.Name.size()); |
| Record.push_back(SymInfo.USR.size()); |
| Stream.EmitRecordWithBlob(AbbrevCode, Record, Blob); |
| } |
| |
| Stream.ExitBlock(); |
| |
| //===--------------------------------------------------------------------===// |
| // DECLOFFSETS_BLOCK_ID |
| //===--------------------------------------------------------------------===// |
| |
| Stream.EnterSubblock(REC_DECLOFFSETS_BLOCK_ID, 3); |
| |
| Abbrev = std::make_shared<BitCodeAbbrev>(); |
| Abbrev->Add(BitCodeAbbrevOp(REC_DECLOFFSETS)); |
| Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32)); // Number of Decls |
| Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob)); // Offsets array |
| AbbrevCode = Stream.EmitAbbrev(std::move(Abbrev)); |
| |
| Record.clear(); |
| Record.push_back(REC_DECLOFFSETS); |
| Record.push_back(DeclOffsets.size()); |
| Stream.EmitRecordWithBlob(AbbrevCode, Record, data(DeclOffsets)); |
| |
| Stream.ExitBlock(); |
| |
| //===--------------------------------------------------------------------===// |
| // DECLOCCURRENCES_BLOCK_ID |
| //===--------------------------------------------------------------------===// |
| |
| Stream.EnterSubblock(REC_DECLOCCURRENCES_BLOCK_ID, 3); |
| |
| Abbrev = std::make_shared<BitCodeAbbrev>(); |
| Abbrev->Add(BitCodeAbbrevOp(REC_DECLOCCURRENCE)); |
| Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // Decl ID |
| Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, SymbolRoleBitNum)); // Roles |
| Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 12)); // Line |
| Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // Column |
| Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // Num related |
| Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); // Related Roles/IDs |
| Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 16)); // Roles or ID |
| AbbrevCode = Stream.EmitAbbrev(std::move(Abbrev)); |
| |
| for (auto &Occur : Occurrences) { |
| Record.clear(); |
| Record.push_back(REC_DECLOCCURRENCE); |
| Record.push_back(Occur.DeclID); |
| Record.push_back(getIndexStoreRoles(Occur.Roles)); |
| Record.push_back(Occur.Line); |
| Record.push_back(Occur.Column); |
| Record.push_back(Occur.Related.size()); |
| for (auto &Rel : Occur.Related) { |
| Record.push_back(getIndexStoreRoles(Rel.first.Roles)); |
| Record.push_back(Rel.second); |
| } |
| Stream.EmitRecordWithAbbrev(AbbrevCode, Record); |
| } |
| Stream.ExitBlock(); |
| } |
| |
| IndexRecordWriter::IndexRecordWriter(StringRef IndexPath) |
| : RecordsPath(IndexPath) { |
| store::appendRecordSubDir(RecordsPath); |
| } |
| |
| IndexRecordWriter::Result |
| IndexRecordWriter::beginRecord(StringRef Filename, hash_code RecordHash, |
| std::string &Error, std::string *OutRecordFile) { |
| using namespace llvm::sys; |
| assert(!Record && "called beginRecord before calling endRecord on previous"); |
| |
| std::string RecordName; |
| { |
| llvm::raw_string_ostream RN(RecordName); |
| RN << path::filename(Filename); |
| RN << "-" << APInt(64, RecordHash).toString(36, /*Signed=*/false); |
| } |
| SmallString<256> RecordPath = RecordsPath.str(); |
| appendInteriorRecordPath(RecordName, RecordPath); |
| |
| if (OutRecordFile) |
| *OutRecordFile = RecordName; |
| |
| if (std::error_code EC = |
| fs::access(RecordPath.c_str(), fs::AccessMode::Exist)) { |
| if (EC != errc::no_such_file_or_directory) { |
| llvm::raw_string_ostream Err(Error); |
| Err << "could not access record '" << RecordPath |
| << "': " << EC.message(); |
| return Result::Failure; |
| } |
| } else { |
| return Result::AlreadyExists; |
| } |
| |
| // Write the record header. |
| auto *State = new RecordState(RecordPath.str()); |
| Record = State; |
| llvm::BitstreamWriter &Stream = State->Stream; |
| Stream.Emit('I', 8); |
| Stream.Emit('D', 8); |
| Stream.Emit('X', 8); |
| Stream.Emit('R', 8); |
| |
| writeBlockInfo(Stream); |
| writeVersionInfo(Stream); |
| |
| return Result::Success; |
| } |
| |
| IndexRecordWriter::Result |
| IndexRecordWriter::endRecord(std::string &Error, |
| writer::SymbolWriterCallback GetSymbolForDecl) { |
| assert(Record && "called endRecord without calling beginRecord"); |
| auto &State = *static_cast<RecordState *>(Record); |
| Record = nullptr; |
| struct ScopedDelete { |
| RecordState *S; |
| ScopedDelete(RecordState *S) : S(S) {} |
| ~ScopedDelete() { delete S; } |
| } Deleter(&State); |
| |
| if (!State.Decls.empty()) { |
| writeDecls(State.Stream, State.Decls, State.Occurrences, GetSymbolForDecl); |
| } |
| |
| if (std::error_code EC = sys::fs::create_directory(sys::path::parent_path(State.RecordPath))) { |
| llvm::raw_string_ostream Err(Error); |
| Err << "failed to create directory '" << sys::path::parent_path(State.RecordPath) << "': " << EC.message(); |
| return Result::Failure; |
| } |
| |
| // Create a unique file to write to so that we can move the result into place |
| // atomically. If this process crashes we don't want to interfere with any |
| // other concurrent processes. |
| SmallString<128> TempPath(State.RecordPath); |
| TempPath += "-temp-%%%%%%%%"; |
| int TempFD; |
| if (sys::fs::createUniqueFile(TempPath.str(), TempFD, TempPath)) { |
| llvm::raw_string_ostream Err(Error); |
| Err << "failed to create temporary file: " << TempPath; |
| return Result::Failure; |
| } |
| |
| raw_fd_ostream OS(TempFD, /*shouldClose=*/true); |
| OS.write(State.Buffer.data(), State.Buffer.size()); |
| OS.close(); |
| |
| // Atomically move the unique file into place. |
| if (std::error_code EC = |
| sys::fs::rename(TempPath.c_str(), State.RecordPath.c_str())) { |
| llvm::raw_string_ostream Err(Error); |
| Err << "failed to rename '" << TempPath << "' to '" << State.RecordPath << "': " << EC.message(); |
| return Result::Failure; |
| } |
| |
| return Result::Success; |
| } |
| |
| void IndexRecordWriter::addOccurrence( |
| OpaqueDecl D, SymbolRoleSet Roles, unsigned Line, unsigned Column, |
| ArrayRef<writer::SymbolRelation> Related) { |
| assert(Record && "called addOccurrence without calling beginRecord"); |
| auto &State = *static_cast<RecordState *>(Record); |
| |
| auto insertDecl = [&](OpaqueDecl D, SymbolRoleSet Roles, |
| SymbolRoleSet RelatedRoles) -> unsigned { |
| auto Insert = |
| State.IndexForDecl.insert(std::make_pair(D, State.Decls.size())); |
| unsigned Index = Insert.first->second; |
| |
| if (Insert.second) { |
| State.Decls.push_back(DeclInfo{D, Roles, RelatedRoles}); |
| } else { |
| State.Decls[Index].Roles |= Roles; |
| State.Decls[Index].RelatedRoles |= RelatedRoles; |
| } |
| return Index + 1; |
| }; |
| |
| unsigned DeclID = insertDecl(D, Roles, SymbolRoleSet()); |
| |
| decltype(OccurrenceInfo::Related) RelatedDecls; |
| RelatedDecls.reserve(Related.size()); |
| for (auto &Rel : Related) { |
| unsigned ID = insertDecl(Rel.RelatedSymbol, SymbolRoleSet(), Rel.Roles); |
| RelatedDecls.emplace_back(Rel, ID); |
| } |
| |
| State.Occurrences.push_back( |
| OccurrenceInfo{DeclID, D, Roles, Line, Column, std::move(RelatedDecls)}); |
| } |