blob: 82812565b0c4c845a8bae811b8a8b139d07a9722 [file] [log] [blame]
//===--- LocalizationFormat.h - Format for Diagnostic Messages --*- C++ -*-===//
// This source file is part of the open source project
// Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors
// Licensed under Apache License v2.0 with Runtime Library Exception
// See for license information
// See for the list of Swift project authors
// This file defines the format for localized diagnostic messages.
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/Hashing.h"
#include "llvm/ADT/Optional.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/Bitstream/BitstreamReader.h"
#include "llvm/Support/EndianStream.h"
#include "llvm/Support/MemoryBuffer.h"
#include "llvm/Support/OnDiskHashTable.h"
#include "llvm/Support/YAMLParser.h"
#include "llvm/Support/YAMLTraits.h"
#include "llvm/Support/raw_ostream.h"
#include <cstdint>
#include <memory>
#include <string>
#include <type_traits>
#include <utility>
#include <vector>
namespace swift {
enum class DiagID : uint32_t;
namespace diag {
using namespace llvm::support;
class DefToYAMLConverter {
llvm::ArrayRef<const char *> IDs;
llvm::ArrayRef<const char *> Messages;
DefToYAMLConverter(llvm::ArrayRef<const char *> ids,
llvm::ArrayRef<const char *> messages)
: IDs(ids), Messages(messages) {
assert(IDs.size() == Messages.size());
void convert(llvm::raw_ostream &out);
class LocalizationWriterInfo {
using key_type = uint32_t;
using key_type_ref = const uint32_t &;
using data_type = std::string;
using data_type_ref = llvm::StringRef;
using hash_value_type = uint32_t;
using offset_type = uint32_t;
hash_value_type ComputeHash(key_type_ref key) { return llvm::hash_code(key); }
std::pair<offset_type, offset_type> EmitKeyDataLength(llvm::raw_ostream &out,
key_type_ref key,
data_type_ref data) {
offset_type dataLength = static_cast<offset_type>(data.size());
endian::write<offset_type>(out, dataLength, little);
// No need to write the key length; it's constant.
return {sizeof(key_type), dataLength};
void EmitKey(llvm::raw_ostream &out, key_type_ref key, unsigned len) {
assert(len == sizeof(key_type));
endian::write<key_type>(out, key, little);
void EmitData(llvm::raw_ostream &out, key_type_ref key, data_type_ref data,
unsigned len) {
out << data;
class LocalizationReaderInfo {
using internal_key_type = uint32_t;
using external_key_type = swift::DiagID;
using data_type = llvm::StringRef;
using hash_value_type = uint32_t;
using offset_type = uint32_t;
internal_key_type GetInternalKey(external_key_type key) {
return static_cast<internal_key_type>(key);
external_key_type GetExternalKey(internal_key_type key) {
return static_cast<external_key_type>(key);
static bool EqualKey(internal_key_type lhs, internal_key_type rhs) {
return lhs == rhs;
hash_value_type ComputeHash(internal_key_type key) {
return llvm::hash_code(key);
static std::pair<offset_type, offset_type>
ReadKeyDataLength(const unsigned char *&data) {
offset_type dataLength =
endian::readNext<offset_type, little, unaligned>(data);
return {sizeof(uint32_t), dataLength};
internal_key_type ReadKey(const unsigned char *data, offset_type length) {
return endian::readNext<internal_key_type, little, unaligned>(data);
data_type ReadData(internal_key_type Key, const unsigned char *data,
offset_type length) {
return data_type((const char *)data, length);
class SerializedLocalizationWriter {
using offset_type = LocalizationWriterInfo::offset_type;
llvm::OnDiskChainedHashTableGenerator<LocalizationWriterInfo> generator;
/// Enqueue the given diagnostic to be included in a serialized translations
/// file.
/// \param id The identifier associated with the given diagnostic message e.g.
/// 'cannot_convert_argument'.
/// \param translation The localized diagnostic message for the given
/// identifier.
void insert(swift::DiagID id, llvm::StringRef translation);
/// Write out previously inserted diagnostic translations into the given
/// location.
/// \param filePath The location of the serialized diagnostics file. It's
/// supposed to be a file with '.db' postfix.
/// \returns true if all diagnostic
/// messages have been successfully serialized, false otherwise.
bool emit(llvm::StringRef filePath);
class LocalizationProducer {
/// If the message isn't available/localized in current context
/// return the fallback default message.
virtual llvm::StringRef getMessageOr(swift::DiagID id,
llvm::StringRef defaultMessage) const {
auto message = getMessage(id);
return message.empty() ? defaultMessage : message;
virtual ~LocalizationProducer() {}
/// Retrieve a message for the given diagnostic id.
/// \returns empty string if message couldn't be found.
virtual llvm::StringRef getMessage(swift::DiagID id) const = 0;
class YAMLLocalizationProducer final : public LocalizationProducer {
std::vector<std::string> diagnostics;
/// The diagnostics IDs that are no longer available in `.def`
std::vector<std::string> unknownIDs;
explicit YAMLLocalizationProducer(llvm::StringRef filePath);
/// Iterate over all of the available (non-empty) translations
/// maintained by this producer, callback gets each translation
/// with its unique identifier.
void forEachAvailable(
llvm::function_ref<void(swift::DiagID, llvm::StringRef)> callback) const;
llvm::StringRef getMessage(swift::DiagID id) const override;
class SerializedLocalizationProducer final : public LocalizationProducer {
using SerializedLocalizationTable =
using offset_type = LocalizationReaderInfo::offset_type;
std::unique_ptr<llvm::MemoryBuffer> Buffer;
std::unique_ptr<SerializedLocalizationTable> SerializedTable;
explicit SerializedLocalizationProducer(
std::unique_ptr<llvm::MemoryBuffer> buffer);
llvm::StringRef getMessage(swift::DiagID id) const override;
class LocalizationInput : public llvm::yaml::Input {
using Input::Input;
/// Read diagnostics in the YAML file iteratively
template <typename T, typename Context>
friend typename std::enable_if<llvm::yaml::has_SequenceTraits<T>::value,
readYAML(llvm::yaml::IO &io, T &Seq, T &unknownIDs, bool, Context &Ctx);
template <typename T>
friend typename std::enable_if<llvm::yaml::has_SequenceTraits<T>::value,
LocalizationInput &>::type
operator>>(LocalizationInput &yin, T &diagnostics);
/// A vector that keeps track of the diagnostics IDs that are available in
/// YAML and not available in `.def` files.
std::vector<std::string> unknownIDs;
/// A diagnostic ID might be present in YAML and not in `.def` file, if that's
/// the case the `id` won't have a `DiagID` value.
/// If the `id` is available in `.def` file this method will return the `id`'s
/// value, otherwise this method won't return a value.
static llvm::Optional<uint32_t> readID(llvm::yaml::IO &io);
} // namespace diag
} // namespace swift