blob: a9b7959c6787dcf1e5712fbc2ee59e6e6c5d26c5 [file] [log] [blame]
// Copyright 2018 The Fuchsia Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include <deque>
#include <fstream>
#include <memory>
#include <random>
#include <set>
#include <string>
#include <unordered_map>
#include <vector>
#include "encoder/envelope_maker.h"
#include "encoder/observation_store.h"
#include "third_party/protobuf/src/google/protobuf/io/zero_copy_stream_impl.h"
#include "third_party/statusor/statusor.h"
#include "util/file_system.h"
#include "util/protected_fields.h"
namespace cobalt {
namespace encoder {
// FileObservationStore is an implementation of ObservationStore that persists
// observations to a file system.
// The store returns FileEnvelopeHolders from calls to TakeNextEnvelopeHolder().
// As long as there are FileEnvelopeHolders that have not been returned or
// deleted, the store should not be destroyed.
// This object is thread safe.
class FileObservationStore : public ObservationStore {
// FileEnvelopeHolder is an implementation of
// ObservationStore::EnvelopeHolder.
// It represents the envelope as a list of filenames. The observations are not
// actually read into memory until a call to GetEnvelope() is made.
// Note: This object is not thread safe.
class FileEnvelopeHolder : public EnvelopeHolder {
// |fs|. An implementation of FileSystem used to interact with the system's
// filesystem.
// |root_directory|. The absolute path to the directory where the
// observation files are written. (e.g. /system/data/cobalt_legacy)
// |file_name|. The file name for the file containing the observations.
FileEnvelopeHolder(util::FileSystem *fs, const std::string &root_directory,
const std::string &file_name)
: fs_(fs),
envelope_read_(false) {}
void MergeWith(std::unique_ptr<EnvelopeHolder> container) override;
const Envelope &GetEnvelope() override;
size_t Size() override;
const std::set<std::string> &file_names() { return file_names_; }
void clear() { file_names_.clear(); }
std::string FullPath(const std::string &filename) const;
util::FileSystem *fs_;
const std::string root_directory_;
// file_names contains a set of file names that contain observations.
// These files should all be read into |envelope| when GetEnvelope is
// called.
std::set<std::string> file_names_;
bool envelope_read_;
Envelope envelope_;
size_t cached_file_size_ = 0;
// |fs|. An implementation of FileSystem used to interact with the system's
// filesystem.
// |root_directory|. The absolute path to the directory where the observation
// files should be written. (e.g. /system/data/cobalt_legacy)
// |name| is used in log messages to distinguish this instance of
// FileObservationStore.
FileObservationStore(size_t max_bytes_per_observation,
size_t max_bytes_per_envelope, size_t max_bytes_total,
std::unique_ptr<util::FileSystem> fs,
std::string root_directory,
std::string name = "FileObservationStore");
StoreStatus AddEncryptedObservation(
std::unique_ptr<EncryptedMessage> message,
std::unique_ptr<ObservationMetadata> metadata) override;
std::unique_ptr<EnvelopeHolder> TakeNextEnvelopeHolder() override;
void ReturnEnvelopeHolder(std::unique_ptr<EnvelopeHolder> envelopes) override;
size_t Size() const override;
bool Empty() const override;
size_t num_observations_added() override;
void ResetObservationCounter() override;
// Delete removes all of the files associated with this FileObservationStore.
// This is useful for cleaning up after testing.
void Delete();
// ListFinalizedFiles lists all files in root directory that match the format
// <13-digit timestamp>-<7 digit random number>.data
std::vector<std::string> ListFinalizedFiles() const;
struct Fields {
bool metadata_written;
// last_written_metadata is a string encoding of the last metadata written
// to the active_file. If another observation comes in with an identical
// metadata, it is not necessary to write it again.
std::string last_written_metadata;
std::ofstream active_fstream;
std::unique_ptr<google::protobuf::io::OstreamOutputStream> active_file;
// files_taken lists the filenames that have been "Taken" from the store.
// These should not be used to construct EnvelopeHolders for
// TakeNextEnvelopeHolder(). If an EnvelopeHolder is returned, the
// associated file names should also be removed from this list.
std::set<std::string> files_taken;
// The total size in bytes of the finalized files. This should be kept up to
// date as files are added to/removed from the store.
size_t finalized_bytes;
util::ProtectedFields<Fields> protected_fields_;
// GetOldestFinalizedFile returns a file name for the oldest file in the
// store.
statusor::StatusOr<std::string> GetOldestFinalizedFile(
util::ProtectedFields<Fields>::LockedFieldsPtr *fields);
// GenerateFinalizedName returns an absolute path that can be used for
// finalizing a file. It is based on the current timestamp and a random number
// to avoid collisions.
std::string GenerateFinalizedName() const;
// FullPath returns the absolute path to the filename by prefixing the file
// name with the root directory.
std::string FullPath(const std::string &filename) const;
bool FinalizeActiveFile(
util::ProtectedFields<Fields>::LockedFieldsPtr *fields);
// GetActiveFile returns a pointer to the current OstreamOutputStream. If the
// file is not yet opened, it will be opened by this function.
google::protobuf::io::OstreamOutputStream *GetActiveFile(
util::ProtectedFields<Fields>::LockedFieldsPtr *fields);
const std::unique_ptr<util::FileSystem> fs_;
const std::string root_directory_;
const std::string active_file_name_;
mutable std::random_device random_dev_;
mutable std::uniform_int_distribution<uint32_t> random_int_;
const std::string name_;
size_t num_observations_added_;
} // namespace encoder
} // namespace cobalt