| /* | |
| Copyright (c) 2014-2015 Microsoft Corporation. All rights reserved. | |
| Released under Apache 2.0 license as described in the file LICENSE. | |
| Authors: Leonardo de Moura, Gabriel Ebner, Sebastian Ullrich | |
| .olean serialization and deserialization. | |
| */ | |
| #include <unordered_map> | |
| #include <vector> | |
| #include <utility> | |
| #include <string> | |
| #include <sstream> | |
| #include <fstream> | |
| #include <algorithm> | |
| #include <sys/stat.h> | |
| #include "runtime/thread.h" | |
| #include "runtime/interrupt.h" | |
| #include "runtime/sstream.h" | |
| #include "runtime/hash.h" | |
| #include "runtime/io.h" | |
| #include "runtime/compact.h" | |
| #include "runtime/buffer.h" | |
| #include "runtime/array_ref.h" | |
| #include "util/io.h" | |
| #include "util/name_map.h" | |
| #include "library/module.h" | |
| #include "library/constants.h" | |
| #include "library/time_task.h" | |
| #include "library/util.h" | |
| #include "githash.h" | |
| #ifdef LEAN_WINDOWS | |
| #include <windows.h> | |
| #include <io.h> | |
| #include <fcntl.h> | |
| #else | |
| #include <sys/mman.h> | |
| #include <unistd.h> | |
| #include <fcntl.h> | |
| #endif | |
| #if defined(__has_feature) | |
| #if __has_feature(address_sanitizer) | |
| #include <sanitizer/lsan_interface.h> | |
| #endif | |
| #endif | |
| namespace lean { | |
| /** Trivial RAII wrapper for file descriptors so we don't have to worry about `close` management. */ | |
| class file_descriptor { | |
| private: | |
| int m_fd; | |
| public: | |
| explicit file_descriptor(int fd) : m_fd(fd) {} | |
| // It should not be copyable | |
| file_descriptor(const file_descriptor&) = delete; | |
| file_descriptor& operator=(const file_descriptor&) = delete; | |
| file_descriptor(file_descriptor&& other) noexcept : m_fd(other.m_fd) { | |
| other.m_fd = -1; | |
| } | |
| ~file_descriptor() { | |
| if (m_fd != -1) { | |
| close(m_fd); | |
| } | |
| } | |
| int get() const { return m_fd; } | |
| operator bool() const { return m_fd != -1; } | |
| }; | |
| /** On-disk format of a .olean file. */ | |
| struct olean_header { | |
| // 5 bytes: magic number | |
| char marker[5] = {'o', 'l', 'e', 'a', 'n'}; | |
| // 1 byte: version, incremented on structural changes to header | |
| uint8_t version = 2; | |
| // 1 byte of flags: | |
| // * bit 0: whether persisted bignums use GMP or Lean-native encoding | |
| // * bit 1-7: reserved | |
| uint8_t flags = | |
| #ifdef LEAN_USE_GMP | |
| 0b1; | |
| #else | |
| 0b0; | |
| #endif | |
| // 33 bytes: Lean version string, padded with '\0' to the right | |
| // e.g. "4.12.0-nightly-2024-10-18". Other suffixes after the version | |
| // triple currently in use are `-rcN` for some `N` and `-pre` for any | |
| // other non-release commit. Not necessarily null-terminated. | |
| char lean_version[33]; | |
| // 81b008650766442a0dfa9faa796e4588c9d7d3a1 | |
| // 40 bytes: build githash, padded with `\0` to the right | |
| char githash[40]; | |
| // address at which the beginning of the file (including header) is attempted to be mmapped | |
| size_t base_addr; | |
| // payload, a serialize Lean object graph; `size_t` has same alignment requirements as Lean objects | |
| size_t data[]; | |
| }; | |
| // make sure we don't have any padding bytes, which also ensures `data` is properly aligned | |
| static_assert(sizeof(olean_header) == 5 + 1 + 1 + 33 + 40 + sizeof(size_t), "olean_header must be packed"); | |
| extern "C" LEAN_EXPORT object * lean_save_module_data_parts(b_obj_arg mod, b_obj_arg oparts, object *) { | |
| #ifdef LEAN_WINDOWS | |
| uint32_t pid = GetCurrentProcessId(); | |
| #else | |
| uint32_t pid = getpid(); | |
| #endif | |
| // Derive a base address that is uniformly distributed by deterministic, and should most likely | |
| // work for `mmap` on all interesting platforms | |
| // NOTE: an overlapping/non-compatible base address does not prevent the module from being imported, | |
| // merely from using `mmap` for that | |
| // Let's start with a hash of the module name. Note that while our string hash is a dubious 32-bit | |
| // algorithm, the mixing of multiple `Name` parts seems to result in a nicely distributed 64-bit | |
| // output | |
| size_t base_addr = name(mod, true).hash(); | |
| // x86-64 user space is currently limited to the lower 47 bits | |
| // https://en.wikipedia.org/wiki/X86-64#Virtual_address_space_details | |
| // On Linux at least, the stack grows down from ~0x7fff... followed by shared libraries, so reserve | |
| // a bit of space for them (0x7fff...-0x7f00... = 1TB) | |
| base_addr = base_addr % 0x7f0000000000; | |
| // `mmap` addresses must be page-aligned. The default (non-huge) page size on x86-64 is 4KB. | |
| // `MapViewOfFileEx` addresses must be aligned to the "memory allocation granularity", which is 64KB. | |
| const size_t ALIGN = 1LL<<16; | |
| base_addr = base_addr & ~(ALIGN - 1); | |
| object_compactor compactor(reinterpret_cast<void *>(base_addr)); | |
| array_ref<pair_ref<string_ref, object_ref>> parts(oparts, true); | |
| std::vector<std::string> tmp_fnames; | |
| for (auto const & part : parts) { | |
| std::string olean_fn = part.fst().to_std_string(); | |
| try { | |
| // we first write to a temp file and then move it to the correct path (possibly deleting an older file) | |
| // so that we neither expose partially-written files nor modify possibly memory-mapped files | |
| std::string olean_tmp_fn = olean_fn + ".tmp." + std::to_string(pid); | |
| tmp_fnames.push_back(olean_tmp_fn); | |
| std::ofstream out(olean_tmp_fn, std::ios_base::binary); | |
| if (compactor.size() % ALIGN != 0) { | |
| compactor.alloc(ALIGN - (compactor.size() % ALIGN)); | |
| } | |
| size_t file_offset = compactor.size(); | |
| compactor.alloc(sizeof(olean_header)); | |
| olean_header header = {}; | |
| // see/sync with file format description above | |
| header.base_addr = base_addr + file_offset; | |
| strncpy(header.lean_version, get_short_version_string().c_str(), sizeof(header.lean_version)); | |
| strncpy(header.githash, LEAN_GITHASH, sizeof(header.githash)); | |
| out.write(reinterpret_cast<char *>(&header), sizeof(header)); | |
| compactor(part.snd().raw()); | |
| if (out.fail()) { | |
| throw exception((sstream() << "failed to create file '" << olean_fn << "'").str()); | |
| } | |
| out.write(static_cast<char const *>(compactor.data()) + file_offset + sizeof(olean_header), compactor.size() - file_offset - sizeof(olean_header)); | |
| out.close(); | |
| } catch (exception & ex) { | |
| return io_result_mk_error((sstream() << "failed to write '" << olean_fn << "': " << ex.what()).str()); | |
| } | |
| } | |
| for (unsigned i = 0; i < parts.size(); i++) { | |
| std::string olean_fn = parts[i].fst().to_std_string(); | |
| while (std::rename(tmp_fnames[i].c_str(), olean_fn.c_str()) != 0) { | |
| #ifdef LEAN_WINDOWS | |
| if (errno == EEXIST) { | |
| // Memory-mapped files can be deleted starting with Windows 10 using "POSIX semantics" | |
| HANDLE h_olean_fn = CreateFile(olean_fn.c_str(), GENERIC_READ | DELETE, FILE_SHARE_READ | FILE_SHARE_DELETE, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL); | |
| if (h_olean_fn == INVALID_HANDLE_VALUE) { | |
| return io_result_mk_error((sstream() << "failed to open '" << olean_fn << "': " << GetLastError()).str()); | |
| } | |
| FILE_DISPOSITION_INFO_EX fdi = { FILE_DISPOSITION_FLAG_DELETE | FILE_DISPOSITION_FLAG_POSIX_SEMANTICS }; | |
| if (SetFileInformationByHandle(h_olean_fn, static_cast<FILE_INFO_BY_HANDLE_CLASS>(21) /* FileDispositionInfoEx */, &fdi, sizeof(fdi)) != 0) { | |
| lean_always_assert(CloseHandle(h_olean_fn)); | |
| continue; | |
| } else { | |
| return io_result_mk_error((sstream() << "failed to delete '" << olean_fn << "': " << GetLastError()).str()); | |
| } | |
| } | |
| #endif | |
| return io_result_mk_error((sstream() << "failed to write '" << olean_fn << "': " << errno << " " << strerror(errno)).str()); | |
| } | |
| } | |
| return io_result_mk_ok(box(0)); | |
| } | |
| struct module_file { | |
| std::string m_fname; | |
| file_descriptor m_fd; | |
| #ifdef LEAN_WINDOWS | |
| HANDLE m_handle; // store the original Windows for mmap | |
| #endif | |
| char * m_base_addr; | |
| size_t m_size; | |
| char * m_buffer; | |
| std::function<void()> m_free_data; | |
| }; | |
| extern "C" LEAN_EXPORT object * lean_read_module_data_parts(b_obj_arg ofnames, object *) { | |
| array_ref<string_ref> fnames(ofnames, true); | |
| // first read in all headers | |
| std::vector<module_file> files; | |
| for (auto const & fname : fnames) { | |
| std::string olean_fn = fname.to_std_string(); | |
| try { | |
| #ifdef LEAN_WINDOWS | |
| // Use CreateFile with proper sharing flags, then convert to POSIX fd for shared code | |
| // `FILE_SHARE_DELETE` is necessary to allow the file to (be marked to) be deleted while in use | |
| HANDLE h_file = CreateFile(olean_fn.c_str(), GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_DELETE, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL); | |
| if (h_file == INVALID_HANDLE_VALUE) { | |
| return io_result_mk_error((sstream() << "failed to open file '" << olean_fn << "': " << GetLastError()).str()); | |
| } | |
| int raw_fd = _open_osfhandle((intptr_t)h_file, _O_RDONLY); | |
| if (raw_fd == -1) { | |
| CloseHandle(h_file); | |
| return io_result_mk_error((sstream() << "failed to convert handle to fd for '" << olean_fn << "'").str()); | |
| } | |
| file_descriptor fd(raw_fd); | |
| #else | |
| file_descriptor fd(open(olean_fn.c_str(), O_RDONLY)); | |
| if (!fd) { | |
| return io_result_mk_error((sstream() << "failed to open file '" << olean_fn << "': " << strerror(errno)).str()); | |
| } | |
| #endif | |
| /* Get file size */ | |
| struct stat st; | |
| if (fstat(fd.get(), &st) == -1) { | |
| return io_result_mk_error((sstream() << "failed to stat file '" << olean_fn << "': " << strerror(errno)).str()); | |
| } | |
| size_t size = st.st_size; | |
| olean_header default_header = {}; | |
| olean_header header; | |
| if (read(fd.get(), &header, sizeof(header)) != sizeof(header) | |
| || memcmp(header.marker, default_header.marker, sizeof(header.marker)) != 0) { | |
| return io_result_mk_error((sstream() << "failed to read file '" << olean_fn << "', invalid header").str()); | |
| } | |
| lseek(fd.get(), 0, SEEK_SET); | |
| if (header.version != default_header.version || header.flags != default_header.flags | |
| #ifdef LEAN_CHECK_OLEAN_VERSION | |
| || strncmp(header.githash, LEAN_GITHASH, sizeof(header.githash)) != 0 | |
| #endif | |
| ) { | |
| return io_result_mk_error((sstream() << "failed to read file '" << olean_fn << "', incompatible header").str()); | |
| } | |
| char * base_addr = reinterpret_cast<char *>(header.base_addr); | |
| #ifdef LEAN_WINDOWS | |
| files.push_back({olean_fn, std::move(fd), h_file, base_addr, size, nullptr, nullptr}); | |
| #else | |
| files.push_back({olean_fn, std::move(fd), base_addr, size, nullptr, nullptr}); | |
| #endif | |
| } catch (exception & ex) { | |
| return io_result_mk_error((sstream() << "failed to read '" << olean_fn << "': " << ex.what()).str()); | |
| } | |
| } | |
| #ifndef LEAN_MMAP | |
| bool is_mmap = false; | |
| #else | |
| // now try mmapping *all* files | |
| bool is_mmap = true; | |
| for (auto & file : files) { | |
| std::string const & olean_fn = file.m_fname; | |
| char * base_addr = file.m_base_addr; | |
| try { | |
| #ifdef LEAN_WINDOWS | |
| // Use the stored handle that was created with proper sharing flags | |
| HANDLE h_olean_fn = file.m_handle; | |
| HANDLE h_map = CreateFileMapping(h_olean_fn, NULL, PAGE_READONLY, 0, 0, NULL); | |
| if (h_map == NULL) { | |
| return io_result_mk_error((sstream() << "failed to map '" << olean_fn << "': " << GetLastError()).str()); | |
| } | |
| char * buffer = static_cast<char *>(MapViewOfFileEx(h_map, FILE_MAP_READ, 0, 0, 0, base_addr)); | |
| lean_always_assert(CloseHandle(h_map)); | |
| // NOTE: no need to close `h_olean_fn` as it's owned by `file.m_fd` | |
| if (!buffer) { | |
| is_mmap = false; | |
| break; | |
| } | |
| file.m_free_data = [=]() { | |
| lean_always_assert(UnmapViewOfFile(base_addr)); | |
| }; | |
| #else | |
| int fd = file.m_fd.get(); | |
| // NOTE: `file.m_fd` does NOT need to outlive `buffer` after this call | |
| char * buffer = static_cast<char *>(mmap(base_addr, file.m_size, PROT_READ, MAP_PRIVATE, fd, 0)); | |
| if (buffer == MAP_FAILED) { | |
| is_mmap = false; | |
| break; | |
| } | |
| size_t size = file.m_size; | |
| file.m_free_data = [=]() { | |
| lean_always_assert(munmap(buffer, size) == 0); | |
| }; | |
| #endif | |
| if (buffer == base_addr) { | |
| file.m_buffer = buffer; | |
| } else { | |
| is_mmap = false; | |
| break; | |
| } | |
| } catch (exception & ex) { | |
| return io_result_mk_error((sstream() << "failed to read '" << olean_fn << "': " << ex.what()).str()); | |
| } | |
| } | |
| #endif | |
| // if *any* file failed to mmap, read all of them into a single big allocation so that offsets | |
| // between them are unchanged | |
| if (!is_mmap) { | |
| for (auto & file : files) { | |
| if (file.m_free_data) { | |
| file.m_free_data(); | |
| file.m_free_data = {}; | |
| } | |
| } | |
| size_t big_size = files[files.size()-1].m_base_addr + files[files.size()-1].m_size - files[0].m_base_addr; | |
| char * big_buffer = static_cast<char *>(malloc(big_size)); | |
| for (auto & file : files) { | |
| std::string const & olean_fn = file.m_fname; | |
| try { | |
| file.m_buffer = big_buffer + (file.m_base_addr - files[0].m_base_addr); | |
| if (read(file.m_fd.get(), file.m_buffer, file.m_size) != static_cast<ssize_t>(file.m_size)) { | |
| return io_result_mk_error((sstream() << "failed to read file '" << olean_fn << "'").str()); | |
| } | |
| } catch (exception & ex) { | |
| return io_result_mk_error((sstream() << "failed to read '" << olean_fn << "': " << ex.what()).str()); | |
| } | |
| } | |
| files[0].m_free_data = [=]() { | |
| free_sized(big_buffer, big_size); | |
| }; | |
| } | |
| std::vector<object_ref> res; | |
| for (auto & file : files) { | |
| compacted_region * region = | |
| new compacted_region(file.m_size - sizeof(olean_header), file.m_buffer + sizeof(olean_header), static_cast<char *>(file.m_base_addr) + sizeof(olean_header), is_mmap, file.m_free_data); | |
| #if defined(__has_feature) | |
| #if __has_feature(address_sanitizer) | |
| // do not report as leak | |
| __lsan_ignore_object(region); | |
| #endif | |
| #endif | |
| object * mod = region->read(); | |
| object * mod_region = alloc_cnstr(0, 2, 0); | |
| cnstr_set(mod_region, 0, mod); | |
| cnstr_set(mod_region, 1, box_size_t(reinterpret_cast<size_t>(region))); | |
| res.push_back(object_ref(mod_region)); | |
| } | |
| return io_result_mk_ok(to_array(res)); | |
| } | |
| } |