| /* | |
| * Copyright (c) Meta Platforms, Inc. and affiliates. | |
| * All rights reserved. | |
| * | |
| * This source code is licensed under both the BSD-style license (found in the | |
| * LICENSE file in the root directory of this source tree) and the GPLv2 (found | |
| * in the COPYING file in the root directory of this source tree). | |
| * You may select, at your option, one of the above-listed licenses. | |
| */ | |
| #include "data.h" | |
| #include <assert.h> | |
| #include <errno.h> | |
| #include <stdio.h> | |
| #include <string.h> | |
| #include <stdlib.h> /* free() */ | |
| #include <sys/stat.h> | |
| #include <curl/curl.h> | |
| #include "mem.h" | |
| #include "util.h" | |
| #define XXH_STATIC_LINKING_ONLY | |
| #include "xxhash.h" | |
| /** | |
| * Data objects | |
| */ | |
| #define REGRESSION_RELEASE(x) \ | |
| "https://github.com/facebook/zstd/releases/download/regression-data/" x | |
| data_t silesia = { | |
| .name = "silesia", | |
| .type = data_type_dir, | |
| .data = | |
| { | |
| .url = REGRESSION_RELEASE("silesia.tar.zst"), | |
| .xxhash64 = 0x48a199f92f93e977LL, | |
| }, | |
| }; | |
| data_t silesia_tar = { | |
| .name = "silesia.tar", | |
| .type = data_type_file, | |
| .data = | |
| { | |
| .url = REGRESSION_RELEASE("silesia.tar.zst"), | |
| .xxhash64 = 0x48a199f92f93e977LL, | |
| }, | |
| }; | |
| data_t github = { | |
| .name = "github", | |
| .type = data_type_dir, | |
| .data = | |
| { | |
| .url = REGRESSION_RELEASE("github.tar.zst"), | |
| .xxhash64 = 0xa9b1b44b020df292LL, | |
| }, | |
| .dict = | |
| { | |
| .url = REGRESSION_RELEASE("github.dict.zst"), | |
| .xxhash64 = 0x1eddc6f737d3cb53LL, | |
| }, | |
| }; | |
| data_t github_tar = { | |
| .name = "github.tar", | |
| .type = data_type_file, | |
| .data = | |
| { | |
| .url = REGRESSION_RELEASE("github.tar.zst"), | |
| .xxhash64 = 0xa9b1b44b020df292LL, | |
| }, | |
| .dict = | |
| { | |
| .url = REGRESSION_RELEASE("github.dict.zst"), | |
| .xxhash64 = 0x1eddc6f737d3cb53LL, | |
| }, | |
| }; | |
| static data_t* g_data[] = { | |
| &silesia, | |
| &silesia_tar, | |
| &github, | |
| &github_tar, | |
| NULL, | |
| }; | |
| data_t const* const* data = (data_t const* const*)g_data; | |
| /** | |
| * data helpers. | |
| */ | |
| int data_has_dict(data_t const* data) { | |
| return data->dict.url != NULL; | |
| } | |
| /** | |
| * data buffer helper functions (documented in header). | |
| */ | |
| data_buffer_t data_buffer_create(size_t const capacity) { | |
| data_buffer_t buffer = {}; | |
| buffer.data = (uint8_t*)malloc(capacity); | |
| if (buffer.data == NULL) | |
| return buffer; | |
| buffer.capacity = capacity; | |
| return buffer; | |
| } | |
| data_buffer_t data_buffer_read(char const* filename) { | |
| data_buffer_t buffer = {}; | |
| uint64_t const size = UTIL_getFileSize(filename); | |
| if (size == UTIL_FILESIZE_UNKNOWN) { | |
| fprintf(stderr, "unknown size for %s\n", filename); | |
| return buffer; | |
| } | |
| buffer.data = (uint8_t*)malloc(size); | |
| if (buffer.data == NULL) { | |
| fprintf(stderr, "malloc failed\n"); | |
| return buffer; | |
| } | |
| buffer.capacity = size; | |
| FILE* file = fopen(filename, "rb"); | |
| if (file == NULL) { | |
| fprintf(stderr, "file null\n"); | |
| goto err; | |
| } | |
| buffer.size = fread(buffer.data, 1, buffer.capacity, file); | |
| fclose(file); | |
| if (buffer.size != buffer.capacity) { | |
| fprintf(stderr, "read %zu != %zu\n", buffer.size, buffer.capacity); | |
| goto err; | |
| } | |
| return buffer; | |
| err: | |
| free(buffer.data); | |
| memset(&buffer, 0, sizeof(buffer)); | |
| return buffer; | |
| } | |
| data_buffer_t data_buffer_get_data(data_t const* data) { | |
| data_buffer_t const kEmptyBuffer = {}; | |
| if (data->type != data_type_file) | |
| return kEmptyBuffer; | |
| return data_buffer_read(data->data.path); | |
| } | |
| data_buffer_t data_buffer_get_dict(data_t const* data) { | |
| data_buffer_t const kEmptyBuffer = {}; | |
| if (!data_has_dict(data)) | |
| return kEmptyBuffer; | |
| return data_buffer_read(data->dict.path); | |
| } | |
| int data_buffer_compare(data_buffer_t buffer1, data_buffer_t buffer2) { | |
| size_t const size = | |
| buffer1.size < buffer2.size ? buffer1.size : buffer2.size; | |
| int const cmp = memcmp(buffer1.data, buffer2.data, size); | |
| if (cmp != 0) | |
| return cmp; | |
| if (buffer1.size < buffer2.size) | |
| return -1; | |
| if (buffer1.size == buffer2.size) | |
| return 0; | |
| assert(buffer1.size > buffer2.size); | |
| return 1; | |
| } | |
| void data_buffer_free(data_buffer_t buffer) { | |
| free(buffer.data); | |
| } | |
| /** | |
| * data filenames helpers. | |
| */ | |
| FileNamesTable* data_filenames_get(data_t const* data) | |
| { | |
| char const* const path = data->data.path; | |
| return UTIL_createExpandedFNT(&path, 1, 0 /* followLinks */ ); | |
| } | |
| /** | |
| * data buffers helpers. | |
| */ | |
| data_buffers_t data_buffers_get(data_t const* data) { | |
| data_buffers_t buffers = {.size = 0}; | |
| FileNamesTable* const filenames = data_filenames_get(data); | |
| if (filenames == NULL) return buffers; | |
| if (filenames->tableSize == 0) { | |
| UTIL_freeFileNamesTable(filenames); | |
| return buffers; | |
| } | |
| data_buffer_t* buffersPtr = | |
| (data_buffer_t*)malloc(filenames->tableSize * sizeof(*buffersPtr)); | |
| if (buffersPtr == NULL) { | |
| UTIL_freeFileNamesTable(filenames); | |
| return buffers; | |
| } | |
| buffers.buffers = (data_buffer_t const*)buffersPtr; | |
| buffers.size = filenames->tableSize; | |
| for (size_t i = 0; i < filenames->tableSize; ++i) { | |
| buffersPtr[i] = data_buffer_read(filenames->fileNames[i]); | |
| if (buffersPtr[i].data == NULL) { | |
| data_buffers_t const kEmptyBuffer = {}; | |
| data_buffers_free(buffers); | |
| UTIL_freeFileNamesTable(filenames); | |
| return kEmptyBuffer; | |
| } | |
| } | |
| UTIL_freeFileNamesTable(filenames); | |
| return buffers; | |
| } | |
| /** | |
| * Frees the data buffers. | |
| */ | |
| void data_buffers_free(data_buffers_t buffers) { | |
| free((data_buffer_t*)buffers.buffers); | |
| } | |
| /** | |
| * Initialization and download functions. | |
| */ | |
| static char* g_data_dir = NULL; | |
| /* mkdir -p */ | |
| static int ensure_directory_exists(char const* indir) { | |
| char* const dir = strdup(indir); | |
| char* end = dir; | |
| int ret = 0; | |
| if (dir == NULL) { | |
| ret = EINVAL; | |
| goto out; | |
| } | |
| do { | |
| /* Find the next directory level. */ | |
| for (++end; *end != '\0' && *end != '/'; ++end) | |
| ; | |
| /* End the string there, make the directory, and restore the string. */ | |
| char const save = *end; | |
| *end = '\0'; | |
| int const isdir = UTIL_isDirectory(dir); | |
| ret = mkdir(dir, S_IRWXU); | |
| *end = save; | |
| /* Its okay if the directory already exists. */ | |
| if (ret == 0 || (errno == EEXIST && isdir)) | |
| continue; | |
| ret = errno; | |
| fprintf(stderr, "mkdir() failed\n"); | |
| goto out; | |
| } while (*end != '\0'); | |
| ret = 0; | |
| out: | |
| free(dir); | |
| return ret; | |
| } | |
| /** Concatenate 3 strings into a new buffer. */ | |
| static char* cat3(char const* str1, char const* str2, char const* str3) { | |
| size_t const size1 = strlen(str1); | |
| size_t const size2 = strlen(str2); | |
| size_t const size3 = str3 == NULL ? 0 : strlen(str3); | |
| size_t const size = size1 + size2 + size3 + 1; | |
| char* const dst = (char*)malloc(size); | |
| if (dst == NULL) | |
| return NULL; | |
| strcpy(dst, str1); | |
| strcpy(dst + size1, str2); | |
| if (str3 != NULL) | |
| strcpy(dst + size1 + size2, str3); | |
| assert(strlen(dst) == size1 + size2 + size3); | |
| return dst; | |
| } | |
| static char* cat2(char const* str1, char const* str2) { | |
| return cat3(str1, str2, NULL); | |
| } | |
| /** | |
| * State needed by the curl callback. | |
| * It takes data from curl, hashes it, and writes it to the file. | |
| */ | |
| typedef struct { | |
| FILE* file; | |
| XXH64_state_t xxhash64; | |
| int error; | |
| } curl_data_t; | |
| /** Create the curl state. */ | |
| static curl_data_t curl_data_create( | |
| data_resource_t const* resource, | |
| data_type_t type) { | |
| curl_data_t cdata = {}; | |
| XXH64_reset(&cdata.xxhash64, 0); | |
| assert(UTIL_isDirectory(g_data_dir)); | |
| if (type == data_type_file) { | |
| /* Decompress the resource and store to the path. */ | |
| char* cmd = cat3("zstd -dqfo '", resource->path, "'"); | |
| if (cmd == NULL) { | |
| cdata.error = ENOMEM; | |
| return cdata; | |
| } | |
| cdata.file = popen(cmd, "w"); | |
| free(cmd); | |
| } else { | |
| /* Decompress and extract the resource to the cache directory. */ | |
| char* cmd = cat3("zstd -dc | tar -x -C '", g_data_dir, "'"); | |
| if (cmd == NULL) { | |
| cdata.error = ENOMEM; | |
| return cdata; | |
| } | |
| cdata.file = popen(cmd, "w"); | |
| free(cmd); | |
| } | |
| if (cdata.file == NULL) { | |
| cdata.error = errno; | |
| } | |
| return cdata; | |
| } | |
| /** Free the curl state. */ | |
| static int curl_data_free(curl_data_t cdata) { | |
| return pclose(cdata.file); | |
| } | |
| /** curl callback. Updates the hash, and writes to the file. */ | |
| static size_t curl_write(void* data, size_t size, size_t count, void* ptr) { | |
| curl_data_t* cdata = (curl_data_t*)ptr; | |
| size_t const written = fwrite(data, size, count, cdata->file); | |
| XXH64_update(&cdata->xxhash64, data, written * size); | |
| return written; | |
| } | |
| static int curl_download_resource( | |
| CURL* curl, | |
| data_resource_t const* resource, | |
| data_type_t type) { | |
| curl_data_t cdata; | |
| /* Download the data. */ | |
| if (curl_easy_setopt(curl, CURLOPT_URL, resource->url) != 0) | |
| return EINVAL; | |
| if (curl_easy_setopt(curl, CURLOPT_WRITEDATA, &cdata) != 0) | |
| return EINVAL; | |
| cdata = curl_data_create(resource, type); | |
| if (cdata.error != 0) | |
| return cdata.error; | |
| int const curl_err = curl_easy_perform(curl); | |
| int const close_err = curl_data_free(cdata); | |
| if (curl_err) { | |
| fprintf( | |
| stderr, | |
| "downloading '%s' for '%s' failed\n", | |
| resource->url, | |
| resource->path); | |
| return EIO; | |
| } | |
| if (close_err) { | |
| fprintf(stderr, "writing data to '%s' failed\n", resource->path); | |
| return EIO; | |
| } | |
| /* check that the file exists. */ | |
| if (type == data_type_file && !UTIL_isRegularFile(resource->path)) { | |
| fprintf(stderr, "output file '%s' does not exist\n", resource->path); | |
| return EIO; | |
| } | |
| if (type == data_type_dir && !UTIL_isDirectory(resource->path)) { | |
| fprintf( | |
| stderr, "output directory '%s' does not exist\n", resource->path); | |
| return EIO; | |
| } | |
| /* Check that the hash matches. */ | |
| if (XXH64_digest(&cdata.xxhash64) != resource->xxhash64) { | |
| fprintf( | |
| stderr, | |
| "checksum does not match: 0x%llxLL != 0x%llxLL\n", | |
| (unsigned long long)XXH64_digest(&cdata.xxhash64), | |
| (unsigned long long)resource->xxhash64); | |
| return EINVAL; | |
| } | |
| return 0; | |
| } | |
| /** Download a single data object. */ | |
| static int curl_download_datum(CURL* curl, data_t const* data) { | |
| int ret; | |
| ret = curl_download_resource(curl, &data->data, data->type); | |
| if (ret != 0) | |
| return ret; | |
| if (data_has_dict(data)) { | |
| ret = curl_download_resource(curl, &data->dict, data_type_file); | |
| if (ret != 0) | |
| return ret; | |
| } | |
| return ret; | |
| } | |
| /** Download all the data. */ | |
| static int curl_download_data(data_t const* const* data) { | |
| if (curl_global_init(CURL_GLOBAL_ALL) != 0) | |
| return EFAULT; | |
| curl_data_t cdata = {}; | |
| CURL* curl = curl_easy_init(); | |
| int err = EFAULT; | |
| if (curl == NULL) | |
| return EFAULT; | |
| if (curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 1L) != 0) | |
| goto out; | |
| if (curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L) != 0) | |
| goto out; | |
| if (curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, curl_write) != 0) | |
| goto out; | |
| assert(data != NULL); | |
| for (; *data != NULL; ++data) { | |
| if (curl_download_datum(curl, *data) != 0) | |
| goto out; | |
| } | |
| err = 0; | |
| out: | |
| curl_easy_cleanup(curl); | |
| curl_global_cleanup(); | |
| return err; | |
| } | |
| /** Fill the path member variable of the data objects. */ | |
| static int data_create_paths(data_t* const* data, char const* dir) { | |
| size_t const dirlen = strlen(dir); | |
| assert(data != NULL); | |
| for (; *data != NULL; ++data) { | |
| data_t* const datum = *data; | |
| datum->data.path = cat3(dir, "/", datum->name); | |
| if (datum->data.path == NULL) | |
| return ENOMEM; | |
| if (data_has_dict(datum)) { | |
| datum->dict.path = cat2(datum->data.path, ".dict"); | |
| if (datum->dict.path == NULL) | |
| return ENOMEM; | |
| } | |
| } | |
| return 0; | |
| } | |
| /** Free the path member variable of the data objects. */ | |
| static void data_free_paths(data_t* const* data) { | |
| assert(data != NULL); | |
| for (; *data != NULL; ++data) { | |
| data_t* datum = *data; | |
| free((void*)datum->data.path); | |
| free((void*)datum->dict.path); | |
| datum->data.path = NULL; | |
| datum->dict.path = NULL; | |
| } | |
| } | |
| static char const kStampName[] = "STAMP"; | |
| static void xxh_update_le(XXH64_state_t* state, uint64_t data) { | |
| if (!MEM_isLittleEndian()) | |
| data = MEM_swap64(data); | |
| XXH64_update(state, &data, sizeof(data)); | |
| } | |
| /** Hash the data to create the stamp. */ | |
| static uint64_t stamp_hash(data_t const* const* data) { | |
| XXH64_state_t state; | |
| XXH64_reset(&state, 0); | |
| assert(data != NULL); | |
| for (; *data != NULL; ++data) { | |
| data_t const* datum = *data; | |
| /* We don't care about the URL that we fetch from. */ | |
| /* The path is derived from the name. */ | |
| XXH64_update(&state, datum->name, strlen(datum->name)); | |
| xxh_update_le(&state, datum->data.xxhash64); | |
| xxh_update_le(&state, datum->dict.xxhash64); | |
| xxh_update_le(&state, datum->type); | |
| } | |
| return XXH64_digest(&state); | |
| } | |
| /** Check if the stamp matches the stamp in the cache directory. */ | |
| static int stamp_check(char const* dir, data_t const* const* data) { | |
| char* stamp = cat3(dir, "/", kStampName); | |
| uint64_t const expected = stamp_hash(data); | |
| XXH64_canonical_t actual; | |
| FILE* stampfile = NULL; | |
| int matches = 0; | |
| if (stamp == NULL) | |
| goto out; | |
| if (!UTIL_isRegularFile(stamp)) { | |
| fprintf(stderr, "stamp does not exist: recreating the data cache\n"); | |
| goto out; | |
| } | |
| stampfile = fopen(stamp, "rb"); | |
| if (stampfile == NULL) { | |
| fprintf(stderr, "could not open stamp: recreating the data cache\n"); | |
| goto out; | |
| } | |
| size_t b; | |
| if ((b = fread(&actual, sizeof(actual), 1, stampfile)) != 1) { | |
| fprintf(stderr, "invalid stamp: recreating the data cache\n"); | |
| goto out; | |
| } | |
| matches = (expected == XXH64_hashFromCanonical(&actual)); | |
| if (matches) | |
| fprintf(stderr, "stamp matches: reusing the cached data\n"); | |
| else | |
| fprintf(stderr, "stamp does not match: recreating the data cache\n"); | |
| out: | |
| free(stamp); | |
| if (stampfile != NULL) | |
| fclose(stampfile); | |
| return matches; | |
| } | |
| /** On success write a new stamp, on failure delete the old stamp. */ | |
| static int | |
| stamp_write(char const* dir, data_t const* const* data, int const data_err) { | |
| char* stamp = cat3(dir, "/", kStampName); | |
| FILE* stampfile = NULL; | |
| int err = EIO; | |
| if (stamp == NULL) | |
| return ENOMEM; | |
| if (data_err != 0) { | |
| err = data_err; | |
| goto out; | |
| } | |
| XXH64_canonical_t hash; | |
| XXH64_canonicalFromHash(&hash, stamp_hash(data)); | |
| stampfile = fopen(stamp, "wb"); | |
| if (stampfile == NULL) | |
| goto out; | |
| if (fwrite(&hash, sizeof(hash), 1, stampfile) != 1) | |
| goto out; | |
| err = 0; | |
| fprintf(stderr, "stamped new data cache\n"); | |
| out: | |
| if (err != 0) | |
| /* Ignore errors. */ | |
| unlink(stamp); | |
| free(stamp); | |
| if (stampfile != NULL) | |
| fclose(stampfile); | |
| return err; | |
| } | |
| int data_init(char const* dir) { | |
| int err; | |
| if (dir == NULL) | |
| return EINVAL; | |
| /* This must be first to simplify logic. */ | |
| err = ensure_directory_exists(dir); | |
| if (err != 0) | |
| return err; | |
| /* Save the cache directory. */ | |
| g_data_dir = strdup(dir); | |
| if (g_data_dir == NULL) | |
| return ENOMEM; | |
| err = data_create_paths(g_data, dir); | |
| if (err != 0) | |
| return err; | |
| /* If the stamp matches then we are good to go. | |
| * This must be called before any modifications to the data cache. | |
| * After this point, we MUST call stamp_write() to update the STAMP, | |
| * since we've updated the data cache. | |
| */ | |
| if (stamp_check(dir, data)) | |
| return 0; | |
| err = curl_download_data(data); | |
| if (err != 0) | |
| goto out; | |
| out: | |
| /* This must be last, since it must know if data_init() succeeded. */ | |
| stamp_write(dir, data, err); | |
| return err; | |
| } | |
| void data_finish(void) { | |
| data_free_paths(g_data); | |
| free(g_data_dir); | |
| g_data_dir = NULL; | |
| } |