| /* | |
| * Copyright (c) Meta Platforms, Inc. and affiliates. | |
| * All rights reserved. | |
| * | |
| * This source code is licensed under both the BSD-style license (found in the | |
| * LICENSE file in the root directory of this source tree) and the GPLv2 (found | |
| * in the COPYING file in the root directory of this source tree). | |
| * You may select, at your option, one of the above-listed licenses. | |
| */ | |
| /*-************************************ | |
| * Compiler specific | |
| **************************************/ | |
| #ifdef _MSC_VER /* Visual Studio */ | |
| # define _CRT_SECURE_NO_WARNINGS /* fgets */ | |
| # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ | |
| # pragma warning(disable : 4204) /* disable: C4204: non-constant aggregate initializer */ | |
| #endif | |
| /*-************************************ | |
| * Includes | |
| **************************************/ | |
| #include <stdlib.h> /* free */ | |
| #include <stdio.h> /* fgets, sscanf */ | |
| #include <string.h> /* strcmp */ | |
| #include <time.h> /* time(), time_t */ | |
| #undef NDEBUG /* always enable assert() */ | |
| #include <assert.h> | |
| #define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressContinue, ZSTD_compressBlock */ | |
| #include "debug.h" /* DEBUG_STATIC_ASSERT */ | |
| #include "fse.h" | |
| #include "huf.h" | |
| #define ZSTD_DISABLE_DEPRECATE_WARNINGS /* No deprecation warnings, we still test some deprecated functions */ | |
| #include "zstd.h" /* ZSTD_VERSION_STRING */ | |
| #include "zstd_errors.h" /* ZSTD_getErrorCode */ | |
| #define ZDICT_STATIC_LINKING_ONLY | |
| #include "zdict.h" /* ZDICT_trainFromBuffer */ | |
| #include "mem.h" | |
| #include "datagen.h" /* RDG_genBuffer */ | |
| #define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */ | |
| #include "xxhash.h" /* XXH64 */ | |
| #include "timefn.h" /* SEC_TO_MICRO, UTIL_time_t, UTIL_TIME_INITIALIZER, UTIL_clockSpanMicro, UTIL_getTime */ | |
| /* must be included after util.h, due to ERROR macro redefinition issue on Visual Studio */ | |
| #include "zstd_internal.h" /* ZSTD_WORKSPACETOOLARGE_MAXDURATION, ZSTD_WORKSPACETOOLARGE_FACTOR, KB, MB */ | |
| #include "threading.h" /* ZSTD_pthread_create, ZSTD_pthread_join */ | |
| #include "pool.h" /* POOL_sizeof */ | |
| #include "compress/hist.h" /* HIST_count_wksp */ | |
| #include "compress/zstd_compress_internal.h" /* ZSTD_get1BlockSummary */ | |
| #include "decompress/zstd_decompress_internal.h" /* ZSTD_DCtx */ | |
| #include "decompress/zstd_ddict.h" /* empty DDict storage invariants */ | |
| #include "bitstream.h" /* BIT_reloadDStream boundary tests */ | |
| #include "dictBuilder/cover.h" /* COVER_selectDict */ | |
| /*-************************************ | |
| * Macros | |
| **************************************/ | |
| #define COUNTOF(array) (sizeof(array) / sizeof(*(array))) | |
| /*-************************************ | |
| * Constants | |
| **************************************/ | |
| #define GB *(1U<<30) | |
| static const int FUZ_compressibility_default = 50; | |
| static const int nbTestsDefault = 30000; | |
| /*-************************************ | |
| * Display Macros | |
| **************************************/ | |
| #define DISPLAY(...) fprintf(stderr, __VA_ARGS__) | |
| #define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); } | |
| static U32 g_displayLevel = 2; | |
| static const U64 g_refreshRate = SEC_TO_MICRO / 6; | |
| static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER; | |
| #define DISPLAYUPDATE(l, ...) \ | |
| if (g_displayLevel>=l) { \ | |
| if ((UTIL_clockSpanMicro(g_displayClock) > g_refreshRate) || (g_displayLevel>=4)) \ | |
| { g_displayClock = UTIL_getTime(); DISPLAY(__VA_ARGS__); \ | |
| if (g_displayLevel>=4) fflush(stderr); } \ | |
| } | |
| /*-******************************************************* | |
| * Compile time test | |
| *********************************************************/ | |
| #undef MIN | |
| #undef MAX | |
| /* Declaring the function, to avoid -Wmissing-prototype */ | |
| void FUZ_bug976(void); | |
| void FUZ_bug976(void) | |
| { /* these constants shall not depend on MIN() macro */ | |
| DEBUG_STATIC_ASSERT(ZSTD_HASHLOG_MAX < 31); | |
| DEBUG_STATIC_ASSERT(ZSTD_CHAINLOG_MAX < 31); | |
| } | |
| /*-******************************************************* | |
| * Internal functions | |
| *********************************************************/ | |
| #define MIN(a,b) ((a)<(b)?(a):(b)) | |
| #define MAX(a,b) ((a)>(b)?(a):(b)) | |
| #define FUZ_rotl32(x,r) ((x << r) | (x >> (32 - r))) | |
| static U32 FUZ_rand(U32* src) | |
| { | |
| static const U32 prime1 = 2654435761U; | |
| static const U32 prime2 = 2246822519U; | |
| U32 rand32 = *src; | |
| rand32 *= prime1; | |
| rand32 += prime2; | |
| rand32 = FUZ_rotl32(rand32, 13); | |
| *src = rand32; | |
| return rand32 >> 5; | |
| } | |
| static U32 FUZ_highbit32(U32 v32) | |
| { | |
| unsigned nbBits = 0; | |
| if (v32==0) return 0; | |
| while (v32) v32 >>= 1, nbBits++; | |
| return nbBits; | |
| } | |
| /*============================================= | |
| * Test macros | |
| =============================================*/ | |
| #define CHECK(fn) { if(!(fn)) { DISPLAYLEVEL(1, "Error : test (%s) failed \n", #fn); exit(1); } } | |
| #define CHECK_Z(f) { \ | |
| size_t const err = f; \ | |
| if (ZSTD_isError(err)) { \ | |
| DISPLAY("Error => %s : %s ", \ | |
| #f, ZSTD_getErrorName(err)); \ | |
| exit(1); \ | |
| } } | |
| #define CHECK_VAR(var, fn) var = fn; if (ZSTD_isError(var)) { DISPLAYLEVEL(1, "%s : fails : %s \n", #fn, ZSTD_getErrorName(var)); exit(1); } | |
| #define CHECK_NEWV(var, fn) size_t const CHECK_VAR(var, fn) | |
| #define CHECKPLUS(var, fn, more) { CHECK_NEWV(var, fn); more; } | |
| #define CHECK_OP(op, lhs, rhs) { \ | |
| if (!((lhs) op (rhs))) { \ | |
| DISPLAY("Error L%u => FAILED %s %s %s ", __LINE__, #lhs, #op, #rhs); \ | |
| exit(1); \ | |
| } \ | |
| } | |
| #define CHECK_EQ(lhs, rhs) CHECK_OP(==, lhs, rhs) | |
| #define CHECK_LT(lhs, rhs) CHECK_OP(<, lhs, rhs) | |
| /*============================================= | |
| * Memory Tests | |
| =============================================*/ | |
| #if defined(__APPLE__) && defined(__MACH__) | |
| #include <malloc/malloc.h> /* malloc_size */ | |
| typedef struct { | |
| unsigned long long totalMalloc; | |
| size_t currentMalloc; | |
| size_t peakMalloc; | |
| unsigned nbMalloc; | |
| unsigned nbFree; | |
| } mallocCounter_t; | |
| static const mallocCounter_t INIT_MALLOC_COUNTER = { 0, 0, 0, 0, 0 }; | |
| static void* FUZ_mallocDebug(void* counter, size_t size) | |
| { | |
| mallocCounter_t* const mcPtr = (mallocCounter_t*)counter; | |
| void* const ptr = malloc(size); | |
| if (ptr==NULL) return NULL; | |
| DISPLAYLEVEL(4, "allocating %u KB => effectively %u KB \n", | |
| (unsigned)(size >> 10), (unsigned)(malloc_size(ptr) >> 10)); /* OS-X specific */ | |
| mcPtr->totalMalloc += size; | |
| mcPtr->currentMalloc += size; | |
| if (mcPtr->currentMalloc > mcPtr->peakMalloc) | |
| mcPtr->peakMalloc = mcPtr->currentMalloc; | |
| mcPtr->nbMalloc += 1; | |
| return ptr; | |
| } | |
| static void FUZ_freeDebug(void* counter, void* address) | |
| { | |
| mallocCounter_t* const mcPtr = (mallocCounter_t*)counter; | |
| DISPLAYLEVEL(4, "freeing %u KB \n", (unsigned)(malloc_size(address) >> 10)); | |
| mcPtr->nbFree += 1; | |
| mcPtr->currentMalloc -= malloc_size(address); /* OS-X specific */ | |
| free(address); | |
| } | |
| static void FUZ_displayMallocStats(mallocCounter_t count) | |
| { | |
| DISPLAYLEVEL(3, "peak:%6u KB, nbMallocs:%2u, total:%6u KB \n", | |
| (unsigned)(count.peakMalloc >> 10), | |
| count.nbMalloc, | |
| (unsigned)(count.totalMalloc >> 10)); | |
| } | |
| static int FUZ_mallocTests_internal(unsigned seed, double compressibility, unsigned part, | |
| void* inBuffer, size_t inSize, void* outBuffer, size_t outSize) | |
| { | |
| /* test only played in verbose mode, as they are long */ | |
| if (g_displayLevel<3) return 0; | |
| /* Create compressible noise */ | |
| if (!inBuffer || !outBuffer) { | |
| DISPLAY("Not enough memory, aborting\n"); | |
| exit(1); | |
| } | |
| RDG_genBuffer(inBuffer, inSize, compressibility, 0. /*auto*/, seed); | |
| /* simple compression tests */ | |
| if (part <= 1) | |
| { int compressionLevel; | |
| for (compressionLevel=1; compressionLevel<=6; compressionLevel++) { | |
| mallocCounter_t malcount = INIT_MALLOC_COUNTER; | |
| ZSTD_customMem const cMem = { FUZ_mallocDebug, FUZ_freeDebug, &malcount }; | |
| ZSTD_CCtx* const cctx = ZSTD_createCCtx_advanced(cMem); | |
| CHECK_Z( ZSTD_compressCCtx(cctx, outBuffer, outSize, inBuffer, inSize, compressionLevel) ); | |
| ZSTD_freeCCtx(cctx); | |
| DISPLAYLEVEL(3, "compressCCtx level %i : ", compressionLevel); | |
| FUZ_displayMallocStats(malcount); | |
| } } | |
| /* streaming compression tests */ | |
| if (part <= 2) | |
| { int compressionLevel; | |
| for (compressionLevel=1; compressionLevel<=6; compressionLevel++) { | |
| mallocCounter_t malcount = INIT_MALLOC_COUNTER; | |
| ZSTD_customMem const cMem = { FUZ_mallocDebug, FUZ_freeDebug, &malcount }; | |
| ZSTD_CCtx* const cstream = ZSTD_createCStream_advanced(cMem); | |
| ZSTD_outBuffer out = { outBuffer, outSize, 0 }; | |
| ZSTD_inBuffer in = { inBuffer, inSize, 0 }; | |
| CHECK_Z( ZSTD_initCStream(cstream, compressionLevel) ); | |
| CHECK_Z( ZSTD_compressStream(cstream, &out, &in) ); | |
| CHECK_Z( ZSTD_endStream(cstream, &out) ); | |
| ZSTD_freeCStream(cstream); | |
| DISPLAYLEVEL(3, "compressStream level %i : ", compressionLevel); | |
| FUZ_displayMallocStats(malcount); | |
| } } | |
| /* advanced MT API test */ | |
| if (part <= 3) | |
| { int nbThreads; | |
| for (nbThreads=1; nbThreads<=4; nbThreads++) { | |
| int compressionLevel; | |
| for (compressionLevel=1; compressionLevel<=6; compressionLevel++) { | |
| mallocCounter_t malcount = INIT_MALLOC_COUNTER; | |
| ZSTD_customMem const cMem = { FUZ_mallocDebug, FUZ_freeDebug, &malcount }; | |
| ZSTD_CCtx* const cctx = ZSTD_createCCtx_advanced(cMem); | |
| CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, compressionLevel) ); | |
| CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, nbThreads) ); | |
| CHECK_Z( ZSTD_compress2(cctx, outBuffer, outSize, inBuffer, inSize) ); | |
| ZSTD_freeCCtx(cctx); | |
| DISPLAYLEVEL(3, "compress_generic,-T%i,end level %i : ", | |
| nbThreads, compressionLevel); | |
| FUZ_displayMallocStats(malcount); | |
| } } } | |
| /* advanced MT streaming API test */ | |
| if (part <= 4) | |
| { int nbThreads; | |
| for (nbThreads=1; nbThreads<=4; nbThreads++) { | |
| int compressionLevel; | |
| for (compressionLevel=1; compressionLevel<=6; compressionLevel++) { | |
| mallocCounter_t malcount = INIT_MALLOC_COUNTER; | |
| ZSTD_customMem const cMem = { FUZ_mallocDebug, FUZ_freeDebug, &malcount }; | |
| ZSTD_CCtx* const cctx = ZSTD_createCCtx_advanced(cMem); | |
| ZSTD_outBuffer out = { outBuffer, outSize, 0 }; | |
| ZSTD_inBuffer in = { inBuffer, inSize, 0 }; | |
| CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, compressionLevel) ); | |
| CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, nbThreads) ); | |
| CHECK_Z( ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_continue) ); | |
| while ( ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end) ) {} | |
| ZSTD_freeCCtx(cctx); | |
| DISPLAYLEVEL(3, "compress_generic,-T%i,continue level %i : ", | |
| nbThreads, compressionLevel); | |
| FUZ_displayMallocStats(malcount); | |
| } } } | |
| return 0; | |
| } | |
| static int FUZ_mallocTests(unsigned seed, double compressibility, unsigned part) | |
| { | |
| size_t const inSize = 64 MB + 16 MB + 4 MB + 1 MB + 256 KB + 64 KB; /* 85.3 MB */ | |
| size_t const outSize = ZSTD_compressBound(inSize); | |
| void* const inBuffer = malloc(inSize); | |
| void* const outBuffer = malloc(outSize); | |
| int result; | |
| /* Create compressible noise */ | |
| if (!inBuffer || !outBuffer) { | |
| DISPLAY("Not enough memory, aborting \n"); | |
| exit(1); | |
| } | |
| result = FUZ_mallocTests_internal(seed, compressibility, part, | |
| inBuffer, inSize, outBuffer, outSize); | |
| free(inBuffer); | |
| free(outBuffer); | |
| return result; | |
| } | |
| #else | |
| static int FUZ_mallocTests(unsigned seed, double compressibility, unsigned part) | |
| { | |
| (void)seed; (void)compressibility; (void)part; | |
| return 0; | |
| } | |
| #endif | |
| static void FUZ_decodeSequences(BYTE* dst, ZSTD_Sequence* seqs, size_t seqsSize, | |
| BYTE* src, size_t size, ZSTD_SequenceFormat_e format) | |
| { | |
| size_t i; | |
| size_t j; | |
| for(i = 0; i < seqsSize; ++i) { | |
| assert(dst + seqs[i].litLength + seqs[i].matchLength <= dst + size); | |
| assert(src + seqs[i].litLength + seqs[i].matchLength <= src + size); | |
| if (format == ZSTD_sf_noBlockDelimiters) { | |
| assert(seqs[i].matchLength != 0 || seqs[i].offset != 0); | |
| } | |
| memcpy(dst, src, seqs[i].litLength); | |
| dst += seqs[i].litLength; | |
| src += seqs[i].litLength; | |
| size -= seqs[i].litLength; | |
| if (seqs[i].offset != 0) { | |
| for (j = 0; j < seqs[i].matchLength; ++j) | |
| dst[j] = dst[(ptrdiff_t)(j - seqs[i].offset)]; | |
| dst += seqs[i].matchLength; | |
| src += seqs[i].matchLength; | |
| size -= seqs[i].matchLength; | |
| } | |
| } | |
| if (format == ZSTD_sf_noBlockDelimiters) { | |
| memcpy(dst, src, size); | |
| } | |
| } | |
| static size_t FUZ_getLitSize(const ZSTD_Sequence* seqs, size_t nbSeqs) | |
| { | |
| size_t n, litSize = 0; | |
| assert(seqs != NULL); | |
| for (n=0; n<nbSeqs; n++) { | |
| litSize += seqs[n].litLength; | |
| } | |
| return litSize; | |
| } | |
| static void | |
| FUZ_transferLiterals(void* dst, size_t dstCapacity, | |
| const void* src, size_t srcSize, | |
| const ZSTD_Sequence* seqs, size_t nbSeqs) | |
| { | |
| size_t n; | |
| const char* ip = (const char*)src; | |
| char* op = (char*)dst; | |
| size_t const litSize = FUZ_getLitSize(seqs, nbSeqs); | |
| assert(litSize <= dstCapacity); | |
| for (n=0; n<nbSeqs; n++) { | |
| size_t const ll = seqs[n].litLength; | |
| memcpy(op, ip, ll); | |
| op += ll; | |
| ip += ll + seqs[n].matchLength; | |
| } | |
| assert((size_t)(ip - (const char*)src) == srcSize); | |
| } | |
| #ifdef ZSTD_MULTITHREAD | |
| typedef struct { | |
| ZSTD_CCtx* cctx; | |
| ZSTD_threadPool* pool; | |
| void* CNBuffer; | |
| size_t CNBuffSize; | |
| void* compressedBuffer; | |
| size_t compressedBufferSize; | |
| void* decodedBuffer; | |
| int err; | |
| } threadPoolTests_compressionJob_payload; | |
| static void* threadPoolTests_compressionJob(void* payload) { | |
| threadPoolTests_compressionJob_payload* args = (threadPoolTests_compressionJob_payload*)payload; | |
| size_t cSize; | |
| if (ZSTD_isError(ZSTD_CCtx_refThreadPool(args->cctx, args->pool))) args->err = 1; | |
| cSize = ZSTD_compress2(args->cctx, args->compressedBuffer, args->compressedBufferSize, args->CNBuffer, args->CNBuffSize); | |
| if (ZSTD_isError(cSize)) args->err = 1; | |
| if (ZSTD_isError(ZSTD_decompress(args->decodedBuffer, args->CNBuffSize, args->compressedBuffer, cSize))) args->err = 1; | |
| return payload; | |
| } | |
| static int threadPoolTests(void) { | |
| int testResult = 0; | |
| size_t err; | |
| size_t const CNBuffSize = 5 MB; | |
| void* const CNBuffer = malloc(CNBuffSize); | |
| size_t const compressedBufferSize = ZSTD_compressBound(CNBuffSize); | |
| void* const compressedBuffer = malloc(compressedBufferSize); | |
| void* const decodedBuffer = malloc(CNBuffSize); | |
| size_t const kPoolNumThreads = 8; | |
| RDG_genBuffer(CNBuffer, CNBuffSize, 0.5, 0.5, 0); | |
| DISPLAYLEVEL(3, "thread pool test : threadPool reuse roundtrips: "); | |
| { | |
| ZSTD_CCtx* cctx = ZSTD_createCCtx(); | |
| ZSTD_threadPool* pool = ZSTD_createThreadPool(kPoolNumThreads); | |
| size_t nbThreads = 1; | |
| for (; nbThreads <= kPoolNumThreads; ++nbThreads) { | |
| ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters); | |
| ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, (int)nbThreads); | |
| err = ZSTD_CCtx_refThreadPool(cctx, pool); | |
| if (ZSTD_isError(err)) { | |
| DISPLAYLEVEL(3, "refThreadPool error!\n"); | |
| ZSTD_freeCCtx(cctx); | |
| goto _output_error; | |
| } | |
| err = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize); | |
| if (ZSTD_isError(err)) { | |
| DISPLAYLEVEL(3, "Compression error!\n"); | |
| ZSTD_freeCCtx(cctx); | |
| goto _output_error; | |
| } | |
| err = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, err); | |
| if (ZSTD_isError(err)) { | |
| DISPLAYLEVEL(3, "Decompression error!\n"); | |
| ZSTD_freeCCtx(cctx); | |
| goto _output_error; | |
| } | |
| } | |
| ZSTD_freeCCtx(cctx); | |
| ZSTD_freeThreadPool(pool); | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "thread pool test : threadPool simultaneous usage: "); | |
| { | |
| void* const decodedBuffer2 = malloc(CNBuffSize); | |
| void* const compressedBuffer2 = malloc(compressedBufferSize); | |
| ZSTD_threadPool* pool = ZSTD_createThreadPool(kPoolNumThreads); | |
| ZSTD_CCtx* cctx1 = ZSTD_createCCtx(); | |
| ZSTD_CCtx* cctx2 = ZSTD_createCCtx(); | |
| ZSTD_pthread_t t1; | |
| ZSTD_pthread_t t2; | |
| threadPoolTests_compressionJob_payload p1 = {cctx1, pool, CNBuffer, CNBuffSize, | |
| compressedBuffer, compressedBufferSize, decodedBuffer, 0 /* err */}; | |
| threadPoolTests_compressionJob_payload p2 = {cctx2, pool, CNBuffer, CNBuffSize, | |
| compressedBuffer2, compressedBufferSize, decodedBuffer2, 0 /* err */}; | |
| ZSTD_CCtx_setParameter(cctx1, ZSTD_c_nbWorkers, 2); | |
| ZSTD_CCtx_setParameter(cctx2, ZSTD_c_nbWorkers, 2); | |
| ZSTD_CCtx_refThreadPool(cctx1, pool); | |
| ZSTD_CCtx_refThreadPool(cctx2, pool); | |
| ZSTD_pthread_create(&t1, NULL, threadPoolTests_compressionJob, &p1); | |
| ZSTD_pthread_create(&t2, NULL, threadPoolTests_compressionJob, &p2); | |
| ZSTD_pthread_join(t1); | |
| ZSTD_pthread_join(t2); | |
| assert(!memcmp(decodedBuffer, decodedBuffer2, CNBuffSize)); | |
| free(decodedBuffer2); | |
| free(compressedBuffer2); | |
| ZSTD_freeThreadPool(pool); | |
| ZSTD_freeCCtx(cctx1); | |
| ZSTD_freeCCtx(cctx2); | |
| if (p1.err || p2.err) goto _output_error; | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "thread pool test : change the pool and nbWorkers of a context: "); | |
| { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); | |
| ZSTD_threadPool* const pool1 = ZSTD_createThreadPool(1); | |
| ZSTD_threadPool* const pool2 = ZSTD_createThreadPool(1); | |
| CHECK(cctx != NULL && pool1 != NULL && pool2 != NULL); | |
| CHECK(!ZSTD_isError( ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 2) )); | |
| CHECK(!ZSTD_isError( ZSTD_CCtx_refThreadPool(cctx, pool1) )); | |
| CHECK(!ZSTD_isError( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize) )); | |
| /* the next frames must use pool2, then an internal pool */ | |
| CHECK(!ZSTD_isError( ZSTD_CCtx_refThreadPool(cctx, pool2) )); | |
| ZSTD_freeThreadPool(pool1); | |
| CHECK(!ZSTD_isError( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize) )); | |
| { size_t const pool2Size = POOL_sizeof(pool2); | |
| /* the context must not resize a pool it shares */ | |
| CHECK(!ZSTD_isError( ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 4) )); | |
| CHECK(!ZSTD_isError( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize) )); | |
| CHECK(POOL_sizeof(pool2) == pool2Size); | |
| } | |
| CHECK(!ZSTD_isError( ZSTD_CCtx_refThreadPool(cctx, NULL) )); | |
| ZSTD_freeThreadPool(pool2); | |
| CHECK(!ZSTD_isError( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize) )); | |
| ZSTD_freeCCtx(cctx); | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| _end: | |
| free(CNBuffer); | |
| free(compressedBuffer); | |
| free(decodedBuffer); | |
| return testResult; | |
| _output_error: | |
| testResult = 1; | |
| DISPLAY("Error detected in Unit tests ! \n"); | |
| goto _end; | |
| } | |
| #endif /* ZSTD_MULTITHREAD */ | |
| /*============================================= | |
| * Unit tests | |
| =============================================*/ | |
| static void test_compressBound(unsigned tnb) | |
| { | |
| DISPLAYLEVEL(3, "test%3u : compressBound : ", tnb); | |
| /* check ZSTD_compressBound == ZSTD_COMPRESSBOUND | |
| * for a large range of known valid values */ | |
| DEBUG_STATIC_ASSERT(sizeof(size_t) >= 4); | |
| { int s; | |
| for (s=0; s<30; s++) { | |
| size_t const w = (size_t)1 << s; | |
| CHECK_EQ(ZSTD_compressBound(w), ZSTD_COMPRESSBOUND(w)); | |
| } } | |
| /* Ensure error if srcSize too big */ | |
| { size_t const w = ZSTD_MAX_INPUT_SIZE + 1; | |
| CHECK(ZSTD_isError(ZSTD_compressBound(w))); /* must fail */ | |
| CHECK_EQ(ZSTD_COMPRESSBOUND(w), 0); | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| } | |
| static void test_decompressBound(unsigned tnb) | |
| { | |
| DISPLAYLEVEL(3, "test%3u : decompressBound : ", tnb); | |
| /* Simple compression, with size : should provide size; */ | |
| { const char example[] = "abcd"; | |
| char cBuffer[ZSTD_COMPRESSBOUND(sizeof(example))]; | |
| size_t const cSize = ZSTD_compress(cBuffer, sizeof(cBuffer), example, sizeof(example), 0); | |
| CHECK_Z(cSize); | |
| CHECK_EQ(ZSTD_decompressBound(cBuffer, cSize), (unsigned long long)sizeof(example)); | |
| } | |
| /* Simple small compression without size : should provide 1 block size */ | |
| { char cBuffer[ZSTD_COMPRESSBOUND(0)]; | |
| ZSTD_outBuffer out = { cBuffer, sizeof(cBuffer), 0 }; | |
| ZSTD_inBuffer in = { NULL, 0, 0 }; | |
| ZSTD_CCtx* const cctx = ZSTD_createCCtx(); | |
| assert(cctx); | |
| CHECK_Z( ZSTD_initCStream(cctx, 0) ); | |
| CHECK_Z( ZSTD_compressStream(cctx, &out, &in) ); | |
| CHECK_EQ( ZSTD_endStream(cctx, &out), 0 ); | |
| CHECK_EQ( ZSTD_decompressBound(cBuffer, out.pos), ZSTD_BLOCKSIZE_MAX ); | |
| ZSTD_freeCCtx(cctx); | |
| } | |
| /* Attempt to overflow 32-bit intermediate multiplication result | |
| * This requires dBound >= 4 GB, aka 2^32. | |
| * This requires 2^32 / 2^17 = 2^15 blocks | |
| * => create 2^15 blocks (can be empty, or just 1 byte). */ | |
| { const char input[] = "a"; | |
| size_t const nbBlocks = (1 << 15) + 1; | |
| size_t blockNb; | |
| size_t const outCapacity = 1 << 18; /* large margin */ | |
| char* const outBuffer = malloc (outCapacity); | |
| ZSTD_outBuffer out = { outBuffer, outCapacity, 0 }; | |
| ZSTD_CCtx* const cctx = ZSTD_createCCtx(); | |
| assert(cctx); | |
| assert(outBuffer); | |
| CHECK_Z( ZSTD_initCStream(cctx, 0) ); | |
| for (blockNb=0; blockNb<nbBlocks; blockNb++) { | |
| ZSTD_inBuffer in = { input, sizeof(input), 0 }; | |
| CHECK_Z( ZSTD_compressStream(cctx, &out, &in) ); | |
| CHECK_EQ( ZSTD_flushStream(cctx, &out), 0 ); | |
| } | |
| CHECK_EQ( ZSTD_endStream(cctx, &out), 0 ); | |
| CHECK( ZSTD_decompressBound(outBuffer, out.pos) > 0x100000000ULL /* 4 GB */ ); | |
| ZSTD_freeCCtx(cctx); | |
| free(outBuffer); | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| } | |
| /* Size-inspection APIs trust the declared content size without decoding the | |
| * blocks. Keep the frame structurally complete while varying that field. */ | |
| static const BYTE FUZ_contentSizeFrame[] = { | |
| 0x28, 0xB5, 0x2F, 0xFD, 0xE0, | |
| 0, 0, 0, 0, 0, 0, 0, 0, | |
| 0x01, 0x00, 0x00 | |
| }; | |
| static void FUZ_writeContentSizeFrame(BYTE* dst, U64 contentSize) | |
| { | |
| memcpy(dst, FUZ_contentSizeFrame, sizeof(FUZ_contentSizeFrame)); | |
| MEM_writeLE64(dst + ZSTD_FRAMEIDSIZE + 1, contentSize); | |
| } | |
| static void test_decompressBoundOverflow(unsigned tnb) | |
| { | |
| static const U64 tails[] = { 0, 1, 0xFC, 0xFD, 0xFE, 0xFF, 0x100, 0x110 }; | |
| static const BYTE skippable[8] = { 0x50, 0x2A, 0x4D, 0x18, 0, 0, 0, 0 }; | |
| static const BYTE unknownSize[9] = { | |
| 0x28, 0xB5, 0x2F, 0xFD, 0x00, 0x00, 0x01, 0x00, 0x00 | |
| }; | |
| U64 const first = 0xFFFFFFFFFFFFFF00ULL; | |
| U64 const maxBound = ZSTD_CONTENTSIZE_ERROR - 1; | |
| size_t const frameSize = sizeof(FUZ_contentSizeFrame); | |
| size_t const skippableEnd = frameSize + sizeof(skippable); | |
| BYTE frames[2 * sizeof(FUZ_contentSizeFrame) + sizeof(skippable)]; | |
| size_t i; | |
| DISPLAYLEVEL(3, "test%3u : decompressBound concatenation overflow : ", tnb); | |
| CHECK_EQ(ZSTD_decompressBound(NULL, 0), 0); | |
| FUZ_writeContentSizeFrame(frames, maxBound); | |
| CHECK_EQ(ZSTD_decompressBound(frames, frameSize), maxBound); | |
| FUZ_writeContentSizeFrame(frames, ZSTD_CONTENTSIZE_ERROR); | |
| CHECK_EQ(ZSTD_decompressBound(frames, frameSize), ZSTD_CONTENTSIZE_ERROR); | |
| FUZ_writeContentSizeFrame(frames, ZSTD_CONTENTSIZE_UNKNOWN); | |
| /* Regardless of how the header parser treats this reserved field value, | |
| * decompressBound() must return a numeric bound or ERROR, never UNKNOWN. */ | |
| CHECK(ZSTD_decompressBound(frames, frameSize) != ZSTD_CONTENTSIZE_UNKNOWN); | |
| FUZ_writeContentSizeFrame(frames, first); | |
| CHECK_EQ(ZSTD_decompressBound(frames, frameSize), first); | |
| for (i = 0; i < COUNTOF(tails); ++i) { | |
| U64 const expected = tails[i] <= maxBound - first | |
| ? first + tails[i] : ZSTD_CONTENTSIZE_ERROR; | |
| FUZ_writeContentSizeFrame(frames + frameSize, tails[i]); | |
| CHECK_EQ(ZSTD_decompressBound(frames, 2 * frameSize), expected); | |
| } | |
| /* The third addition must use the accumulated total, not the first size. */ | |
| { BYTE threeFrames[3 * sizeof(FUZ_contentSizeFrame)]; | |
| FUZ_writeContentSizeFrame(threeFrames, first); | |
| FUZ_writeContentSizeFrame(threeFrames + frameSize, 0x80); | |
| FUZ_writeContentSizeFrame(threeFrames + 2 * frameSize, 0x7D); | |
| CHECK_EQ(ZSTD_decompressBound(threeFrames, sizeof(threeFrames)), maxBound); | |
| FUZ_writeContentSizeFrame(threeFrames + 2 * frameSize, 0x7E); | |
| CHECK_EQ(ZSTD_decompressBound(threeFrames, sizeof(threeFrames)), ZSTD_CONTENTSIZE_ERROR); | |
| } | |
| /* A skippable frame contributes zero, including at the largest result. */ | |
| memcpy(frames + frameSize, skippable, sizeof(skippable)); | |
| FUZ_writeContentSizeFrame(frames + skippableEnd, maxBound - first); | |
| CHECK_EQ(ZSTD_decompressBound(frames, sizeof(frames)), maxBound); | |
| FUZ_writeContentSizeFrame(frames + skippableEnd, 0x100); | |
| CHECK_EQ(ZSTD_decompressBound(frames, sizeof(frames)), ZSTD_CONTENTSIZE_ERROR); | |
| CHECK_EQ(ZSTD_decompressBound(skippable, sizeof(skippable)), 0); | |
| /* Unknown-size frames have a numeric bound, not an UNKNOWN sentinel. */ | |
| CHECK_EQ(ZSTD_getFrameContentSize(unknownSize, sizeof(unknownSize)), ZSTD_CONTENTSIZE_UNKNOWN); | |
| CHECK_EQ(ZSTD_decompressBound(unknownSize, sizeof(unknownSize)), 1024); | |
| FUZ_writeContentSizeFrame(frames, maxBound - 1024); | |
| memcpy(frames + frameSize, unknownSize, sizeof(unknownSize)); | |
| CHECK_EQ(ZSTD_decompressBound(frames, frameSize + sizeof(unknownSize)), maxBound); | |
| FUZ_writeContentSizeFrame(frames, maxBound - 1023); | |
| CHECK_EQ(ZSTD_decompressBound(frames, frameSize + sizeof(unknownSize)), ZSTD_CONTENTSIZE_ERROR); | |
| /* Zero-sized frames remain valid; malformed trailing data does not. */ | |
| FUZ_writeContentSizeFrame(frames, 0); | |
| memcpy(frames + frameSize, skippable, sizeof(skippable)); | |
| CHECK_EQ(ZSTD_decompressBound(frames, skippableEnd), 0); | |
| CHECK_EQ(ZSTD_decompressBound(frames, skippableEnd - 1), ZSTD_CONTENTSIZE_ERROR); | |
| frames[skippableEnd] = 0; | |
| CHECK_EQ(ZSTD_decompressBound(frames, skippableEnd + 1), ZSTD_CONTENTSIZE_ERROR); | |
| DISPLAYLEVEL(3, "OK \n"); | |
| } | |
| /* Write a compressed block with Huffman literals and no sequences. An empty | |
| * table header selects repeat mode, otherwise this installs a new table. */ | |
| static size_t FUZ_writeHuffmanLiteralBlock(BYTE* block, size_t capacity, | |
| const BYTE* literals, size_t literalSize, const HUF_CElt* ctable, | |
| const BYTE* tableHeader, size_t tableHeaderSize, int fourStreams) | |
| { | |
| size_t hufSize; | |
| size_t cLiteralSize; | |
| CHECK((tableHeader == NULL) == (tableHeaderSize == 0)); | |
| CHECK(capacity > 4 + tableHeaderSize); | |
| if (tableHeaderSize != 0) | |
| memcpy(block + 3, tableHeader, tableHeaderSize); | |
| hufSize = fourStreams | |
| ? HUF_compress4X_usingCTable(block + 3 + tableHeaderSize, | |
| capacity - 4 - tableHeaderSize, literals, literalSize, ctable, 0) | |
| : HUF_compress1X_usingCTable(block + 3 + tableHeaderSize, | |
| capacity - 4 - tableHeaderSize, literals, literalSize, ctable, 0); | |
| CHECK_Z(hufSize); | |
| CHECK(hufSize != 0); | |
| cLiteralSize = tableHeaderSize + hufSize; | |
| CHECK(literalSize < 1024 && cLiteralSize < 1024); | |
| MEM_writeLE24(block, (U32)(tableHeaderSize ? set_compressed : set_repeat) | |
| | ((U32)fourStreams << 2) | |
| | ((U32)literalSize << 4) | ((U32)cLiteralSize << 14)); | |
| block[3 + cLiteralSize] = 0; /* no sequences */ | |
| return 4 + cLiteralSize; | |
| } | |
| /* The malformed cases regress the forced-X2 dispatch bug. Default and | |
| * forced-X1 builds already reject them and serve as compatibility controls. */ | |
| static void test_decompressHuffmanTable(unsigned tnb) | |
| { | |
| enum { dictionaryHeaderSize = 8, fseTableCount = 3, fseNCountSize = 2, | |
| dictionaryContentSize = 16 }; | |
| BYTE literals[32]; | |
| BYTE output[128]; | |
| BYTE tableHeader[HUF_CTABLEBOUND]; | |
| BYTE block[128]; | |
| BYTE repeatBlock[128]; | |
| BYTE frame[256]; | |
| /* Magic/id, Huffman table, three FSE tables, repcodes, and content. */ | |
| BYTE dictionary[dictionaryHeaderSize + HUF_CTABLEBOUND | |
| + fseTableCount * fseNCountSize + ZSTD_REP_NUM * 4 | |
| + dictionaryContentSize]; | |
| unsigned counts[HUF_SYMBOLVALUE_MAX + 1] = { 0 }; | |
| HUF_CREATE_STATIC_CTABLE(ctable, HUF_SYMBOLVALUE_MAX); | |
| HUF_CREATE_STATIC_DTABLEX2(dtable, HUF_TABLELOG_MAX); | |
| U64 workspace[HUF_WORKSPACE_SIZE_U64]; | |
| ZSTD_DCtx* const dctx = ZSTD_createDCtx(); | |
| ZSTD_DDict* ddict; | |
| size_t tableLog; | |
| size_t tableHeaderSize; | |
| size_t dictionarySize; | |
| size_t repeatSize; | |
| size_t i; | |
| unsigned fourStreams; | |
| DISPLAYLEVEL(3, "test%3u : require a table for compressed Huffman literals : ", tnb); | |
| CHECK(dctx != NULL); | |
| for (i = 0; i < sizeof(literals); ++i) { | |
| literals[i] = (BYTE)(i & 1); | |
| ++counts[literals[i]]; | |
| } | |
| CHECK_VAR(tableLog, HUF_buildCTable_wksp(ctable, counts, 1, | |
| HUF_TABLELOG_DEFAULT, workspace, sizeof(workspace))); | |
| CHECK_VAR(tableHeaderSize, HUF_writeCTable_wksp(tableHeader, | |
| sizeof(tableHeader), ctable, 1, (unsigned)tableLog, | |
| workspace, sizeof(workspace))); | |
| CHECK(tableHeaderSize != 0); | |
| /* A formatted dictionary with this Huffman table, minimal FSE tables, | |
| * repcodes of 4, and 16 bytes of content. */ | |
| MEM_writeLE32(dictionary, ZSTD_MAGIC_DICTIONARY); | |
| MEM_writeLE32(dictionary + 4, 1); | |
| memcpy(dictionary + dictionaryHeaderSize, tableHeader, tableHeaderSize); | |
| dictionarySize = dictionaryHeaderSize + tableHeaderSize; | |
| for (i = 0; i < fseTableCount; ++i) { | |
| /* FSE NCount: tableLog 5, normalized count 32 for symbol 0. */ | |
| MEM_writeLE16(dictionary + dictionarySize, 0x03F0); | |
| dictionarySize += fseNCountSize; | |
| } | |
| for (i = 0; i < ZSTD_REP_NUM; ++i) { | |
| MEM_writeLE32(dictionary + dictionarySize, 4); | |
| dictionarySize += 4; | |
| } | |
| memset(dictionary + dictionarySize, 0, dictionaryContentSize); | |
| dictionarySize += dictionaryContentSize; | |
| ddict = ZSTD_createDDict(dictionary, dictionarySize); | |
| CHECK(ddict != NULL); | |
| repeatSize = FUZ_writeHuffmanLiteralBlock(repeatBlock, sizeof(repeatBlock), | |
| literals, sizeof(literals), ctable, NULL, 0, 0); | |
| for (i = 0; i < 2; ++i) { | |
| size_t const cLiteralSize = i == 0 ? 1 : 16; | |
| size_t const blockSize = 4 + cLiteralSize; | |
| size_t frameSize = 6; | |
| unsigned withDict; | |
| /* These are the generic HUF decoder's RLE and raw shortcuts, but | |
| * neither is a valid Zstd compressed-literals Huffman table. */ | |
| MEM_writeLE24(block, set_compressed | (16U << 4) | |
| | ((U32)cLiteralSize << 14)); | |
| memset(block + 3, 'A', cLiteralSize); | |
| block[3 + cLiteralSize] = 0; | |
| for (withDict = 0; withDict < 2; ++withDict) { | |
| CHECK_Z(ZSTD_decompressBegin_usingDDict(dctx, withDict ? ddict : NULL)); | |
| CHECK_EQ(ZSTD_getErrorCode(ZSTD_decompressBlock(dctx, output, | |
| sizeof(output), block, blockSize)), ZSTD_error_corruption_detected); | |
| } | |
| /* The originally reported two-block frame must stop at the malformed | |
| * block instead of reaching its repeat block with invalid state. */ | |
| MEM_writeLE32(frame, ZSTD_MAGICNUMBER); | |
| frame[4] = 0; | |
| frame[5] = 0; | |
| MEM_writeLE24(frame + frameSize, (U32)(blockSize << 3) | (bt_compressed << 1)); | |
| frameSize += 3; | |
| memcpy(frame + frameSize, block, blockSize); | |
| frameSize += blockSize; | |
| MEM_writeLE24(frame + frameSize, (U32)(repeatSize << 3) | (bt_compressed << 1) | 1); | |
| frameSize += 3; | |
| memcpy(frame + frameSize, repeatBlock, repeatSize); | |
| frameSize += repeatSize; | |
| CHECK_EQ(ZSTD_getErrorCode(ZSTD_decompressDCtx(dctx, output, | |
| sizeof(output), frame, frameSize)), ZSTD_error_corruption_detected); | |
| } | |
| /* Valid one- and four-stream tables must still install reusable state. | |
| * Also reuse the context after the preceding errors and test a DDict's | |
| * caller-owned table without installing an in-frame Huffman table. */ | |
| for (fourStreams = 0; fourStreams < 2; ++fourStreams) { | |
| size_t const blockSize = FUZ_writeHuffmanLiteralBlock(block, sizeof(block), | |
| literals, sizeof(literals), ctable, tableHeader, tableHeaderSize, fourStreams); | |
| repeatSize = FUZ_writeHuffmanLiteralBlock(repeatBlock, sizeof(repeatBlock), | |
| literals, sizeof(literals), ctable, NULL, 0, fourStreams); | |
| CHECK_Z(ZSTD_decompressBegin(dctx)); | |
| CHECK_EQ(ZSTD_decompressBlock(dctx, output, sizeof(output), block, blockSize), sizeof(literals)); | |
| CHECK_EQ(memcmp(output, literals, sizeof(literals)), 0); | |
| CHECK_EQ(ZSTD_decompressBlock(dctx, output, sizeof(output), repeatBlock, repeatSize), sizeof(literals)); | |
| CHECK_EQ(memcmp(output, literals, sizeof(literals)), 0); | |
| CHECK_Z(ZSTD_decompressBegin_usingDDict(dctx, ddict)); | |
| CHECK_EQ(ZSTD_decompressBlock(dctx, output, sizeof(output), repeatBlock, repeatSize), sizeof(literals)); | |
| CHECK_EQ(memcmp(output, literals, sizeof(literals)), 0); | |
| } | |
| /* The generic HUF API still promises its raw/RLE shortcuts. */ | |
| CHECK_EQ(HUF_decompress1X_DCtx_wksp(dtable, output, sizeof(literals), | |
| "A", 1, workspace, sizeof(workspace), 0), sizeof(literals)); | |
| for (i = 0; i < sizeof(literals); ++i) CHECK_EQ(output[i], 'A'); | |
| CHECK_EQ(HUF_decompress1X_DCtx_wksp(dtable, output, sizeof(literals), | |
| literals, sizeof(literals), workspace, sizeof(workspace), 0), sizeof(literals)); | |
| CHECK_EQ(memcmp(output, literals, sizeof(literals)), 0); | |
| ZSTD_freeDDict(ddict); | |
| ZSTD_freeDCtx(dctx); | |
| DISPLAYLEVEL(3, "OK \n"); | |
| } | |
| /* Size of the literals of the block built by FUZ_writeSplitLitBlock(). | |
| * It is larger than the decoder's 64 KB extra literal buffer, so the first | |
| * kSplitLitInDst literals are stored at the end of the block's area in dst. */ | |
| #define kSplitLitSize (64 KB + 112) | |
| #define kSplitLitInDst (kSplitLitSize - 64 KB) | |
| #define kSplitLitBlockMaxSize (kSplitLitSize + 3 + 9) | |
| /* Writes a compressed block (without block header) with raw literals and 2 sequences, | |
| * which decodes to 48 literals, a match of 65539+seq1MLExtra bytes at offset 1, | |
| * 49 literals, a match of 3 bytes at offset 1, and the remaining literals. | |
| * The block decodes to more than ZSTD_BLOCKSIZE_MAX bytes, so the first match runs | |
| * up to (or past) the literals that are stored in dst. | |
| * Writes the expected output to expected, which must hold | |
| * FUZ_splitLitBlockDecodedSize(seq1MLExtra) bytes. | |
| * @return : size of the block */ | |
| static size_t FUZ_writeSplitLitBlock(BYTE* block, BYTE* expected, U32 seq1MLExtra) | |
| { | |
| U32 const seq1ML = 65539 + seq1MLExtra; | |
| BYTE* op = block; | |
| size_t i; | |
| /* Literals section header: raw literals, 3 byte header */ | |
| MEM_writeLE24(op, (U32)(kSplitLitSize << 4) | (3 << 2)); | |
| op += 3; | |
| for (i = 0; i < kSplitLitSize; i++) { | |
| *op++ = i < kSplitLitInDst ? (BYTE)(i + 1) : 0xCC; | |
| } | |
| /* Sequences section header: 2 sequences, LL & OF RLE, ML predefined. | |
| * LL code 24 (base 48, 4 extra bits), OF code 0 (repcode 1 = offset 1). */ | |
| *op++ = 2; | |
| *op++ = 0x50; | |
| *op++ = 24; | |
| *op++ = 0; | |
| /* Bitstream, read from the top down: ML initial state 57 (ML code 52: base 65539, | |
| * 16 extra bits), sequence 1 ML extra bits, LL extra bits 0 (LL 48), | |
| * ML state update to state 0 (ML 3), sequence 2 LL extra bits 1 (LL 49). */ | |
| { U64 const bitStream = (1ULL << 36) | (57ULL << 30) | ((U64)seq1MLExtra << 14) | 1; | |
| MEM_writeLE32(op, (U32)bitStream); | |
| op[4] = (BYTE)(bitStream >> 32); | |
| op += 5; | |
| } | |
| { BYTE* eop = expected; | |
| memcpy(eop, block + 3, 48); | |
| eop += 48; | |
| memset(eop, 48, seq1ML); | |
| eop += seq1ML; | |
| memcpy(eop, block + 3 + 48, 49); | |
| eop += 49; | |
| memset(eop, 97, 3); | |
| eop += 3; | |
| memcpy(eop, block + 3 + 97, kSplitLitSize - 97); | |
| } | |
| assert((size_t)(op - block) <= kSplitLitBlockMaxSize); | |
| return (size_t)(op - block); | |
| } | |
| static size_t FUZ_splitLitBlockDecodedSize(U32 seq1MLExtra) | |
| { | |
| return kSplitLitSize + 65539 + seq1MLExtra + 3; | |
| } | |
| static void test_decompressSplitLitBuffer(unsigned tnb) | |
| { | |
| /* The literals are split, rather than stored after the block's output, because | |
| * dstCapacity <= ZSTD_BLOCKSIZE_MAX + litSize + 2 * WILDCOPY_OVERLENGTH. */ | |
| size_t const dstCapacity = ZSTD_BLOCKSIZE_MAX + kSplitLitSize + 64; | |
| size_t const srcCapacity = ZSTD_FRAMEHEADERSIZE_MAX + ZSTD_blockHeaderSize + kSplitLitBlockMaxSize; | |
| /* seq1MLExtra = 65371 : The first match ends 18 bytes before the next literals, | |
| * so the literals are copied to an overlapping position. | |
| * seq1MLExtra = 65460 : The first match overwrites the next literals. */ | |
| U32 const seq1MLExtras[] = { 65371, 65460 }; | |
| BYTE* const src = (BYTE*)malloc(srcCapacity); | |
| BYTE* const dst = (BYTE*)malloc(dstCapacity); | |
| BYTE* const expected = (BYTE*)malloc(dstCapacity); | |
| ZSTD_DCtx* const dctx = ZSTD_createDCtx(); | |
| size_t i; | |
| int anySucceeded = 0; | |
| CHECK(src != NULL && dst != NULL && expected != NULL && dctx != NULL); | |
| DISPLAYLEVEL(3, "test%3u : decompress block with split literals overrun by output : ", tnb); | |
| for (i = 0; i < COUNTOF(seq1MLExtras); i++) { | |
| size_t const decodedSize = FUZ_splitLitBlockDecodedSize(seq1MLExtras[i]); | |
| size_t blockSize; | |
| size_t r; | |
| CHECK(decodedSize <= dstCapacity); | |
| /* The block API has no ZSTD_BLOCKSIZE_MAX limit on the decoded size, | |
| * so the decoder must either reject the block or decode it correctly. */ | |
| blockSize = FUZ_writeSplitLitBlock(src, expected, seq1MLExtras[i]); | |
| CHECK_Z( ZSTD_decompressBegin(dctx) ); | |
| r = ZSTD_decompressBlock(dctx, dst, dstCapacity, src, blockSize); | |
| if (!ZSTD_isError(r)) { | |
| CHECK_EQ(r, decodedSize); | |
| CHECK(!memcmp(dst, expected, decodedSize)); | |
| anySucceeded = 1; | |
| } | |
| /* Wrap the block in a frame with windowLog 17, so blockSizeMax = ZSTD_BLOCKSIZE_MAX. | |
| * The frame must be rejected, because the block decodes to more than blockSizeMax. */ | |
| MEM_writeLE32(src, ZSTD_MAGICNUMBER); | |
| src[4] = 0x00; /* Frame header descriptor: no content size, no checksum, no dictID */ | |
| src[5] = 0x38; /* Window descriptor: windowLog 17 */ | |
| blockSize = FUZ_writeSplitLitBlock(src + 9, expected, seq1MLExtras[i]); | |
| MEM_writeLE24(src + 6, 1 /* last block */ | ((U32)bt_compressed << 1) | (U32)(blockSize << 3)); | |
| r = ZSTD_decompress(dst, dstCapacity, src, 9 + blockSize); | |
| CHECK(ZSTD_isError(r)); | |
| } | |
| CHECK(anySucceeded); /* Must not reject all blocks */ | |
| DISPLAYLEVEL(3, "OK \n"); | |
| ZSTD_freeDCtx(dctx); | |
| free(expected); | |
| free(dst); | |
| free(src); | |
| } | |
| static void test_decompressContinueWithoutBegin(unsigned tnb) | |
| { | |
| char src[1000]; | |
| char compressed[ZSTD_COMPRESSBOUND(sizeof(src))]; | |
| char decoded[sizeof(src)]; | |
| ZSTD_DCtx* const dctx = ZSTD_createDCtx(); | |
| size_t cSize, cPos = 0, dPos = 0; | |
| CHECK(dctx != NULL); | |
| DISPLAYLEVEL(3, "test%3u : ZSTD_decompressContinue() without ZSTD_decompressBegin() : ", tnb); | |
| memset(src, 'a', sizeof(src)); | |
| CHECK_VAR(cSize, ZSTD_compress(compressed, sizeof(compressed), src, sizeof(src), 1)); | |
| /* new context */ | |
| CHECK_EQ(ZSTD_nextSrcSizeToDecompress(dctx), 0); | |
| CHECK_EQ(ZSTD_getErrorCode(ZSTD_decompressContinue(dctx, decoded, sizeof(decoded), NULL, 0)), ZSTD_error_stage_wrong); | |
| /* finished frame */ | |
| CHECK_Z(ZSTD_decompressBegin(dctx)); | |
| while (ZSTD_nextSrcSizeToDecompress(dctx) != 0) { | |
| size_t const inSize = ZSTD_nextSrcSizeToDecompress(dctx); | |
| CHECK_NEWV(outSize, ZSTD_decompressContinue(dctx, decoded + dPos, sizeof(decoded) - dPos, compressed + cPos, inSize)); | |
| cPos += inSize; | |
| dPos += outSize; | |
| } | |
| CHECK_EQ(cPos, cSize); | |
| CHECK_EQ(dPos, sizeof(src)); | |
| CHECK_EQ(ZSTD_getErrorCode(ZSTD_decompressContinue(dctx, decoded, sizeof(decoded), NULL, 0)), ZSTD_error_stage_wrong); | |
| DISPLAYLEVEL(3, "OK \n"); | |
| ZSTD_freeDCtx(dctx); | |
| } | |
| typedef struct { | |
| const void* prefixStart; | |
| const void* virtualStart; | |
| const void* dictEnd; | |
| const void* previousDstEnd; | |
| } FUZ_DCtxHistory; | |
| static FUZ_DCtxHistory FUZ_getDCtxHistory(const ZSTD_DCtx* dctx) | |
| { | |
| FUZ_DCtxHistory history; | |
| history.prefixStart = dctx->prefixStart; | |
| history.virtualStart = dctx->virtualStart; | |
| history.dictEnd = dctx->dictEnd; | |
| history.previousDstEnd = dctx->previousDstEnd; | |
| return history; | |
| } | |
| static void FUZ_checkDCtxHistory(const ZSTD_DCtx* dctx, FUZ_DCtxHistory history) | |
| { | |
| CHECK_EQ(dctx->prefixStart, history.prefixStart); | |
| CHECK_EQ(dctx->virtualStart, history.virtualStart); | |
| CHECK_EQ(dctx->dictEnd, history.dictEnd); | |
| CHECK_EQ(dctx->previousDstEnd, history.previousDstEnd); | |
| } | |
| typedef enum { | |
| /* Keep the three dictionary-configuration cases first. */ | |
| historyNone, | |
| historyDictionary, | |
| historyDDict, | |
| historyInsertedPrefix, | |
| historyDecodedPrefix, | |
| historyCount | |
| } FUZ_HistoryKind; | |
| static void FUZ_beginHistory(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict, | |
| FUZ_HistoryKind historyKind, | |
| const void* dictionary, size_t dictionarySize) | |
| { | |
| if (historyKind == historyDDict) { | |
| CHECK_Z(ZSTD_decompressBegin_usingDDict(dctx, ddict)); | |
| } else if (historyKind == historyDictionary) { | |
| CHECK_Z(ZSTD_decompressBegin_usingDict(dctx, dictionary, dictionarySize)); | |
| } else { | |
| CHECK_Z(ZSTD_decompressBegin(dctx)); | |
| } | |
| } | |
| static void test_decompressZeroByteHistory(unsigned tnb) | |
| { | |
| enum { | |
| emptyInsertion, | |
| emptyDecompression, | |
| emptyOperationCount | |
| } emptyOperation; | |
| /* Magic, no checksum/content size/dictionary ID, and a 1 KiB window. */ | |
| static const BYTE frameHeader[] = { 0x28, 0xB5, 0x2F, 0xFD, 0, 0 }; | |
| static const struct { | |
| BYTE bytes[5]; | |
| size_t size; | |
| } emptyBlocks[] = { | |
| { { 0, 0, 0 }, 3 }, /* empty raw block */ | |
| { { 2, 0, 0, 'A' }, 4 }, /* empty RLE block */ | |
| { { 0x14, 0, 0, 0, 0 }, 5 } /* compressed block: no literals or sequences */ | |
| }; | |
| static const BYTE matchHeader[] = { 0x3D, 0, 0 }; | |
| /* No literals, one four-byte match at repeat offset 4. */ | |
| static const BYTE matchBlock[] = { 0, 1, 0x54, 0, 0, 1, 1 }; | |
| static const BYTE emptyCompressedBlock[] = { 0, 0 }; | |
| static const BYTE literalsBlock[] = { 0x20, 'w', 'x', 'y', 'z', 0 }; | |
| static const char dictionary[] = "0123456789abcdef"; | |
| static const char insertedPrefix[] = "wxyz"; | |
| BYTE output1[64]; | |
| BYTE output2[64]; | |
| BYTE decodedPrefix[64]; | |
| void* const emptyDsts[] = { NULL, output1, output2 }; | |
| void* const outputDsts[] = { output1, output2 }; | |
| ZSTD_DCtx* const dctx = ZSTD_createDCtx(); | |
| ZSTD_DDict* const ddict = ZSTD_createDDict(dictionary, sizeof(dictionary) - 1); | |
| size_t blockType, emptyDstNb, outputDstNb; | |
| FUZ_HistoryKind historyKind; | |
| CHECK(dctx != NULL && ddict != NULL); | |
| DISPLAYLEVEL(3, "test%3u : zero-byte decompression preserves history : ", tnb); | |
| /* Bufferless frame decoding. Headers and empty raw/RLE/compressed blocks | |
| * may be supplied with any zero-capacity destination, including NULL. | |
| * They must neither invent history nor discard a real dictionary. */ | |
| for (historyKind = historyNone; historyKind < historyInsertedPrefix; ++historyKind) { | |
| for (blockType = 0; blockType < COUNTOF(emptyBlocks); ++blockType) { | |
| BYTE prefix[sizeof(frameHeader) + sizeof(emptyBlocks[0].bytes)]; | |
| size_t const prefixSize = sizeof(frameHeader) + emptyBlocks[blockType].size; | |
| memcpy(prefix, frameHeader, sizeof(frameHeader)); | |
| memcpy(prefix + sizeof(frameHeader), emptyBlocks[blockType].bytes, emptyBlocks[blockType].size); | |
| for (emptyDstNb = 0; emptyDstNb < COUNTOF(emptyDsts); ++emptyDstNb) { | |
| for (outputDstNb = 0; outputDstNb < COUNTOF(outputDsts); ++outputDstNb) { | |
| FUZ_DCtxHistory history; | |
| size_t pos = 0; | |
| size_t result; | |
| FUZ_beginHistory(dctx, ddict, historyKind, dictionary, sizeof(dictionary) - 1); | |
| history = FUZ_getDCtxHistory(dctx); | |
| while (pos < prefixSize) { | |
| size_t const next = ZSTD_nextSrcSizeToDecompress(dctx); | |
| CHECK(next != 0 && next <= prefixSize - pos); | |
| CHECK_EQ(ZSTD_decompressContinue(dctx, emptyDsts[emptyDstNb], 0, | |
| prefix + pos, next), 0); | |
| pos += next; | |
| } | |
| FUZ_checkDCtxHistory(dctx, history); | |
| CHECK_EQ(ZSTD_nextSrcSizeToDecompress(dctx), sizeof(matchHeader)); | |
| CHECK_EQ(ZSTD_decompressContinue(dctx, outputDsts[outputDstNb], sizeof(output1), | |
| matchHeader, sizeof(matchHeader)), 0); | |
| result = ZSTD_decompressContinue(dctx, outputDsts[outputDstNb], sizeof(output1), | |
| matchBlock, sizeof(matchBlock)); | |
| if (historyKind != historyNone) { | |
| CHECK_EQ(result, 4); | |
| CHECK(!memcmp(outputDsts[outputDstNb], "cdef", 4)); | |
| } else { | |
| CHECK_EQ(ZSTD_getErrorCode(result), ZSTD_error_corruption_detected); | |
| } | |
| } | |
| } | |
| } | |
| } | |
| /* The block APIs have the same invariant. Exercise both empty insertion | |
| * and empty compressed blocks with no history, raw and prepared dictionaries, | |
| * an inserted raw prefix, and a previously decoded prefix. */ | |
| for (historyKind = historyNone; historyKind < historyCount; ++historyKind) { | |
| for (emptyOperation = emptyInsertion; emptyOperation < emptyOperationCount; ++emptyOperation) { | |
| for (emptyDstNb = 0; emptyDstNb < COUNTOF(emptyDsts); ++emptyDstNb) { | |
| for (outputDstNb = 0; outputDstNb < COUNTOF(outputDsts); ++outputDstNb) { | |
| FUZ_DCtxHistory history; | |
| size_t result; | |
| FUZ_beginHistory(dctx, ddict, historyKind, dictionary, sizeof(dictionary) - 1); | |
| if (historyKind == historyInsertedPrefix) | |
| CHECK_EQ(ZSTD_insertBlock(dctx, insertedPrefix, sizeof(insertedPrefix) - 1), 4); | |
| if (historyKind == historyDecodedPrefix) | |
| CHECK_EQ(ZSTD_decompressBlock(dctx, decodedPrefix, sizeof(decodedPrefix), | |
| literalsBlock, sizeof(literalsBlock)), 4); | |
| history = FUZ_getDCtxHistory(dctx); | |
| if (emptyOperation == emptyInsertion) { | |
| CHECK_EQ(ZSTD_insertBlock(dctx, emptyDsts[emptyDstNb], 0), 0); | |
| } else { | |
| CHECK_EQ(ZSTD_decompressBlock(dctx, emptyDsts[emptyDstNb], 0, | |
| emptyCompressedBlock, sizeof(emptyCompressedBlock)), 0); | |
| } | |
| FUZ_checkDCtxHistory(dctx, history); | |
| result = ZSTD_decompressBlock(dctx, outputDsts[outputDstNb], sizeof(output1), | |
| matchBlock, sizeof(matchBlock)); | |
| if (historyKind != historyNone) { | |
| CHECK_EQ(result, 4); | |
| CHECK(!memcmp(outputDsts[outputDstNb], | |
| historyKind == historyDictionary || historyKind == historyDDict ? "cdef" : "wxyz", 4)); | |
| } else { | |
| CHECK_EQ(ZSTD_getErrorCode(result), ZSTD_error_corruption_detected); | |
| } | |
| } | |
| } | |
| } | |
| } | |
| ZSTD_freeDCtx(dctx); | |
| ZSTD_freeDDict(ddict); | |
| DISPLAYLEVEL(3, "OK \n"); | |
| } | |
| /* Decodes src with ZSTD_decompressStream() using a single input buffer, whose | |
| * size is first limited to `split` bytes and then extended to srcSize, so the | |
| * second call continues at in.pos == split. Returns the last streaming result. | |
| * Output is written to dst (of capacity dstCap), and its size to *dstSize. */ | |
| static size_t FUZ_decompressStreamSplit(ZSTD_DCtx* dctx, const void* src, size_t srcSize, | |
| size_t split, void* dst, size_t dstCap, size_t* dstSize) | |
| { | |
| ZSTD_inBuffer in = { src, split, 0 }; | |
| ZSTD_outBuffer out = { dst, dstCap, 0 }; | |
| size_t result; | |
| CHECK_Z(ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters)); | |
| result = ZSTD_decompressStream(dctx, &out, &in); | |
| if (!ZSTD_isError(result) && split < srcSize) { | |
| in.size = srcSize; | |
| result = ZSTD_decompressStream(dctx, &out, &in); | |
| } | |
| *dstSize = out.pos; | |
| return result; | |
| } | |
| /* Checks that streaming decompression does not depend on where the input is | |
| * split between two calls (in particular, in the middle of the frame header). | |
| * Decodes with every split point and compares against decoding with no split. | |
| * If expected == NULL, the stream must be rejected, else it must decode to expected. */ | |
| static void FUZ_checkStreamSplits(const void* src, size_t srcSize, | |
| const void* expected, size_t expectedSize) | |
| { | |
| /* Large enough for the content size claimed by embeddedFrame below, since | |
| * the single-pass fast path is only taken if the output has room for it. */ | |
| size_t const outCapacity = 128 KB; | |
| ZSTD_DCtx* const dctx = ZSTD_createDCtx(); | |
| BYTE* const refOut = (BYTE*)malloc(outCapacity); | |
| BYTE* const out = (BYTE*)malloc(outCapacity); | |
| size_t refSize, outSize, split, ref; | |
| CHECK(dctx != NULL && refOut != NULL && out != NULL); | |
| ref = FUZ_decompressStreamSplit(dctx, src, srcSize, srcSize, refOut, outCapacity, &refSize); | |
| if (expected == NULL) { | |
| CHECK(ZSTD_isError(ref)); | |
| } else { | |
| CHECK_EQ(ref, 0); | |
| CHECK_EQ(refSize, expectedSize); | |
| CHECK(memcmp(refOut, expected, expectedSize) == 0); | |
| } | |
| for (split = 0; split < srcSize; ++split) { | |
| size_t const result = FUZ_decompressStreamSplit(dctx, src, srcSize, split, out, outCapacity, &outSize); | |
| CHECK_EQ(ZSTD_getErrorCode(result), ZSTD_getErrorCode(ref)); | |
| if (!ZSTD_isError(result)) CHECK_EQ(result, ref); | |
| CHECK_EQ(outSize, refSize); | |
| CHECK(memcmp(out, refOut, refSize) == 0); | |
| } | |
| free(out); | |
| free(refOut); | |
| ZSTD_freeDCtx(dctx); | |
| } | |
| /* Guards against ZSTD_decompressStream() mishandling a frame header that is | |
| * split across calls. The streaming decoder has fast paths (decode the whole | |
| * frame in one pass, detect legacy frames) that inspect the current input | |
| * buffer from its start. They are only valid when the frame header starts in | |
| * this call. After a split header, they would instead look at the middle of | |
| * the frame, so bytes of a malformed frame that look like the start of another | |
| * frame could be decoded as if they were a frame. | |
| * | |
| * Each case is decoded at every split point, and must behave exactly like | |
| * decoding without a split. The malformed frames are crafted so that a wrong | |
| * fast path would accept them, which makes the comparison fail. A valid frame | |
| * checks that normal frames are unaffected. */ | |
| static void test_decompressStreamSplitHeader(unsigned tnb) | |
| { | |
| /* Malformed: the header says the frame uses a dictionary (dictID field) | |
| * that is not provided. The 12 bytes after the first 5 bytes (magic + FHD | |
| * 0xE3, which selects 4-byte dictID and 8-byte content size) also encode | |
| * a complete valid empty frame: magic, FHD 0x20, content size 0, and an | |
| * empty last block. Decoding that from a split at byte 5 is wrong. */ | |
| static const BYTE embeddedFrame[] = { | |
| 0x28, 0xB5, 0x2F, 0xFD, 0xE3, | |
| 0x28, 0xB5, 0x2F, 0xFD, 0x20, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00 | |
| }; | |
| /* Malformed: FHD 0x08 sets a reserved bit, which is reported once the | |
| * 6-byte header is available. The bytes after the first 5 start with the | |
| * magic number of a legacy (v0.7) frame. Only matters when the library | |
| * is built with legacy support, and must not be detected as a legacy | |
| * frame since the frame did not start there. */ | |
| static const BYTE embeddedLegacy[] = { | |
| 0x28, 0xB5, 0x2F, 0xFD, 0x08, | |
| 0x27, 0xB5, 0x2F, 0xFD, 0x00, 0x00, 0x00, 0x00 | |
| }; | |
| BYTE source[257]; | |
| BYTE compressed[ZSTD_COMPRESSBOUND(sizeof(source))]; | |
| size_t compressedSize, i; | |
| DISPLAYLEVEL(3, "test%3u : streaming frame header split across calls : ", tnb); | |
| FUZ_checkStreamSplits(embeddedFrame, sizeof(embeddedFrame), NULL, 0); | |
| FUZ_checkStreamSplits(embeddedLegacy, sizeof(embeddedLegacy), NULL, 0); | |
| for (i = 0; i < sizeof(source); ++i) source[i] = (BYTE)(i * 23); | |
| CHECK_VAR(compressedSize, ZSTD_compress(compressed, sizeof(compressed), source, sizeof(source), 1)); | |
| FUZ_checkStreamSplits(compressed, compressedSize, source, sizeof(source)); | |
| DISPLAYLEVEL(3, "OK \n"); | |
| } | |
| static unsigned test_histCountWksp(unsigned seed, unsigned testNb) | |
| { | |
| static const unsigned symLowLimits[] = { 0, 27, 0, 0, 27, 42, 0, 0, 27, 42, 27, 42 }; | |
| static const unsigned symHighLimits[] = { 255, 255, 210, 110, 42, 42, 210, 110, 42, 42, 42, 42 }; | |
| static const unsigned symMaxLimits[] = { 255, 255, 255, 255, 255, 255, 230, 130, 99, 99, 42, 42 }; | |
| static const size_t inputSizes[] = { 3367, 1761, 893, 117 }; | |
| unsigned workspace[HIST_WKSP_SIZE_U32]; | |
| size_t res, i, is, il; | |
| DISPLAYLEVEL(3, "test%3u : HIST_count_wksp with empty source : ", testNb++); | |
| { | |
| /* With NULL source UBSan of older Clang could fail: applying zero offset to null pointer. */ | |
| static const unsigned char source[4] = { 0 }; | |
| unsigned count[1] = { 0 }; | |
| unsigned maxSym = 0; | |
| res = HIST_count_wksp(count, &maxSym, source, 0, workspace, sizeof(workspace)); | |
| CHECK_EQ(res, 0); | |
| CHECK_EQ(maxSym, 0); | |
| CHECK_EQ(count[0], 0); | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| #if HIST_WKSP_SIZE_U32 | |
| DISPLAYLEVEL(3, "test%3u : HIST_count_wksp with small workspace : ", testNb++); | |
| { | |
| unsigned count[1] = { 0 }; | |
| unsigned maxSym = 0; | |
| res = HIST_count_wksp(count, &maxSym, NULL, 0, workspace, sizeof(workspace) - 1); | |
| CHECK_EQ(res, ERROR(workSpace_tooSmall)); | |
| CHECK_EQ(maxSym, 0); | |
| CHECK_EQ(count[0], 0); | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3u : HIST_count_wksp with wrong workspace alignment : ", testNb++); | |
| { | |
| unsigned count[1] = { 0 }; | |
| unsigned maxSym = 0; | |
| res = HIST_count_wksp(count, &maxSym, NULL, 0, (unsigned*)(void*)((char*)workspace + 1), sizeof(workspace)); | |
| CHECK_EQ(res, ERROR(GENERIC)); | |
| CHECK_EQ(maxSym, 0); | |
| CHECK_EQ(count[0], 0); | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| #endif | |
| DISPLAYLEVEL(3, "test%3u : HIST_count_wksp with symbol out of range, small size : ", testNb++); | |
| { | |
| /* For less elements HIST_count_parallel_wksp would fail. */ | |
| static const unsigned char source[4] = { 1, 4, 0, 2 }; | |
| static const unsigned expected[6] = { 0 }; | |
| unsigned count[6] = { 0 }; | |
| unsigned maxSym = 2; | |
| res = HIST_count_wksp(count, &maxSym, source, sizeof(source), workspace, sizeof(workspace)); | |
| CHECK_EQ(res, ERROR(maxSymbolValue_tooSmall)); | |
| CHECK_EQ(maxSym, 2); | |
| for (i = 0; i < COUNTOF(expected); ++i) CHECK_EQ(count[i], expected[i]); | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3u : HIST_count_wksp with symbol out of range, medium size : ", testNb++); | |
| { | |
| unsigned char source[3407]; | |
| unsigned count[6] = { 0 }; | |
| unsigned maxSym = 2; | |
| for (i = 0; i < COUNTOF(source); ++i) { | |
| source[i] = (48271 * (i + 1)) & 3; | |
| } | |
| res = HIST_count_wksp(count, &maxSym, source, sizeof(source), workspace, sizeof(workspace)); | |
| CHECK_EQ(res, ERROR(maxSymbolValue_tooSmall)); | |
| CHECK_EQ(maxSym, 2); | |
| for (i = 0; i < COUNTOF(count); ++i) CHECK_EQ(count[i], 0); | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| for (il = 0; il < COUNTOF(symMaxLimits); ++il) { | |
| unsigned symMax = symMaxLimits[il]; | |
| unsigned symLow = symLowLimits[il]; | |
| unsigned symHigh = symHighLimits[il]; | |
| unsigned symRange = symHigh - symLow + 1; | |
| for (is = 0; is < COUNTOF(inputSizes); ++is) { | |
| unsigned char source[4000]; | |
| size_t inputSize = inputSizes[is]; | |
| assert(inputSize <= sizeof(source)); | |
| DISPLAYLEVEL(3, "test%3u : HIST_count_wksp test in [%u..%u], symMax: %u, inputSize: %u : ", | |
| testNb++, symLow, symHigh, symMax, (unsigned)inputSize); | |
| { | |
| unsigned count[260] = { 0 }; | |
| unsigned expected[COUNTOF(count)] = { 0 }; | |
| unsigned maxSym = symMax; | |
| unsigned realMaxSym = symMax; | |
| unsigned maxCount = 0; | |
| for (i = 0; i < inputSize; ++i) { | |
| unsigned prng = (48271 * (i + seed)) % symRange + symLow; | |
| source[i] = (unsigned char)prng; | |
| ++expected[prng]; | |
| } | |
| /* for basic buffer overwrite checks */ | |
| for (i = maxSym + 1; i < COUNTOF(count); ++i) expected[i] = count[i] = ~0u; | |
| for (i = 0; i <= maxSym; ++i) maxCount = MAX(maxCount, expected[i]); | |
| for (i = realMaxSym; i > 0; --i) { | |
| if (expected[i]) break; | |
| --realMaxSym; | |
| } | |
| res = HIST_count_wksp(count, &maxSym, source, inputSize, workspace, sizeof(workspace)); | |
| CHECK_EQ(res, maxCount); | |
| CHECK_EQ(maxSym, realMaxSym); | |
| for (i = 0; i < COUNTOF(expected); ++i) CHECK_EQ(count[i], expected[i]); | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| } | |
| } | |
| return testNb; | |
| } | |
| static void test_histCountOverflowCase(const BYTE* source, size_t sourceSize, | |
| unsigned maxSymbolValue) | |
| { | |
| unsigned workspace[HIST_WKSP_SIZE_U32]; | |
| unsigned expected[260] = { 0 }; | |
| unsigned expectedMaxSymbol = 0; | |
| unsigned expectedLargest = 0; | |
| unsigned fast; | |
| size_t index; | |
| for (index = 0; index < sourceSize; ++index) { | |
| unsigned const symbol = source[index]; | |
| CHECK(symbol <= maxSymbolValue); | |
| ++expected[symbol]; | |
| expectedMaxSymbol = MAX(expectedMaxSymbol, symbol); | |
| expectedLargest = MAX(expectedLargest, expected[symbol]); | |
| } | |
| /* Entries past maxSymbolValue must not be written. */ | |
| for (index = maxSymbolValue + 1; index < COUNTOF(expected); ++index) | |
| expected[index] = ~0U; | |
| for (fast = 0; fast < 2; ++fast) { | |
| unsigned count[COUNTOF(expected)]; | |
| unsigned maxSymbol = maxSymbolValue; | |
| size_t result; | |
| memset(count, 0xFF, sizeof(count)); | |
| result = fast | |
| ? HIST_countFast_wksp(count, &maxSymbol, source, sourceSize, workspace, sizeof(workspace)) | |
| : HIST_count_wksp(count, &maxSymbol, source, sourceSize, workspace, sizeof(workspace)); | |
| CHECK_EQ(result, expectedLargest); | |
| CHECK_EQ(maxSymbol, expectedMaxSymbol); | |
| for (index = 0; index < COUNTOF(expected); ++index) | |
| CHECK_EQ(count[index], expected[index]); | |
| } | |
| } | |
| static void test_histCountOverflow(unsigned testNb) | |
| { | |
| /* The SVE2 histogram counts in chunks of 65520 bytes. Cover the sizes | |
| * around 1 and 2 chunks, 16-bit wraparound, and a 4-chunk input. */ | |
| static const size_t sizes[] = { | |
| 65519, 65520, 65521, 65536, 131040, 131041, 131072, 196625 | |
| }; | |
| static const unsigned symbols[] = { 0, 63, 64, 159, 160, 255 }; | |
| size_t const capacity = sizes[COUNTOF(sizes) - 1]; | |
| BYTE* const source = (BYTE*)malloc(capacity); | |
| unsigned workspace[HIST_WKSP_SIZE_U32]; | |
| unsigned symbol; | |
| size_t sizeIndex, symbolIndex, index; | |
| CHECK(source != NULL); | |
| DISPLAYLEVEL(3, "test%3u : histogram counter overflow : ", testNb); | |
| /* Every symbol, so every counter lane overflows. */ | |
| for (symbol = 0; symbol <= 255; ++symbol) { | |
| memset(source, (int)symbol, 131072); | |
| test_histCountOverflowCase(source, 65536, symbol); | |
| test_histCountOverflowCase(source, 131072, 255); | |
| } | |
| /* Every size, for symbols in each pass, plus a differing tail. */ | |
| for (symbolIndex = 0; symbolIndex < COUNTOF(symbols); ++symbolIndex) { | |
| symbol = symbols[symbolIndex]; | |
| memset(source, (int)symbol, capacity); | |
| for (sizeIndex = 0; sizeIndex < COUNTOF(sizes); ++sizeIndex) { | |
| test_histCountOverflowCase(source, sizes[sizeIndex], symbol); | |
| test_histCountOverflowCase(source, sizes[sizeIndex], 255); | |
| } | |
| for (index = 65536; index < capacity; ++index) | |
| source[index] = (BYTE)(symbol + 1 + (index - 65536) % 255); | |
| test_histCountOverflowCase(source, capacity, 255); | |
| } | |
| /* Per-chunk max symbols that rise and then fall. */ | |
| memset(source, 1, 65520); | |
| memset(source + 65520, 255, 65520); | |
| memset(source + 131040, 0, capacity - 131040); | |
| test_histCountOverflowCase(source, capacity, 255); | |
| /* A symbol out of range in the last chunk is an error. */ | |
| { unsigned count[260]; | |
| unsigned maxSymbol = 159; | |
| memset(source, 1, capacity); | |
| source[capacity - 1] = 160; | |
| memset(count, 0xFF, sizeof(count)); | |
| CHECK_EQ(HIST_count_wksp(count, &maxSymbol, source, capacity, workspace, sizeof(workspace)), | |
| ERROR(maxSymbolValue_tooSmall)); | |
| CHECK_EQ(maxSymbol, 159); | |
| for (index = 160; index < COUNTOF(count); ++index) | |
| CHECK_EQ(count[index], ~0U); | |
| } | |
| free(source); | |
| DISPLAYLEVEL(3, "OK \n"); | |
| } | |
| static void test_setCParams(unsigned tnb) | |
| { | |
| ZSTD_CCtx* const cctx = ZSTD_createCCtx(); | |
| ZSTD_compressionParameters cparams; | |
| assert(cctx); | |
| DISPLAYLEVEL(3, "test%3u : ZSTD_CCtx_setCParams : ", tnb); | |
| /* valid cparams */ | |
| cparams = ZSTD_getCParams(1, 0, 0); | |
| CHECK_Z(ZSTD_CCtx_setCParams(cctx, cparams)); | |
| /* invalid cparams (must fail) */ | |
| cparams.windowLog = 99; | |
| CHECK(ZSTD_isError(ZSTD_CCtx_setCParams(cctx, cparams))); | |
| free(cctx); | |
| DISPLAYLEVEL(3, "OK \n"); | |
| } | |
| static void test_blockSplitter_incompressibleExpansionProtection(unsigned testNb, unsigned seed) | |
| { | |
| DISPLAYLEVEL(3, "test%3i : Check block splitter doesn't oversplit incompressible data (seed %u): ", testNb, seed); | |
| { ZSTD_CCtx* cctx = ZSTD_createCCtx(); | |
| size_t const srcSize = 256 * 1024; /* needs to be at least 2 blocks */ | |
| void* incompressible = malloc(srcSize); | |
| size_t const dstCapacity = ZSTD_compressBound(srcSize); | |
| void* cBuffer = malloc(dstCapacity); | |
| size_t const chunkSize = 8 KB; | |
| size_t const nbChunks = srcSize / chunkSize; | |
| size_t chunkNb, cSizeNoSplit, cSizeWithSplit; | |
| assert(cctx != NULL); | |
| assert(incompressible != NULL); | |
| assert(cBuffer != NULL); | |
| /* let's fill input with random noise (incompressible) */ | |
| RDG_genBuffer(incompressible, srcSize, 0.0, 0.0, seed); | |
| /* this pattern targets the fastest _byChunk variant's sampling (level 3). | |
| * manually checked that, without the @savings protection, it would over-split. | |
| */ | |
| for (chunkNb=0; chunkNb<nbChunks; chunkNb++) { | |
| BYTE* const p = (BYTE*)incompressible + chunkNb * chunkSize; | |
| size_t const samplingRate = 43; | |
| int addOrRemove = chunkNb % 2; | |
| size_t n; | |
| for (n=0; n<chunkSize; n+=samplingRate) { | |
| if (addOrRemove) { | |
| p[n] &= 0x80; | |
| } else { | |
| p[n] |= 0x80; | |
| } | |
| } | |
| } | |
| /* run first without splitting */ | |
| ZSTD_CCtx_setParameter(cctx, ZSTD_c_blockSplitterLevel, 1 /* no split */); | |
| cSizeNoSplit = ZSTD_compress2(cctx, cBuffer, dstCapacity, incompressible, srcSize); | |
| /* run with sample43 splitter, check it's still the same */ | |
| ZSTD_CCtx_setParameter(cctx, ZSTD_c_blockSplitterLevel, 3 /* sample43, fastest _byChunk variant */); | |
| cSizeWithSplit = ZSTD_compress2(cctx, cBuffer, dstCapacity, incompressible, srcSize); | |
| if (cSizeWithSplit != cSizeNoSplit) { | |
| DISPLAYLEVEL(1, "invalid compressed size: cSizeWithSplit %u != %u cSizeNoSplit \n", | |
| (unsigned)cSizeWithSplit, (unsigned)cSizeNoSplit); | |
| abort(); | |
| } | |
| DISPLAYLEVEL(4, "compressed size: cSizeWithSplit %u == %u cSizeNoSplit : ", | |
| (unsigned)cSizeWithSplit, (unsigned)cSizeNoSplit); | |
| free(incompressible); | |
| free(cBuffer); | |
| ZSTD_freeCCtx(cctx); | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| } | |
| static void test_targetCBlockSize_dstCapacity(unsigned testNb) | |
| { | |
| static const size_t srcSizes[] = { 1300, 8 KB }; | |
| size_t const srcCapacity = 8 KB; | |
| size_t const dstAllocation = 2 * ZSTD_compressBound(srcCapacity); | |
| BYTE* const src = (BYTE*)malloc(srcCapacity); | |
| BYTE* const dst = (BYTE*)malloc(dstAllocation); | |
| BYTE* const decoded = (BYTE*)malloc(srcCapacity); | |
| ZSTD_CCtx* const cctx = ZSTD_createCCtx(); | |
| ZSTD_DCtx* const dctx = ZSTD_createDCtx(); | |
| size_t input; | |
| DISPLAYLEVEL(3, "test%3u : targetCBlockSize with short destinations : ", testNb); | |
| CHECK(src != NULL && dst != NULL && decoded != NULL); | |
| CHECK(cctx != NULL && dctx != NULL); | |
| /* Generate Huffman and FSE table descriptions across multiple sub-blocks. */ | |
| RDG_genBuffer(src, srcCapacity, 0.5, 0. /*auto*/, 0); | |
| CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 1)); | |
| CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_targetCBlockSize, 1340)); | |
| for (input = 0; input < COUNTOF(srcSizes); ++input) { | |
| size_t const srcSize = srcSizes[input]; | |
| size_t const bound = ZSTD_compressBound(srcSize); | |
| size_t capacity; | |
| for (capacity = 0; capacity <= bound; ++capacity) { | |
| size_t cSize, i; | |
| memset(dst, 0xA5, dstAllocation); | |
| cSize = ZSTD_compress2(cctx, dst, capacity, src, srcSize); | |
| /* Check before handling errors: the original bug returned | |
| * dstSize_tooSmall after already overwriting the destination. */ | |
| for (i = capacity; i < dstAllocation; ++i) { | |
| if (dst[i] != 0xA5) { | |
| DISPLAY("targetCBlockSize wrote past capacity %zu " | |
| "at offset %zu (srcSize=%zu)\n", capacity, i, srcSize); | |
| exit(1); | |
| } | |
| } | |
| if (ZSTD_isError(cSize)) { | |
| CHECK_EQ(ZSTD_getErrorCode(cSize), ZSTD_error_dstSize_tooSmall); | |
| CHECK_LT(capacity, bound); | |
| } else { | |
| CHECK(cSize <= capacity); | |
| CHECK_EQ(ZSTD_decompressDCtx(dctx, decoded, srcCapacity, dst, cSize), srcSize); | |
| CHECK_EQ(memcmp(decoded, src, srcSize), 0); | |
| } | |
| } | |
| } | |
| ZSTD_freeDCtx(dctx); | |
| ZSTD_freeCCtx(cctx); | |
| free(decoded); | |
| free(dst); | |
| free(src); | |
| DISPLAYLEVEL(3, "OK \n"); | |
| } | |
| size_t convertSequences_noRepcodes(SeqDef* dstSeqs, const ZSTD_Sequence* inSeqs, | |
| size_t nbSequences); | |
| static size_t convertSequences_noRepcodes_ref( | |
| SeqDef* dstSeqs, | |
| const ZSTD_Sequence* inSeqs, | |
| size_t nbSequences) | |
| { | |
| size_t longLen = 0; | |
| size_t n; | |
| for (n=0; n<nbSequences; n++) { | |
| dstSeqs[n].offBase = OFFSET_TO_OFFBASE(inSeqs[n].offset); | |
| dstSeqs[n].litLength = (U16)inSeqs[n].litLength; | |
| dstSeqs[n].mlBase = (U16)(inSeqs[n].matchLength - MINMATCH); | |
| /* Check for long length > 65535. */ | |
| if (UNLIKELY(inSeqs[n].matchLength > 65535+MINMATCH)) { | |
| assert(longLen == 0); | |
| longLen = n + 1; | |
| } | |
| if (UNLIKELY(inSeqs[n].litLength > 65535)) { | |
| assert(longLen == 0); | |
| longLen = n + nbSequences + 1; | |
| } | |
| } | |
| return longLen; | |
| } | |
| static unsigned test_convertSequences_noRepcodes(unsigned seed, unsigned testNb) | |
| { | |
| ZSTD_Sequence nsrc[12]; | |
| SeqDef ndst[12], rdst[12]; | |
| size_t ref, ret, i, j; | |
| seed += 0xDEADBEEF; | |
| for (i = 0; i < COUNTOF(nsrc); ++i) { | |
| seed = 48271 * ((unsigned)i + seed); | |
| nsrc[i].offset = (seed & 0xFFFF) | 1; /* Offset shall not be zero. */ | |
| seed = 48271 * ((unsigned)i + seed); | |
| nsrc[i].litLength = seed & 0xFFFF; | |
| seed = 48271 * ((unsigned)i + seed); | |
| nsrc[i].matchLength = (seed & 0xFFFFFF) % (65536 + MINMATCH); | |
| seed = 48271 * ((unsigned)i + seed); | |
| nsrc[i].rep = seed & 0xFF; | |
| } | |
| /* For near overflow and proper negative value handling. */ | |
| nsrc[5].matchLength = 65535 + MINMATCH; | |
| nsrc[6].litLength = 65535; | |
| nsrc[6].matchLength = 0; | |
| nsrc[7].litLength = 0; | |
| nsrc[7].matchLength = MINMATCH; | |
| for (i = 0; i <= COUNTOF(nsrc); ++i) { | |
| DISPLAYLEVEL(3, "test%3u : convertSequences_noRepcodes with %u inputs : ", | |
| testNb++, (unsigned)i); | |
| memset(ndst, 0, sizeof(ndst)); | |
| memset(rdst, 0, sizeof(rdst)); | |
| ref = convertSequences_noRepcodes_ref(rdst, nsrc, i); | |
| ret = convertSequences_noRepcodes(ndst, nsrc, i); | |
| CHECK_EQ(ret, ref); | |
| CHECK_EQ(memcmp(rdst, ndst, sizeof(ndst)), 0); | |
| DISPLAYLEVEL(3, "OK \n"); | |
| } | |
| nsrc[7].matchLength = 65536 + MINMATCH; | |
| for (i = 8; i <= COUNTOF(nsrc); ++i) { | |
| DISPLAYLEVEL(3, "test%3u : convertSequences_noRepcodes with %u inputs and " | |
| "matchLength overflow : ", | |
| testNb++, (unsigned)i); | |
| memset(ndst, 0, sizeof(ndst)); | |
| memset(rdst, 0, sizeof(rdst)); | |
| ref = convertSequences_noRepcodes_ref(rdst, nsrc, i); | |
| ret = convertSequences_noRepcodes(ndst, nsrc, i); | |
| CHECK_EQ(ret, ref); | |
| CHECK_EQ(memcmp(rdst, ndst, sizeof(ndst)), 0); | |
| DISPLAYLEVEL(3, "OK \n"); | |
| assert(COUNTOF(nsrc) > 8); | |
| for (j = 4; j < 8; ++j) { | |
| DISPLAYLEVEL(3, "test%3u : convertSequences_noRepcodes with %u inputs and " | |
| "matchLength overflow #%u : ", | |
| testNb++, (unsigned)i, (unsigned)(i - j)); | |
| memset(ndst, 0, sizeof(ndst)); | |
| memset(rdst, 0, sizeof(rdst)); | |
| ref = convertSequences_noRepcodes_ref(rdst, nsrc + j, i - j); | |
| ret = convertSequences_noRepcodes(ndst, nsrc + j, i - j); | |
| CHECK_EQ(ret, ref); | |
| CHECK_EQ(memcmp(rdst, ndst, sizeof(ndst)), 0); | |
| DISPLAYLEVEL(3, "OK \n"); | |
| } | |
| } | |
| nsrc[7].matchLength = 1; | |
| nsrc[7].litLength = 65536; | |
| for (i = 8; i <= COUNTOF(nsrc); ++i) { | |
| DISPLAYLEVEL(3, "test%3u : convertSequences_noRepcodes with %u inputs and " | |
| "litLength overflow: ", | |
| testNb++, (unsigned)i); | |
| memset(ndst, 0, sizeof(ndst)); | |
| memset(rdst, 0, sizeof(rdst)); | |
| ref = convertSequences_noRepcodes_ref(rdst, nsrc, i); | |
| ret = convertSequences_noRepcodes(ndst, nsrc, i); | |
| CHECK_EQ(ret, ref); | |
| CHECK_EQ(memcmp(rdst, ndst, sizeof(ndst)), 0); | |
| DISPLAYLEVEL(3, "OK \n"); | |
| assert(COUNTOF(nsrc) > 8); | |
| for (j = 4; j < 8; ++j) { | |
| DISPLAYLEVEL(3, "test%3u : convertSequences_noRepcodes with %u inputs and " | |
| "litLength overflow #%u: ", | |
| testNb++, (unsigned)i, (unsigned)(i - j)); | |
| memset(ndst, 0, sizeof(ndst)); | |
| memset(rdst, 0, sizeof(rdst)); | |
| ref = convertSequences_noRepcodes_ref(rdst, nsrc + j, i - j); | |
| ret = convertSequences_noRepcodes(ndst, nsrc + j, i - j); | |
| CHECK_EQ(ret, ref); | |
| CHECK_EQ(memcmp(rdst, ndst, sizeof(ndst)), 0); | |
| DISPLAYLEVEL(3, "OK \n"); | |
| } | |
| } | |
| return testNb; | |
| } | |
| static U32 FUZ_blockSplitHash(U32 x) | |
| { | |
| x ^= x >> 16; | |
| x *= 0x7feb352dU; | |
| x ^= x >> 15; | |
| x *= 0x846ca68bU; | |
| return x ^ (x >> 16); | |
| } | |
| /* The input is all zeroes, so all these offsets are valid matches. | |
| * Groups of 150 sequences have distinct offset-code distributions, making | |
| * enough dyadic splits beneficial to exhaust the block splitter's budget. */ | |
| static size_t FUZ_blockSplitSequenceProducer(void* state, | |
| ZSTD_Sequence* out, size_t capacity, | |
| const void* src, size_t srcSize, const void* dict, size_t dictSize, | |
| int compressionLevel, size_t windowSize) | |
| { | |
| size_t i; | |
| (void)state; (void)src; (void)dict; (void)dictSize; | |
| (void)compressionLevel; (void)windowSize; | |
| CHECK_EQ(srcSize, 128000); | |
| CHECK(capacity >= 38401); | |
| for (i = 0; i < 38400; ++i) { | |
| U32 const code = 3 + FUZ_blockSplitHash((U32)(i / 150)) % 10; | |
| out[i].offset = (1U << code) + (U32)(i % 4); | |
| out[i].litLength = i == 0 ? 12800 : 0; | |
| out[i].matchLength = 3; | |
| out[i].rep = 0; | |
| } | |
| memset(out + i, 0, sizeof(*out)); | |
| return i + 1; | |
| } | |
| static unsigned test_blockSplitLimit(unsigned testNb) | |
| { | |
| size_t const srcSize = 128000; | |
| size_t const dstCapacity = ZSTD_compressBound(srcSize); | |
| BYTE* const src = (BYTE*)calloc(1, srcSize); | |
| BYTE* const dst = (BYTE*)malloc(dstCapacity); | |
| BYTE* const decoded = (BYTE*)malloc(srcSize); | |
| ZSTD_CCtx* const cctx = ZSTD_createCCtx(); | |
| ZSTD_FrameHeader header; | |
| size_t cSize, pos, nbBlocks = 0; | |
| DISPLAYLEVEL(3, "test%3u : external sequences exhaust the block split budget : ", testNb++); | |
| CHECK(src != NULL && dst != NULL && decoded != NULL && cctx != NULL); | |
| CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 19)); | |
| CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_minMatch, 3)); | |
| CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_splitAfterSequences, ZSTD_ps_enable)); | |
| CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_validateSequences, 1)); | |
| ZSTD_registerSequenceProducer(cctx, NULL, FUZ_blockSplitSequenceProducer); | |
| cSize = ZSTD_compress2(cctx, dst, dstCapacity, src, srcSize); | |
| CHECK_Z(cSize); | |
| CHECK_EQ(ZSTD_getFrameHeader(&header, dst, cSize), 0); | |
| pos = header.headerSize; | |
| for (;;) { | |
| U32 blockHeader; | |
| CHECK(pos + ZSTD_blockHeaderSize <= cSize); | |
| blockHeader = MEM_readLE24(dst + pos); | |
| pos += ZSTD_blockHeaderSize; | |
| pos += ((blockHeader >> 1) & 3) == bt_rle ? 1 : blockHeader >> 3; | |
| CHECK(pos <= cSize); | |
| ++nbBlocks; | |
| if (blockHeader & 1) break; | |
| } | |
| CHECK_EQ(pos, cSize); | |
| CHECK_EQ(nbBlocks, ZSTD_MAX_NB_BLOCK_SPLITS + 1); | |
| CHECK_EQ(cctx->blockSplitCtx.partitions[ZSTD_MAX_NB_BLOCK_SPLITS], 38400); | |
| CHECK_EQ(ZSTD_decompress(decoded, srcSize, dst, cSize), srcSize); | |
| CHECK_EQ(memcmp(src, decoded, srcSize), 0); | |
| ZSTD_freeCCtx(cctx); | |
| free(decoded); | |
| free(dst); | |
| free(src); | |
| DISPLAYLEVEL(3, "OK \n"); | |
| return testNb; | |
| } | |
| static unsigned test_get1BlockSummary(unsigned testNb) | |
| { | |
| static const ZSTD_Sequence nseqs[] = { | |
| { 10, 2, 4, 1 }, | |
| { 20, 3, 5, 2 }, | |
| { 30, 6, 8, 3 }, | |
| { 40, 7, 9, 4 }, | |
| { 50, 10, 12, 5 }, | |
| { 60, 11, 13, 6 }, | |
| { 0, 14, 0, 7 }, | |
| { 70, 15, 17, 8 }, | |
| { 80, 16, 18, 9 }, | |
| { 90, 19, 21, 1 }, | |
| { 99, 20, 22, 2 }, | |
| }; | |
| static const BlockSummary blocks[] = { | |
| { 7, 104, 53 }, | |
| { 6, 98, 51 }, | |
| { 5, 90, 48 }, | |
| { 4, 76, 42 }, | |
| { 3, 60, 35 }, | |
| { 2, 38, 25 }, | |
| { 1, 14, 14 }, | |
| }; | |
| size_t i; | |
| DISPLAYLEVEL(3, "test%3u : ZSTD_get1BlockSummary with empty array : ", testNb++); | |
| { | |
| BlockSummary bs = ZSTD_get1BlockSummary(nseqs, 0); | |
| CHECK_EQ(bs.nbSequences, ERROR(externalSequences_invalid)); | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3u : ZSTD_get1BlockSummary with 1 literal only : ", testNb++); | |
| { | |
| static const ZSTD_Sequence seqs[] = { { 0, 5, 0, 0 } }; | |
| BlockSummary bs = ZSTD_get1BlockSummary(seqs, 1); | |
| CHECK_EQ(bs.nbSequences, 1); | |
| CHECK_EQ(bs.litSize, 5); | |
| CHECK_EQ(bs.blockSize, 5); | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3u : ZSTD_get1BlockSummary with no terminator : ", testNb++); | |
| { | |
| static const ZSTD_Sequence seqs[] = { { 10, 2, 4, 0 }, { 20, 3, 5, 0 } }; | |
| BlockSummary bs = ZSTD_get1BlockSummary(seqs, 2); | |
| CHECK_EQ(bs.nbSequences, ERROR(externalSequences_invalid)); | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3u : ZSTD_get1BlockSummary with rep ignored : ", testNb++); | |
| { | |
| static const ZSTD_Sequence seqs[] = { | |
| { 10, 2, 4, 2 }, | |
| { 10, 3, 5, 2 }, | |
| { 0, 7, 0, 3 }, | |
| }; | |
| BlockSummary bs = ZSTD_get1BlockSummary(seqs, 3); | |
| CHECK_EQ(bs.nbSequences, 3); | |
| CHECK_EQ(bs.litSize, 2 + 3 + 7); | |
| CHECK_EQ(bs.blockSize, (4 + 5) + (2 + 3 + 7)); | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| assert(COUNTOF(nseqs) > COUNTOF(blocks)); | |
| for (i = 0; i < COUNTOF(blocks); ++i) { | |
| BlockSummary bs; | |
| DISPLAYLEVEL(3, "test%3u : ZSTD_get1BlockSummary with %u inputs : ", | |
| testNb++, (unsigned)(COUNTOF(nseqs) - i)); | |
| bs = ZSTD_get1BlockSummary(nseqs + i, COUNTOF(nseqs) - i); | |
| CHECK_EQ(bs.nbSequences, blocks[i].nbSequences); | |
| CHECK_EQ(bs.litSize, blocks[i].litSize); | |
| CHECK_EQ(bs.blockSize, blocks[i].blockSize); | |
| DISPLAYLEVEL(3, "OK \n"); | |
| } | |
| return testNb; | |
| } | |
| /* ============================================================= */ | |
| typedef struct { | |
| int fail; | |
| size_t outstanding; | |
| } FUZ_dictAllocState; | |
| static void* FUZ_dictMalloc(void* opaque, size_t size) | |
| { | |
| FUZ_dictAllocState* const state = (FUZ_dictAllocState*)opaque; | |
| void* allocation; | |
| if (state->fail) return NULL; | |
| allocation = malloc(size); | |
| if (allocation != NULL) ++state->outstanding; | |
| return allocation; | |
| } | |
| static void FUZ_dictFree(void* opaque, void* allocation) | |
| { | |
| FUZ_dictAllocState* const state = (FUZ_dictAllocState*)opaque; | |
| if (allocation != NULL) { | |
| CHECK(state->outstanding != 0); | |
| --state->outstanding; | |
| } | |
| free(allocation); | |
| } | |
| /* Writes an FSE table description (NCount) for symbols [0, maxSymbol] where | |
| * symbol 1 is absent (zero count), symbols 2..maxSymbol have probability | |
| * 1/2^tableLog, and symbol 0 takes the rest. Returns the bytes written. */ | |
| static size_t FUZ_writeNCount(BYTE* dst, size_t capacity, | |
| unsigned maxSymbol, unsigned tableLog) | |
| { | |
| short norm[MaxSeq + 1] = { 0 }; | |
| size_t n; | |
| unsigned s; | |
| assert(maxSymbol >= 2); | |
| for (s = 2; s <= maxSymbol; ++s) | |
| norm[s] = 1; | |
| norm[0] = (short)((1U << tableLog) - (maxSymbol - 1)); | |
| CHECK_VAR(n, FSE_writeNCount(dst, capacity, norm, maxSymbol, tableLog)); | |
| return n; | |
| } | |
| /* Guards against the optimal parser (btopt/btultra/btultra2) reading | |
| * uninitialized or out-of-range costs when it seeds its statistics from a | |
| * dictionary's entropy tables. | |
| * | |
| * The dictionary format allows the match length and literal length FSE tables | |
| * to omit the high symbols (they are implicitly absent), and to contain absent | |
| * symbols inside the listed range. The compressor used to build its tables | |
| * only up to the last listed symbol, leaving the costs of the omitted symbols | |
| * uninitialized. An absent symbol at the maximum table log costs tableLog+1, | |
| * which is not below the frequency scale, and a Huffman table deeper than 11 | |
| * bits is likewise too deep for the scale. Both tripped an assert and, in | |
| * release builds, made an invalid shift (#4681). | |
| * | |
| * The dictionary is hand-built to follow the format, rather than trained, so | |
| * the test does not depend on dictionary training. The only observable | |
| * contract is that compression with this valid dictionary succeeds and | |
| * round-trips, so that is what is checked. The bug itself is detected by | |
| * asserts (and sanitizers), so this test is most effective in debug builds. */ | |
| static void test_optimalParserDictEntropy(unsigned tnb) | |
| { | |
| BYTE dict[1024]; | |
| BYTE src[1024]; | |
| BYTE compressed[ZSTD_COMPRESSBOUND(sizeof(src))]; | |
| BYTE decoded[sizeof(src)]; | |
| size_t pos = 8; | |
| unsigned i; | |
| ZSTD_strategy strategy; | |
| ZSTD_CCtx* const cctx = ZSTD_createCCtx(); | |
| ZSTD_DCtx* const dctx = ZSTD_createDCtx(); | |
| DISPLAYLEVEL(3, "test%3u : optimal parser with sparse dictionary entropy tables : ", tnb); | |
| CHECK(cctx != NULL && dctx != NULL); | |
| /* Dictionary: magic, ID, skewed 12-bit Huffman table with every literal | |
| * present, FSE tables at the maximum table log with symbol 1 absent | |
| * (offset: symbols 0-MaxOff, ML & LL: symbols 0-2 only), repcodes, and | |
| * content. */ | |
| MEM_writeLE32(dict, ZSTD_MAGIC_DICTIONARY); | |
| MEM_writeLE32(dict + 4, 1); | |
| { unsigned counts[MaxLit + 1]; | |
| HUF_CREATE_STATIC_CTABLE(hufTable, MaxLit); | |
| U64 workspace[HUF_WORKSPACE_SIZE_U64]; | |
| size_t n; | |
| for (i = 0; i <= MaxLit; ++i) | |
| counts[i] = i < 16 ? 1U << (16 - i) : 1; | |
| CHECK_Z(HUF_buildCTable_wksp(hufTable, counts, MaxLit, 12, workspace, sizeof(workspace))); | |
| CHECK_VAR(n, HUF_writeCTable_wksp(dict + pos, sizeof(dict) - pos, | |
| hufTable, MaxLit, 12, workspace, sizeof(workspace))); | |
| pos += n; | |
| } | |
| pos += FUZ_writeNCount(dict + pos, sizeof(dict) - pos, MaxOff, OffFSELog); | |
| pos += FUZ_writeNCount(dict + pos, sizeof(dict) - pos, 2, MLFSELog); | |
| pos += FUZ_writeNCount(dict + pos, sizeof(dict) - pos, 2, LLFSELog); | |
| for (i = 0; i < ZSTD_REP_NUM; ++i, pos += 4) | |
| MEM_writeLE32(dict + pos, 1 + i); | |
| for (i = 0; i < 256; ++i) | |
| dict[pos++] = (BYTE)i; | |
| CHECK(pos <= sizeof(dict)); | |
| for (i = 0; i < sizeof(src); ++i) | |
| src[i] = (BYTE)(i % 251); | |
| for (strategy = ZSTD_btopt; strategy <= ZSTD_btultra2; ++strategy) { | |
| size_t cSize, dSize; | |
| CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_strategy, (int)strategy)); | |
| CHECK_Z(ZSTD_CCtx_loadDictionary(cctx, dict, pos)); | |
| CHECK_VAR(cSize, ZSTD_compress2(cctx, compressed, sizeof(compressed), src, sizeof(src))); | |
| CHECK_VAR(dSize, ZSTD_decompress_usingDict(dctx, decoded, sizeof(decoded), | |
| compressed, cSize, dict, pos)); | |
| CHECK(dSize == sizeof(src) && !memcmp(decoded, src, sizeof(src))); | |
| } | |
| ZSTD_freeCCtx(cctx); | |
| ZSTD_freeDCtx(dctx); | |
| DISPLAYLEVEL(3, "OK \n"); | |
| } | |
| static void test_dictionaryLoadErrors(const void* dict, size_t dictSize) | |
| { | |
| FUZ_dictAllocState state = { 0, 0 }; | |
| ZSTD_customMem const customMem = { FUZ_dictMalloc, FUZ_dictFree, &state }; | |
| ZSTD_CCtx* const cctx = ZSTD_createCCtx_advanced(customMem); | |
| ZSTD_CCtx_params cdictParams; | |
| BYTE const src[32] = { 0 }; | |
| BYTE compressed[ZSTD_COMPRESSBOUND(sizeof(src))]; | |
| size_t const baseline = state.outstanding; | |
| size_t cSize; | |
| CHECK(cctx != NULL); | |
| CHECK_Z(ZSTD_CCtxParams_init(&cdictParams, 1)); | |
| CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 1)); | |
| CHECK_Z(ZSTD_CCtx_loadDictionary_byReference(cctx, dict, dictSize)); | |
| /* Actual allocation failures must keep their distinct error code. */ | |
| state.fail = 1; | |
| CHECK(ZSTD_createCDict_advanced2(dict, dictSize, ZSTD_dlm_byRef, | |
| ZSTD_dct_fullDict, &cdictParams, customMem) == NULL); | |
| CHECK_EQ(ZSTD_getErrorCode(ZSTD_compress2(cctx, compressed, sizeof(compressed), | |
| src, sizeof(src))), ZSTD_error_memory_allocation); | |
| CHECK_EQ(state.outstanding, baseline); | |
| state.fail = 0; | |
| /* A failed CDict initialization must free its partial allocation and | |
| * leave the CCtx able to retry. The first eight bytes retain the magic | |
| * number and dictionary ID, but omit the required entropy tables. */ | |
| CHECK(dictSize > 8); | |
| CHECK_Z(ZSTD_CCtx_loadDictionary_byReference(cctx, dict, 8)); | |
| CHECK(ZSTD_createCDict_advanced2(dict, 8, ZSTD_dlm_byRef, | |
| ZSTD_dct_fullDict, &cdictParams, customMem) == NULL); | |
| CHECK_EQ(state.outstanding, baseline); | |
| CHECK_EQ(ZSTD_getErrorCode(ZSTD_compress2(cctx, compressed, sizeof(compressed), | |
| src, sizeof(src))), ZSTD_error_dictionary_corrupted); | |
| CHECK_EQ(state.outstanding, baseline); | |
| CHECK_EQ(ZSTD_getErrorCode(ZSTD_compress2(cctx, compressed, sizeof(compressed), | |
| src, sizeof(src))), ZSTD_error_dictionary_corrupted); | |
| CHECK_EQ(state.outstanding, baseline); | |
| CHECK_Z(ZSTD_CCtx_loadDictionary_byReference(cctx, dict, dictSize)); | |
| CHECK_VAR(cSize, ZSTD_compress2(cctx, compressed, sizeof(compressed), src, sizeof(src))); | |
| { ZSTD_DCtx* const dctx = ZSTD_createDCtx(); | |
| BYTE decoded[sizeof(src)]; | |
| CHECK(dctx != NULL); | |
| { CHECK_NEWV(dSize, ZSTD_decompress_usingDict(dctx, decoded, sizeof(decoded), | |
| compressed, cSize, dict, dictSize)); | |
| CHECK(dSize == sizeof(src) && memcmp(src, decoded, sizeof(src)) == 0); | |
| } | |
| ZSTD_freeDCtx(dctx); | |
| } | |
| ZSTD_freeCCtx(cctx); | |
| CHECK_EQ(state.outstanding, 0); | |
| } | |
| #if ZSTD_WINDOW_OVERFLOW_CORRECT_FREQUENTLY | |
| /* dict has a valid full-dictionary header followed by at least 3 MB + 64 bytes of content. */ | |
| static void test_dictionaryRepcodesAfterOverflow(BYTE* dict, size_t headerSize) | |
| { | |
| /* Reaching the normal overflow threshold requires compressing a few GB. | |
| * Only run this regression in frequent-correction mode, where a smaller | |
| * dictionary and forceMaxWindow reproduce the same state cheaply. Loading | |
| * the dictionary into the reused CCtx retains only 3 MB of its content. */ | |
| size_t const contentSize = 3 MB + 64; | |
| size_t const preloadSize = 256 KB - 64; | |
| size_t const fullDictSize = headerSize + contentSize; | |
| U32 const rep = (U32)contentSize - 16; | |
| size_t const compressedCapacity = ZSTD_compressBound(128 KB); | |
| void* const compressed = malloc(compressedCapacity); | |
| ZSTD_CCtx* const fresh = ZSTD_createCCtx(); | |
| ZSTD_CCtx* const reused = ZSTD_createCCtx(); | |
| ZSTD_DCtx* const dctx = ZSTD_createDCtx(); | |
| ZSTD_parameters params = ZSTD_getParams(1, ZSTD_CONTENTSIZE_UNKNOWN, 0); | |
| ZSTD_CCtx_params cctxParams; | |
| BYTE src[4 KB]; | |
| BYTE decoded[sizeof(src)]; | |
| size_t remaining = preloadSize; | |
| U32 u; | |
| CHECK(compressed != NULL && fresh != NULL && reused != NULL && dctx != NULL); | |
| for (u = 0; u < ZSTD_REP_NUM; ++u) | |
| MEM_writeLE32(dict + headerSize - 12 + 4 * u, repStartValue[u]); | |
| MEM_writeLE32(dict + headerSize - 12, rep); | |
| memcpy(src, dict + fullDictSize - rep, sizeof(src)); | |
| params.cParams.windowLog = 20; | |
| params.cParams.hashLog = 18; | |
| params.cParams.chainLog = 17; | |
| CHECK_Z(ZSTD_CCtxParams_init_advanced(&cctxParams, params)); | |
| CHECK_Z(ZSTD_CCtxParams_setParameter(&cctxParams, ZSTD_c_forceMaxWindow, 1)); | |
| /* First prove that exactly this dictionary and these parameters work. */ | |
| CHECK_Z(ZSTD_compressBegin_advanced_internal(fresh, dict, fullDictSize, | |
| ZSTD_dct_fullDict, ZSTD_dtlm_fast, NULL, &cctxParams, ZSTD_CONTENTSIZE_UNKNOWN)); | |
| CHECK_EQ(fresh->blockState.matchState.window.nbOverflowCorrections, 0); | |
| { CHECK_NEWV(cSize, ZSTD_compressEnd(fresh, compressed, compressedCapacity, src, sizeof(src))); | |
| { CHECK_NEWV(dSize, ZSTD_decompress_usingDict(dctx, decoded, sizeof(decoded), | |
| compressed, cSize, dict, fullDictSize)); | |
| CHECK(dSize == sizeof(src) && memcmp(src, decoded, sizeof(src)) == 0); | |
| } | |
| } | |
| /* Advance the indices using actual compression, without replacing the | |
| * workspace or manufacturing internal match-state indices. */ | |
| CHECK_Z(ZSTD_compressBegin_advanced_internal(reused, NULL, 0, | |
| ZSTD_dct_auto, ZSTD_dtlm_fast, NULL, &cctxParams, ZSTD_CONTENTSIZE_UNKNOWN)); | |
| while (remaining != 0) { | |
| size_t const chunkSize = MIN(remaining, 128 KB); | |
| CHECK_Z(ZSTD_compressContinue(reused, compressed, compressedCapacity, | |
| dict + headerSize, chunkSize)); | |
| remaining -= chunkSize; | |
| } | |
| CHECK_Z(ZSTD_compressEnd(reused, compressed, compressedCapacity, NULL, 0)); | |
| CHECK_EQ(reused->blockState.matchState.window.nbOverflowCorrections, 0); | |
| CHECK_EQ((size_t)(reused->blockState.matchState.window.nextSrc | |
| - reused->blockState.matchState.window.base), | |
| preloadSize + ZSTD_WINDOW_START_INDEX); | |
| { size_t const result = ZSTD_compressBegin_advanced_internal(reused, dict, fullDictSize, | |
| ZSTD_dct_fullDict, ZSTD_dtlm_fast, NULL, &cctxParams, ZSTD_CONTENTSIZE_UNKNOWN); | |
| ZSTD_window_t const* const window = &reused->blockState.matchState.window; | |
| U32 const loadedDictSize = (U32)(window->nextSrc - window->base) - window->lowLimit; | |
| CHECK_EQ(window->nbOverflowCorrections, 1); | |
| CHECK_EQ(loadedDictSize, 3 MB); | |
| CHECK_LT(loadedDictSize, rep); | |
| CHECK_Z(result); | |
| } | |
| { CHECK_NEWV(cSize, ZSTD_compressEnd(reused, compressed, compressedCapacity, src, sizeof(src))); | |
| { CHECK_NEWV(dSize, ZSTD_decompress_usingDict(dctx, decoded, sizeof(decoded), | |
| compressed, cSize, dict, fullDictSize)); | |
| CHECK(dSize == sizeof(src) && memcmp(src, decoded, sizeof(src)) == 0); | |
| } | |
| } | |
| ZSTD_freeDCtx(dctx); | |
| ZSTD_freeCCtx(reused); | |
| ZSTD_freeCCtx(fresh); | |
| free(compressed); | |
| } | |
| #endif | |
| static void test_largeDictionaryRepcodes(const void* dict, size_t dictSize) | |
| { | |
| U32 const shortCacheMaxDictSize = (1U << (32 - ZSTD_SHORT_CACHE_TAG_BITS)) - ZSTD_WINDOW_START_INDEX; | |
| struct DictCase { size_t contentSize; U32 rep; }; | |
| struct DictCase const cases[] = { | |
| { 20 MB, shortCacheMaxDictSize + 1 }, | |
| { 20 MB, 20 MB - 16 }, | |
| { 20 MB, shortCacheMaxDictSize }, | |
| { 10 MB, 10 MB - 16 }, | |
| }; | |
| int const levels[] = { -5, 1, 2, 3, 4, 5 }; | |
| size_t const headerSize = ZDICT_getDictHeaderSize(dict, dictSize); | |
| BYTE* largeDict; | |
| ZSTD_CCtx* const cctx = ZSTD_createCCtx(); | |
| ZSTD_DCtx* const dctx = ZSTD_createDCtx(); | |
| BYTE src[4 KB]; | |
| BYTE compressed[8 KB]; | |
| BYTE decoded[sizeof(src)]; | |
| size_t ci, li; | |
| U32 r; | |
| CHECK_Z(headerSize); | |
| CHECK(headerSize >= 12); | |
| largeDict = (BYTE*)malloc(headerSize + 20 MB); | |
| CHECK(largeDict != NULL && cctx != NULL && dctx != NULL); | |
| memcpy(largeDict, dict, headerSize); | |
| RDG_genBuffer(largeDict + headerSize, 20 MB, 0.5, 0., 0); | |
| #if ZSTD_WINDOW_OVERFLOW_CORRECT_FREQUENTLY | |
| test_dictionaryRepcodesAfterOverflow(largeDict, headerSize); | |
| #endif | |
| for (ci = 0; ci < COUNTOF(cases); ++ci) { | |
| size_t const largeDictSize = headerSize + cases[ci].contentSize; | |
| /* Make the large repcode a possible match in the full dictionary. */ | |
| memcpy(src, largeDict + largeDictSize - cases[ci].rep, sizeof(src)); | |
| for (li = 0; li < COUNTOF(levels); ++li) { | |
| int const level = levels[li]; | |
| ZSTD_CCtx_params cdictParams; | |
| ZSTD_compressionParameters cParams; | |
| int reject; | |
| CHECK_Z(ZSTD_CCtxParams_init(&cdictParams, level)); | |
| cParams = ZSTD_getCParamsFromCCtxParams(&cdictParams, | |
| ZSTD_CONTENTSIZE_UNKNOWN, largeDictSize, ZSTD_cpm_createCDict); | |
| reject = (cParams.strategy == ZSTD_fast || cParams.strategy == ZSTD_dfast) | |
| && cases[ci].rep > shortCacheMaxDictSize; | |
| for (r = 0; r < ZSTD_REP_NUM; ++r) { | |
| ZSTD_CDict* cdict; | |
| U32 u; | |
| size_t cSize; | |
| ZSTD_inBuffer in = { src, sizeof(src), 0 }; | |
| ZSTD_outBuffer out = { compressed, sizeof(compressed), 0 }; | |
| for (u = 0; u < ZSTD_REP_NUM; ++u) | |
| MEM_writeLE32(largeDict + headerSize - 12 + 4 * u, repStartValue[u]); | |
| MEM_writeLE32(largeDict + headerSize - 12 + 4 * r, cases[ci].rep); | |
| cdict = ZSTD_createCDict_byReference(largeDict, largeDictSize, level); | |
| if (reject) { | |
| CHECK(cdict == NULL); | |
| /* The copy constructor and advanced constructor share the check. */ | |
| CHECK(ZSTD_createCDict(largeDict, largeDictSize, level) == NULL); | |
| CHECK(ZSTD_createCDict_advanced(largeDict, largeDictSize, | |
| ZSTD_dlm_byRef, ZSTD_dct_fullDict, | |
| cParams, ZSTD_defaultCMem) == NULL); | |
| /* Local CDict creation is deferred until compression starts. */ | |
| CHECK_Z(ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters)); | |
| CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, level)); | |
| CHECK_Z(ZSTD_CCtx_loadDictionary(cctx, largeDict, largeDictSize)); | |
| CHECK_EQ(ZSTD_getErrorCode(ZSTD_compress2(cctx, compressed, | |
| sizeof(compressed), src, sizeof(src))), ZSTD_error_dictionary_corrupted); | |
| CHECK_Z(ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters)); | |
| CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, level)); | |
| CHECK_Z(ZSTD_CCtx_loadDictionary_byReference(cctx, largeDict, largeDictSize)); | |
| CHECK_EQ(ZSTD_getErrorCode(ZSTD_compressStream2(cctx, &out, &in, | |
| ZSTD_e_continue)), ZSTD_error_dictionary_corrupted); | |
| /* The dictionary is format-valid. The non-CDict path retains | |
| * all of its content, so it must still round-trip. */ | |
| CHECK_VAR(cSize, ZSTD_compress_usingDict(cctx, compressed, sizeof(compressed), | |
| src, sizeof(src), largeDict, largeDictSize, level)); | |
| } else { | |
| CHECK(cdict != NULL); | |
| CHECK_VAR(cSize, ZSTD_compress_usingCDict(cctx, compressed, sizeof(compressed), | |
| src, sizeof(src), cdict)); | |
| { CHECK_NEWV(dSize, ZSTD_decompress_usingDict(dctx, decoded, sizeof(decoded), | |
| compressed, cSize, largeDict, largeDictSize)); | |
| CHECK(dSize == sizeof(src) && memcmp(src, decoded, sizeof(src)) == 0); | |
| } | |
| /* Start with unknown input size to exercise attached streaming. */ | |
| CHECK_Z(ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters)); | |
| CHECK_Z(ZSTD_CCtx_refCDict(cctx, cdict)); | |
| CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_continue)); | |
| CHECK(in.pos == in.size); | |
| CHECK(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end) == 0); | |
| cSize = out.pos; | |
| ZSTD_freeCDict(cdict); | |
| } | |
| { CHECK_NEWV(dSize, ZSTD_decompress_usingDict(dctx, decoded, sizeof(decoded), | |
| compressed, cSize, largeDict, largeDictSize)); | |
| CHECK(dSize == sizeof(src) && memcmp(src, decoded, sizeof(src)) == 0); | |
| } | |
| } | |
| } | |
| } | |
| ZSTD_freeDCtx(dctx); | |
| ZSTD_freeCCtx(cctx); | |
| free(largeDict); | |
| } | |
| /* Multi-DDict fixtures deliberately share entropy tables but have different | |
| * content. The handcrafted frame always copies eight dictionary bytes: unlike | |
| * compressor-generated data, it cannot turn into a literals-only block and | |
| * accidentally pass when the decoder selects the wrong dictionary. */ | |
| enum { | |
| FUZ_mdict_knownSize = 1, | |
| FUZ_mdict_checksum = 2 | |
| }; | |
| static void FUZ_mdict_makeDict(BYTE dst[44], U32 id, int fill) | |
| { | |
| static const BYTE header[28] = { | |
| 0x37,0xa4,0x30,0xec, 0,0,0,0, /* magic and dictionary ID */ | |
| 128,0x10, /* Huffman table */ | |
| 0xf0,0x03, 0xf0,0x03, 0xf0,0x03, /* FSE tables */ | |
| 4,0,0,0, 4,0,0,0, 4,0,0,0 /* repeat offsets */ | |
| }; | |
| memcpy(dst, header, sizeof(header)); | |
| MEM_writeLE32(dst + 4, id); | |
| memset(dst + sizeof(header), fill, 16); | |
| } | |
| /* ID 0 creates a raw-content DDict; other IDs create formatted DDicts. */ | |
| static ZSTD_DDict* FUZ_mdict_createDict(U32 id, int fill) | |
| { | |
| BYTE bytes[44]; | |
| ZSTD_DDict* ddict; | |
| FUZ_mdict_makeDict(bytes, id, fill); | |
| ddict = id ? ZSTD_createDDict(bytes, sizeof(bytes)) | |
| : ZSTD_createDDict(bytes + 28, 16); | |
| CHECK(ddict != NULL && ZSTD_getDictID_fromDDict(ddict) == id); | |
| return ddict; | |
| } | |
| /* One compressed block, no literals, one match of length 8 at offset 4. | |
| * ID 0 omits the frame's dictID. A known content size permits the streaming | |
| * shortcut; the optional checksum is for the expected output "AAAAAAAA". */ | |
| static size_t FUZ_mdict_makeFrame(BYTE* dst, U32 id, unsigned flags) | |
| { | |
| static const BYTE block[] = { | |
| 0x3d,0,0, /* last compressed block, seven-byte payload */ | |
| 0,1,0x54, /* no literals; one sequence; RLE sequence tables */ | |
| 0,0,5,1 /* LL=0, repeat offset, ML=8, bitstream */ | |
| }; | |
| int const knownSize = (flags & FUZ_mdict_knownSize) != 0; | |
| size_t n = 4; | |
| MEM_writeLE32(dst, ZSTD_MAGICNUMBER); | |
| dst[n++] = (BYTE)((id ? 3 : 0) | (knownSize ? 32 : 0) | |
| | ((flags & FUZ_mdict_checksum) ? 4 : 0)); | |
| if (!knownSize) dst[n++] = 0; /* 1 KiB window */ | |
| if (id) { MEM_writeLE32(dst + n, id); n += 4; } | |
| if (knownSize) dst[n++] = 8; | |
| memcpy(dst + n, block, sizeof(block)); n += sizeof(block); | |
| if (flags & FUZ_mdict_checksum) { | |
| MEM_writeLE32(dst + n, (U32)XXH64("AAAAAAAA", 8, 0)); n += 4; | |
| } | |
| return n; | |
| } | |
| static size_t FUZ_mdict_makeSkip(BYTE* dst, unsigned variant) | |
| { | |
| MEM_writeLE32(dst, ZSTD_MAGIC_SKIPPABLE_START + variant); | |
| MEM_writeLE32(dst + 4, 0); /* empty skippable frame */ | |
| return 8; | |
| } | |
| /* Register one or two DDicts in a new multiple-DDict context. */ | |
| static ZSTD_DCtx* FUZ_mdict_createContext(const ZSTD_DDict* first, | |
| const ZSTD_DDict* second) | |
| { | |
| ZSTD_DCtx* const dctx = ZSTD_createDCtx(); | |
| CHECK(dctx != NULL); | |
| CHECK_Z(ZSTD_DCtx_setParameter(dctx, ZSTD_d_refMultipleDDicts, 1)); | |
| if (first) CHECK_Z(ZSTD_DCtx_refDDict(dctx, first)); | |
| if (second) CHECK_Z(ZSTD_DCtx_refDDict(dctx, second)); | |
| return dctx; | |
| } | |
| /* Feed complete frames in chunks, returning the first error or total output. | |
| * These tests provide enough output space, so every call must make progress. */ | |
| static size_t FUZ_mdict_decompressStream(ZSTD_DCtx* dctx, void* dst, size_t capacity, | |
| const BYTE* src, size_t size, size_t chunk) | |
| { | |
| ZSTD_outBuffer out = {dst, capacity, 0}; | |
| size_t pos = 0, ret = 1; | |
| CHECK(chunk != 0); | |
| while (pos < size) { | |
| size_t const n = MIN(size - pos, chunk); | |
| ZSTD_inBuffer in = {src + pos, n, 0}; | |
| while (in.pos < in.size) { | |
| size_t const oldIn = in.pos, oldOut = out.pos; | |
| ret = ZSTD_decompressStream(dctx, &out, &in); | |
| if (ZSTD_isError(ret)) return ret; | |
| CHECK(in.pos > oldIn || out.pos > oldOut); | |
| } | |
| pos += n; | |
| } | |
| CHECK_EQ(ret, 0); | |
| return out.pos; | |
| } | |
| /* The bufferless API requires exactly the input size requested by the decoder. */ | |
| static size_t FUZ_mdict_decompressContinue(ZSTD_DCtx* dctx, BYTE* dst, size_t capacity, | |
| const BYTE* src, size_t size) | |
| { | |
| size_t inPos = 0, outPos = 0; | |
| for (;;) { | |
| size_t const n = ZSTD_nextSrcSizeToDecompress(dctx); | |
| size_t r; | |
| if (n == 0) break; | |
| CHECK(inPos <= size && n <= size - inPos); | |
| CHECK(outPos <= capacity); | |
| r = ZSTD_decompressContinue(dctx, outPos != 0 ? dst + outPos : dst, capacity - outPos, | |
| src + inPos, n); | |
| if (ZSTD_isError(r)) return r; | |
| inPos += n; | |
| outPos += r; | |
| } | |
| CHECK_EQ(inPos, size); | |
| return outPos; | |
| } | |
| static void FUZ_mdict_checkOutput(size_t r, const void* out, const char* expected) | |
| { | |
| CHECK_Z(r); | |
| CHECK_EQ(r, strlen(expected)); | |
| CHECK_EQ(memcmp(out, expected, r), 0); | |
| } | |
| /* Layout of FUZ_mdict_makeDict(): 28-byte header, then 16 bytes of content. */ | |
| #define FUZ_DH_HEADER 28 | |
| #define FUZ_DH_CONTENT 16 | |
| #define FUZ_DH_DICT (FUZ_DH_HEADER + FUZ_DH_CONTENT) | |
| /* A frame whose only sequence is a four-byte match at `offset`. */ | |
| static size_t FUZ_ddictHeader_makeFrame(BYTE* frame, U32 dictID, unsigned offset) | |
| { | |
| BYTE* op = frame + 17; | |
| unsigned const ofCode = ZSTD_highbit32(offset + ZSTD_REP_NUM); | |
| MEM_writeLE32(frame, ZSTD_MAGICNUMBER); | |
| frame[4] = 0x83; frame[5] = 0x38; /* 4-byte dictID and size, 128 KiB window */ | |
| MEM_writeLE32(frame + 6, dictID); | |
| MEM_writeLE32(frame + 10, 4); | |
| *op++ = 0; /* no literals */ | |
| *op++ = 1; *op++ = 0x54; /* one sequence, three RLE tables */ | |
| *op++ = 0; *op++ = (BYTE)ofCode; *op++ = 1; /* LL=0, ML=4 */ | |
| *op++ = (BYTE)((1U << ofCode) | (offset + 3 - (1U << ofCode))); | |
| MEM_writeLE24(frame + 14, (U32)((op - (frame + 17)) << 3) | (bt_compressed << 1) | 1); | |
| return (size_t)(op - frame); | |
| } | |
| /* Matches into the header must be rejected; all others must decode. */ | |
| static void FUZ_ddictHeader_check(size_t r, const BYTE* out, const BYTE* dict, | |
| unsigned offset, int raw) | |
| { | |
| if (!raw && offset > FUZ_DH_CONTENT) { | |
| CHECK_EQ(ZSTD_getErrorCode(r), ZSTD_error_corruption_detected); | |
| } else { | |
| CHECK_EQ(r, 4); | |
| CHECK_EQ(memcmp(out, dict + FUZ_DH_DICT - offset, 4), 0); | |
| } | |
| } | |
| static void FUZ_ddictHeader_testOffset(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict, | |
| const BYTE* dict, unsigned offset, int raw) | |
| { | |
| ZSTD_dictContentType_e const type = raw ? ZSTD_dct_rawContent : ZSTD_dct_fullDict; | |
| BYTE frame[32], out[4]; | |
| size_t const size = FUZ_ddictHeader_makeFrame(frame, ZSTD_getDictID_fromDDict(ddict), offset); | |
| int mode; | |
| /* Every way of installing a dictionary reaches the same decoder state. */ | |
| for (mode = 0; mode < (raw ? 5 : 6); ++mode) { | |
| size_t r; | |
| CHECK_Z(ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters)); | |
| switch (mode) { | |
| case 0: r = ZSTD_decompress_usingDDict(dctx, out, sizeof(out), frame, size, ddict); break; | |
| case 1: case 2: | |
| CHECK_Z(ZSTD_DCtx_refDDict(dctx, ddict)); | |
| r = FUZ_mdict_decompressStream(dctx, out, sizeof(out), frame, size, mode == 1 ? 1 : size); | |
| break; | |
| case 3: | |
| CHECK_Z(ZSTD_DCtx_loadDictionary_advanced(dctx, dict, FUZ_DH_DICT, ZSTD_dlm_byCopy, type)); | |
| r = ZSTD_decompressDCtx(dctx, out, sizeof(out), frame, size); | |
| break; | |
| case 4: | |
| CHECK_Z(ZSTD_DCtx_refPrefix_advanced(dctx, dict, FUZ_DH_DICT, type)); | |
| r = ZSTD_decompressDCtx(dctx, out, sizeof(out), frame, size); | |
| break; | |
| default: /* Control: the non-DDict API, which already rejected these. */ | |
| r = ZSTD_decompress_usingDict(dctx, out, sizeof(out), frame, size, dict, FUZ_DH_DICT); | |
| break; | |
| } | |
| FUZ_ddictHeader_check(r, out, dict, offset, raw); | |
| } | |
| } | |
| static void test_decompressDDictHeader(unsigned tnb) | |
| { | |
| /* The last valid offset, the first header byte, and the dictionary magic. */ | |
| static const unsigned offsets[] = { FUZ_DH_CONTENT, FUZ_DH_CONTENT + 1, FUZ_DH_DICT }; | |
| BYTE dict[FUZ_DH_DICT]; | |
| ZSTD_DCtx* const dctx = ZSTD_createDCtx(); | |
| ZSTD_DDict* ddicts[2]; | |
| int raw; | |
| size_t i; | |
| DISPLAYLEVEL(3, "test%3u : formatted dictionary headers are not match history : ", tnb); | |
| FUZ_mdict_makeDict(dict, 1, 0); | |
| for (i = 0; i < FUZ_DH_CONTENT; ++i) dict[FUZ_DH_HEADER + i] = (BYTE)(0x80 + i); | |
| ddicts[0] = ZSTD_createDDict(dict, sizeof(dict)); | |
| ddicts[1] = ZSTD_createDDict_advanced(dict, sizeof(dict), | |
| ZSTD_dlm_byCopy, ZSTD_dct_rawContent, ZSTD_defaultCMem); | |
| CHECK(dctx != NULL && ddicts[0] != NULL && ddicts[1] != NULL); | |
| for (raw = 0; raw < 2; ++raw) { | |
| /* Legacy decoders need the complete dictionary, header included. */ | |
| CHECK_EQ(ZSTD_DDict_dictSize(ddicts[raw]), sizeof(dict)); | |
| CHECK_EQ(memcmp(ZSTD_DDict_dictContent(ddicts[raw]), dict, sizeof(dict)), 0); | |
| for (i = 0; i < COUNTOF(offsets); ++i) | |
| FUZ_ddictHeader_testOffset(dctx, ddicts[raw], dict, offsets[i], raw); | |
| } | |
| ZSTD_freeDDict(ddicts[0]); | |
| ZSTD_freeDDict(ddicts[1]); | |
| ZSTD_freeDCtx(dctx); | |
| DISPLAYLEVEL(3, "OK \n"); | |
| } | |
| static void FUZ_mdict_testOneShot(void) | |
| { | |
| ZSTD_DDict* const a = FUZ_mdict_createDict(1111, 'A'); | |
| ZSTD_DDict* const b = FUZ_mdict_createDict(2222, 'B'); | |
| ZSTD_DDict* const raw = FUZ_mdict_createDict(0, 'R'); | |
| BYTE frame[96], out[32], tmp[32]; | |
| size_t n, r; | |
| int magicless, invalid; | |
| /* B is registered last. Every frame must nevertheless install its own | |
| * exact match, including after reset and within concatenated input. */ | |
| for (magicless = 0; magicless <= 1; ++magicless) { | |
| ZSTD_DCtx* const dctx = FUZ_mdict_createContext(raw, a); | |
| size_t const skip = magicless ? 4 : 0; | |
| static const U32 ids[] = {1111, 2222, 0}; | |
| size_t i; | |
| CHECK_Z(ZSTD_DCtx_refDDict(dctx, b)); | |
| if (magicless) CHECK_Z(ZSTD_DCtx_setParameter(dctx, ZSTD_d_format, ZSTD_f_zstd1_magicless)); | |
| n = FUZ_mdict_makeFrame(frame, 1111, 0); | |
| r = ZSTD_decompressDCtx(dctx, out, sizeof(out), frame + skip, n - skip); | |
| FUZ_mdict_checkOutput(r, out, "AAAAAAAA"); | |
| CHECK_Z(ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only)); | |
| n = FUZ_mdict_makeFrame(frame, 2222, 0); | |
| r = ZSTD_decompressDCtx(dctx, out, sizeof(out), frame + skip, n - skip); | |
| FUZ_mdict_checkOutput(r, out, "BBBBBBBB"); | |
| n = magicless ? 0 : FUZ_mdict_makeSkip(frame, 7); | |
| for (i = 0; i < COUNTOF(ids); ++i) { | |
| size_t const fSize = FUZ_mdict_makeFrame(tmp, ids[i], i == 1 ? FUZ_mdict_knownSize : 0); | |
| memcpy(frame + n, tmp + skip, fSize - skip); | |
| n += fSize - skip; | |
| } | |
| r = ZSTD_decompressDCtx(dctx, out, sizeof(out), frame, n); | |
| FUZ_mdict_checkOutput(r, out, "AAAAAAAABBBBBBBBRRRRRRRR"); | |
| CHECK_Z(ZSTD_freeDCtx(dctx)); | |
| } | |
| /* Failed headers must not prevent the next frame from finding its match. */ | |
| for (invalid = 0; invalid < 2; ++invalid) { | |
| ZSTD_DCtx* const dctx = FUZ_mdict_createContext(raw, a); | |
| FUZ_mdict_makeFrame(frame, 1111, 0); | |
| if (invalid) memset(frame, 0xff, 5); | |
| r = ZSTD_decompressDCtx(dctx, out, sizeof(out), frame, 5); | |
| CHECK(ZSTD_isError(r)); | |
| n = FUZ_mdict_makeFrame(frame, 1111, 0); | |
| r = ZSTD_decompressDCtx(dctx, out, sizeof(out), frame, n); | |
| FUZ_mdict_checkOutput(r, out, "AAAAAAAA"); | |
| CHECK_Z(ZSTD_freeDCtx(dctx)); | |
| } | |
| CHECK_Z(ZSTD_freeDDict(a)); CHECK_Z(ZSTD_freeDDict(b)); | |
| CHECK_Z(ZSTD_freeDDict(raw)); | |
| } | |
| static void FUZ_mdict_testExplicitDictionary(void) | |
| { | |
| ZSTD_DDict* const a = FUZ_mdict_createDict(1111, 'A'); | |
| ZSTD_DDict* const b = FUZ_mdict_createDict(2222, 'B'); | |
| BYTE frame[32], out[32], dict[44]; | |
| size_t n, r; | |
| int local; | |
| for (local = 0; local < 2; ++local) { | |
| ZSTD_DCtx* const dctx = FUZ_mdict_createContext(b, NULL); | |
| FUZ_mdict_makeDict(dict, 2222, 'B'); | |
| if (local) CHECK_Z(ZSTD_DCtx_loadDictionary(dctx, dict, sizeof(dict))); | |
| n = FUZ_mdict_makeFrame(frame, 1111, 0); | |
| /* The set must not override the explicit argument, even though A is | |
| * absent from the set and strict lookup would reject this frame. */ | |
| r = ZSTD_decompress_usingDDict(dctx, out, sizeof(out), frame, n, b); | |
| CHECK_EQ(ZSTD_getErrorCode(r), ZSTD_error_dictionary_wrong); | |
| r = ZSTD_decompress_usingDDict(dctx, out, sizeof(out), frame, n, a); | |
| FUZ_mdict_checkOutput(r, out, "AAAAAAAA"); | |
| r = ZSTD_decompress_usingDict(dctx, out, sizeof(out), frame, n, dict, sizeof(dict)); | |
| CHECK_EQ(ZSTD_getErrorCode(r), ZSTD_error_dictionary_wrong); | |
| FUZ_mdict_makeDict(dict, 1111, 'A'); | |
| r = ZSTD_decompress_usingDict(dctx, out, sizeof(out), frame, n, dict, sizeof(dict)); | |
| FUZ_mdict_checkOutput(r, out, "AAAAAAAA"); | |
| /* Header validation must likewise honor the bufferless API's DDict. */ | |
| CHECK_Z(ZSTD_decompressBegin_usingDDict(dctx, b)); | |
| r = FUZ_mdict_decompressContinue(dctx, out, sizeof(out), frame, n); | |
| CHECK_EQ(ZSTD_getErrorCode(r), ZSTD_error_dictionary_wrong); | |
| CHECK_Z(ZSTD_decompressBegin_usingDDict(dctx, a)); | |
| r = FUZ_mdict_decompressContinue(dctx, out, sizeof(out), frame, n); | |
| FUZ_mdict_checkOutput(r, out, "AAAAAAAA"); | |
| /* Explicit dictionaries must not replace the configured set or the | |
| * context-owned B. Disabling the set must not retain borrowed B. */ | |
| n = FUZ_mdict_makeFrame(frame, 2222, 0); | |
| r = ZSTD_decompressDCtx(dctx, out, sizeof(out), frame, n); | |
| FUZ_mdict_checkOutput(r, out, "BBBBBBBB"); | |
| if (!local) { CHECK_Z(ZSTD_DCtx_setParameter(dctx, ZSTD_d_refMultipleDDicts, 0)); } | |
| n = FUZ_mdict_makeFrame(frame, 0, 0); | |
| r = ZSTD_decompressDCtx(dctx, out, sizeof(out), frame, n); | |
| if (local) { FUZ_mdict_checkOutput(r, out, "BBBBBBBB"); } | |
| else { CHECK_EQ(ZSTD_getErrorCode(r), ZSTD_error_corruption_detected); } | |
| CHECK_Z(ZSTD_freeDCtx(dctx)); | |
| } | |
| CHECK_Z(ZSTD_freeDDict(a)); CHECK_Z(ZSTD_freeDDict(b)); | |
| } | |
| static void FUZ_mdict_testPrefixCompatibility(void) | |
| { | |
| ZSTD_DDict* const b = FUZ_mdict_createDict(2222, 'B'); | |
| BYTE prefix[16], frame[64], out[32]; | |
| size_t n, first, r; | |
| int multiple; | |
| memset(prefix, 'A', sizeof(prefix)); | |
| for (multiple = 0; multiple < 2; ++multiple) { | |
| ZSTD_DCtx* const dctx = multiple ? FUZ_mdict_createContext(b, NULL) : ZSTD_createDCtx(); | |
| CHECK(dctx != NULL); | |
| CHECK_Z(ZSTD_DCtx_refPrefix(dctx, prefix, sizeof(prefix))); | |
| /* Preserve the existing single-DDict one-shot behavior: a prefix is | |
| * resolved once per call, even across concatenated frames. This also | |
| * applies when the multi-DDict flag is enabled but the set is empty. */ | |
| first = FUZ_mdict_makeFrame(frame, 0, 0); | |
| n = first + FUZ_mdict_makeFrame(frame + first, 0, 0); | |
| r = ZSTD_decompressDCtx(dctx, out, sizeof(out), frame, n); | |
| FUZ_mdict_checkOutput(r, out, "AAAAAAAAAAAAAAAA"); | |
| r = ZSTD_decompressDCtx(dctx, out, sizeof(out), frame, first); | |
| CHECK_EQ(ZSTD_getErrorCode(r), ZSTD_error_corruption_detected); | |
| /* Streaming still consumes the prefix on its first frame. */ | |
| CHECK_Z(ZSTD_DCtx_refPrefix(dctx, prefix, sizeof(prefix))); | |
| r = FUZ_mdict_decompressStream(dctx, out, sizeof(out), frame, first, 1); | |
| FUZ_mdict_checkOutput(r, out, "AAAAAAAA"); | |
| r = FUZ_mdict_decompressStream(dctx, out, sizeof(out), frame, first, 1); | |
| CHECK_EQ(ZSTD_getErrorCode(r), ZSTD_error_corruption_detected); | |
| CHECK_Z(ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only)); | |
| /* Do not change the established skippable-frame prefix behavior in | |
| * this multi-DDict fix. One-shot keeps its dictionary for the call; | |
| * streaming consumes the prefix even on a skippable frame. */ | |
| CHECK_Z(ZSTD_DCtx_refPrefix(dctx, prefix, sizeof(prefix))); | |
| first = FUZ_mdict_makeSkip(frame, 7); | |
| n = first + FUZ_mdict_makeFrame(frame + first, 0, 0); | |
| r = ZSTD_decompressDCtx(dctx, out, sizeof(out), frame, n); | |
| FUZ_mdict_checkOutput(r, out, "AAAAAAAA"); | |
| CHECK_Z(ZSTD_DCtx_refPrefix(dctx, prefix, sizeof(prefix))); | |
| r = ZSTD_decompressDCtx(dctx, out, sizeof(out), frame, first); | |
| CHECK_EQ(r, 0); | |
| r = ZSTD_decompressDCtx(dctx, out, sizeof(out), frame + first, n - first); | |
| CHECK_EQ(ZSTD_getErrorCode(r), ZSTD_error_corruption_detected); | |
| CHECK_Z(ZSTD_DCtx_refPrefix(dctx, prefix, sizeof(prefix))); | |
| r = FUZ_mdict_decompressStream(dctx, out, sizeof(out), frame, n, 1); | |
| CHECK_EQ(ZSTD_getErrorCode(r), ZSTD_error_corruption_detected); | |
| CHECK_Z(ZSTD_freeDCtx(dctx)); | |
| } | |
| CHECK_Z(ZSTD_freeDDict(b)); | |
| } | |
| /* --------------------------------------------------------------------------- | |
| * Prefix lifetime and session reset of the one-shot decompression functions. | |
| * These test the contracts documented in zstd.h for ZSTD_DCtx_refPrefix(), | |
| * ZSTD_DCtx_reset() and ZSTD_decompressDCtx() / _usingDict() / _usingDDict(). | |
| * | |
| * Prefix tests use the fixture below: the frame copies 8 bytes out of the | |
| * prefix, so it decodes to "AAAAAAAA" if and only if the prefix is in use. | |
| * Unlike compressor-generated data it cannot degrade to a literals-only block | |
| * and pass by accident when the prefix has been lost. | |
| * ------------------------------------------------------------------------- */ | |
| typedef struct { | |
| ZSTD_DCtx* dctx; | |
| BYTE dict[44]; /* formatted dictionary "A"; its last 16 bytes are the raw content */ | |
| BYTE frame[32]; /* decodes to "AAAAAAAA" only when dictionary A is available */ | |
| size_t frameSize; | |
| int asFullDict; /* reference the formatted dictionary instead of its raw content */ | |
| } FUZ_pfx; | |
| static void FUZ_pfx_init(FUZ_pfx* p, unsigned asFullDict) | |
| { | |
| p->dctx = ZSTD_createDCtx(); | |
| CHECK(p->dctx != NULL); | |
| FUZ_mdict_makeDict(p->dict, 1111, 'A'); | |
| p->frameSize = FUZ_mdict_makeFrame(p->frame, 0, 0); | |
| p->asFullDict = (int)asFullDict; | |
| } | |
| static void FUZ_pfx_ref(FUZ_pfx* p) | |
| { | |
| if (p->asFullDict) { | |
| CHECK_Z(ZSTD_DCtx_refPrefix_advanced(p->dctx, p->dict, 44, ZSTD_dct_fullDict)); | |
| } else { | |
| CHECK_Z(ZSTD_DCtx_refPrefix(p->dctx, p->dict + 28, 16)); | |
| } | |
| } | |
| /* Is the prefix still referenced? Probes with ZSTD_decompressDCtx(), | |
| * so it consumes the prefix: call it last, and reference again before reuse. | |
| * A frame that needs a missing dictionary must not decode to "AAAAAAAA". */ | |
| static int FUZ_pfx_isSet(FUZ_pfx* p) | |
| { | |
| BYTE out[8]; | |
| size_t const r = ZSTD_decompressDCtx(p->dctx, out, sizeof(out), p->frame, p->frameSize); | |
| return r == 8 && memcmp(out, "AAAAAAAA", 8) == 0; | |
| } | |
| /* Begin streaming a frame, but stop after `size` bytes of input. */ | |
| static void FUZ_pfx_startStream(ZSTD_DCtx* dctx, const BYTE* src, size_t size) | |
| { | |
| BYTE out[32]; | |
| ZSTD_inBuffer in = {src, size, 0}; | |
| ZSTD_outBuffer ob = {out, sizeof(out), 0}; | |
| size_t const r = ZSTD_decompressStream(dctx, &ob, &in); | |
| CHECK_Z(r); | |
| CHECK(r != 0); /* frame not finished */ | |
| } | |
| /* ZSTD_decompressDCtx() uses the prefix for exactly one call, whatever the | |
| * input is. The explicit dictionary functions use their own dictionary and | |
| * leave the prefix alone. */ | |
| static void FUZ_pfx_testOneShot(unsigned mode) | |
| { | |
| FUZ_pfx p; | |
| ZSTD_DDict* const b = FUZ_mdict_createDict(0, 'B'); | |
| BYTE rawB[16], in[64], out[32]; | |
| size_t n, r; | |
| unsigned i; | |
| FUZ_pfx_init(&p, mode); | |
| memset(rawB, 'B', sizeof(rawB)); | |
| /* Every frame of a single call, including after a skippable frame, uses the prefix. */ | |
| n = FUZ_mdict_makeSkip(in, 0); | |
| memcpy(in + n, p.frame, p.frameSize); n += p.frameSize; | |
| memcpy(in + n, p.frame, p.frameSize); n += p.frameSize; | |
| FUZ_pfx_ref(&p); | |
| r = ZSTD_decompressDCtx(p.dctx, out, sizeof(out), in, n); | |
| FUZ_mdict_checkOutput(r, out, "AAAAAAAAAAAAAAAA"); | |
| CHECK(!FUZ_pfx_isSet(&p)); | |
| /* Consumed even when there is no frame to decode, or the call fails. */ | |
| for (i = 0; i < 3; ++i) { | |
| if (i == 0) n = 0; | |
| else if (i == 1) n = FUZ_mdict_makeSkip(in, 0); | |
| else { memset(in, 0xff, 8); n = 8; } | |
| FUZ_pfx_ref(&p); | |
| r = ZSTD_decompressDCtx(p.dctx, out, sizeof(out), in, n); | |
| CHECK((int)ZSTD_isError(r) == (int)(i == 2)); | |
| CHECK(!FUZ_pfx_isSet(&p)); | |
| } | |
| /* Explicit dictionaries take precedence and do not consume the prefix. */ | |
| FUZ_pfx_ref(&p); | |
| r = ZSTD_decompress_usingDict(p.dctx, out, sizeof(out), p.frame, p.frameSize, rawB, sizeof(rawB)); | |
| FUZ_mdict_checkOutput(r, out, "BBBBBBBB"); | |
| r = ZSTD_decompress_usingDDict(p.dctx, out, sizeof(out), p.frame, p.frameSize, b); | |
| FUZ_mdict_checkOutput(r, out, "BBBBBBBB"); | |
| CHECK(FUZ_pfx_isSet(&p)); | |
| CHECK_Z(ZSTD_freeDDict(b)); | |
| CHECK_Z(ZSTD_freeDCtx(p.dctx)); | |
| } | |
| /* ZSTD_decompressStream() uses the prefix until the first frame is complete | |
| * (returns 0), including frames with no output. Input is fed in chunks of every | |
| * size, so the header and the payload are split at every possible position. */ | |
| static void FUZ_pfx_testStreaming(unsigned mode) | |
| { | |
| static const BYTE skipEmpty[] = {0x50,0x2a,0x4d,0x18, 0,0,0,0}; | |
| static const BYTE skipData[] = {0x5f,0x2a,0x4d,0x18, 5,0,0,0, 1,2,3,4,5}; | |
| static const BYTE emptyFrame[] = {0x28,0xb5,0x2f,0xfd,0x20,0,1,0,0}; | |
| static const struct { const BYTE* src; size_t size; } noOutput[] = { | |
| {skipEmpty, sizeof(skipEmpty)}, | |
| {skipData, sizeof(skipData)}, | |
| {emptyFrame, sizeof(emptyFrame)} /* zstd frame with no content */ | |
| }; | |
| FUZ_pfx p; | |
| BYTE frame[32], out[32]; | |
| size_t size, chunk, r; | |
| unsigned i; | |
| FUZ_pfx_init(&p, mode); | |
| /* Frames with and without a known size, which can be decoded in one pass. */ | |
| for (i = 0; i < 2; ++i) { | |
| size = FUZ_mdict_makeFrame(frame, 0, i ? FUZ_mdict_knownSize : 0); | |
| for (chunk = 1; chunk <= size; ++chunk) { | |
| FUZ_pfx_ref(&p); | |
| r = FUZ_mdict_decompressStream(p.dctx, out, sizeof(out), frame, size, chunk); | |
| FUZ_mdict_checkOutput(r, out, "AAAAAAAA"); | |
| CHECK(!FUZ_pfx_isSet(&p)); | |
| } | |
| } | |
| /* Frames that produce no output also consume the prefix. */ | |
| for (i = 0; i < COUNTOF(noOutput); ++i) { | |
| for (chunk = 1; chunk <= noOutput[i].size; ++chunk) { | |
| FUZ_pfx_ref(&p); | |
| CHECK_EQ(FUZ_mdict_decompressStream(p.dctx, out, sizeof(out), | |
| noOutput[i].src, noOutput[i].size, chunk), 0); | |
| CHECK(!FUZ_pfx_isSet(&p)); | |
| } | |
| } | |
| CHECK_Z(ZSTD_freeDCtx(p.dctx)); | |
| } | |
| /* Interaction with ZSTD_DCtx_reset() and with one-shot calls after a stream is | |
| * started and abandoned. This guards the bug where a prefix belonging to an | |
| * abandoned stream was silently applied to the next, unrelated frame. */ | |
| static void FUZ_pfx_testReset(unsigned mode) | |
| { | |
| FUZ_pfx p; | |
| ZSTD_DDict* const b = FUZ_mdict_createDict(0, 'B'); | |
| BYTE rawB[16], out[32]; | |
| size_t r; | |
| unsigned how; | |
| FUZ_pfx_init(&p, mode); | |
| memset(rawB, 'B', sizeof(rawB)); | |
| /* A session reset keeps a prefix that streaming has not touched yet. */ | |
| FUZ_pfx_ref(&p); | |
| CHECK_Z(ZSTD_DCtx_reset(p.dctx, ZSTD_reset_session_only)); | |
| CHECK(FUZ_pfx_isSet(&p)); | |
| /* Once streaming has started, even with an incomplete header, abandoning the | |
| * frame discards the prefix: either explicitly, or implicitly by calling any | |
| * one-shot function (which must not leak the prefix to the next frame). */ | |
| for (how = 0; how < 5; ++how) { | |
| size_t const sizes[] = {1, p.frameSize - 1}; | |
| size_t i; | |
| for (i = 0; i < COUNTOF(sizes); ++i) { | |
| FUZ_pfx_ref(&p); | |
| FUZ_pfx_startStream(p.dctx, p.frame, sizes[i]); | |
| switch (how) { | |
| case 0: CHECK_Z(ZSTD_DCtx_reset(p.dctx, ZSTD_reset_session_only)); break; | |
| case 1: CHECK_Z(ZSTD_DCtx_reset(p.dctx, ZSTD_reset_session_and_parameters)); break; | |
| case 2: break; /* FUZ_pfx_isSet() is itself a one-shot call */ | |
| case 3: | |
| r = ZSTD_decompress_usingDict(p.dctx, out, sizeof(out), p.frame, p.frameSize, rawB, sizeof(rawB)); | |
| FUZ_mdict_checkOutput(r, out, "BBBBBBBB"); | |
| break; | |
| default: | |
| r = ZSTD_decompress_usingDDict(p.dctx, out, sizeof(out), p.frame, p.frameSize, b); | |
| FUZ_mdict_checkOutput(r, out, "BBBBBBBB"); | |
| break; | |
| } | |
| CHECK(!FUZ_pfx_isSet(&p)); | |
| } | |
| } | |
| /* Resetting the parameters discards the prefix when no frame is active ... */ | |
| FUZ_pfx_ref(&p); | |
| CHECK_Z(ZSTD_DCtx_reset(p.dctx, ZSTD_reset_parameters)); | |
| CHECK(!FUZ_pfx_isSet(&p)); | |
| /* ... and is rejected, with no effect, while a frame is being decompressed. */ | |
| FUZ_pfx_ref(&p); | |
| FUZ_pfx_startStream(p.dctx, p.frame, 1); | |
| CHECK_EQ(ZSTD_getErrorCode(ZSTD_DCtx_reset(p.dctx, ZSTD_reset_parameters)), ZSTD_error_stage_wrong); | |
| r = FUZ_mdict_decompressStream(p.dctx, out, sizeof(out), p.frame + 1, p.frameSize - 1, p.frameSize); | |
| FUZ_mdict_checkOutput(r, out, "AAAAAAAA"); | |
| CHECK_Z(ZSTD_freeDDict(b)); | |
| CHECK_Z(ZSTD_freeDCtx(p.dctx)); | |
| } | |
| /* Referencing any prefix or dictionary (including NULL or empty) replaces the | |
| * previous prefix, and so does ZSTD_initDStream(). */ | |
| static void FUZ_pfx_testReplacement(unsigned mode) | |
| { | |
| FUZ_pfx p; | |
| ZSTD_DDict* const b = FUZ_mdict_createDict(0, 'B'); | |
| BYTE rawB[16], out[32]; | |
| size_t r; | |
| unsigned i; | |
| FUZ_pfx_init(&p, mode); | |
| memset(rawB, 'B', sizeof(rawB)); | |
| /* Replaced by something else: only the new dictionary is used. */ | |
| for (i = 0; i < 3; ++i) { | |
| FUZ_pfx_ref(&p); | |
| switch (i) { | |
| case 0: CHECK_Z(ZSTD_DCtx_refPrefix(p.dctx, rawB, sizeof(rawB))); break; | |
| case 1: CHECK_Z(ZSTD_DCtx_loadDictionary(p.dctx, rawB, sizeof(rawB))); break; | |
| default: CHECK_Z(ZSTD_DCtx_refDDict(p.dctx, b)); break; | |
| } | |
| r = ZSTD_decompressDCtx(p.dctx, out, sizeof(out), p.frame, p.frameSize); | |
| FUZ_mdict_checkOutput(r, out, "BBBBBBBB"); | |
| } | |
| /* Replaced by nothing. */ | |
| for (i = 0; i < 4; ++i) { | |
| FUZ_pfx_ref(&p); | |
| switch (i) { | |
| case 0: CHECK_Z(ZSTD_DCtx_refPrefix(p.dctx, NULL, 0)); break; | |
| case 1: CHECK_Z(ZSTD_DCtx_loadDictionary(p.dctx, NULL, 0)); break; | |
| case 2: CHECK_Z(ZSTD_DCtx_refDDict(p.dctx, NULL)); break; | |
| default: CHECK_Z(ZSTD_initDStream(p.dctx)); break; | |
| } | |
| CHECK(!FUZ_pfx_isSet(&p)); | |
| } | |
| CHECK_Z(ZSTD_freeDDict(b)); | |
| CHECK_Z(ZSTD_freeDCtx(p.dctx)); | |
| } | |
| /* One-shot functions reset the streaming session first: they work on a DCtx | |
| * left mid-stream or after a streaming error, and leave it ready to stream a | |
| * new frame without any explicit ZSTD_DCtx_reset(). Before the fix they failed | |
| * (or decoded garbage) on a dirty DCtx, and the single-pass streaming shortcut | |
| * was entangled with this. | |
| * `api` selects the one-shot function: | |
| * 0: decompressDCtx, 1: usingDict, 2: usingDDict, 3: decompressDCtx with | |
| * ZSTD_d_refMultipleDDicts (a separate code path). The DCtx always | |
| * references dictionary A, so streaming can decode the frame as well. */ | |
| static void FUZ_reset_testDirtyStream(unsigned api) | |
| { | |
| ZSTD_DDict* const ddict = FUZ_mdict_createDict(1111, 'A'); | |
| ZSTD_DCtx* const dctx = api == 3 ? FUZ_mdict_createContext(ddict, NULL) : ZSTD_createDCtx(); | |
| BYTE dict[44], frame[32], bad[32], out[32]; | |
| size_t const frameSize = FUZ_mdict_makeFrame(frame, 1111, 0); | |
| unsigned dirty; | |
| CHECK(dctx != NULL); | |
| if (api != 3) CHECK_Z(ZSTD_DCtx_refDDict(dctx, ddict)); | |
| FUZ_mdict_makeDict(dict, 1111, 'A'); | |
| memcpy(bad, frame, frameSize); | |
| bad[frameSize - 10] = 0x3f; /* block header (last 10 bytes): reserved block type */ | |
| for (dirty = 0; dirty < 3; ++dirty) { | |
| /* 0: incomplete frame header, 1: inside the block, 2: streaming error */ | |
| const BYTE* const src = dirty == 2 ? bad : frame; | |
| size_t const size = dirty == 0 ? 3 : dirty == 1 ? frameSize - 1 : frameSize; | |
| ZSTD_inBuffer in = {src, size, 0}; | |
| ZSTD_outBuffer ob = {out, sizeof(out), 0}; | |
| size_t r = ZSTD_decompressStream(dctx, &ob, &in); | |
| CHECK((int)ZSTD_isError(r) == (int)(dirty == 2)); | |
| switch (api) { | |
| case 1: r = ZSTD_decompress_usingDict(dctx, out, sizeof(out), frame, frameSize, dict, sizeof(dict)); break; | |
| case 2: r = ZSTD_decompress_usingDDict(dctx, out, sizeof(out), frame, frameSize, ddict); break; | |
| default: r = ZSTD_decompressDCtx(dctx, out, sizeof(out), frame, frameSize); break; | |
| } | |
| FUZ_mdict_checkOutput(r, out, "AAAAAAAA"); | |
| r = FUZ_mdict_decompressStream(dctx, out, sizeof(out), frame, frameSize, frameSize); | |
| FUZ_mdict_checkOutput(r, out, "AAAAAAAA"); | |
| } | |
| CHECK_Z(ZSTD_freeDCtx(dctx)); | |
| CHECK_Z(ZSTD_freeDDict(ddict)); | |
| } | |
| static unsigned test_sessionReset(unsigned testNb) | |
| { | |
| static const struct { const char* name; void (*run)(unsigned); unsigned nbParams; } tests[] = { | |
| {"prefix lifetime: one-shot calls", FUZ_pfx_testOneShot, 2}, | |
| {"prefix lifetime: streaming", FUZ_pfx_testStreaming, 2}, | |
| {"prefix lifetime: reset and abandoned streams", FUZ_pfx_testReset, 2}, | |
| {"prefix lifetime: replacement", FUZ_pfx_testReplacement, 2}, | |
| {"one-shot decompression resets the streaming session", FUZ_reset_testDirtyStream, 4}, | |
| }; | |
| size_t i; | |
| unsigned param; | |
| for (i = 0; i < COUNTOF(tests); ++i) { | |
| DISPLAYLEVEL(3, "test%3u : %s : ", testNb++, tests[i].name); | |
| for (param = 0; param < tests[i].nbParams; ++param) tests[i].run(param); | |
| DISPLAYLEVEL(3, "OK \n"); | |
| } | |
| return testNb; | |
| } | |
| static void FUZ_mdict_testZeroID(void) | |
| { | |
| U32 const zero = 0; | |
| U32 id = 1; | |
| ZSTD_DDict* const raw = FUZ_mdict_createDict(0, 'R'); | |
| ZSTD_DDict* full; | |
| ZSTD_DCtx* const dctx = FUZ_mdict_createContext(raw, NULL); | |
| BYTE frame[32], out[32]; | |
| size_t n, r; | |
| /* Find a nonzero ID in raw dictionary 0's initial 64-slot hash bucket. */ | |
| while ((XXH64(&id, sizeof(id), 0) & 63) != (XXH64(&zero, sizeof(zero), 0) & 63)) | |
| ++id; | |
| full = FUZ_mdict_createDict(id, 'A'); | |
| n = FUZ_mdict_makeFrame(frame, id, 0); | |
| /* An occupied ID-0 slot is not a match for the missing nonzero ID. */ | |
| r = ZSTD_decompressDCtx(dctx, out, sizeof(out), frame, n); | |
| CHECK_EQ(ZSTD_getErrorCode(r), ZSTD_error_dictionary_wrong); | |
| CHECK_Z(ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only)); | |
| r = FUZ_mdict_decompressStream(dctx, out, sizeof(out), frame, n, 1); | |
| CHECK_EQ(ZSTD_getErrorCode(r), ZSTD_error_dictionary_wrong); | |
| CHECK_Z(ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only)); | |
| /* After insertion, lookup must probe past raw dictionary 0 to find A. | |
| * An actual ID-less frame must still find the raw dictionary itself. */ | |
| CHECK_Z(ZSTD_DCtx_refDDict(dctx, full)); | |
| CHECK_Z(ZSTD_DCtx_refDDict(dctx, raw)); | |
| r = ZSTD_decompressDCtx(dctx, out, sizeof(out), frame, n); | |
| FUZ_mdict_checkOutput(r, out, "AAAAAAAA"); | |
| r = FUZ_mdict_decompressStream(dctx, out, sizeof(out), frame, n, 1); | |
| FUZ_mdict_checkOutput(r, out, "AAAAAAAA"); | |
| n = FUZ_mdict_makeFrame(frame, 0, 0); | |
| r = ZSTD_decompressDCtx(dctx, out, sizeof(out), frame, n); | |
| FUZ_mdict_checkOutput(r, out, "RRRRRRRR"); | |
| r = FUZ_mdict_decompressStream(dctx, out, sizeof(out), frame, n, 1); | |
| FUZ_mdict_checkOutput(r, out, "RRRRRRRR"); | |
| CHECK_Z(ZSTD_freeDCtx(dctx)); | |
| CHECK_Z(ZSTD_freeDDict(raw)); CHECK_Z(ZSTD_freeDDict(full)); | |
| } | |
| static void FUZ_mdict_testClearReferences(void) | |
| { | |
| int mode; | |
| /* Register A, clear its references, free A, then register B and request A. | |
| * The result must be dictionary_wrong, without consulting freed A. */ | |
| for (mode = 0; mode < 5; ++mode) { | |
| ZSTD_DDict* const a = FUZ_mdict_createDict(1111, 'A'); | |
| ZSTD_DDict* const b = FUZ_mdict_createDict(2222, 'B'); | |
| ZSTD_DCtx* const dctx = FUZ_mdict_createContext(a, NULL); | |
| BYTE frame[32], out[32], dummy = 0; | |
| size_t const n = FUZ_mdict_makeFrame(frame, 1111, 0); | |
| size_t r; | |
| switch (mode) { | |
| case 0: CHECK_Z(ZSTD_DCtx_reset(dctx, ZSTD_reset_parameters)); break; | |
| case 1: CHECK_Z(ZSTD_DCtx_loadDictionary(dctx, NULL, 0)); break; | |
| case 2: CHECK_Z(ZSTD_DCtx_loadDictionary(dctx, &dummy, 0)); break; | |
| case 3: CHECK_Z(ZSTD_DCtx_refDDict(dctx, NULL)); break; | |
| default: | |
| /* Even a dictionary already selected for decoding is released. */ | |
| r = ZSTD_decompressDCtx(dctx, out, sizeof(out), frame, n); | |
| FUZ_mdict_checkOutput(r, out, "AAAAAAAA"); | |
| CHECK_Z(ZSTD_DCtx_setParameter(dctx, ZSTD_d_refMultipleDDicts, 0)); | |
| break; | |
| } | |
| CHECK_Z(ZSTD_freeDDict(a)); | |
| r = ZSTD_decompressDCtx(dctx, out, sizeof(out), frame, n); | |
| CHECK_EQ(ZSTD_getErrorCode(r), ZSTD_error_dictionary_wrong); | |
| CHECK_Z(ZSTD_DCtx_setParameter(dctx, ZSTD_d_refMultipleDDicts, 1)); | |
| CHECK_Z(ZSTD_DCtx_refDDict(dctx, b)); | |
| r = ZSTD_decompressDCtx(dctx, out, sizeof(out), frame, n); | |
| CHECK_EQ(ZSTD_getErrorCode(r), ZSTD_error_dictionary_wrong); | |
| CHECK_Z(ZSTD_freeDCtx(dctx)); CHECK_Z(ZSTD_freeDDict(b)); | |
| } | |
| } | |
| static void FUZ_mdict_testDisableSet(void) | |
| { | |
| int mode; | |
| for (mode = 0; mode < 4; ++mode) { | |
| ZSTD_DDict* const a = FUZ_mdict_createDict(1111, 'A'); | |
| ZSTD_DDict* const b = FUZ_mdict_createDict(2222, 'B'); | |
| ZSTD_DDict* const raw = FUZ_mdict_createDict(0, 'R'); | |
| ZSTD_DDict* const d = FUZ_mdict_createDict(3333, 'D'); | |
| ZSTD_DCtx* const dctx = FUZ_mdict_createContext(a, b); | |
| BYTE frame[32], out[32]; | |
| size_t n, r; | |
| CHECK_Z(ZSTD_DCtx_refDDict(dctx, raw)); | |
| n = FUZ_mdict_makeFrame(frame, 1111, mode == 3 ? FUZ_mdict_knownSize : 0); | |
| if (mode != 0) { | |
| r = mode == 1 ? ZSTD_decompressDCtx(dctx, out, sizeof(out), frame, n) | |
| : FUZ_mdict_decompressStream(dctx, out, sizeof(out), frame, n, | |
| mode == 2 ? 1 : sizeof(frame)); | |
| FUZ_mdict_checkOutput(r, out, "AAAAAAAA"); | |
| } | |
| /* All registered DDicts can be freed immediately, including the most | |
| * recently registered one and any DDict already used for decoding. */ | |
| CHECK_Z(ZSTD_DCtx_setParameter(dctx, ZSTD_d_refMultipleDDicts, 0)); | |
| CHECK_Z(ZSTD_freeDDict(a)); CHECK_Z(ZSTD_freeDDict(b)); | |
| CHECK_Z(ZSTD_freeDDict(raw)); | |
| n = FUZ_mdict_makeFrame(frame, 0, 0); | |
| r = ZSTD_decompressDCtx(dctx, out, sizeof(out), frame, n); | |
| CHECK_EQ(ZSTD_getErrorCode(r), ZSTD_error_corruption_detected); | |
| CHECK_Z(ZSTD_DCtx_setParameter(dctx, ZSTD_d_refMultipleDDicts, 1)); | |
| n = FUZ_mdict_makeFrame(frame, 1111, 0); | |
| r = ZSTD_decompressDCtx(dctx, out, sizeof(out), frame, n); | |
| CHECK_EQ(ZSTD_getErrorCode(r), ZSTD_error_dictionary_wrong); | |
| CHECK_Z(ZSTD_DCtx_refDDict(dctx, d)); | |
| r = ZSTD_decompressDCtx(dctx, out, sizeof(out), frame, n); | |
| CHECK_EQ(ZSTD_getErrorCode(r), ZSTD_error_dictionary_wrong); | |
| n = FUZ_mdict_makeFrame(frame, 3333, 0); | |
| r = ZSTD_decompressDCtx(dctx, out, sizeof(out), frame, n); | |
| FUZ_mdict_checkOutput(r, out, "DDDDDDDD"); | |
| CHECK_Z(ZSTD_freeDCtx(dctx)); CHECK_Z(ZSTD_freeDDict(d)); | |
| } | |
| /* With no set, toggling the opt-in parameter does not discard a separately | |
| * configured local dictionary, prefix, or single borrowed DDict. */ | |
| for (mode = 0; mode < 3; ++mode) { | |
| ZSTD_DDict* const a = FUZ_mdict_createDict(1111, 'A'); | |
| ZSTD_DCtx* const dctx = ZSTD_createDCtx(); | |
| BYTE dict[44], frame[32], out[32]; | |
| size_t n, r; | |
| CHECK(dctx != NULL); | |
| FUZ_mdict_makeDict(dict, 1111, 'A'); | |
| if (mode == 0) { CHECK_Z(ZSTD_DCtx_loadDictionary(dctx, dict, sizeof(dict))); } | |
| if (mode == 1) { CHECK_Z(ZSTD_DCtx_refPrefix_advanced(dctx, dict, sizeof(dict), ZSTD_dct_fullDict)); } | |
| if (mode == 2) { CHECK_Z(ZSTD_DCtx_refDDict(dctx, a)); } | |
| CHECK_Z(ZSTD_DCtx_setParameter(dctx, ZSTD_d_refMultipleDDicts, 1)); | |
| CHECK_Z(ZSTD_DCtx_setParameter(dctx, ZSTD_d_refMultipleDDicts, 0)); | |
| n = FUZ_mdict_makeFrame(frame, 1111, 0); | |
| r = ZSTD_decompressDCtx(dctx, out, sizeof(out), frame, n); | |
| FUZ_mdict_checkOutput(r, out, "AAAAAAAA"); | |
| CHECK_Z(ZSTD_freeDCtx(dctx)); CHECK_Z(ZSTD_freeDDict(a)); | |
| } | |
| } | |
| static void FUZ_mdict_testDictionaryReplacement(void) | |
| { | |
| int mode, used; | |
| for (mode = 0; mode < 5; ++mode) { | |
| for (used = 0; used < 2; ++used) { | |
| ZSTD_DDict* const a = FUZ_mdict_createDict(1111, 'A'); | |
| ZSTD_DDict* const b = FUZ_mdict_createDict(2222, 'B'); | |
| ZSTD_DDict* const d = FUZ_mdict_createDict(4444, 'D'); | |
| ZSTD_DCtx* const dctx = FUZ_mdict_createContext(a, b); | |
| BYTE* const dict = (BYTE*)malloc(44); | |
| BYTE frame[32], out[32]; | |
| size_t n, r; | |
| CHECK(dict != NULL); | |
| FUZ_mdict_makeDict(dict, 3333, 'C'); | |
| switch (mode) { | |
| case 0: CHECK_Z(ZSTD_DCtx_loadDictionary(dctx, dict, 44)); break; | |
| case 1: CHECK_Z(ZSTD_DCtx_loadDictionary_byReference(dctx, dict, 44)); break; | |
| case 2: CHECK_Z(ZSTD_DCtx_loadDictionary_advanced(dctx, dict + 28, 16, | |
| ZSTD_dlm_byCopy, ZSTD_dct_rawContent)); break; | |
| case 3: CHECK_Z(ZSTD_DCtx_refPrefix(dctx, dict + 28, 16)); break; | |
| default: CHECK_Z(ZSTD_DCtx_refPrefix_advanced(dctx, dict, 44, ZSTD_dct_fullDict)); break; | |
| } | |
| /* Every local dictionary/prefix replacement releases all borrowed | |
| * DDicts, even though multiple-DDict mode remains enabled. */ | |
| CHECK_Z(ZSTD_freeDDict(a)); CHECK_Z(ZSTD_freeDDict(b)); | |
| if (used) { | |
| n = FUZ_mdict_makeFrame(frame, 0, 0); | |
| r = ZSTD_decompressDCtx(dctx, out, sizeof(out), frame, n); | |
| FUZ_mdict_checkOutput(r, out, "CCCCCCCC"); | |
| } | |
| /* Going back to the set must discard both an unused and an already | |
| * consumed prefix, as well as a sticky local dictionary. */ | |
| CHECK_Z(ZSTD_DCtx_refDDict(dctx, d)); | |
| free(dict); | |
| n = FUZ_mdict_makeFrame(frame, 1111, 0); | |
| r = ZSTD_decompressDCtx(dctx, out, sizeof(out), frame, n); | |
| CHECK_EQ(ZSTD_getErrorCode(r), ZSTD_error_dictionary_wrong); | |
| n = FUZ_mdict_makeFrame(frame, 4444, 0); | |
| r = ZSTD_decompressDCtx(dctx, out, sizeof(out), frame, n); | |
| FUZ_mdict_checkOutput(r, out, "DDDDDDDD"); | |
| CHECK_Z(ZSTD_freeDCtx(dctx)); CHECK_Z(ZSTD_freeDDict(d)); | |
| } | |
| } | |
| } | |
| static void FUZ_mdict_testStreaming(void) | |
| { | |
| ZSTD_DDict* const a = FUZ_mdict_createDict(1111, 'A'); | |
| ZSTD_DDict* const b = FUZ_mdict_createDict(7, 'B'); | |
| BYTE data[64], out[32]; | |
| static const struct { unsigned flags; size_t chunk; } cases[] = { | |
| {0, 1}, /* fragmented/buffered */ | |
| {FUZ_mdict_knownSize, sizeof(data)}, /* single-pass shortcut */ | |
| {FUZ_mdict_knownSize | FUZ_mdict_checksum, sizeof(data)} | |
| }; | |
| size_t i, n, r; | |
| /* Skippable frames do not require a dictionary, including an ID-0 entry. | |
| * The following ordinary frame must still select its exact match. */ | |
| for (i = 0; i < COUNTOF(cases); ++i) { | |
| ZSTD_DCtx* const dctx = FUZ_mdict_createContext(b, a); | |
| n = FUZ_mdict_makeSkip(data, 7); | |
| r = FUZ_mdict_decompressStream(dctx, out, sizeof(out), data, n, cases[i].chunk); | |
| CHECK_EQ(r, 0); | |
| n = FUZ_mdict_makeFrame(data, 1111, cases[i].flags); | |
| r = FUZ_mdict_decompressStream(dctx, out, sizeof(out), data, n, cases[i].chunk); | |
| FUZ_mdict_checkOutput(r, out, "AAAAAAAA"); | |
| CHECK_Z(ZSTD_freeDCtx(dctx)); | |
| } | |
| /* Ordinary frames select an exact match on both buffered and shortcut paths. */ | |
| for (i = 0; i < COUNTOF(cases); ++i) { | |
| ZSTD_DCtx* const dctx = FUZ_mdict_createContext(a, b); | |
| n = FUZ_mdict_makeFrame(data, 1111, cases[i].flags); | |
| r = FUZ_mdict_decompressStream(dctx, out, sizeof(out), data, n, cases[i].chunk); | |
| FUZ_mdict_checkOutput(r, out, "AAAAAAAA"); | |
| CHECK_Z(ZSTD_freeDCtx(dctx)); | |
| } | |
| /* Incomplete and rejected headers must not trigger an ID-0 lookup. | |
| * This set deliberately has no ID-0 entry. */ | |
| for (i = 0; i < 2; ++i) { | |
| ZSTD_DCtx* const dctx = FUZ_mdict_createContext(a, b); | |
| ZSTD_inBuffer in = {data, 5, 0}; | |
| ZSTD_outBuffer ob = {out, sizeof(out), 0}; | |
| FUZ_mdict_makeFrame(data, 1111, 0); | |
| if (i) memset(data, 0xff, 5); | |
| r = ZSTD_decompressStream(dctx, &ob, &in); | |
| if (i) { CHECK_EQ(ZSTD_getErrorCode(r), ZSTD_error_prefix_unknown); } | |
| else { CHECK_Z(r); CHECK(r != 0); CHECK_EQ(in.pos, in.size); } | |
| CHECK_Z(ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only)); | |
| n = FUZ_mdict_makeFrame(data, 1111, 0); | |
| r = FUZ_mdict_decompressStream(dctx, out, sizeof(out), data, n, 1); | |
| FUZ_mdict_checkOutput(r, out, "AAAAAAAA"); | |
| CHECK_Z(ZSTD_freeDCtx(dctx)); | |
| } | |
| CHECK_Z(ZSTD_freeDDict(a)); CHECK_Z(ZSTD_freeDDict(b)); | |
| } | |
| static void FUZ_mdict_testStrictMatching(void) | |
| { | |
| ZSTD_DDict* const a = FUZ_mdict_createDict(1111, 'A'); | |
| ZSTD_DDict* const b = FUZ_mdict_createDict(2222, 'B'); | |
| ZSTD_DDict* const raw = FUZ_mdict_createDict(0, 'R'); | |
| static const U32 missingIDs[] = {0, 3333}; | |
| BYTE frame[32], out[32]; | |
| size_t i, n, r; | |
| int mode; | |
| for (mode = 0; mode < 4; ++mode) { | |
| int const magicless = mode >> 1; | |
| int const shortcut = mode & 1; | |
| ZSTD_DCtx* const dctx = FUZ_mdict_createContext(a, b); | |
| unsigned const flags = shortcut ? FUZ_mdict_knownSize : 0; | |
| size_t const chunk = shortcut ? sizeof(frame) : 1; | |
| size_t const skip = magicless ? 4 : 0; | |
| if (magicless) { CHECK_Z(ZSTD_DCtx_setParameter(dctx, ZSTD_d_format, ZSTD_f_zstd1_magicless)); } | |
| /* Neither a missing nonzero ID nor an omitted ID may fall back to B. */ | |
| for (i = 0; i < COUNTOF(missingIDs); ++i) { | |
| ZSTD_ErrorCode const expected = missingIDs[i] == 0 | |
| ? ZSTD_error_corruption_detected : ZSTD_error_dictionary_wrong; | |
| n = FUZ_mdict_makeFrame(frame, 2222, flags); | |
| r = ZSTD_decompressDCtx(dctx, out, sizeof(out), frame + skip, n - skip); | |
| FUZ_mdict_checkOutput(r, out, "BBBBBBBB"); | |
| n = FUZ_mdict_makeFrame(frame, missingIDs[i], flags); | |
| r = ZSTD_decompressDCtx(dctx, out, sizeof(out), frame + skip, n - skip); | |
| CHECK_EQ(ZSTD_getErrorCode(r), expected); | |
| CHECK_Z(ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only)); | |
| n = FUZ_mdict_makeFrame(frame, 2222, flags); | |
| r = FUZ_mdict_decompressStream(dctx, out, sizeof(out), frame + skip, n - skip, chunk); | |
| FUZ_mdict_checkOutput(r, out, "BBBBBBBB"); | |
| n = FUZ_mdict_makeFrame(frame, missingIDs[i], flags); | |
| r = FUZ_mdict_decompressStream(dctx, out, sizeof(out), frame + skip, n - skip, chunk); | |
| CHECK_EQ(ZSTD_getErrorCode(r), expected); | |
| CHECK_Z(ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only)); | |
| } | |
| /* ID 0 is a real key: registering raw R makes an ID-less frame use R, | |
| * even when a different DDict was most recently referenced or used. */ | |
| CHECK_Z(ZSTD_DCtx_refDDict(dctx, raw)); | |
| CHECK_Z(ZSTD_DCtx_refDDict(dctx, a)); | |
| n = FUZ_mdict_makeFrame(frame, 0, flags); | |
| r = ZSTD_decompressDCtx(dctx, out, sizeof(out), frame + skip, n - skip); | |
| FUZ_mdict_checkOutput(r, out, "RRRRRRRR"); | |
| CHECK_Z(ZSTD_DCtx_refDDict(dctx, a)); | |
| r = FUZ_mdict_decompressStream(dctx, out, sizeof(out), frame + skip, n - skip, chunk); | |
| FUZ_mdict_checkOutput(r, out, "RRRRRRRR"); | |
| n = FUZ_mdict_makeFrame(frame, 1111, flags); | |
| r = FUZ_mdict_decompressStream(dctx, out, sizeof(out), frame + skip, n - skip, chunk); | |
| FUZ_mdict_checkOutput(r, out, "AAAAAAAA"); | |
| n = FUZ_mdict_makeFrame(frame, 0, flags); | |
| r = FUZ_mdict_decompressStream(dctx, out, sizeof(out), frame + skip, n - skip, chunk); | |
| FUZ_mdict_checkOutput(r, out, "RRRRRRRR"); | |
| CHECK_Z(ZSTD_freeDCtx(dctx)); | |
| } | |
| /* Ordinary frames compressed without a dictionary work with or without | |
| * an explicitly registered empty ID-0 DDict. */ | |
| { ZSTD_DDict* const empty = ZSTD_createDDict(NULL, 0); | |
| ZSTD_DCtx* const dctx = FUZ_mdict_createContext(a, NULL); | |
| BYTE plain[128]; | |
| CHECK(empty != NULL && ZSTD_getDictID_fromDDict(empty) == 0); | |
| n = ZSTD_compress(plain, sizeof(plain), "no dictionary", 13, 1); | |
| CHECK_Z(n); | |
| r = ZSTD_decompressDCtx(dctx, out, sizeof(out), plain, n); | |
| FUZ_mdict_checkOutput(r, out, "no dictionary"); | |
| CHECK_Z(ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only)); | |
| r = FUZ_mdict_decompressStream(dctx, out, sizeof(out), plain, n, 1); | |
| FUZ_mdict_checkOutput(r, out, "no dictionary"); | |
| CHECK_Z(ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only)); | |
| CHECK_Z(ZSTD_DCtx_refDDict(dctx, empty)); | |
| CHECK_Z(ZSTD_DCtx_refDDict(dctx, a)); | |
| r = ZSTD_decompressDCtx(dctx, out, sizeof(out), plain, n); | |
| FUZ_mdict_checkOutput(r, out, "no dictionary"); | |
| CHECK_Z(ZSTD_DCtx_refDDict(dctx, a)); | |
| r = FUZ_mdict_decompressStream(dctx, out, sizeof(out), plain, n, 1); | |
| FUZ_mdict_checkOutput(r, out, "no dictionary"); | |
| CHECK_Z(ZSTD_freeDCtx(dctx)); CHECK_Z(ZSTD_freeDDict(empty)); | |
| } | |
| CHECK_Z(ZSTD_freeDDict(a)); CHECK_Z(ZSTD_freeDDict(b)); | |
| CHECK_Z(ZSTD_freeDDict(raw)); | |
| } | |
| static void FUZ_mdict_testEmptyDDict(void) | |
| { | |
| ZSTD_DDict* const a = FUZ_mdict_createDict(1111, 'A'); | |
| BYTE dummy = 0, plain[128], emptyFrame[32], frame[32], out[32]; | |
| ZSTD_DDict* const empty[] = { | |
| ZSTD_createDDict(NULL, 0), | |
| ZSTD_createDDict_byReference(NULL, 0), | |
| ZSTD_createDDict(&dummy, 0), | |
| ZSTD_createDDict_byReference(&dummy, 0) | |
| }; | |
| size_t const plainSize = ZSTD_compress(plain, sizeof(plain), "no dictionary", 13, 1); | |
| size_t const emptySize = ZSTD_compress(emptyFrame, sizeof(emptyFrame), NULL, 0, 1); | |
| size_t i; | |
| CHECK_Z(plainSize); | |
| CHECK_Z(emptySize); | |
| for (i = 0; i < COUNTOF(empty); ++i) { | |
| ZSTD_DCtx* dctx; | |
| size_t n, r; | |
| CHECK(empty[i] != NULL && ZSTD_getDictID_fromDDict(empty[i]) == 0); | |
| dctx = FUZ_mdict_createContext(empty[i], a); | |
| /* Installing an empty DDict must replace the previous frame's history, | |
| * without doing arithmetic on a possibly NULL dictionary pointer. */ | |
| n = FUZ_mdict_makeFrame(frame, 1111, 0); | |
| r = ZSTD_decompressDCtx(dctx, out, sizeof(out), frame, n); | |
| FUZ_mdict_checkOutput(r, out, "AAAAAAAA"); | |
| r = ZSTD_decompressDCtx(dctx, out, sizeof(out), plain, plainSize); | |
| FUZ_mdict_checkOutput(r, out, "no dictionary"); | |
| r = FUZ_mdict_decompressStream(dctx, out, sizeof(out), plain, plainSize, 1); | |
| FUZ_mdict_checkOutput(r, out, "no dictionary"); | |
| r = FUZ_mdict_decompressStream(dctx, out, sizeof(out), plain, plainSize, plainSize); | |
| FUZ_mdict_checkOutput(r, out, "no dictionary"); | |
| /* The explicit and bufferless APIs must accept the same DDict. */ | |
| r = ZSTD_decompress_usingDDict(dctx, out, sizeof(out), plain, plainSize, empty[i]); | |
| FUZ_mdict_checkOutput(r, out, "no dictionary"); | |
| CHECK_Z(ZSTD_decompressBegin_usingDDict(dctx, empty[i])); | |
| r = FUZ_mdict_decompressContinue(dctx, out, sizeof(out), plain, plainSize); | |
| FUZ_mdict_checkOutput(r, out, "no dictionary"); | |
| /* Empty frames also permit a NULL, zero-capacity output buffer. */ | |
| CHECK_EQ(ZSTD_decompressDCtx(dctx, NULL, 0, emptyFrame, emptySize), 0); | |
| CHECK_EQ(FUZ_mdict_decompressStream(dctx, NULL, 0, emptyFrame, emptySize, 1), 0); | |
| CHECK_EQ(FUZ_mdict_decompressStream(dctx, NULL, 0, emptyFrame, emptySize, emptySize), 0); | |
| CHECK_EQ(ZSTD_decompress_usingDDict(dctx, NULL, 0, emptyFrame, emptySize, empty[i]), 0); | |
| CHECK_Z(ZSTD_decompressBegin_usingDDict(dctx, empty[i])); | |
| CHECK_EQ(FUZ_mdict_decompressContinue(dctx, NULL, 0, emptyFrame, emptySize), 0); | |
| CHECK_Z(ZSTD_DCtx_setParameter(dctx, ZSTD_d_refMultipleDDicts, 0)); | |
| CHECK_Z(ZSTD_DCtx_refDDict(dctx, empty[i])); | |
| r = ZSTD_decompressDCtx(dctx, out, sizeof(out), plain, plainSize); | |
| FUZ_mdict_checkOutput(r, out, "no dictionary"); | |
| CHECK_Z(ZSTD_freeDCtx(dctx)); | |
| CHECK_Z(ZSTD_freeDDict(empty[i])); | |
| } | |
| CHECK_Z(ZSTD_freeDDict(a)); | |
| } | |
| static void FUZ_mdict_testNoDictionary(void) | |
| { | |
| ZSTD_DDict* const a = FUZ_mdict_createDict(1111, 'A'); | |
| ZSTD_DDict* const b = FUZ_mdict_createDict(2222, 'B'); | |
| BYTE plain[128], frame[32], data[256], out[128]; | |
| size_t const plainSize = ZSTD_compress(plain, sizeof(plain), "no dictionary", 13, 1); | |
| int mode; | |
| CHECK_Z(plainSize); | |
| CHECK_EQ(ZSTD_getDictID_fromFrame(plain, plainSize), 0); | |
| CHECK_EQ(ZSTD_getFrameContentSize(plain, plainSize), 13); | |
| for (mode = 0; mode < 4; ++mode) { | |
| int const magicless = mode >> 1; | |
| int const shortcut = mode & 1; | |
| ZSTD_DCtx* const dctx = FUZ_mdict_createContext(a, b); | |
| unsigned const flags = shortcut ? FUZ_mdict_knownSize : 0; | |
| size_t const chunk = shortcut ? sizeof(data) : 1; | |
| size_t const skip = magicless ? 4 : 0; | |
| static const U32 ids[] = {2222, 1111}; | |
| size_t i, n = 0, r; | |
| if (magicless) { CHECK_Z(ZSTD_DCtx_setParameter(dctx, ZSTD_d_format, ZSTD_f_zstd1_magicless)); } | |
| /* Dictionary-free frames neither inherit the preceding dictionary nor | |
| * prevent the following frame from selecting its own exact match. */ | |
| for (i = 0; i < COUNTOF(ids); ++i) { | |
| size_t const frameSize = FUZ_mdict_makeFrame(frame, ids[i], flags); | |
| memcpy(data + n, frame + skip, frameSize - skip); n += frameSize - skip; | |
| memcpy(data + n, plain + skip, plainSize - skip); n += plainSize - skip; | |
| } | |
| r = ZSTD_decompressDCtx(dctx, out, sizeof(out), data, n); | |
| FUZ_mdict_checkOutput(r, out, "BBBBBBBBno dictionaryAAAAAAAAno dictionary"); | |
| r = FUZ_mdict_decompressStream(dctx, out, sizeof(out), data, n, chunk); | |
| FUZ_mdict_checkOutput(r, out, "BBBBBBBBno dictionaryAAAAAAAAno dictionary"); | |
| CHECK_Z(ZSTD_freeDCtx(dctx)); | |
| } | |
| CHECK_Z(ZSTD_freeDDict(a)); CHECK_Z(ZSTD_freeDDict(b)); | |
| } | |
| /* The DDicts use the default allocator. Counting the DCtx's first 128-slot | |
| * table lets the test observe growth without inspecting private structs. */ | |
| static void* FUZ_mdict_alloc(void* opaque, size_t size) | |
| { | |
| if (size == 128 * sizeof(ZSTD_DDict*)) ++*(unsigned*)opaque; | |
| return malloc(size); | |
| } | |
| static void FUZ_mdict_free(void* opaque, void* ptr) | |
| { | |
| (void)opaque; | |
| free(ptr); | |
| } | |
| static void FUZ_mdict_testGrowth(void) | |
| { | |
| unsigned expansions = 0; | |
| ZSTD_customMem const mem = {FUZ_mdict_alloc, FUZ_mdict_free, &expansions}; | |
| ZSTD_DCtx* const dctx = ZSTD_createDCtx_advanced(mem); | |
| ZSTD_DDict* ddicts[49]; | |
| BYTE frame[32], out[32]; | |
| size_t i, n, r; | |
| CHECK(dctx != NULL); | |
| expansions = 0; /* Do not count allocation of the DCtx itself. */ | |
| CHECK_Z(ZSTD_DCtx_setParameter(dctx, ZSTD_d_refMultipleDDicts, 1)); | |
| for (i = 0; i < COUNTOF(ddicts); ++i) { | |
| ddicts[i] = FUZ_mdict_createDict(10000 + (U32)i, 'A' + (int)(i % 26)); | |
| CHECK_Z(ZSTD_DCtx_refDDict(dctx, ddicts[i])); | |
| /* A 64-slot table grows when inserting entry 49, after reaching 75%. | |
| * The old expression incorrectly grew it when inserting entry 17. */ | |
| CHECK_EQ(expansions, (unsigned)(i == 48)); | |
| } | |
| n = FUZ_mdict_makeFrame(frame, 10000, 0); | |
| r = FUZ_mdict_decompressStream(dctx, out, sizeof(out), frame, n, 1); | |
| FUZ_mdict_checkOutput(r, out, "AAAAAAAA"); /* Rehashing retained entry 1. */ | |
| CHECK_Z(ZSTD_freeDCtx(dctx)); | |
| for (i = 0; i < COUNTOF(ddicts); ++i) CHECK_Z(ZSTD_freeDDict(ddicts[i])); | |
| } | |
| static unsigned test_multipleDDicts(unsigned testNb) | |
| { | |
| static const struct { const char* name; void (*run)(void); } tests[] = { | |
| {"one-shot frame selection", FUZ_mdict_testOneShot}, | |
| {"explicit dictionary APIs", FUZ_mdict_testExplicitDictionary}, | |
| {"single-DDict prefix compatibility", FUZ_mdict_testPrefixCompatibility}, | |
| {"raw dictionary ID zero", FUZ_mdict_testZeroID}, | |
| {"clearing dictionary references", FUZ_mdict_testClearReferences}, | |
| {"disabling the dictionary set", FUZ_mdict_testDisableSet}, | |
| {"local dictionary replacement", FUZ_mdict_testDictionaryReplacement}, | |
| {"streaming frame selection", FUZ_mdict_testStreaming}, | |
| {"strict dictionary-ID matching", FUZ_mdict_testStrictMatching}, | |
| {"empty dictionaries", FUZ_mdict_testEmptyDDict}, | |
| {"dictionary-free frames", FUZ_mdict_testNoDictionary}, | |
| {"hash-table growth", FUZ_mdict_testGrowth}, | |
| }; | |
| size_t i; | |
| for (i = 0; i < COUNTOF(tests); ++i) { | |
| DISPLAYLEVEL(3, "test%3u : multi-DDict %s : ", testNb++, tests[i].name); | |
| tests[i].run(); | |
| DISPLAYLEVEL(3, "OK \n"); | |
| } | |
| return testNb; | |
| } | |
| static void FUZ_checkEmptyDDict(const ZSTD_DDict* ddict) | |
| { | |
| static const BYTE frame[] = { | |
| 0x28, 0xB5, 0x2F, 0xFD, 0x20, 4, 0x21, 0, 0, 'T', 'E', 'S', 'T' | |
| }; | |
| BYTE dst[4]; | |
| ZSTD_DCtx* const dctx = ZSTD_createDCtx(); | |
| CHECK(ddict != NULL && dctx != NULL); | |
| CHECK(ZSTD_DDict_dictContent(ddict) == NULL); | |
| CHECK_EQ(ZSTD_DDict_dictSize(ddict), 0); | |
| CHECK_Z(ZSTD_decompressBegin_usingDDict(dctx, ddict)); | |
| CHECK(dctx->prefixStart == NULL && dctx->virtualStart == NULL); | |
| CHECK(dctx->dictEnd == NULL && dctx->previousDstEnd == NULL); | |
| ZSTD_copyDDictParameters(dctx, ddict); | |
| CHECK(dctx->dictEnd == NULL && dctx->previousDstEnd == NULL); | |
| CHECK_EQ(ZSTD_decompress_usingDDict(dctx, dst, sizeof(dst), frame, sizeof(frame), ddict), 4); | |
| CHECK(!memcmp(dst, "TEST", 4)); | |
| CHECK_Z(ZSTD_DCtx_refDDict(dctx, ddict)); | |
| { ZSTD_inBuffer in = { frame, sizeof(frame), 0 }; | |
| ZSTD_outBuffer out = { dst, sizeof(dst), 0 }; | |
| CHECK_EQ(ZSTD_decompressStream(dctx, &out, &in), 0); | |
| CHECK_EQ(in.pos, sizeof(frame)); | |
| CHECK_EQ(out.pos, sizeof(dst)); | |
| CHECK(!memcmp(dst, "TEST", 4)); | |
| } | |
| ZSTD_freeDCtx(dctx); | |
| } | |
| static void FUZ_checkHufBoundaries(HUF_DTable* dtable, HUF_CElt const* ctable) | |
| { | |
| BYTE src[512]; | |
| BYTE compressed[1024]; | |
| BYTE decoded[sizeof(src)]; | |
| size_t i; | |
| int const bmi2 = ZSTD_cpuSupportsBmi2() ? HUF_flags_bmi2 : 0; | |
| int const flags[] = { 0, HUF_flags_disableAsm, HUF_flags_disableFast }; | |
| unsigned flagNb; | |
| for (i = 0; i < sizeof(src); ++i) src[i] = (BYTE)(i & 1); | |
| for (i = 1; i <= 17; ++i) { | |
| size_t const cSize = HUF_compress1X_usingCTable(compressed, sizeof(compressed), src, i, ctable, 0); | |
| CHECK_Z(cSize); | |
| CHECK(cSize != 0); | |
| CHECK_EQ(HUF_decompress1X_usingDTable(decoded, i, compressed, cSize, dtable, bmi2), i); | |
| CHECK(!memcmp(src, decoded, i)); | |
| } | |
| /* The two-symbol table is expanded to the fast decoder's table log. | |
| * Each of these four streams is at least eight compressed bytes long. */ | |
| { size_t const cSize = HUF_compress4X_usingCTable(compressed, sizeof(compressed), src, sizeof(src), ctable, 0); | |
| CHECK_Z(cSize); | |
| CHECK(cSize != 0); | |
| for (flagNb = 0; flagNb < COUNTOF(flags); ++flagNb) { | |
| CHECK_EQ(HUF_decompress4X_usingDTable(decoded, sizeof(decoded), compressed, cSize, dtable, bmi2 | flags[flagNb]), sizeof(decoded)); | |
| CHECK(!memcmp(src, decoded, sizeof(src))); | |
| } | |
| } | |
| } | |
| static void test_decoderPointerBoundaries(unsigned tnb) | |
| { | |
| static const BYTE emptyFrame[] = { | |
| 0x28, 0xB5, 0x2F, 0xFD, 0, 0, 3, 0, 0, 'A' | |
| }; | |
| ZSTD_dictLoadMethod_e const methods[] = { ZSTD_dlm_byCopy, ZSTD_dlm_byRef }; | |
| unsigned methodNb; | |
| unsigned nullInput; | |
| DISPLAYLEVEL(3, "test%3u : empty dictionaries and decoder pointer boundaries : ", tnb); | |
| for (methodNb = 0; methodNb < COUNTOF(methods); ++methodNb) { | |
| for (nullInput = 0; nullInput < 2; ++nullInput) { | |
| BYTE* const input = (BYTE*)malloc(16); | |
| ZSTD_DDict* ddict; | |
| const ZSTD_DDict* staticDDict; | |
| size_t const workspaceSize = ZSTD_estimateDDictSize(0, methods[methodNb]); | |
| void* const workspace = malloc(workspaceSize); | |
| CHECK(input != NULL && workspace != NULL); | |
| memset(input, 0xA5, 16); | |
| ddict = ZSTD_createDDict_advanced(nullInput ? NULL : input, 0, | |
| methods[methodNb], ZSTD_dct_auto, ZSTD_defaultCMem); | |
| staticDDict = ZSTD_initStaticDDict(workspace, workspaceSize, nullInput ? NULL : input, 0, | |
| methods[methodNb], ZSTD_dct_auto); | |
| free(input); | |
| FUZ_checkEmptyDDict(ddict); | |
| FUZ_checkEmptyDDict(staticDDict); | |
| ZSTD_freeDDict(ddict); | |
| free(workspace); | |
| } | |
| /* Do not change nonempty raw-dictionary copying/reference semantics. */ | |
| { BYTE input[16]; | |
| ZSTD_DDict* ddict; | |
| memset(input, 0x5A, sizeof(input)); | |
| ddict = ZSTD_createDDict_advanced(input, sizeof(input), methods[methodNb], | |
| ZSTD_dct_rawContent, ZSTD_defaultCMem); | |
| CHECK(ddict != NULL); | |
| CHECK_EQ(ZSTD_DDict_dictSize(ddict), sizeof(input)); | |
| CHECK(!memcmp(ZSTD_DDict_dictContent(ddict), input, sizeof(input))); | |
| CHECK((ZSTD_DDict_dictContent(ddict) == input) == (methods[methodNb] == ZSTD_dlm_byRef)); | |
| ZSTD_freeDDict(ddict); | |
| } | |
| } | |
| { ZSTD_DCtx* const dctx = ZSTD_createDCtx(); | |
| size_t pos = 0; | |
| CHECK(dctx != NULL); | |
| CHECK_Z(ZSTD_decompressBegin(dctx)); | |
| while (ZSTD_nextSrcSizeToDecompress(dctx) != 0) { | |
| size_t const next = ZSTD_nextSrcSizeToDecompress(dctx); | |
| CHECK(next <= sizeof(emptyFrame) - pos); | |
| CHECK_EQ(ZSTD_decompressContinue(dctx, NULL, 0, emptyFrame + pos, next), 0); | |
| pos += next; | |
| } | |
| CHECK_EQ(pos, sizeof(emptyFrame)); | |
| CHECK_Z(ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters)); | |
| CHECK_Z(ZSTD_DCtx_setParameter(dctx, ZSTD_d_stableOutBuffer, 1)); | |
| { ZSTD_inBuffer in1 = { emptyFrame, 6, 0 }; | |
| ZSTD_inBuffer in2 = { emptyFrame + 6, sizeof(emptyFrame) - 6, 0 }; | |
| ZSTD_outBuffer out = { NULL, 0, 0 }; | |
| CHECK_Z(ZSTD_decompressStream(dctx, &out, &in1)); | |
| CHECK_EQ(ZSTD_decompressStream(dctx, &out, &in2), 0); | |
| CHECK_EQ(out.pos, 0); | |
| } | |
| ZSTD_freeDCtx(dctx); | |
| } | |
| { BYTE input[2 * sizeof(BitContainerType)]; | |
| size_t distance; | |
| U32 nbBytes; | |
| memset(input, 0xA5, sizeof(input)); | |
| for (distance = 1; distance < sizeof(BitContainerType); ++distance) { | |
| for (nbBytes = 0; nbBytes <= sizeof(BitContainerType); ++nbBytes) { | |
| BIT_DStream_t bit; | |
| size_t const consumed = MIN(distance, (size_t)nbBytes); | |
| CHECK_Z(BIT_initDStream(&bit, input, sizeof(input))); | |
| bit.ptr = bit.start + distance; | |
| bit.bitsConsumed = nbBytes * 8; | |
| CHECK_EQ(BIT_reloadDStream(&bit), nbBytes > distance ? BIT_DStream_endOfBuffer : BIT_DStream_unfinished); | |
| CHECK(bit.ptr == bit.start + distance - consumed); | |
| CHECK_EQ(bit.bitsConsumed, (nbBytes - consumed) * 8); | |
| } | |
| } | |
| } | |
| { static const BYTE header[] = { 128, 0x10 }; | |
| HUF_CREATE_STATIC_CTABLE(ctable, HUF_SYMBOLVALUE_MAX); | |
| U32 workspace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32]; | |
| unsigned maxSymbolValue = HUF_SYMBOLVALUE_MAX; | |
| unsigned hasZeroWeights = 0; | |
| CHECK_EQ(HUF_readCTable(ctable, &maxSymbolValue, header, sizeof(header), &hasZeroWeights), sizeof(header)); | |
| #ifndef HUF_FORCE_DECOMPRESS_X2 | |
| { HUF_CREATE_STATIC_DTABLEX1(dtable, HUF_TABLELOG_MAX); | |
| CHECK_EQ(HUF_readDTableX1_wksp(dtable, header, sizeof(header), workspace, sizeof(workspace), 0), sizeof(header)); | |
| FUZ_checkHufBoundaries(dtable, ctable); | |
| } | |
| #endif | |
| #ifndef HUF_FORCE_DECOMPRESS_X1 | |
| { HUF_CREATE_STATIC_DTABLEX2(dtable, HUF_TABLELOG_MAX); | |
| CHECK_EQ(HUF_readDTableX2_wksp(dtable, header, sizeof(header), workspace, sizeof(workspace), 0), sizeof(header)); | |
| FUZ_checkHufBoundaries(dtable, ctable); | |
| } | |
| #endif | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| } | |
| static int basicUnitTests(U32 const seed, double compressibility) | |
| { | |
| size_t const CNBuffSize = 5 MB; | |
| void* const CNBuffer = malloc(CNBuffSize); | |
| size_t const compressedBufferSize = ZSTD_compressBound(CNBuffSize); | |
| void* const compressedBuffer = malloc(compressedBufferSize); | |
| void* const decodedBuffer = malloc(CNBuffSize); | |
| int testResult = 0; | |
| unsigned testNb=0; | |
| size_t cSize; | |
| /* Create compressible noise */ | |
| if (!CNBuffer || !compressedBuffer || !decodedBuffer) { | |
| DISPLAY("Not enough memory, aborting\n"); | |
| testResult = 1; | |
| goto _end; | |
| } | |
| RDG_genBuffer(CNBuffer, CNBuffSize, compressibility, 0., seed); | |
| /* Basic tests */ | |
| DISPLAYLEVEL(3, "test%3u : ZSTD_getErrorName : ", testNb++); | |
| { const char* errorString = ZSTD_getErrorName(0); | |
| DISPLAYLEVEL(3, "OK : %s \n", errorString); | |
| } | |
| DISPLAYLEVEL(3, "test%3u : ZSTD_getErrorName with wrong value : ", testNb++); | |
| { const char* errorString = ZSTD_getErrorName(499); | |
| DISPLAYLEVEL(3, "OK : %s \n", errorString); | |
| } | |
| DISPLAYLEVEL(3, "test%3u : min compression level : ", testNb++); | |
| { int const mcl = ZSTD_minCLevel(); | |
| DISPLAYLEVEL(3, "%i (OK) \n", mcl); | |
| } | |
| DISPLAYLEVEL(3, "test%3u : default compression level : ", testNb++); | |
| { int const defaultCLevel = ZSTD_defaultCLevel(); | |
| if (defaultCLevel != ZSTD_CLEVEL_DEFAULT) goto _output_error; | |
| DISPLAYLEVEL(3, "%i (OK) \n", defaultCLevel); | |
| } | |
| DISPLAYLEVEL(3, "test%3u : ZSTD_versionNumber : ", testNb++); | |
| { unsigned const vn = ZSTD_versionNumber(); | |
| DISPLAYLEVEL(3, "%u (OK) \n", vn); | |
| } | |
| test_compressBound(testNb++); | |
| test_decompressBound(testNb++); | |
| test_decompressBoundOverflow(testNb++); | |
| test_decompressHuffmanTable(testNb++); | |
| test_decompressDDictHeader(testNb++); | |
| test_decoderPointerBoundaries(testNb++); | |
| test_decompressSplitLitBuffer(testNb++); | |
| test_decompressContinueWithoutBegin(testNb++); | |
| test_decompressZeroByteHistory(testNb++); | |
| test_decompressStreamSplitHeader(testNb++); | |
| test_setCParams(testNb++); | |
| test_optimalParserDictEntropy(testNb++); | |
| DISPLAYLEVEL(3, "test%3u : ZSTD_adjustCParams : ", testNb++); | |
| { | |
| ZSTD_compressionParameters params; | |
| memset(¶ms, 0, sizeof(params)); | |
| params.windowLog = 10; | |
| params.hashLog = 19; | |
| params.chainLog = 19; | |
| params = ZSTD_adjustCParams(params, 1000, 100000); | |
| if (params.hashLog != 18) goto _output_error; | |
| if (params.chainLog != 17) goto _output_error; | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| testNb = test_histCountWksp(seed, testNb); | |
| test_histCountOverflow(testNb++); | |
| DISPLAYLEVEL(3, "test%3u : compress %u bytes : ", testNb++, (unsigned)CNBuffSize); | |
| { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); | |
| if (cctx==NULL) goto _output_error; | |
| CHECK_VAR(cSize, ZSTD_compressCCtx(cctx, | |
| compressedBuffer, compressedBufferSize, | |
| CNBuffer, CNBuffSize, 1) ); | |
| DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100); | |
| DISPLAYLEVEL(3, "test%3i : size of cctx for level 1 : ", testNb++); | |
| { size_t const cctxSize = ZSTD_sizeof_CCtx(cctx); | |
| DISPLAYLEVEL(3, "%u bytes \n", (unsigned)cctxSize); | |
| } | |
| ZSTD_freeCCtx(cctx); | |
| } | |
| DISPLAYLEVEL(3, "test%3i : decompress skippable frame -8 size : ", testNb++); | |
| { | |
| char const skippable8[] = "\x50\x2a\x4d\x18\xf8\xff\xff\xff"; | |
| size_t const size = ZSTD_decompress(NULL, 0, skippable8, 8); | |
| if (!ZSTD_isError(size)) goto _output_error; | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : ZSTD_getFrameContentSize test : ", testNb++); | |
| { unsigned long long const rSize = ZSTD_getFrameContentSize(compressedBuffer, cSize); | |
| if (rSize != CNBuffSize) goto _output_error; | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : ZSTD_getDecompressedSize test : ", testNb++); | |
| { unsigned long long const rSize = ZSTD_getDecompressedSize(compressedBuffer, cSize); | |
| if (rSize != CNBuffSize) goto _output_error; | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : ZSTD_findDecompressedSize test : ", testNb++); | |
| { unsigned long long const rSize = ZSTD_findDecompressedSize(compressedBuffer, cSize); | |
| if (rSize != CNBuffSize) goto _output_error; | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : tight ZSTD_decompressBound test : ", testNb++); | |
| { | |
| unsigned long long bound = ZSTD_decompressBound(compressedBuffer, cSize); | |
| if (bound != CNBuffSize) goto _output_error; | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : ZSTD_decompressBound test with invalid srcSize : ", testNb++); | |
| { | |
| unsigned long long bound = ZSTD_decompressBound(compressedBuffer, cSize - 1); | |
| if (bound != ZSTD_CONTENTSIZE_ERROR) goto _output_error; | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : decompress %u bytes : ", testNb++, (unsigned)CNBuffSize); | |
| { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize); | |
| if (r != CNBuffSize) goto _output_error; } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : decompress %u bytes with Huffman assembly disabled : ", testNb++, (unsigned)CNBuffSize); | |
| { | |
| ZSTD_DCtx* dctx = ZSTD_createDCtx(); | |
| size_t r; | |
| CHECK_Z(ZSTD_DCtx_setParameter(dctx, ZSTD_d_disableHuffmanAssembly, 1)); | |
| r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize); | |
| if (r != CNBuffSize || memcmp(decodedBuffer, CNBuffer, CNBuffSize)) goto _output_error; | |
| ZSTD_freeDCtx(dctx); | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : check decompressed result : ", testNb++); | |
| { size_t u; | |
| for (u=0; u<CNBuffSize; u++) { | |
| if (((BYTE*)decodedBuffer)[u] != ((BYTE*)CNBuffer)[u]) goto _output_error; | |
| } } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3u : invalid endDirective : ", testNb++); | |
| { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); | |
| ZSTD_inBuffer inb = { CNBuffer, CNBuffSize, 0 }; | |
| ZSTD_outBuffer outb = { compressedBuffer, compressedBufferSize, 0 }; | |
| if (cctx==NULL) goto _output_error; | |
| CHECK( ZSTD_isError( ZSTD_compressStream2(cctx, &outb, &inb, (ZSTD_EndDirective) 3) ) ); /* must fail */ | |
| CHECK( ZSTD_isError( ZSTD_compressStream2(cctx, &outb, &inb, (ZSTD_EndDirective)-1) ) ); /* must fail */ | |
| ZSTD_freeCCtx(cctx); | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : ZSTD_checkCParams : ", testNb++); | |
| { | |
| ZSTD_parameters params = ZSTD_getParams(3, 0, 0); | |
| assert(!ZSTD_checkCParams(params.cParams)); | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : ZSTD_createDCtx_advanced and ZSTD_sizeof_DCtx: ", testNb++); | |
| { | |
| ZSTD_DCtx* const dctx = ZSTD_createDCtx_advanced(ZSTD_defaultCMem); | |
| assert(dctx != NULL); | |
| assert(ZSTD_sizeof_DCtx(dctx) != 0); | |
| ZSTD_freeDCtx(dctx); | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : misc unaccounted for zstd symbols : ", testNb++); | |
| { | |
| /* %p takes a void*. In ISO C, it's illegal to cast a function pointer | |
| * to a data pointer. (Although in POSIX you're required to be allowed | |
| * to do it...) So we have to fall back to our trusty friend memcpy. */ | |
| unsigned (* const funcptr_getDictID)(const ZSTD_DDict* ddict) = | |
| ZSTD_getDictID_fromDDict; | |
| ZSTD_DStream* (* const funcptr_createDStream)( | |
| ZSTD_customMem customMem) = ZSTD_createDStream_advanced; | |
| ZSTD_nextInputType_e (* const funcptr_nextInputType)(ZSTD_DCtx* dctx) = | |
| ZSTD_nextInputType; | |
| const void *voidptr_getDictID; | |
| const void *voidptr_createDStream; | |
| const void *voidptr_nextInputType; | |
| DEBUG_STATIC_ASSERT(sizeof(funcptr_getDictID) == sizeof(voidptr_getDictID)); | |
| memcpy( | |
| (void*)&voidptr_getDictID, | |
| (const void*)&funcptr_getDictID, | |
| sizeof(void*)); | |
| memcpy( | |
| (void*)&voidptr_createDStream, | |
| (const void*)&funcptr_createDStream, | |
| sizeof(void*)); | |
| memcpy( | |
| (void*)&voidptr_nextInputType, | |
| (const void*)&funcptr_nextInputType, | |
| sizeof(void*)); | |
| DISPLAYLEVEL(3, "%p ", voidptr_getDictID); | |
| DISPLAYLEVEL(3, "%p ", voidptr_createDStream); | |
| DISPLAYLEVEL(3, "%p ", voidptr_nextInputType); | |
| } | |
| DISPLAYLEVEL(3, ": OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : decompress with null dict : ", testNb++); | |
| { ZSTD_DCtx* const dctx = ZSTD_createDCtx(); assert(dctx != NULL); | |
| { size_t const r = ZSTD_decompress_usingDict(dctx, | |
| decodedBuffer, CNBuffSize, | |
| compressedBuffer, cSize, | |
| NULL, 0); | |
| if (r != CNBuffSize) goto _output_error; | |
| } | |
| ZSTD_freeDCtx(dctx); | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : decompress with null DDict : ", testNb++); | |
| { ZSTD_DCtx* const dctx = ZSTD_createDCtx(); assert(dctx != NULL); | |
| { size_t const r = ZSTD_decompress_usingDDict(dctx, | |
| decodedBuffer, CNBuffSize, | |
| compressedBuffer, cSize, | |
| NULL); | |
| if (r != CNBuffSize) goto _output_error; | |
| } | |
| ZSTD_freeDCtx(dctx); | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : decompress with 1 missing byte : ", testNb++); | |
| { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize-1); | |
| if (!ZSTD_isError(r)) goto _output_error; | |
| if (ZSTD_getErrorCode((size_t)r) != ZSTD_error_srcSize_wrong) goto _output_error; } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : decompress with 1 too much byte : ", testNb++); | |
| { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize+1); | |
| if (!ZSTD_isError(r)) goto _output_error; | |
| if (ZSTD_getErrorCode(r) != ZSTD_error_srcSize_wrong) goto _output_error; } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : decompress too large input : ", testNb++); | |
| { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, compressedBufferSize); | |
| if (!ZSTD_isError(r)) goto _output_error; | |
| if (ZSTD_getErrorCode(r) != ZSTD_error_srcSize_wrong) goto _output_error; } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : decompress into NULL buffer : ", testNb++); | |
| { size_t const r = ZSTD_decompress(NULL, 0, compressedBuffer, compressedBufferSize); | |
| if (!ZSTD_isError(r)) goto _output_error; | |
| if (ZSTD_getErrorCode(r) != ZSTD_error_dstSize_tooSmall) goto _output_error; } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : decompress with corrupted checksum : ", testNb++); | |
| { /* create compressed buffer with checksumming enabled */ | |
| ZSTD_CCtx* const cctx = ZSTD_createCCtx(); | |
| if (!cctx) { | |
| DISPLAY("Not enough memory, aborting\n"); | |
| testResult = 1; | |
| goto _end; | |
| } | |
| CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1) ); | |
| CHECK_VAR(cSize, ZSTD_compress2(cctx, | |
| compressedBuffer, compressedBufferSize, | |
| CNBuffer, CNBuffSize) ); | |
| ZSTD_freeCCtx(cctx); | |
| } | |
| { /* copy the compressed buffer and corrupt the checksum */ | |
| size_t r; | |
| ZSTD_DCtx* const dctx = ZSTD_createDCtx(); | |
| if (!dctx) { | |
| DISPLAY("Not enough memory, aborting\n"); | |
| testResult = 1; | |
| goto _end; | |
| } | |
| ((char*)compressedBuffer)[cSize-1] += 1; | |
| r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize); | |
| if (!ZSTD_isError(r)) goto _output_error; | |
| if (ZSTD_getErrorCode(r) != ZSTD_error_checksum_wrong) goto _output_error; | |
| CHECK_Z(ZSTD_DCtx_setParameter(dctx, ZSTD_d_forceIgnoreChecksum, ZSTD_d_ignoreChecksum)); | |
| r = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize-1); | |
| if (!ZSTD_isError(r)) goto _output_error; /* wrong checksum size should still throw error */ | |
| r = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize); | |
| if (ZSTD_isError(r)) goto _output_error; | |
| ZSTD_freeDCtx(dctx); | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : ZSTD_decompressBound test with content size missing : ", testNb++); | |
| { /* create compressed buffer with content size missing */ | |
| ZSTD_CCtx* const cctx = ZSTD_createCCtx(); | |
| CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_contentSizeFlag, 0) ); | |
| CHECK_VAR(cSize, ZSTD_compress2(cctx, | |
| compressedBuffer, compressedBufferSize, | |
| CNBuffer, CNBuffSize) ); | |
| ZSTD_freeCCtx(cctx); | |
| } | |
| { /* ensure frame content size is missing */ | |
| ZSTD_FrameHeader zfh; | |
| size_t const ret = ZSTD_getFrameHeader(&zfh, compressedBuffer, compressedBufferSize); | |
| if (ret != 0 || zfh.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN) goto _output_error; | |
| } | |
| { /* ensure CNBuffSize <= decompressBound */ | |
| unsigned long long const bound = ZSTD_decompressBound(compressedBuffer, compressedBufferSize); | |
| if (CNBuffSize > bound) goto _output_error; | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3d: check DCtx size is reduced after many oversized calls : ", testNb++); | |
| { | |
| size_t const largeFrameSrcSize = 200; | |
| size_t const smallFrameSrcSize = 10; | |
| size_t const nbFrames = 256; | |
| size_t i = 0, consumed = 0, produced = 0, prevDCtxSize = 0; | |
| int sizeReduced = 0; | |
| BYTE* const dst = (BYTE*)compressedBuffer; | |
| ZSTD_DCtx* dctx = ZSTD_createDCtx(); | |
| /* create a large frame and then a bunch of small frames */ | |
| size_t srcSize = ZSTD_compress((void*)dst, | |
| compressedBufferSize, CNBuffer, largeFrameSrcSize, 3); | |
| for (i = 0; i < nbFrames; i++) | |
| srcSize += ZSTD_compress((void*)(dst + srcSize), | |
| compressedBufferSize - srcSize, CNBuffer, | |
| smallFrameSrcSize, 3); | |
| /* decompressStream and make sure that dctx size was reduced at least once */ | |
| while (consumed < srcSize) { | |
| ZSTD_inBuffer in = {(void*)(dst + consumed), MIN(1, srcSize - consumed), 0}; | |
| ZSTD_outBuffer out = {(BYTE*)CNBuffer + produced, CNBuffSize - produced, 0}; | |
| ZSTD_decompressStream(dctx, &out, &in); | |
| consumed += in.pos; | |
| produced += out.pos; | |
| /* success! size was reduced from the previous frame */ | |
| if (prevDCtxSize > ZSTD_sizeof_DCtx(dctx)) | |
| sizeReduced = 1; | |
| prevDCtxSize = ZSTD_sizeof_DCtx(dctx); | |
| } | |
| assert(sizeReduced); | |
| ZSTD_freeDCtx(dctx); | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| { | |
| ZSTD_CCtx* const cctx = ZSTD_createCCtx(); | |
| ZSTD_CDict* const cdict = ZSTD_createCDict(CNBuffer, 100, 1); | |
| ZSTD_parameters const params = ZSTD_getParams(1, 0, 0); | |
| CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_format, ZSTD_f_zstd1_magicless) ); | |
| DISPLAYLEVEL(3, "test%3i : ZSTD_compressCCtx() doesn't use advanced parameters", testNb++); | |
| CHECK_Z(ZSTD_compressCCtx(cctx, compressedBuffer, compressedBufferSize, NULL, 0, 1)); | |
| if (MEM_readLE32(compressedBuffer) != ZSTD_MAGICNUMBER) goto _output_error; | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : ZSTD_compress_usingDict() doesn't use advanced parameters: ", testNb++); | |
| CHECK_Z(ZSTD_compress_usingDict(cctx, compressedBuffer, compressedBufferSize, NULL, 0, NULL, 0, 1)); | |
| if (MEM_readLE32(compressedBuffer) != ZSTD_MAGICNUMBER) goto _output_error; | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : ZSTD_compress_usingCDict() doesn't use advanced parameters: ", testNb++); | |
| CHECK_Z(ZSTD_compress_usingCDict(cctx, compressedBuffer, compressedBufferSize, NULL, 0, cdict)); | |
| if (MEM_readLE32(compressedBuffer) != ZSTD_MAGICNUMBER) goto _output_error; | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : ZSTD_compress_advanced() doesn't use advanced parameters: ", testNb++); | |
| CHECK_Z(ZSTD_compress_advanced(cctx, compressedBuffer, compressedBufferSize, NULL, 0, NULL, 0, params)); | |
| if (MEM_readLE32(compressedBuffer) != ZSTD_MAGICNUMBER) goto _output_error; | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : ZSTD_compress_usingCDict_advanced() doesn't use advanced parameters: ", testNb++); | |
| CHECK_Z(ZSTD_compress_usingCDict_advanced(cctx, compressedBuffer, compressedBufferSize, NULL, 0, cdict, params.fParams)); | |
| if (MEM_readLE32(compressedBuffer) != ZSTD_MAGICNUMBER) goto _output_error; | |
| DISPLAYLEVEL(3, "OK \n"); | |
| ZSTD_freeCDict(cdict); | |
| ZSTD_freeCCtx(cctx); | |
| } | |
| DISPLAYLEVEL(3, "test%3i : maxBlockSize = 2K : ", testNb++); | |
| { | |
| ZSTD_CCtx* cctx = ZSTD_createCCtx(); | |
| ZSTD_DCtx* dctx = ZSTD_createDCtx(); | |
| CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1)); | |
| CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_maxBlockSize, 2048)); | |
| CHECK_Z(ZSTD_DCtx_setParameter(dctx, ZSTD_d_maxBlockSize, 2048)); | |
| cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize); | |
| CHECK_Z(cSize); | |
| CHECK_Z(ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize)); | |
| CHECK_Z(ZSTD_DCtx_setParameter(dctx, ZSTD_d_maxBlockSize, 1024)); | |
| CHECK(ZSTD_isError(ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize))); | |
| ZSTD_freeDCtx(dctx); | |
| ZSTD_freeCCtx(cctx); | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : maxBlockSize rejects raw & RLE blocks : ", testNb++); | |
| { | |
| ZSTD_CCtx* cctx = ZSTD_createCCtx(); | |
| ZSTD_DCtx* dctx = ZSTD_createDCtx(); | |
| size_t const srcSize = 8 KB; | |
| BYTE* const src = (BYTE*)malloc(srcSize); | |
| int i; | |
| CHECK(src != NULL); | |
| CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_maxBlockSize, 2048)); | |
| CHECK_Z(ZSTD_DCtx_setParameter(dctx, ZSTD_d_maxBlockSize, 1024)); | |
| /* i = 0 : incompressible data, which is stored in raw blocks | |
| * i = 1 : a single repeated byte, which is stored in RLE blocks */ | |
| for (i = 0; i < 2; i++) { | |
| if (i == 0) { | |
| RDG_genBuffer(src, srcSize, 0.0, 0.0, seed); | |
| } else { | |
| memset(src, 'a', srcSize); | |
| } | |
| cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, src, srcSize); | |
| CHECK_Z(cSize); | |
| CHECK(ZSTD_isError(ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize))); | |
| } | |
| free(src); | |
| ZSTD_freeDCtx(dctx); | |
| ZSTD_freeCCtx(cctx); | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : ldm fill dict out-of-bounds check", testNb++); | |
| { | |
| ZSTD_CCtx* const cctx = ZSTD_createCCtx(); | |
| size_t const size = (1U << 10); | |
| size_t const dstCapacity = ZSTD_compressBound(size); | |
| void* dict = (void*)malloc(size); | |
| void* src = (void*)malloc(size); | |
| void* dst = (void*)malloc(dstCapacity); | |
| RDG_genBuffer(dict, size, 0.5, 0.5, seed); | |
| RDG_genBuffer(src, size, 0.5, 0.5, seed); | |
| CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable)); | |
| assert(!ZSTD_isError(ZSTD_compress_usingDict(cctx, dst, dstCapacity, src, size, dict, size, 3))); | |
| ZSTD_freeCCtx(cctx); | |
| free(dict); | |
| free(src); | |
| free(dst); | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : ldm hashRateLog > windowLog underflow check : ", testNb++); | |
| { | |
| /* Test that when windowLog < hashRateLog, we don't get excessive memory usage | |
| * due to underflow in hashLog calculation (windowLog - hashRateLog). */ | |
| ZSTD_CCtx* const cctx = ZSTD_createCCtx(); | |
| size_t const size = (1U << 10); // 1 KB | |
| size_t const dstCapacity = ZSTD_compressBound(size); | |
| void* src = (void*)malloc(size); | |
| void* dst = (void*)malloc(dstCapacity); | |
| RDG_genBuffer(src, size, 0.5, 0.5, seed); | |
| CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable)); | |
| CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_ldmHashLog, 0)); | |
| CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, 12)); | |
| CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_ldmHashRateLog, 13)); | |
| CHECK_Z(ZSTD_compress2(cctx, dst, dstCapacity, src, size)); | |
| { size_t const cctxSize = ZSTD_sizeof_CCtx(cctx); | |
| DISPLAYLEVEL(3, "CCtx size: %u bytes ", (unsigned)cctxSize); | |
| CHECK_LT(cctxSize, 50 MB); | |
| } | |
| ZSTD_freeCCtx(cctx); | |
| free(src); | |
| free(dst); | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : testing dict compression with enableLdm and forceMaxWindow : ", testNb++); | |
| { | |
| ZSTD_CCtx* const cctx = ZSTD_createCCtx(); | |
| ZSTD_DCtx* const dctx = ZSTD_createDCtx(); | |
| void* dict = (void*)malloc(CNBuffSize); | |
| int nbWorkers; | |
| for (nbWorkers = 0; nbWorkers < 3; ++nbWorkers) { | |
| RDG_genBuffer(dict, CNBuffSize, 0.5, 0.5, seed); | |
| RDG_genBuffer(CNBuffer, CNBuffSize, 0.6, 0.6, seed); | |
| CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, nbWorkers)); | |
| CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1)); | |
| CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_forceMaxWindow, 1)); | |
| CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable)); | |
| CHECK_Z(ZSTD_CCtx_refPrefix(cctx, dict, CNBuffSize)); | |
| cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize); | |
| CHECK_Z(cSize); | |
| CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dict, CNBuffSize)); | |
| } | |
| ZSTD_freeCCtx(cctx); | |
| ZSTD_freeDCtx(dctx); | |
| free(dict); | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : testing dict compression for determinism : ", testNb++); | |
| { | |
| size_t const testSize = 1024; | |
| ZSTD_CCtx* const cctx = ZSTD_createCCtx(); | |
| ZSTD_DCtx* const dctx = ZSTD_createDCtx(); | |
| char* dict = (char*)malloc(2 * testSize); | |
| int ldmEnabled, level; | |
| RDG_genBuffer(dict, testSize, 0.5, 0.5, seed); | |
| RDG_genBuffer(CNBuffer, testSize, 0.6, 0.6, seed); | |
| memcpy(dict + testSize, CNBuffer, testSize); | |
| for (level = 1; level <= 5; ++level) { | |
| for (ldmEnabled = ZSTD_ps_enable; ldmEnabled <= ZSTD_ps_disable; ++ldmEnabled) { | |
| size_t cSize0; | |
| XXH64_hash_t compressedChecksum0; | |
| CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1)); | |
| CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, level)); | |
| CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ldmEnabled)); | |
| CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_deterministicRefPrefix, 1)); | |
| CHECK_Z(ZSTD_CCtx_refPrefix(cctx, dict, testSize)); | |
| cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, testSize); | |
| CHECK_Z(cSize); | |
| CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, testSize, compressedBuffer, cSize, dict, testSize)); | |
| cSize0 = cSize; | |
| compressedChecksum0 = XXH64(compressedBuffer, cSize, 0); | |
| CHECK_Z(ZSTD_CCtx_refPrefix(cctx, dict, testSize)); | |
| cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, dict + testSize, testSize); | |
| CHECK_Z(cSize); | |
| if (cSize != cSize0) goto _output_error; | |
| if (XXH64(compressedBuffer, cSize, 0) != compressedChecksum0) goto _output_error; | |
| } | |
| } | |
| ZSTD_freeCCtx(cctx); | |
| ZSTD_freeDCtx(dctx); | |
| free(dict); | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : LDM + opt parser with small uncompressible block ", testNb++); | |
| { ZSTD_CCtx* cctx = ZSTD_createCCtx(); | |
| ZSTD_DCtx* dctx = ZSTD_createDCtx(); | |
| size_t const srcSize = 300 KB; | |
| size_t const flushSize = 128 KB + 5; | |
| size_t const dstSize = ZSTD_compressBound(srcSize); | |
| char* src = (char*)CNBuffer; | |
| char* dst = (char*)compressedBuffer; | |
| ZSTD_outBuffer out = { dst, dstSize, 0 }; | |
| ZSTD_inBuffer in = { src, flushSize, 0 }; | |
| if (!cctx || !dctx) { | |
| DISPLAY("Not enough memory, aborting\n"); | |
| testResult = 1; | |
| goto _end; | |
| } | |
| RDG_genBuffer(src, srcSize, 0.5, 0.5, seed); | |
| /* Force an LDM to exist that crosses block boundary into uncompressible block */ | |
| memcpy(src + 125 KB, src, 3 KB + 5); | |
| /* Enable MT, LDM, and opt parser */ | |
| CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 1)); | |
| CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable)); | |
| CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1)); | |
| CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 19)); | |
| /* Flushes a block of 128 KB and block of 5 bytes */ | |
| CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush)); | |
| /* Compress the rest */ | |
| in.size = 300 KB; | |
| CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end)); | |
| CHECK_Z(ZSTD_decompress(decodedBuffer, CNBuffSize, dst, out.pos)); | |
| ZSTD_freeCCtx(cctx); | |
| ZSTD_freeDCtx(dctx); | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : testing ldm dictionary gets invalidated : ", testNb++); | |
| { | |
| ZSTD_CCtx* const cctx = ZSTD_createCCtx(); | |
| ZSTD_DCtx* const dctx = ZSTD_createDCtx(); | |
| void* dict = (void*)malloc(CNBuffSize); | |
| size_t const kWindowLog = 10; | |
| size_t const kWindowSize = (size_t)1 << kWindowLog; | |
| size_t const dictSize = kWindowSize * 10; | |
| size_t const srcSize1 = kWindowSize / 2; | |
| size_t const srcSize2 = kWindowSize * 10; | |
| CHECK(cctx!=NULL); | |
| CHECK(dctx!=NULL); | |
| CHECK(dict!=NULL); | |
| if (CNBuffSize < dictSize) goto _output_error; | |
| RDG_genBuffer(dict, dictSize, 0.5, 0.5, seed); | |
| RDG_genBuffer(CNBuffer, srcSize1 + srcSize2, 0.5, 0.5, seed); | |
| /* Enable checksum to verify round trip. */ | |
| CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1)); | |
| /* Disable content size to skip single-pass decompression. */ | |
| CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_contentSizeFlag, 0)); | |
| CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, (int)kWindowLog)); | |
| CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable)); | |
| CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_ldmMinMatch, 32)); | |
| CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_ldmHashRateLog, 1)); | |
| CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_ldmHashLog, 16)); | |
| CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_ldmBucketSizeLog, 3)); | |
| /* Round trip once with a dictionary. */ | |
| CHECK_Z(ZSTD_CCtx_refPrefix(cctx, dict, dictSize)); | |
| cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, srcSize1); | |
| CHECK_Z(cSize); | |
| CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dict, dictSize)); | |
| cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, srcSize2); | |
| /* Streaming decompression to catch out of bounds offsets. */ | |
| { | |
| ZSTD_inBuffer in = {compressedBuffer, cSize, 0}; | |
| ZSTD_outBuffer out = {decodedBuffer, CNBuffSize, 0}; | |
| size_t const dSize = ZSTD_decompressStream(dctx, &out, &in); | |
| CHECK_Z(dSize); | |
| if (dSize != 0) goto _output_error; | |
| } | |
| CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 2)); | |
| /* Round trip once with a dictionary. */ | |
| CHECK_Z(ZSTD_CCtx_refPrefix(cctx, dict, dictSize)); | |
| { ZSTD_inBuffer in = {CNBuffer, srcSize1, 0}; | |
| ZSTD_outBuffer out = {compressedBuffer, compressedBufferSize, 0}; | |
| CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush)); | |
| CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end)); | |
| cSize = out.pos; | |
| } | |
| CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dict, dictSize)); | |
| { ZSTD_inBuffer in = {CNBuffer, srcSize2, 0}; | |
| ZSTD_outBuffer out = {compressedBuffer, compressedBufferSize, 0}; | |
| CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush)); | |
| CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end)); | |
| cSize = out.pos; | |
| } | |
| /* Streaming decompression to catch out of bounds offsets. */ | |
| { ZSTD_inBuffer in = {compressedBuffer, cSize, 0}; | |
| ZSTD_outBuffer out = {decodedBuffer, CNBuffSize, 0}; | |
| size_t const dSize = ZSTD_decompressStream(dctx, &out, &in); | |
| CHECK_Z(dSize); | |
| if (dSize != 0) goto _output_error; | |
| } | |
| ZSTD_freeCCtx(cctx); | |
| ZSTD_freeDCtx(dctx); | |
| free(dict); | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| /* Note: this test takes 0.5 seconds to run */ | |
| DISPLAYLEVEL(3, "test%3i : testing refPrefx vs refPrefx + ldm (size comparison) : ", testNb++); | |
| { | |
| /* test a big buffer so that ldm can take effect */ | |
| size_t const size = 100 MB; | |
| int const windowLog = 27; | |
| size_t const dstSize = ZSTD_compressBound(size); | |
| void* dict = (void*)malloc(size); | |
| void* src = (void*)malloc(size); | |
| void* dst = (void*)malloc(dstSize); | |
| void* recon = (void*)malloc(size); | |
| size_t refPrefixCompressedSize = 0; | |
| size_t refPrefixLdmCompressedSize = 0; | |
| size_t reconSize = 0; | |
| ZSTD_CCtx* const cctx = ZSTD_createCCtx(); | |
| ZSTD_DCtx* const dctx = ZSTD_createDCtx(); | |
| /* make dict and src the same uncompressible data */ | |
| RDG_genBuffer(src, size, 0, 0, seed); | |
| memcpy(dict, src, size); | |
| assert(!memcmp(dict, src, size)); | |
| /* set level 1 and windowLog to cover src */ | |
| CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 1)); | |
| CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, windowLog)); | |
| /* compress on level 1 using just refPrefix and no ldm */ | |
| ZSTD_CCtx_refPrefix(cctx, dict, size); | |
| refPrefixCompressedSize = ZSTD_compress2(cctx, dst, dstSize, src, size); | |
| assert(!ZSTD_isError(refPrefixCompressedSize)); | |
| /* test round trip just refPrefix */ | |
| ZSTD_DCtx_refPrefix(dctx, dict, size); | |
| reconSize = ZSTD_decompressDCtx(dctx, recon, size, dst, refPrefixCompressedSize); | |
| assert(!ZSTD_isError(reconSize)); | |
| assert(reconSize == size); | |
| assert(!memcmp(recon, src, size)); | |
| /* compress on level 1 using refPrefix and ldm */ | |
| ZSTD_CCtx_refPrefix(cctx, dict, size);; | |
| CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable)) | |
| refPrefixLdmCompressedSize = ZSTD_compress2(cctx, dst, dstSize, src, size); | |
| assert(!ZSTD_isError(refPrefixLdmCompressedSize)); | |
| /* test round trip refPrefix + ldm*/ | |
| ZSTD_DCtx_refPrefix(dctx, dict, size); | |
| reconSize = ZSTD_decompressDCtx(dctx, recon, size, dst, refPrefixLdmCompressedSize); | |
| assert(!ZSTD_isError(reconSize)); | |
| assert(reconSize == size); | |
| assert(!memcmp(recon, src, size)); | |
| /* make sure that refPrefixCompressedSize is significantly greater */ | |
| assert(refPrefixCompressedSize > 10 * refPrefixLdmCompressedSize); | |
| /* make sure the ldm compressed size is less than 1% of original */ | |
| assert((double)refPrefixLdmCompressedSize / (double)size < 0.01); | |
| ZSTD_freeDCtx(dctx); | |
| ZSTD_freeCCtx(cctx); | |
| free(recon); | |
| free(dict); | |
| free(src); | |
| free(dst); | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : in-place decompression : ", testNb++); | |
| cSize = ZSTD_compress(compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize, -ZSTD_BLOCKSIZE_MAX); | |
| CHECK_Z(cSize); | |
| CHECK_LT(CNBuffSize, cSize); | |
| { | |
| size_t const margin = ZSTD_decompressionMargin(compressedBuffer, cSize); | |
| size_t const outputSize = (CNBuffSize + margin); | |
| char* output = malloc(outputSize); | |
| char* input = output + outputSize - cSize; | |
| CHECK_LT(cSize, CNBuffSize + margin); | |
| CHECK(output != NULL); | |
| CHECK_Z(margin); | |
| CHECK(margin <= ZSTD_DECOMPRESSION_MARGIN(CNBuffSize, ZSTD_BLOCKSIZE_MAX)); | |
| memcpy(input, compressedBuffer, cSize); | |
| { | |
| size_t const dSize = ZSTD_decompress(output, outputSize, input, cSize); | |
| CHECK_Z(dSize); | |
| CHECK_EQ(dSize, CNBuffSize); | |
| } | |
| CHECK(!memcmp(output, CNBuffer, CNBuffSize)); | |
| free(output); | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : in-place decompression with 2 frames : ", testNb++); | |
| cSize = ZSTD_compress(compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize / 3, -ZSTD_BLOCKSIZE_MAX); | |
| CHECK_Z(cSize); | |
| { | |
| size_t const cSize2 = ZSTD_compress((char*)compressedBuffer + cSize, compressedBufferSize - cSize, (char const*)CNBuffer + (CNBuffSize / 3), CNBuffSize / 3, -ZSTD_BLOCKSIZE_MAX); | |
| CHECK_Z(cSize2); | |
| cSize += cSize2; | |
| } | |
| { | |
| size_t const srcSize = (CNBuffSize / 3) * 2; | |
| size_t const margin = ZSTD_decompressionMargin(compressedBuffer, cSize); | |
| size_t const outputSize = (CNBuffSize + margin); | |
| char* output = malloc(outputSize); | |
| char* input = output + outputSize - cSize; | |
| CHECK_LT(cSize, CNBuffSize + margin); | |
| CHECK(output != NULL); | |
| CHECK_Z(margin); | |
| memcpy(input, compressedBuffer, cSize); | |
| { | |
| size_t const dSize = ZSTD_decompress(output, outputSize, input, cSize); | |
| CHECK_Z(dSize); | |
| CHECK_EQ(dSize, srcSize); | |
| } | |
| CHECK(!memcmp(output, CNBuffer, srcSize)); | |
| free(output); | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : Check block splitter with 64K literal length : ", testNb++); | |
| { ZSTD_CCtx* cctx = ZSTD_createCCtx(); | |
| size_t const srcSize = 256 * 1024; | |
| U32 const compressibleLenU32 = 32 * 1024 / 4; | |
| U32 const blockSizeU32 = 128 * 1024 / 4; | |
| U32 const litLenU32 = 64 * 1024 / 4; | |
| U32* data = (U32*)malloc(srcSize); | |
| size_t dSize; | |
| if (data == NULL || cctx == NULL) goto _output_error; | |
| /* Generate data without any matches */ | |
| RDG_genBuffer(data, srcSize, 0.0, 0.01, 2654435761U); | |
| /* Generate 32K of compressible data */ | |
| RDG_genBuffer(data, compressibleLenU32 * 4, 0.5, 0.5, 0xcafebabe); | |
| /* Add a match of offset=12, length=8 at idx=16, 32, 48, 64 */ | |
| data[compressibleLenU32 + 0] = 0xFFFFFFFF; | |
| data[compressibleLenU32 + 1] = 0xEEEEEEEE; | |
| data[compressibleLenU32 + 4] = 0xFFFFFFFF; | |
| data[compressibleLenU32 + 5] = 0xEEEEEEEE; | |
| /* Add a match of offset=16, length=8 at idx=64K + 64. | |
| * This generates a sequence with llen=64K, and repeat code 1. | |
| * The block splitter thought this was ll0, and corrupted the | |
| * repeat offset history. | |
| */ | |
| data[compressibleLenU32 + litLenU32 + 2 + 0] = 0xDDDDDDDD; | |
| data[compressibleLenU32 + litLenU32 + 2 + 1] = 0xCCCCCCCC; | |
| data[compressibleLenU32 + litLenU32 + 2 + 4] = 0xDDDDDDDD; | |
| data[compressibleLenU32 + litLenU32 + 2 + 5] = 0xCCCCCCCC; | |
| /* Add a match of offset=16, length=8 at idx=128K + 16. | |
| * This should generate a sequence with repeat code = 1. | |
| * But the block splitters mistake caused zstd to generate | |
| * repeat code = 2, corrupting the data. | |
| */ | |
| data[blockSizeU32] = 0xBBBBBBBB; | |
| data[blockSizeU32 + 1] = 0xAAAAAAAA; | |
| data[blockSizeU32 + 4] = 0xBBBBBBBB; | |
| data[blockSizeU32 + 5] = 0xAAAAAAAA; | |
| /* Generate a golden file from this data in case datagen changes and | |
| * doesn't generate the exact same data. We will also test this golden file. | |
| */ | |
| if (0) { | |
| FILE* f = fopen("golden-compression/PR-3517-block-splitter-corruption-test", "wb"); | |
| fwrite(data, 1, srcSize, f); | |
| fclose(f); | |
| } | |
| CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 19)); | |
| CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_minMatch, 7)); | |
| CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_splitAfterSequences, ZSTD_ps_enable)); | |
| cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, data, srcSize); | |
| CHECK_Z(cSize); | |
| dSize = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize); | |
| CHECK_Z(dSize); | |
| CHECK_EQ(dSize, srcSize); | |
| CHECK(!memcmp(decodedBuffer, data, srcSize)); | |
| free(data); | |
| ZSTD_freeCCtx(cctx); | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| test_blockSplitter_incompressibleExpansionProtection(testNb++, seed); | |
| test_targetCBlockSize_dstCapacity(testNb++); | |
| DISPLAYLEVEL(3, "test%3d : superblock uncompressible data: too many nocompress superblocks : ", testNb++); | |
| { | |
| ZSTD_CCtx* const cctx = ZSTD_createCCtx(); | |
| const BYTE* src = (BYTE*)CNBuffer; BYTE* dst = (BYTE*)compressedBuffer; | |
| size_t srcSize = 321656; size_t dstCapacity = ZSTD_compressBound(srcSize); | |
| /* This is the number of bytes to stream before ending. This value | |
| * was obtained by trial and error :/. */ | |
| const size_t streamCompressThreshold = 161792; | |
| const size_t streamCompressDelta = 1024; | |
| /* The first 1/5 of the buffer is compressible and the last 4/5 is | |
| * uncompressible. This is an approximation of the type of data | |
| * the fuzzer generated to catch this bug. Streams like this were making | |
| * zstd generate noCompress superblocks (which are larger than the src | |
| * they come from). Do this enough times, and we'll run out of room | |
| * and throw a dstSize_tooSmall error. */ | |
| const size_t compressiblePartSize = srcSize/5; | |
| const size_t uncompressiblePartSize = srcSize-compressiblePartSize; | |
| RDG_genBuffer(CNBuffer, compressiblePartSize, 0.5, 0.5, seed); | |
| RDG_genBuffer((BYTE*)CNBuffer+compressiblePartSize, uncompressiblePartSize, 0, 0, seed); | |
| /* Setting target block size so that superblock is used */ | |
| assert(cctx != NULL); | |
| ZSTD_CCtx_setParameter(cctx, ZSTD_c_targetCBlockSize, 81); | |
| { size_t read; | |
| for (read = 0; read < streamCompressThreshold; read += streamCompressDelta) { | |
| ZSTD_inBuffer in = {src, streamCompressDelta, 0}; | |
| ZSTD_outBuffer out = {dst, dstCapacity, 0}; | |
| CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_continue)); | |
| CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end)); | |
| src += streamCompressDelta; srcSize -= streamCompressDelta; | |
| dst += out.pos; dstCapacity -= out.pos; | |
| } } | |
| /* This is trying to catch a dstSize_tooSmall error */ | |
| { ZSTD_inBuffer in = {src, srcSize, 0}; | |
| ZSTD_outBuffer out = {dst, dstCapacity, 0}; | |
| CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end)); | |
| } | |
| ZSTD_freeCCtx(cctx); | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3d: superblock with no literals : ", testNb++); | |
| /* Generate the same data 20 times over */ | |
| { size_t const avgChunkSize = CNBuffSize / 20; | |
| size_t b; | |
| for (b = 0; b < CNBuffSize; b += avgChunkSize) { | |
| size_t const chunkSize = MIN(CNBuffSize - b, avgChunkSize); | |
| RDG_genBuffer((char*)CNBuffer + b, chunkSize, compressibility, 0. /* auto */, seed); | |
| } } | |
| { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); | |
| size_t const normalCSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize); | |
| size_t const allowedExpansion = (CNBuffSize * 3 / 1000); | |
| size_t superCSize; | |
| CHECK_Z(normalCSize); | |
| ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 19); | |
| ZSTD_CCtx_setParameter(cctx, ZSTD_c_targetCBlockSize, 1000); | |
| superCSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize); | |
| CHECK_Z(superCSize); | |
| if (superCSize > normalCSize + allowedExpansion) { | |
| DISPLAYLEVEL(1, "Superblock too big: %u > %u + %u \n", (U32)superCSize, (U32)normalCSize, (U32)allowedExpansion); | |
| goto _output_error; | |
| } | |
| ZSTD_freeCCtx(cctx); | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| RDG_genBuffer(CNBuffer, CNBuffSize, compressibility, 0. /*auto*/, seed); | |
| DISPLAYLEVEL(3, "test%3d: superblock enough room for checksum : ", testNb++) | |
| /* This tests whether or not we leave enough room for the checksum at the end | |
| * of the dst buffer. The bug that motivated this test was found by the | |
| * stream_round_trip fuzzer but this crashes for the same reason and is | |
| * far more compact than re-creating the stream_round_trip fuzzer's code path */ | |
| { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); | |
| ZSTD_CCtx_setParameter(cctx, ZSTD_c_targetCBlockSize, 64); | |
| assert(!ZSTD_isError(ZSTD_compress2(cctx, compressedBuffer, 1339, CNBuffer, 1278))); | |
| ZSTD_freeCCtx(cctx); | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : compress a NULL input with each level : ", testNb++); | |
| { int level = -1; | |
| ZSTD_CCtx* const cctx = ZSTD_createCCtx(); | |
| if (!cctx) goto _output_error; | |
| for (level = -1; level <= ZSTD_maxCLevel(); ++level) { | |
| CHECK_Z( ZSTD_compress(compressedBuffer, compressedBufferSize, NULL, 0, level) ); | |
| CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, level) ); | |
| CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, NULL, 0) ); | |
| } | |
| ZSTD_freeCCtx(cctx); | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3d : check CCtx size after compressing empty input : ", testNb++); | |
| { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); | |
| size_t const r = ZSTD_compressCCtx(cctx, compressedBuffer, compressedBufferSize, NULL, 0, 19); | |
| if (ZSTD_isError(r)) goto _output_error; | |
| if (ZSTD_sizeof_CCtx(cctx) > (1U << 20)) goto _output_error; | |
| ZSTD_freeCCtx(cctx); | |
| cSize = r; | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3d : decompress empty frame into NULL : ", testNb++); | |
| { size_t const r = ZSTD_decompress(NULL, 0, compressedBuffer, cSize); | |
| if (ZSTD_isError(r)) goto _output_error; | |
| if (r != 0) goto _output_error; | |
| } | |
| { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); | |
| ZSTD_outBuffer output; | |
| if (cctx==NULL) goto _output_error; | |
| output.dst = compressedBuffer; | |
| output.size = compressedBufferSize; | |
| output.pos = 0; | |
| CHECK_Z( ZSTD_initCStream(cctx, 1) ); /* content size unknown */ | |
| CHECK_Z( ZSTD_flushStream(cctx, &output) ); /* ensure no possibility to "concatenate" and determine the content size */ | |
| CHECK_Z( ZSTD_endStream(cctx, &output) ); | |
| ZSTD_freeCCtx(cctx); | |
| /* single scan decompression */ | |
| { size_t const r = ZSTD_decompress(NULL, 0, compressedBuffer, output.pos); | |
| if (ZSTD_isError(r)) goto _output_error; | |
| if (r != 0) goto _output_error; | |
| } | |
| /* streaming decompression */ | |
| { ZSTD_DCtx* const dstream = ZSTD_createDStream(); | |
| ZSTD_inBuffer dinput; | |
| ZSTD_outBuffer doutput; | |
| size_t ipos; | |
| if (dstream==NULL) goto _output_error; | |
| dinput.src = compressedBuffer; | |
| dinput.size = 0; | |
| dinput.pos = 0; | |
| doutput.dst = NULL; | |
| doutput.size = 0; | |
| doutput.pos = 0; | |
| CHECK_Z ( ZSTD_initDStream(dstream) ); | |
| for (ipos=1; ipos<=output.pos; ipos++) { | |
| dinput.size = ipos; | |
| CHECK_Z ( ZSTD_decompressStream(dstream, &doutput, &dinput) ); | |
| } | |
| if (doutput.pos != 0) goto _output_error; | |
| ZSTD_freeDStream(dstream); | |
| } | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3d : reuse CCtx with expanding block size : ", testNb++); | |
| { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); | |
| ZSTD_parameters const params = ZSTD_getParams(1, ZSTD_CONTENTSIZE_UNKNOWN, 0); | |
| assert(params.fParams.contentSizeFlag == 1); /* block size will be adapted if pledgedSrcSize is enabled */ | |
| CHECK_Z( ZSTD_compressBegin_advanced(cctx, NULL, 0, params, 1 /*pledgedSrcSize*/) ); | |
| CHECK_Z( ZSTD_compressEnd(cctx, compressedBuffer, compressedBufferSize, CNBuffer, 1) ); /* creates a block size of 1 */ | |
| CHECK_Z( ZSTD_compressBegin_advanced(cctx, NULL, 0, params, ZSTD_CONTENTSIZE_UNKNOWN) ); /* reuse same parameters */ | |
| { size_t const inSize = 2* 128 KB; | |
| size_t const outSize = ZSTD_compressBound(inSize); | |
| CHECK_Z( ZSTD_compressEnd(cctx, compressedBuffer, outSize, CNBuffer, inSize) ); | |
| /* will fail if blockSize is not resized */ | |
| } | |
| ZSTD_freeCCtx(cctx); | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3d : re-using a CCtx should compress the same : ", testNb++); | |
| { size_t const sampleSize = 30; | |
| int i; | |
| for (i=0; i<20; i++) | |
| ((char*)CNBuffer)[i] = (char)i; /* ensure no match during initial section */ | |
| memcpy((char*)CNBuffer + 20, CNBuffer, 10); /* create one match, starting from beginning of sample, which is the difficult case (see #1241) */ | |
| for (i=1; i<=19; i++) { | |
| ZSTD_CCtx* const cctx = ZSTD_createCCtx(); | |
| size_t size1, size2; | |
| DISPLAYLEVEL(5, "l%i ", i); | |
| size1 = ZSTD_compressCCtx(cctx, compressedBuffer, compressedBufferSize, CNBuffer, sampleSize, i); | |
| CHECK_Z(size1); | |
| size2 = ZSTD_compressCCtx(cctx, compressedBuffer, compressedBufferSize, CNBuffer, sampleSize, i); | |
| CHECK_Z(size2); | |
| CHECK_EQ(size1, size2); | |
| CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, i) ); | |
| size2 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, sampleSize); | |
| CHECK_Z(size2); | |
| CHECK_EQ(size1, size2); | |
| size2 = ZSTD_compress2(cctx, compressedBuffer, ZSTD_compressBound(sampleSize) - 1, CNBuffer, sampleSize); /* force streaming, as output buffer is not large enough to guarantee success */ | |
| CHECK_Z(size2); | |
| CHECK_EQ(size1, size2); | |
| { ZSTD_inBuffer inb; | |
| ZSTD_outBuffer outb; | |
| inb.src = CNBuffer; | |
| inb.pos = 0; | |
| inb.size = sampleSize; | |
| outb.dst = compressedBuffer; | |
| outb.pos = 0; | |
| outb.size = ZSTD_compressBound(sampleSize) - 1; /* force streaming, as output buffer is not large enough to guarantee success */ | |
| CHECK_Z( ZSTD_compressStream2(cctx, &outb, &inb, ZSTD_e_end) ); | |
| assert(inb.pos == inb.size); | |
| CHECK_EQ(size1, outb.pos); | |
| } | |
| ZSTD_freeCCtx(cctx); | |
| } | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3d : btultra2 & 1st block : ", testNb++); | |
| { size_t const sampleSize = 1024; | |
| ZSTD_CCtx* const cctx = ZSTD_createCCtx(); | |
| ZSTD_inBuffer inb; | |
| ZSTD_outBuffer outb; | |
| inb.src = CNBuffer; | |
| inb.pos = 0; | |
| inb.size = 0; | |
| outb.dst = compressedBuffer; | |
| outb.pos = 0; | |
| outb.size = compressedBufferSize; | |
| CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, ZSTD_maxCLevel()) ); | |
| inb.size = sampleSize; /* start with something, so that context is already used */ | |
| CHECK_Z( ZSTD_compressStream2(cctx, &outb, &inb, ZSTD_e_end) ); /* will break internal assert if stats_init is not disabled */ | |
| assert(inb.pos == inb.size); | |
| outb.pos = 0; /* cancel output */ | |
| CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(cctx, sampleSize) ); | |
| inb.size = 4; /* too small size : compression will be skipped */ | |
| inb.pos = 0; | |
| CHECK_Z( ZSTD_compressStream2(cctx, &outb, &inb, ZSTD_e_flush) ); | |
| assert(inb.pos == inb.size); | |
| inb.size += 5; /* too small size : compression will be skipped */ | |
| CHECK_Z( ZSTD_compressStream2(cctx, &outb, &inb, ZSTD_e_flush) ); | |
| assert(inb.pos == inb.size); | |
| inb.size += 11; /* small enough to attempt compression */ | |
| CHECK_Z( ZSTD_compressStream2(cctx, &outb, &inb, ZSTD_e_flush) ); | |
| assert(inb.pos == inb.size); | |
| assert(inb.pos < sampleSize); | |
| inb.size = sampleSize; /* large enough to trigger stats_init, but no longer at beginning */ | |
| CHECK_Z( ZSTD_compressStream2(cctx, &outb, &inb, ZSTD_e_end) ); /* will break internal assert if stats_init is not disabled */ | |
| assert(inb.pos == inb.size); | |
| ZSTD_freeCCtx(cctx); | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3d : ZSTD_CCtx_getParameter() : ", testNb++); | |
| { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); | |
| ZSTD_outBuffer out = {NULL, 0, 0}; | |
| ZSTD_inBuffer in = {NULL, 0, 0}; | |
| int value; | |
| CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_compressionLevel, &value)); | |
| CHECK_EQ(value, 3); | |
| CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value)); | |
| CHECK_EQ(value, 0); | |
| CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_hashLog, ZSTD_HASHLOG_MIN)); | |
| CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_compressionLevel, &value)); | |
| CHECK_EQ(value, 3); | |
| CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value)); | |
| CHECK_EQ(value, ZSTD_HASHLOG_MIN); | |
| CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 7)); | |
| CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_compressionLevel, &value)); | |
| CHECK_EQ(value, 7); | |
| CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value)); | |
| CHECK_EQ(value, ZSTD_HASHLOG_MIN); | |
| /* Start a compression job */ | |
| ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_continue); | |
| CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_compressionLevel, &value)); | |
| CHECK_EQ(value, 7); | |
| CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value)); | |
| CHECK_EQ(value, ZSTD_HASHLOG_MIN); | |
| /* Reset the CCtx */ | |
| ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only); | |
| CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_compressionLevel, &value)); | |
| CHECK_EQ(value, 7); | |
| CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value)); | |
| CHECK_EQ(value, ZSTD_HASHLOG_MIN); | |
| /* Reset the parameters */ | |
| ZSTD_CCtx_reset(cctx, ZSTD_reset_parameters); | |
| CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_compressionLevel, &value)); | |
| CHECK_EQ(value, 3); | |
| CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value)); | |
| CHECK_EQ(value, 0); | |
| ZSTD_freeCCtx(cctx); | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3d : ZSTD_CCtx_setCParams() : ", testNb++); | |
| { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); | |
| int value; | |
| ZSTD_compressionParameters cparams = ZSTD_getCParams(1, 0, 0); | |
| cparams.strategy = (ZSTD_strategy)-1; /* set invalid value, on purpose */ | |
| /* Set invalid cParams == error out, and no change. */ | |
| CHECK(ZSTD_isError(ZSTD_CCtx_setCParams(cctx, cparams))); | |
| CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_windowLog, &value)); | |
| CHECK_EQ(value, 0); | |
| CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_chainLog, &value)); | |
| CHECK_EQ(value, 0); | |
| CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value)); | |
| CHECK_EQ(value, 0); | |
| CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_searchLog, &value)); | |
| CHECK_EQ(value, 0); | |
| CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_minMatch, &value)); | |
| CHECK_EQ(value, 0); | |
| CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_targetLength, &value)); | |
| CHECK_EQ(value, 0); | |
| CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_strategy, &value)); | |
| CHECK_EQ(value, 0); | |
| cparams = ZSTD_getCParams(12, 0, 0); | |
| CHECK_Z(ZSTD_CCtx_setCParams(cctx, cparams)); | |
| CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_windowLog, &value)); | |
| CHECK_EQ(value, (int)cparams.windowLog); | |
| CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_chainLog, &value)); | |
| CHECK_EQ(value, (int)cparams.chainLog); | |
| CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value)); | |
| CHECK_EQ(value, (int)cparams.hashLog); | |
| CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_searchLog, &value)); | |
| CHECK_EQ(value, (int)cparams.searchLog); | |
| CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_minMatch, &value)); | |
| CHECK_EQ(value, (int)cparams.minMatch); | |
| CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_targetLength, &value)); | |
| CHECK_EQ(value, (int)cparams.targetLength); | |
| CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_strategy, &value)); | |
| CHECK_EQ(value, (int)cparams.strategy); | |
| ZSTD_freeCCtx(cctx); | |
| } | |
| DISPLAYLEVEL(3, "test%3d : ZSTD_CCtx_setFParams() : ", testNb++); | |
| { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); | |
| int value; | |
| ZSTD_frameParameters fparams = {0, 1, 1}; | |
| CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_contentSizeFlag, &value)); | |
| CHECK_EQ(value, 1); | |
| CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_checksumFlag, &value)); | |
| CHECK_EQ(value, 0); | |
| CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_dictIDFlag, &value)); | |
| CHECK_EQ(value, 1); | |
| CHECK_Z(ZSTD_CCtx_setFParams(cctx, fparams)); | |
| CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_contentSizeFlag, &value)); | |
| CHECK_EQ(value, fparams.contentSizeFlag); | |
| CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_checksumFlag, &value)); | |
| CHECK_EQ(value, fparams.checksumFlag); | |
| CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_dictIDFlag, &value)); | |
| CHECK_EQ(value, !fparams.noDictIDFlag); | |
| ZSTD_freeCCtx(cctx); | |
| } | |
| DISPLAYLEVEL(3, "test%3d : ZSTD_CCtx_setParams() : ", testNb++); | |
| { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); | |
| int value; | |
| ZSTD_parameters params = ZSTD_getParams(1, 0, 0); | |
| params.cParams.strategy = (ZSTD_strategy)-1; /* set invalid value, on purpose */ | |
| /* Set invalid params == error out, and no change. */ | |
| CHECK(ZSTD_isError(ZSTD_CCtx_setParams(cctx, params))); | |
| CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_windowLog, &value)); | |
| CHECK_EQ(value, 0); | |
| CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_chainLog, &value)); | |
| CHECK_EQ(value, 0); | |
| CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value)); | |
| CHECK_EQ(value, 0); | |
| CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_searchLog, &value)); | |
| CHECK_EQ(value, 0); | |
| CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_minMatch, &value)); | |
| CHECK_EQ(value, 0); | |
| CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_targetLength, &value)); | |
| CHECK_EQ(value, 0); | |
| CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_strategy, &value)); | |
| CHECK_EQ(value, 0); | |
| CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_contentSizeFlag, &value)); | |
| CHECK_EQ(value, 1); | |
| CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_checksumFlag, &value)); | |
| CHECK_EQ(value, 0); | |
| CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_dictIDFlag, &value)); | |
| CHECK_EQ(value, 1); | |
| params = ZSTD_getParams(12, 0, 0); | |
| params.fParams.contentSizeFlag = 0; | |
| params.fParams.checksumFlag = 1; | |
| params.fParams.noDictIDFlag = 1; | |
| CHECK_Z(ZSTD_CCtx_setParams(cctx, params)); | |
| CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_windowLog, &value)); | |
| CHECK_EQ(value, (int)params.cParams.windowLog); | |
| CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_chainLog, &value)); | |
| CHECK_EQ(value, (int)params.cParams.chainLog); | |
| CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value)); | |
| CHECK_EQ(value, (int)params.cParams.hashLog); | |
| CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_searchLog, &value)); | |
| CHECK_EQ(value, (int)params.cParams.searchLog); | |
| CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_minMatch, &value)); | |
| CHECK_EQ(value, (int)params.cParams.minMatch); | |
| CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_targetLength, &value)); | |
| CHECK_EQ(value, (int)params.cParams.targetLength); | |
| CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_strategy, &value)); | |
| CHECK_EQ(value, (int)params.cParams.strategy); | |
| CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_contentSizeFlag, &value)); | |
| CHECK_EQ(value, params.fParams.contentSizeFlag); | |
| CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_checksumFlag, &value)); | |
| CHECK_EQ(value, params.fParams.checksumFlag); | |
| CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_dictIDFlag, &value)); | |
| CHECK_EQ(value, !params.fParams.noDictIDFlag); | |
| ZSTD_freeCCtx(cctx); | |
| } | |
| DISPLAYLEVEL(3, "test%3d : ldm conditionally enabled by default doesn't change cctx params: ", testNb++); | |
| { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); | |
| ZSTD_outBuffer out = {NULL, 0, 0}; | |
| ZSTD_inBuffer in = {NULL, 0, 0}; | |
| int value; | |
| /* Even if LDM will be enabled by default in the applied params (since wlog >= 27 and strategy >= btopt), | |
| * we should not modify the actual parameter specified by the user within the CCtx | |
| */ | |
| CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, 27)); | |
| CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_strategy, ZSTD_btopt)); | |
| CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_continue)); | |
| CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_enableLongDistanceMatching, &value)); | |
| CHECK_EQ(value, 0); | |
| ZSTD_freeCCtx(cctx); | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| /* this test is really too long, and should be made faster */ | |
| DISPLAYLEVEL(3, "test%3d : overflow protection with large windowLog : ", testNb++); | |
| { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); | |
| ZSTD_parameters params = ZSTD_getParams(-999, ZSTD_CONTENTSIZE_UNKNOWN, 0); | |
| size_t const nbCompressions = ((1U << 31) / CNBuffSize) + 2; /* ensure U32 overflow protection is triggered */ | |
| size_t cnb; | |
| assert(cctx != NULL); | |
| params.fParams.contentSizeFlag = 0; | |
| params.cParams.windowLog = ZSTD_WINDOWLOG_MAX; | |
| for (cnb = 0; cnb < nbCompressions; ++cnb) { | |
| DISPLAYLEVEL(6, "run %u / %u \n", (unsigned)cnb, (unsigned)nbCompressions); | |
| CHECK_Z( ZSTD_compressBegin_advanced(cctx, NULL, 0, params, ZSTD_CONTENTSIZE_UNKNOWN) ); /* reuse same parameters */ | |
| CHECK_Z( ZSTD_compressEnd(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize) ); | |
| } | |
| ZSTD_freeCCtx(cctx); | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3d : size down context : ", testNb++); | |
| { ZSTD_CCtx* const largeCCtx = ZSTD_createCCtx(); | |
| assert(largeCCtx != NULL); | |
| CHECK_Z( ZSTD_compressBegin(largeCCtx, 19) ); /* streaming implies ZSTD_CONTENTSIZE_UNKNOWN, which maximizes memory usage */ | |
| CHECK_Z( ZSTD_compressEnd(largeCCtx, compressedBuffer, compressedBufferSize, CNBuffer, 1) ); | |
| { size_t const largeCCtxSize = ZSTD_sizeof_CCtx(largeCCtx); /* size of context must be measured after compression */ | |
| { ZSTD_CCtx* const smallCCtx = ZSTD_createCCtx(); | |
| assert(smallCCtx != NULL); | |
| CHECK_Z(ZSTD_compressCCtx(smallCCtx, compressedBuffer, compressedBufferSize, CNBuffer, 1, 1)); | |
| { size_t const smallCCtxSize = ZSTD_sizeof_CCtx(smallCCtx); | |
| DISPLAYLEVEL(5, "(large) %uKB > 32*%uKB (small) : ", | |
| (unsigned)(largeCCtxSize>>10), (unsigned)(smallCCtxSize>>10)); | |
| assert(largeCCtxSize > 32* smallCCtxSize); /* note : "too large" definition is handled within zstd_compress.c . | |
| * make this test case extreme, so that it doesn't depend on a possibly fluctuating definition */ | |
| } | |
| ZSTD_freeCCtx(smallCCtx); | |
| } | |
| { U32 const maxNbAttempts = 1100; /* nb of usages before triggering size down is handled within zstd_compress.c. | |
| * currently defined as 128x, but could be adjusted in the future. | |
| * make this test long enough so that it's not too much tied to the current definition within zstd_compress.c */ | |
| unsigned u; | |
| for (u=0; u<maxNbAttempts; u++) { | |
| CHECK_Z(ZSTD_compressCCtx(largeCCtx, compressedBuffer, compressedBufferSize, CNBuffer, 1, 1)); | |
| if (ZSTD_sizeof_CCtx(largeCCtx) < largeCCtxSize) break; /* sized down */ | |
| } | |
| DISPLAYLEVEL(5, "size down after %u attempts : ", u); | |
| if (u==maxNbAttempts) goto _output_error; /* no sizedown happened */ | |
| } | |
| } | |
| ZSTD_freeCCtx(largeCCtx); | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : static DCtx dictionary allocation : ", testNb++); | |
| { | |
| static const char dict[] = "static DCtx dictionary regression test"; | |
| size_t const dictSize = sizeof(dict); | |
| size_t const dctxSize = ZSTD_estimateDCtxSize(); | |
| size_t const dstreamSize = ZSTD_estimateDStreamSize(1 KB); | |
| size_t const ddictSize = ZSTD_estimateDDictSize(dictSize, ZSTD_dlm_byRef); | |
| void* const workspace = malloc(dstreamSize); | |
| void* const ddictWorkspace = malloc(ddictSize); | |
| ZSTD_CCtx* const cctx = ZSTD_createCCtx(); | |
| const ZSTD_DDict* ddict; | |
| char compressed[ZSTD_COMPRESSBOUND(sizeof(dict))]; | |
| char decoded[sizeof(dict)]; | |
| size_t compressedSize; | |
| int fill; | |
| int stream; | |
| CHECK(workspace != NULL && ddictWorkspace != NULL && cctx != NULL); | |
| ddict = ZSTD_initStaticDDict(ddictWorkspace, ddictSize, dict, dictSize, | |
| ZSTD_dlm_byRef, ZSTD_dct_rawContent); | |
| CHECK(ddict != NULL); | |
| CHECK_VAR(compressedSize, ZSTD_compress_usingDict(cctx, compressed, sizeof(compressed), | |
| dict, dictSize, dict, dictSize, 1)); | |
| CHECK_Z(ZSTD_freeCCtx(cctx)); | |
| /* Both zeroed and reused workspaces must reject internal DDict | |
| * creation without invoking an allocator. */ | |
| for (fill = 0; fill < 2; ++fill) { | |
| for (stream = 0; stream < 2; ++stream) { | |
| ZSTD_DCtx* dctx; | |
| memset(workspace, fill ? 0xA5 : 0, dstreamSize); | |
| dctx = stream ? ZSTD_initStaticDStream(workspace, dstreamSize) | |
| : ZSTD_initStaticDCtx(workspace, dctxSize); | |
| CHECK(dctx != NULL); | |
| CHECK_EQ(ZSTD_DCtx_loadDictionary(dctx, dict, dictSize), ERROR(memory_allocation)); | |
| CHECK_EQ(ZSTD_DCtx_loadDictionary_byReference(dctx, dict, dictSize), ERROR(memory_allocation)); | |
| CHECK_EQ(ZSTD_DCtx_loadDictionary_advanced(dctx, dict, dictSize, ZSTD_dlm_byCopy, ZSTD_dct_rawContent), ERROR(memory_allocation)); | |
| CHECK_EQ(ZSTD_DCtx_loadDictionary_advanced(dctx, dict, dictSize, ZSTD_dlm_byRef, ZSTD_dct_rawContent), ERROR(memory_allocation)); | |
| CHECK_EQ(ZSTD_DCtx_refPrefix(dctx, dict, dictSize), ERROR(memory_allocation)); | |
| CHECK_EQ(ZSTD_DCtx_refPrefix_advanced(dctx, dict, dictSize, ZSTD_dct_auto), ERROR(memory_allocation)); | |
| CHECK_EQ(ZSTD_initDStream_usingDict(dctx, dict, dictSize), ERROR(memory_allocation)); | |
| CHECK_EQ(dctx->customMem.customAlloc, NULL); | |
| CHECK_EQ(dctx->customMem.customFree, NULL); | |
| CHECK_EQ(dctx->customMem.opaque, NULL); | |
| /* Clearing a dictionary does not allocate. */ | |
| CHECK_Z(ZSTD_DCtx_loadDictionary(dctx, NULL, dictSize)); | |
| CHECK_Z(ZSTD_DCtx_loadDictionary(dctx, dict, 0)); | |
| CHECK_Z(ZSTD_DCtx_loadDictionary_byReference(dctx, NULL, dictSize)); | |
| CHECK_Z(ZSTD_DCtx_loadDictionary_byReference(dctx, dict, 0)); | |
| CHECK_Z(ZSTD_DCtx_refPrefix(dctx, NULL, dictSize)); | |
| CHECK_Z(ZSTD_DCtx_refPrefix(dctx, dict, 0)); | |
| CHECK_Z(ZSTD_initDStream_usingDict(dctx, NULL, dictSize)); | |
| CHECK_Z(ZSTD_initDStream_usingDict(dctx, dict, 0)); | |
| /* A caller-owned DDict remains usable without allocation. */ | |
| CHECK_Z(ZSTD_DCtx_refDDict(dctx, ddict)); | |
| CHECK_EQ(ZSTD_decompressDCtx(dctx, decoded, sizeof(decoded), compressed, compressedSize), dictSize); | |
| CHECK_EQ(memcmp(decoded, dict, dictSize), 0); | |
| CHECK_Z(ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters)); | |
| if (stream) { | |
| ZSTD_inBuffer input = { compressed, compressedSize, 0 }; | |
| ZSTD_outBuffer output = { decoded, sizeof(decoded), 0 }; | |
| CHECK_Z(ZSTD_initDStream_usingDDict(dctx, ddict)); | |
| CHECK_EQ(ZSTD_decompressStream(dctx, &output, &input), 0); | |
| CHECK_EQ(input.pos, compressedSize); | |
| CHECK_EQ(output.pos, dictSize); | |
| CHECK_EQ(memcmp(decoded, dict, dictSize), 0); | |
| } | |
| CHECK_Z(ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters)); | |
| } | |
| } | |
| free(ddictWorkspace); | |
| free(workspace); | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| /* Static CCtx tests */ | |
| #define STATIC_CCTX_LEVEL 4 | |
| DISPLAYLEVEL(3, "test%3i : create static CCtx for level %u : ", testNb++, STATIC_CCTX_LEVEL); | |
| { size_t const staticCStreamSize = ZSTD_estimateCStreamSize(STATIC_CCTX_LEVEL); | |
| void* const staticCCtxBuffer = malloc(staticCStreamSize); | |
| size_t const staticDCtxSize = ZSTD_estimateDCtxSize(); | |
| void* const staticDCtxBuffer = malloc(staticDCtxSize); | |
| DISPLAYLEVEL(4, "CStream size = %u, ", (U32)staticCStreamSize); | |
| if (staticCCtxBuffer==NULL || staticDCtxBuffer==NULL) { | |
| free(staticCCtxBuffer); | |
| free(staticDCtxBuffer); | |
| DISPLAY("Not enough memory, aborting\n"); | |
| testResult = 1; | |
| goto _end; | |
| } | |
| { size_t const smallInSize = 32 KB; | |
| ZSTD_compressionParameters const cparams_small = ZSTD_getCParams(STATIC_CCTX_LEVEL, smallInSize, 0); | |
| size_t const smallCCtxSize = ZSTD_estimateCCtxSize_usingCParams(cparams_small); | |
| size_t const staticCCtxSize = ZSTD_estimateCCtxSize(STATIC_CCTX_LEVEL); | |
| ZSTD_CCtx* staticCCtx = ZSTD_initStaticCCtx(staticCCtxBuffer, smallCCtxSize); | |
| ZSTD_DCtx* const staticDCtx = ZSTD_initStaticDCtx(staticDCtxBuffer, staticDCtxSize); | |
| DISPLAYLEVEL(4, "Full CCtx size = %u, ", (U32)staticCCtxSize); | |
| DISPLAYLEVEL(4, "CCtx for 32 KB = %u, ", (U32)smallCCtxSize); | |
| if ((staticCCtx==NULL) || (staticDCtx==NULL)) goto _output_error; | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : compress small input with small static CCtx : ", testNb++); | |
| CHECK_VAR(cSize, ZSTD_compressCCtx(staticCCtx, | |
| compressedBuffer, compressedBufferSize, | |
| CNBuffer, smallInSize, STATIC_CCTX_LEVEL) ); | |
| DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", | |
| (unsigned)cSize, (double)cSize/smallInSize*100); | |
| DISPLAYLEVEL(3, "test%3i : compress large input with small static CCtx (must fail) : ", testNb++); | |
| { size_t const r = ZSTD_compressCCtx(staticCCtx, | |
| compressedBuffer, compressedBufferSize, | |
| CNBuffer, CNBuffSize, STATIC_CCTX_LEVEL); | |
| if (ZSTD_getErrorCode((size_t)r) != ZSTD_error_memory_allocation) goto _output_error; | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : resize context to full CCtx size : ", testNb++); | |
| staticCCtx = ZSTD_initStaticCStream(staticCCtxBuffer, staticCCtxSize); | |
| DISPLAYLEVEL(4, "staticCCtxBuffer = %p, staticCCtx = %p , ", staticCCtxBuffer, (void*)staticCCtx); | |
| if (staticCCtx == NULL) goto _output_error; | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : compress large input with static CCtx : ", testNb++); | |
| CHECK_VAR(cSize, ZSTD_compressCCtx(staticCCtx, | |
| compressedBuffer, compressedBufferSize, | |
| CNBuffer, CNBuffSize, STATIC_CCTX_LEVEL) ); | |
| DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", | |
| (unsigned)cSize, (double)cSize/CNBuffSize*100); | |
| DISPLAYLEVEL(3, "test%3i : compress small input often enough to trigger context reduce : ", testNb++); | |
| { int nbc; | |
| assert(staticCCtxSize > smallCCtxSize * ZSTD_WORKSPACETOOLARGE_FACTOR); /* ensure size down scenario */ | |
| assert(CNBuffSize > smallInSize + ZSTD_WORKSPACETOOLARGE_MAXDURATION + 3); | |
| for (nbc=0; nbc<ZSTD_WORKSPACETOOLARGE_MAXDURATION+2; nbc++) { | |
| CHECK_Z(ZSTD_compressCCtx(staticCCtx, | |
| compressedBuffer, compressedBufferSize, | |
| (char*)CNBuffer + nbc, smallInSize, | |
| STATIC_CCTX_LEVEL) ); | |
| } } | |
| DISPLAYLEVEL(3, "OK \n") | |
| DISPLAYLEVEL(3, "test%3i : init CCtx for level %u : ", testNb++, STATIC_CCTX_LEVEL); | |
| CHECK_Z( ZSTD_compressBegin(staticCCtx, STATIC_CCTX_LEVEL) ); | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : compression again with static CCtx : ", testNb++); | |
| CHECK_VAR(cSize, ZSTD_compressCCtx(staticCCtx, | |
| compressedBuffer, compressedBufferSize, | |
| CNBuffer, CNBuffSize, STATIC_CCTX_LEVEL) ); | |
| DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", | |
| (unsigned)cSize, (double)cSize/CNBuffSize*100); | |
| DISPLAYLEVEL(3, "test%3i : simple decompression test with static DCtx : ", testNb++); | |
| { size_t const r = ZSTD_decompressDCtx(staticDCtx, | |
| decodedBuffer, CNBuffSize, | |
| compressedBuffer, cSize); | |
| if (r != CNBuffSize) goto _output_error; } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : check decompressed result : ", testNb++); | |
| if (memcmp(decodedBuffer, CNBuffer, CNBuffSize)) goto _output_error; | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : init CCtx for too large level (must fail) : ", testNb++); | |
| { size_t const r = ZSTD_compressBegin(staticCCtx, ZSTD_maxCLevel()); | |
| if (!ZSTD_isError(r)) goto _output_error; } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : init CCtx for small level %u (should work again) : ", testNb++, 1); | |
| CHECK_Z( ZSTD_compressBegin(staticCCtx, 1) ); | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : use CStream on CCtx-sized static context (should fail) : ", testNb++); | |
| CHECK_Z( ZSTD_initCStream(staticCCtx, STATIC_CCTX_LEVEL) ); /* note : doesn't allocate */ | |
| { ZSTD_outBuffer output = { compressedBuffer, compressedBufferSize, 0 }; | |
| ZSTD_inBuffer input = { CNBuffer, CNBuffSize, 0 }; | |
| size_t const r = ZSTD_compressStream(staticCCtx, &output, &input); /* now allocates, should fail */ | |
| if (!ZSTD_isError(r)) goto _output_error; | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : resize context to CStream size, then stream compress : ", testNb++); | |
| staticCCtx = ZSTD_initStaticCStream(staticCCtxBuffer, staticCStreamSize); | |
| assert(staticCCtx != NULL); | |
| CHECK_Z( ZSTD_initCStream(staticCCtx, STATIC_CCTX_LEVEL) ); /* note : doesn't allocate */ | |
| { ZSTD_outBuffer output = { compressedBuffer, compressedBufferSize, 0 }; | |
| ZSTD_inBuffer input = { CNBuffer, CNBuffSize, 0 }; | |
| CHECK_Z( ZSTD_compressStream(staticCCtx, &output, &input) ); | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : CStream for small level %u : ", testNb++, 1); | |
| CHECK_Z( ZSTD_initCStream(staticCCtx, 1) ); /* note : doesn't allocate */ | |
| { ZSTD_outBuffer output = { compressedBuffer, compressedBufferSize, 0 }; | |
| ZSTD_inBuffer input = { CNBuffer, CNBuffSize, 0 }; | |
| CHECK_Z( ZSTD_compressStream(staticCCtx, &output, &input) ); | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : init static CStream with dictionary (should fail) : ", testNb++); | |
| { size_t const r = ZSTD_initCStream_usingDict(staticCCtx, CNBuffer, 64 KB, 1); | |
| if (!ZSTD_isError(r)) goto _output_error; } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : use DStream on DCtx-sized static context (should fail) : ", testNb++); | |
| CHECK_Z( ZSTD_initDStream(staticDCtx) ); | |
| { ZSTD_outBuffer output = { decodedBuffer, CNBuffSize, 0 }; | |
| ZSTD_inBuffer input = { compressedBuffer, ZSTD_FRAMEHEADERSIZE_MAX+1, 0 }; | |
| size_t const r = ZSTD_decompressStream(staticDCtx, &output, &input); | |
| if (!ZSTD_isError(r)) goto _output_error; | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : test estimation functions with default cctx params : ", testNb++); | |
| { | |
| // Test ZSTD_estimateCCtxSize_usingCCtxParams | |
| { | |
| ZSTD_CCtx_params* params = ZSTD_createCCtxParams(); | |
| size_t const cctxSizeDefault = ZSTD_estimateCCtxSize_usingCCtxParams(params); | |
| staticCCtx = ZSTD_initStaticCCtx(staticCCtxBuffer, cctxSizeDefault); | |
| CHECK_VAR(cSize, ZSTD_compressCCtx(staticCCtx, | |
| compressedBuffer, compressedBufferSize, | |
| CNBuffer, CNBuffSize, 3)); | |
| { | |
| size_t const r = ZSTD_decompressDCtx(staticDCtx, | |
| decodedBuffer, CNBuffSize, | |
| compressedBuffer, cSize); | |
| if (r != CNBuffSize) goto _output_error; | |
| if (memcmp(decodedBuffer, CNBuffer, CNBuffSize)) goto _output_error; | |
| } | |
| ZSTD_freeCCtxParams(params); | |
| } | |
| // Test ZSTD_estimateCStreamSize_usingCCtxParams | |
| { | |
| ZSTD_CCtx_params* params = ZSTD_createCCtxParams(); | |
| size_t const cctxSizeDefault = ZSTD_estimateCStreamSize_usingCCtxParams(params); | |
| staticCCtx = ZSTD_initStaticCCtx(staticCCtxBuffer, cctxSizeDefault); | |
| CHECK_VAR(cSize, ZSTD_compressCCtx(staticCCtx, | |
| compressedBuffer, compressedBufferSize, | |
| CNBuffer, CNBuffSize, 3) ); | |
| { | |
| size_t const r = ZSTD_decompressDCtx(staticDCtx, | |
| decodedBuffer, CNBuffSize, | |
| compressedBuffer, cSize); | |
| if (r != CNBuffSize) goto _output_error; | |
| if (memcmp(decodedBuffer, CNBuffer, CNBuffSize)) goto _output_error; | |
| } | |
| ZSTD_freeCCtxParams(params); | |
| } | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : test estimation functions with maxBlockSize = 0 : ", testNb++); | |
| { | |
| // Test ZSTD_estimateCCtxSize_usingCCtxParams | |
| { | |
| ZSTD_CCtx_params* params = ZSTD_createCCtxParams(); | |
| size_t cctxSizeDefault; | |
| CHECK_Z(ZSTD_CCtxParams_setParameter(params, ZSTD_c_maxBlockSize, 0)); | |
| cctxSizeDefault = ZSTD_estimateCCtxSize_usingCCtxParams(params); | |
| staticCCtx = ZSTD_initStaticCCtx(staticCCtxBuffer, cctxSizeDefault); | |
| CHECK_VAR(cSize, ZSTD_compressCCtx(staticCCtx, | |
| compressedBuffer, compressedBufferSize, | |
| CNBuffer, CNBuffSize, 3) ); | |
| { | |
| size_t const r = ZSTD_decompressDCtx(staticDCtx, | |
| decodedBuffer, CNBuffSize, | |
| compressedBuffer, cSize); | |
| if (r != CNBuffSize) goto _output_error; | |
| if (memcmp(decodedBuffer, CNBuffer, CNBuffSize)) goto _output_error; | |
| } | |
| ZSTD_freeCCtxParams(params); | |
| } | |
| // Test ZSTD_estimateCStreamSize_usingCCtxParams | |
| { | |
| ZSTD_CCtx_params* params = ZSTD_createCCtxParams(); | |
| size_t cctxSizeDefault; | |
| CHECK_Z(ZSTD_CCtxParams_setParameter(params, ZSTD_c_maxBlockSize, 0)); | |
| cctxSizeDefault = ZSTD_estimateCStreamSize_usingCCtxParams(params); | |
| staticCCtx = ZSTD_initStaticCCtx(staticCCtxBuffer, cctxSizeDefault); | |
| CHECK_VAR(cSize, ZSTD_compressCCtx(staticCCtx, | |
| compressedBuffer, compressedBufferSize, | |
| CNBuffer, CNBuffSize, 3) ); | |
| { | |
| size_t const r = ZSTD_decompressDCtx(staticDCtx, | |
| decodedBuffer, CNBuffSize, | |
| compressedBuffer, cSize); | |
| if (r != CNBuffSize) goto _output_error; | |
| if (memcmp(decodedBuffer, CNBuffer, CNBuffSize)) goto _output_error; | |
| } | |
| ZSTD_freeCCtxParams(params); | |
| } | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : estimation functions with LDM enabled (issue #4590) : ", testNb++); | |
| { | |
| /* ZSTD_estimateCCtxSize_usingCCtxParams must adjust zeroed-out | |
| * LDM parameters when LDM is enabled, to avoid division by zero | |
| * in ZSTD_ldm_getMaxNbSeq. */ | |
| ZSTD_CCtx_params* params = ZSTD_createCCtxParams(); | |
| size_t cctxSize; | |
| CHECK_Z(ZSTD_CCtxParams_setParameter(params, ZSTD_c_compressionLevel, 22)); | |
| CHECK_Z(ZSTD_CCtxParams_setParameter(params, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable)); | |
| cctxSize = ZSTD_estimateCCtxSize_usingCCtxParams(params); | |
| if (ZSTD_isError(cctxSize)) goto _output_error; | |
| if (cctxSize == 0) goto _output_error; | |
| cctxSize = ZSTD_estimateCStreamSize_usingCCtxParams(params); | |
| if (ZSTD_isError(cctxSize)) goto _output_error; | |
| if (cctxSize == 0) goto _output_error; | |
| ZSTD_freeCCtxParams(params); | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| } | |
| free(staticCCtxBuffer); | |
| free(staticDCtxBuffer); | |
| } | |
| DISPLAYLEVEL(3, "test%3i : Static context sizes for negative levels : ", testNb++); | |
| { size_t const cctxSizeN1 = ZSTD_estimateCCtxSize(-1); | |
| size_t const cctxSizeP1 = ZSTD_estimateCCtxSize(1); | |
| size_t const cstreamSizeN1 = ZSTD_estimateCStreamSize(-1); | |
| size_t const cstreamSizeP1 = ZSTD_estimateCStreamSize(1); | |
| if (!(0 < cctxSizeN1 && cctxSizeN1 <= cctxSizeP1)) goto _output_error; | |
| if (!(0 < cstreamSizeN1 && cstreamSizeN1 <= cstreamSizeP1)) goto _output_error; | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| /* ZSTDMT simple MT compression test */ | |
| DISPLAYLEVEL(3, "test%3i : create ZSTDMT CCtx : ", testNb++); | |
| { ZSTD_CCtx* const mtctx = ZSTD_createCCtx(); | |
| if (mtctx==NULL) { | |
| DISPLAY("mtctx : not enough memory, aborting \n"); | |
| testResult = 1; | |
| goto _end; | |
| } | |
| CHECK_Z( ZSTD_CCtx_setParameter(mtctx, ZSTD_c_nbWorkers, 2) ); | |
| CHECK_Z( ZSTD_CCtx_setParameter(mtctx, ZSTD_c_compressionLevel, 1) ); | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3u : compress %u bytes with 2 threads : ", testNb++, (unsigned)CNBuffSize); | |
| CHECK_VAR(cSize, ZSTD_compress2(mtctx, | |
| compressedBuffer, compressedBufferSize, | |
| CNBuffer, CNBuffSize) ); | |
| DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100); | |
| DISPLAYLEVEL(3, "test%3i : decompressed size test : ", testNb++); | |
| { unsigned long long const rSize = ZSTD_getFrameContentSize(compressedBuffer, cSize); | |
| if (rSize != CNBuffSize) { | |
| DISPLAY("ZSTD_getFrameContentSize incorrect : %u != %u \n", (unsigned)rSize, (unsigned)CNBuffSize); | |
| goto _output_error; | |
| } } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : decompress %u bytes : ", testNb++, (unsigned)CNBuffSize); | |
| { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize); | |
| if (r != CNBuffSize) goto _output_error; } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : check decompressed result : ", testNb++); | |
| { size_t u; | |
| for (u=0; u<CNBuffSize; u++) { | |
| if (((BYTE*)decodedBuffer)[u] != ((BYTE*)CNBuffer)[u]) goto _output_error; | |
| } } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : compress -T2 with checksum : ", testNb++); | |
| CHECK_Z( ZSTD_CCtx_setParameter(mtctx, ZSTD_c_checksumFlag, 1) ); | |
| CHECK_Z( ZSTD_CCtx_setParameter(mtctx, ZSTD_c_contentSizeFlag, 1) ); | |
| CHECK_Z( ZSTD_CCtx_setParameter(mtctx, ZSTD_c_overlapLog, 3) ); | |
| CHECK_VAR(cSize, ZSTD_compress2(mtctx, | |
| compressedBuffer, compressedBufferSize, | |
| CNBuffer, CNBuffSize) ); | |
| DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100); | |
| DISPLAYLEVEL(3, "test%3i : decompress %u bytes : ", testNb++, (unsigned)CNBuffSize); | |
| { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize); | |
| if (r != CNBuffSize) goto _output_error; } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| ZSTD_freeCCtx(mtctx); | |
| } | |
| DISPLAYLEVEL(3, "test%3u : compress empty string and decompress with small window log : ", testNb++); | |
| { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); | |
| ZSTD_DCtx* const dctx = ZSTD_createDCtx(); | |
| char out[32]; | |
| if (cctx == NULL || dctx == NULL) goto _output_error; | |
| CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_contentSizeFlag, 0) ); | |
| CHECK_VAR(cSize, ZSTD_compress2(cctx, out, sizeof(out), NULL, 0) ); | |
| DISPLAYLEVEL(3, "OK (%u bytes)\n", (unsigned)cSize); | |
| CHECK_Z( ZSTD_DCtx_setParameter(dctx, ZSTD_d_windowLogMax, 10) ); | |
| { char const* outPtr = out; | |
| ZSTD_inBuffer inBuffer = { outPtr, cSize, 0 }; | |
| ZSTD_outBuffer outBuffer = { NULL, 0, 0 }; | |
| size_t dSize; | |
| CHECK_VAR(dSize, ZSTD_decompressStream(dctx, &outBuffer, &inBuffer) ); | |
| if (dSize != 0) goto _output_error; | |
| } | |
| ZSTD_freeDCtx(dctx); | |
| ZSTD_freeCCtx(cctx); | |
| } | |
| DISPLAYLEVEL(3, "test%3i : compress with block splitting : ", testNb++) | |
| { ZSTD_CCtx* cctx = ZSTD_createCCtx(); | |
| CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_splitAfterSequences, ZSTD_ps_enable) ); | |
| cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize); | |
| CHECK_Z(cSize); | |
| ZSTD_freeCCtx(cctx); | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : compress -T2 with/without literals compression : ", testNb++) | |
| { ZSTD_CCtx* cctx = ZSTD_createCCtx(); | |
| size_t cSize1, cSize2; | |
| CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 1) ); | |
| CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 2) ); | |
| cSize1 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize); | |
| CHECK_Z(cSize1); | |
| CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_literalCompressionMode, ZSTD_ps_disable) ); | |
| cSize2 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize); | |
| CHECK_Z(cSize2); | |
| CHECK_LT(cSize1, cSize2); | |
| ZSTD_freeCCtx(cctx); | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : Multithreaded ZSTD_compress2() with rsyncable : ", testNb++) | |
| { ZSTD_CCtx* cctx = ZSTD_createCCtx(); | |
| /* Set rsyncable and don't give the ZSTD_compressBound(CNBuffSize) so | |
| * ZSTDMT is forced to not take the shortcut. | |
| */ | |
| CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 1) ); | |
| CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 1) ); | |
| CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_rsyncable, 1) ); | |
| CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize - 1, CNBuffer, CNBuffSize) ); | |
| ZSTD_freeCCtx(cctx); | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : setting multithreaded parameters : ", testNb++) | |
| { ZSTD_CCtx_params* params = ZSTD_createCCtxParams(); | |
| int const jobSize = 512 KB; | |
| int value; | |
| /* Check that the overlap log and job size are unset. */ | |
| CHECK_Z( ZSTD_CCtxParams_getParameter(params, ZSTD_c_overlapLog, &value) ); | |
| CHECK_EQ(value, 0); | |
| CHECK_Z( ZSTD_CCtxParams_getParameter(params, ZSTD_c_jobSize, &value) ); | |
| CHECK_EQ(value, 0); | |
| /* Set and check the overlap log and job size. */ | |
| CHECK_Z( ZSTD_CCtxParams_setParameter(params, ZSTD_c_overlapLog, 5) ); | |
| CHECK_Z( ZSTD_CCtxParams_setParameter(params, ZSTD_c_jobSize, jobSize) ); | |
| CHECK_Z( ZSTD_CCtxParams_getParameter(params, ZSTD_c_overlapLog, &value) ); | |
| CHECK_EQ(value, 5); | |
| CHECK_Z( ZSTD_CCtxParams_getParameter(params, ZSTD_c_jobSize, &value) ); | |
| CHECK_EQ(value, jobSize); | |
| /* Set the number of workers and check the overlap log and job size. */ | |
| CHECK_Z( ZSTD_CCtxParams_setParameter(params, ZSTD_c_nbWorkers, 2) ); | |
| CHECK_Z( ZSTD_CCtxParams_getParameter(params, ZSTD_c_overlapLog, &value) ); | |
| CHECK_EQ(value, 5); | |
| CHECK_Z( ZSTD_CCtxParams_getParameter(params, ZSTD_c_jobSize, &value) ); | |
| CHECK_EQ(value, jobSize); | |
| ZSTD_freeCCtxParams(params); | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| /* Simple API multiframe test */ | |
| DISPLAYLEVEL(3, "test%3i : compress multiple frames : ", testNb++); | |
| { size_t off = 0; | |
| int i; | |
| int const segs = 4; | |
| /* only use the first half so we don't push against size limit of compressedBuffer */ | |
| size_t const segSize = (CNBuffSize / 2) / segs; | |
| const U32 skipLen = 129 KB; | |
| char* const skipBuff = (char*)malloc(skipLen); | |
| assert(skipBuff != NULL); | |
| memset(skipBuff, 0, skipLen); | |
| for (i = 0; i < segs; i++) { | |
| CHECK_NEWV(r, ZSTD_compress( | |
| (BYTE*)compressedBuffer + off, CNBuffSize - off, | |
| (BYTE*)CNBuffer + segSize * (size_t)i, segSize, | |
| 5) ); | |
| off += r; | |
| if (i == segs/2) { | |
| /* insert skippable frame */ | |
| size_t const skippableSize = | |
| ZSTD_writeSkippableFrame((BYTE*)compressedBuffer + off, compressedBufferSize, | |
| skipBuff, skipLen, seed % 15); | |
| CHECK_Z(skippableSize); | |
| off += skippableSize; | |
| } | |
| } | |
| cSize = off; | |
| free(skipBuff); | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : get decompressed size of multiple frames : ", testNb++); | |
| { unsigned long long const r = ZSTD_findDecompressedSize(compressedBuffer, cSize); | |
| if (r != CNBuffSize / 2) goto _output_error; } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : get tight decompressed bound of multiple frames : ", testNb++); | |
| { unsigned long long const bound = ZSTD_decompressBound(compressedBuffer, cSize); | |
| if (bound != CNBuffSize / 2) goto _output_error; } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : decompress multiple frames : ", testNb++); | |
| { CHECK_NEWV(r, ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize)); | |
| if (r != CNBuffSize / 2) goto _output_error; } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : check decompressed result : ", testNb++); | |
| if (memcmp(decodedBuffer, CNBuffer, CNBuffSize / 2) != 0) goto _output_error; | |
| DISPLAYLEVEL(3, "OK \n"); | |
| /* Simple API skippable frame test */ | |
| DISPLAYLEVEL(3, "test%3i : read/write a skippable frame : ", testNb++); | |
| { U32 i; | |
| unsigned readMagic; | |
| unsigned long long receivedSize; | |
| size_t skippableSize; | |
| const U32 skipLen = 129 KB; | |
| char* const skipBuff = (char*)malloc(skipLen); | |
| assert(skipBuff != NULL); | |
| for (i = 0; i < skipLen; i++) | |
| skipBuff[i] = (char) ((seed + i) % 256); | |
| skippableSize = ZSTD_writeSkippableFrame( | |
| compressedBuffer, compressedBufferSize, | |
| skipBuff, skipLen, seed % 15); | |
| CHECK_Z(skippableSize); | |
| CHECK_EQ(1, ZSTD_isSkippableFrame(compressedBuffer, skippableSize)); | |
| receivedSize = ZSTD_readSkippableFrame(decodedBuffer, CNBuffSize, &readMagic, compressedBuffer, skippableSize); | |
| CHECK_EQ(skippableSize, receivedSize + ZSTD_SKIPPABLEHEADERSIZE); | |
| CHECK_EQ(seed % 15, readMagic); | |
| if (memcmp(decodedBuffer, skipBuff, skipLen) != 0) goto _output_error; | |
| free(skipBuff); | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : read/write an empty skippable frame : ", testNb++); | |
| { | |
| unsigned readMagic; | |
| unsigned long long receivedSize; | |
| size_t skippableSize; | |
| skippableSize = ZSTD_writeSkippableFrame( | |
| compressedBuffer, compressedBufferSize, | |
| CNBuffer, 0, seed % 15); | |
| CHECK_EQ(ZSTD_SKIPPABLEHEADERSIZE, skippableSize); | |
| CHECK_EQ(1, ZSTD_isSkippableFrame(compressedBuffer, skippableSize)); | |
| receivedSize = ZSTD_readSkippableFrame(NULL, 0, &readMagic, compressedBuffer, skippableSize); | |
| CHECK_EQ(skippableSize, receivedSize + ZSTD_SKIPPABLEHEADERSIZE); | |
| CHECK_EQ(seed % 15, readMagic); | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| /* Dictionary and CCtx Duplication tests */ | |
| { ZSTD_CCtx* const ctxOrig = ZSTD_createCCtx(); | |
| ZSTD_DCtx* const dctx = ZSTD_createDCtx(); | |
| static const size_t dictSize = 551; | |
| assert(dctx != NULL); assert(ctxOrig != NULL); | |
| DISPLAYLEVEL(3, "test%3i : load dictionary into context : ", testNb++); | |
| CHECK_Z( ZSTD_compressBegin_usingDict(ctxOrig, CNBuffer, dictSize, 2) ); | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : compress with flat dictionary : ", testNb++); | |
| cSize = 0; | |
| CHECKPLUS(r, ZSTD_compressEnd(ctxOrig, | |
| compressedBuffer, compressedBufferSize, | |
| (const char*)CNBuffer + dictSize, CNBuffSize - dictSize), | |
| cSize += r); | |
| DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100); | |
| DISPLAYLEVEL(3, "test%3i : frame built with flat dictionary should be decompressible : ", testNb++); | |
| CHECKPLUS(r, ZSTD_decompress_usingDict(dctx, | |
| decodedBuffer, CNBuffSize, | |
| compressedBuffer, cSize, | |
| CNBuffer, dictSize), | |
| if (r != CNBuffSize - dictSize) goto _output_error); | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : decompress with DDict : ", testNb++); | |
| { ZSTD_DDict* const ddict = ZSTD_createDDict(CNBuffer, dictSize); | |
| size_t const r = ZSTD_decompress_usingDDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, ddict); | |
| if (r != CNBuffSize - dictSize) goto _output_error; | |
| DISPLAYLEVEL(3, "OK (size of DDict : %u) \n", (unsigned)ZSTD_sizeof_DDict(ddict)); | |
| ZSTD_freeDDict(ddict); | |
| } | |
| DISPLAYLEVEL(3, "test%3i : decompress with static DDict : ", testNb++); | |
| { size_t const ddictBufferSize = ZSTD_estimateDDictSize(dictSize, ZSTD_dlm_byCopy); | |
| void* const ddictBuffer = malloc(ddictBufferSize); | |
| if (ddictBuffer == NULL) goto _output_error; | |
| { const ZSTD_DDict* const ddict = ZSTD_initStaticDDict(ddictBuffer, ddictBufferSize, CNBuffer, dictSize, ZSTD_dlm_byCopy, ZSTD_dct_auto); | |
| size_t const r = ZSTD_decompress_usingDDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, ddict); | |
| if (r != CNBuffSize - dictSize) goto _output_error; | |
| } | |
| free(ddictBuffer); | |
| DISPLAYLEVEL(3, "OK (size of static DDict : %u) \n", (unsigned)ddictBufferSize); | |
| } | |
| #if !defined(ZSTD_EXCLUDE_DFAST_BLOCK_COMPRESSOR) \ | |
| && !defined(ZSTD_EXCLUDE_GREEDY_BLOCK_COMPRESSOR) \ | |
| && !defined(ZSTD_EXCLUDE_DFAST_BLOCK_COMPRESSOR) \ | |
| && !defined(ZSTD_EXCLUDE_LAZY_BLOCK_COMPRESSOR) \ | |
| && !defined(ZSTD_EXCLUDE_LAZY2_BLOCK_COMPRESSOR) \ | |
| && !defined(ZSTD_EXCLUDE_BTLAZY2_BLOCK_COMPRESSOR) \ | |
| && !defined(ZSTD_EXCLUDE_BTOPT_BLOCK_COMPRESSOR) \ | |
| && !defined(ZSTD_EXCLUDE_BTULTRA_BLOCK_COMPRESSOR) | |
| /* Note : these tests should be replaced by proper regression tests, | |
| * but existing ones do not focus on small data + dictionary + all levels. | |
| */ | |
| if ((int)(compressibility * 100 + 0.1) == FUZ_compressibility_default) { /* test only valid with known input */ | |
| size_t const flatdictSize = 22 KB; | |
| size_t const contentSize = 9 KB; | |
| const void* const dict = (const char*)CNBuffer; | |
| const void* const contentStart = (const char*)dict + flatdictSize; | |
| /* These upper bounds are generally within a few bytes of the compressed size */ | |
| size_t target_nodict_cSize[22+1] = { 3840, 3770, 3870, 3830, 3770, | |
| 3770, 3770, 3770, 3750, 3750, | |
| 3742, 3675, 3674, 3665, 3664, | |
| 3663, 3662, 3661, 3660, 3660, | |
| 3660, 3660, 3660 }; | |
| size_t const target_wdict_cSize[22+1] = { 2830, 2896, 2893, 2840, 2950, | |
| 2950, 2950, 2925, 2900, 2892, | |
| 2910, 2910, 2910, 2780, 2775, | |
| 2765, 2760, 2755, 2754, 2753, | |
| 2753, 2753, 2753 }; | |
| int l = 1; | |
| int const maxLevel = ZSTD_maxCLevel(); | |
| /* clevels with strategies that support rowhash on small inputs */ | |
| int rowLevel = 4; | |
| int const rowLevelEnd = 8; | |
| DISPLAYLEVEL(3, "test%3i : flat-dictionary efficiency test : \n", testNb++); | |
| assert(maxLevel == 22); | |
| RDG_genBuffer(CNBuffer, flatdictSize + contentSize, compressibility, 0., seed); | |
| DISPLAYLEVEL(4, "content hash : %016llx; dict hash : %016llx \n", | |
| (unsigned long long)XXH64(contentStart, contentSize, 0), | |
| (unsigned long long)XXH64(dict, flatdictSize, 0)); | |
| for ( ; l <= maxLevel; l++) { | |
| size_t const nodict_cSize = ZSTD_compress(compressedBuffer, compressedBufferSize, | |
| contentStart, contentSize, l); | |
| if (nodict_cSize > target_nodict_cSize[l]) { | |
| DISPLAYLEVEL(1, "error : compression at level %i worse than expected (%u > %u) \n", | |
| l, (unsigned)nodict_cSize, (unsigned)target_nodict_cSize[l]); | |
| goto _output_error; | |
| } | |
| DISPLAYLEVEL(4, "level %i : max expected %u >= reached %u \n", | |
| l, (unsigned)target_nodict_cSize[l], (unsigned)nodict_cSize); | |
| } | |
| for ( l=1 ; l <= maxLevel; l++) { | |
| size_t const wdict_cSize = ZSTD_compress_usingDict(ctxOrig, | |
| compressedBuffer, compressedBufferSize, | |
| contentStart, contentSize, | |
| dict, flatdictSize, | |
| l); | |
| if (wdict_cSize > target_wdict_cSize[l]) { | |
| DISPLAYLEVEL(1, "error : compression with dictionary at level %i worse than expected (%u > %u) \n", | |
| l, (unsigned)wdict_cSize, (unsigned)target_wdict_cSize[l]); | |
| goto _output_error; | |
| } | |
| DISPLAYLEVEL(4, "level %i with dictionary : max expected %u >= reached %u \n", | |
| l, (unsigned)target_wdict_cSize[l], (unsigned)wdict_cSize); | |
| } | |
| /* Compression with ZSTD_compress2 and row match finder force enabled. | |
| * Give some slack for force-enabled row matchfinder since we're on a small input (9KB) | |
| */ | |
| for ( ; rowLevel <= rowLevelEnd; ++rowLevel) target_nodict_cSize[rowLevel] += 5; | |
| for (l=1 ; l <= maxLevel; l++) { | |
| ZSTD_CCtx* const cctx = ZSTD_createCCtx(); | |
| size_t nodict_cSize; | |
| ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, l); | |
| ZSTD_CCtx_setParameter(cctx, ZSTD_c_useRowMatchFinder, ZSTD_ps_enable); | |
| nodict_cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, | |
| contentStart, contentSize); | |
| if (nodict_cSize > target_nodict_cSize[l]) { | |
| DISPLAYLEVEL(1, "error : compression with compress2 at level %i worse than expected (%u > %u) \n", | |
| l, (unsigned)nodict_cSize, (unsigned)target_nodict_cSize[l]); | |
| ZSTD_freeCCtx(cctx); | |
| goto _output_error; | |
| } | |
| DISPLAYLEVEL(4, "level %i with compress2 : max expected %u >= reached %u \n", | |
| l, (unsigned)target_nodict_cSize[l], (unsigned)nodict_cSize); | |
| ZSTD_freeCCtx(cctx); | |
| } | |
| /* Dict compression with DMS */ | |
| for ( l=1 ; l <= maxLevel; l++) { | |
| size_t wdict_cSize; | |
| CHECK_Z( ZSTD_CCtx_loadDictionary(ctxOrig, dict, flatdictSize) ); | |
| CHECK_Z( ZSTD_CCtx_setParameter(ctxOrig, ZSTD_c_compressionLevel, l) ); | |
| CHECK_Z( ZSTD_CCtx_setParameter(ctxOrig, ZSTD_c_enableDedicatedDictSearch, 0) ); | |
| CHECK_Z( ZSTD_CCtx_setParameter(ctxOrig, ZSTD_c_forceAttachDict, ZSTD_dictForceAttach) ); | |
| CHECK_Z( ZSTD_CCtx_setParameter(ctxOrig, ZSTD_c_prefetchCDictTables, seed % 3) ); | |
| wdict_cSize = ZSTD_compress2(ctxOrig, compressedBuffer, compressedBufferSize, contentStart, contentSize); | |
| if (wdict_cSize > target_wdict_cSize[l]) { | |
| DISPLAYLEVEL(1, "error : compression with dictionary and compress2 at level %i worse than expected (%u > %u) \n", | |
| l, (unsigned)wdict_cSize, (unsigned)target_wdict_cSize[l]); | |
| goto _output_error; | |
| } | |
| DISPLAYLEVEL(4, "level %i with dictionary and compress2 : max expected %u >= reached %u \n", | |
| l, (unsigned)target_wdict_cSize[l], (unsigned)wdict_cSize); | |
| } | |
| DISPLAYLEVEL(4, "compression efficiency tests OK \n"); | |
| } | |
| #endif | |
| ZSTD_freeCCtx(ctxOrig); | |
| ZSTD_freeDCtx(dctx); | |
| } | |
| /* Dictionary and dictBuilder tests */ | |
| { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); | |
| size_t const dictBufferCapacity = 16 KB; | |
| void* const dictBuffer = malloc(dictBufferCapacity); | |
| size_t const totalSampleSize = 1 MB; | |
| size_t const sampleUnitSize = 8 KB; | |
| U32 const nbSamples = (U32)(totalSampleSize / sampleUnitSize); | |
| size_t* const samplesSizes = (size_t*) malloc(nbSamples * sizeof(size_t)); | |
| size_t dictSize; | |
| U32 dictID; | |
| size_t dictHeaderSize; | |
| size_t dictBufferFixedSize = 144; | |
| unsigned char const dictBufferFixed[144] = {0x37, 0xa4, 0x30, 0xec, 0x63, 0x00, 0x00, 0x00, 0x08, 0x10, 0x00, 0x1f, | |
| 0x0f, 0x00, 0x28, 0xe5, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, | |
| 0x00, 0x80, 0x0f, 0x9e, 0x0f, 0x00, 0x00, 0x24, 0x40, 0x80, 0x00, 0x01, | |
| 0x02, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0xde, 0x08, | |
| 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, | |
| 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, | |
| 0x08, 0x08, 0x08, 0x08, 0xbc, 0xe1, 0x4b, 0x92, 0x0e, 0xb4, 0x7b, 0x18, | |
| 0x86, 0x61, 0x18, 0xc6, 0x18, 0x63, 0x8c, 0x31, 0xc6, 0x18, 0x63, 0x8c, | |
| 0x31, 0x66, 0x66, 0x66, 0x66, 0xb6, 0x6d, 0x01, 0x00, 0x00, 0x00, 0x04, | |
| 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x20, 0x73, 0x6f, 0x64, 0x61, | |
| 0x6c, 0x65, 0x73, 0x20, 0x74, 0x6f, 0x72, 0x74, 0x6f, 0x72, 0x20, 0x65, | |
| 0x6c, 0x65, 0x69, 0x66, 0x65, 0x6e, 0x64, 0x2e, 0x20, 0x41, 0x6c, 0x69}; | |
| if (dictBuffer==NULL || samplesSizes==NULL) { | |
| free(dictBuffer); | |
| free(samplesSizes); | |
| goto _output_error; | |
| } | |
| DISPLAYLEVEL(3, "test%3i : dictBuilder on cyclic data : ", testNb++); | |
| assert(compressedBufferSize >= totalSampleSize); | |
| { U32 u; for (u=0; u<totalSampleSize; u++) ((BYTE*)decodedBuffer)[u] = (BYTE)u; } | |
| { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; } | |
| { size_t const sDictSize = ZDICT_trainFromBuffer(dictBuffer, dictBufferCapacity, | |
| decodedBuffer, samplesSizes, nbSamples); | |
| if (ZDICT_isError(sDictSize)) goto _output_error; | |
| DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)sDictSize); | |
| } | |
| DISPLAYLEVEL(3, "test%3i : dictBuilder : ", testNb++); | |
| { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; } | |
| dictSize = ZDICT_trainFromBuffer(dictBuffer, dictBufferCapacity, | |
| CNBuffer, samplesSizes, nbSamples); | |
| if (ZDICT_isError(dictSize)) goto _output_error; | |
| DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)dictSize); | |
| DISPLAYLEVEL(3, "test%3i : FASTCOVER dictBuilder samples smaller than a dmer (issue #4499) : ", testNb++); | |
| { const char smallData[8] = { 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H' }; | |
| const size_t smallSizes[8] = { 1, 1, 1, 1, 1, 1, 1, 1 }; | |
| size_t const r = ZDICT_trainFromBuffer(dictBuffer, dictBufferCapacity, | |
| smallData, smallSizes, 8); | |
| if (!ZDICT_isError(r)) goto _output_error; | |
| if (ZSTD_getErrorCode(r) != ZSTD_error_srcSize_wrong) goto _output_error; | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : Multithreaded COVER dictBuilder : ", testNb++); | |
| { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; } | |
| { ZDICT_cover_params_t coverParams; | |
| memset(&coverParams, 0, sizeof(coverParams)); | |
| coverParams.steps = 8; | |
| coverParams.nbThreads = 4; | |
| dictSize = ZDICT_optimizeTrainFromBuffer_cover( | |
| dictBuffer, dictBufferCapacity, | |
| CNBuffer, samplesSizes, nbSamples/8, /* less samples for faster tests */ | |
| &coverParams); | |
| if (ZDICT_isError(dictSize)) goto _output_error; | |
| } | |
| DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)dictSize); | |
| DISPLAYLEVEL(3, "test%3i : COVER dictBuilder with shrinkDict: ", testNb++); | |
| { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; } | |
| { ZDICT_cover_params_t coverParams; | |
| memset(&coverParams, 0, sizeof(coverParams)); | |
| coverParams.steps = 8; | |
| coverParams.nbThreads = 4; | |
| coverParams.shrinkDict = 1; | |
| coverParams.shrinkDictMaxRegression = 1; | |
| dictSize = ZDICT_optimizeTrainFromBuffer_cover( | |
| dictBuffer, dictBufferCapacity, | |
| CNBuffer, samplesSizes, nbSamples/8, /* less samples for faster tests */ | |
| &coverParams); | |
| if (ZDICT_isError(dictSize)) goto _output_error; | |
| } | |
| DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)dictSize); | |
| DISPLAYLEVEL(3, "test%3i : Multithreaded FASTCOVER dictBuilder : ", testNb++); | |
| { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; } | |
| { ZDICT_fastCover_params_t fastCoverParams; | |
| memset(&fastCoverParams, 0, sizeof(fastCoverParams)); | |
| fastCoverParams.steps = 8; | |
| fastCoverParams.nbThreads = 4; | |
| dictSize = ZDICT_optimizeTrainFromBuffer_fastCover( | |
| dictBuffer, dictBufferCapacity, | |
| CNBuffer, samplesSizes, nbSamples, | |
| &fastCoverParams); | |
| if (ZDICT_isError(dictSize)) goto _output_error; | |
| } | |
| DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)dictSize); | |
| DISPLAYLEVEL(3, "test%3i : FASTCOVER dictBuilder with shrinkDict: ", testNb++); | |
| { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; } | |
| { ZDICT_fastCover_params_t fastCoverParams; | |
| memset(&fastCoverParams, 0, sizeof(fastCoverParams)); | |
| fastCoverParams.steps = 8; | |
| fastCoverParams.nbThreads = 4; | |
| fastCoverParams.shrinkDict = 1; | |
| fastCoverParams.shrinkDictMaxRegression = 1; | |
| dictSize = ZDICT_optimizeTrainFromBuffer_fastCover( | |
| dictBuffer, dictBufferCapacity, | |
| CNBuffer, samplesSizes, nbSamples, | |
| &fastCoverParams); | |
| if (ZDICT_isError(dictSize)) goto _output_error; | |
| } | |
| DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)dictSize); | |
| DISPLAYLEVEL(3, "test%3i : COVER dictBuilder orders equal dmers by position : ", testNb++); | |
| /* COVER sorts its suffix array with qsort(), breaking ties between | |
| * equal dmers by sample position; COVER_group() relies on that order | |
| * to count how many samples contain a dmer. A tie-break that depends on | |
| * the sort's internal state instead (the former address comparison) | |
| * still produces a dictionary, just a different one. Low-entropy | |
| * samples make ties frequent, so train on them and compare against the | |
| * dictionary the position order produces, for both comparator paths. */ | |
| { size_t const dictCapacity = 16 KB; | |
| void* const coverDict = malloc(dictCapacity); | |
| unsigned const dValues[2] = { 8, 16 }; /* COVER_cmp8() and COVER_cmp() */ | |
| XXH64_hash_t const expectedHash[2] = { 0x77EB0C1D595FF7A3ULL, 0xE02EC0FD3A5BBF6DULL }; | |
| unsigned dIdx; | |
| if (coverDict==NULL) goto _output_error; | |
| RDG_genBuffer(decodedBuffer, totalSampleSize, 0.5, 0.05, 1); /* low-entropy: many repeated dmers */ | |
| { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; } | |
| for (dIdx = 0; dIdx < 2; dIdx++) { | |
| ZDICT_cover_params_t coverParams; | |
| size_t coverDictSize; | |
| XXH64_hash_t hash; | |
| memset(&coverParams, 0, sizeof(coverParams)); | |
| coverParams.k = 256; | |
| coverParams.d = dValues[dIdx]; | |
| coverDictSize = ZDICT_trainFromBuffer_cover(coverDict, dictCapacity, | |
| decodedBuffer, samplesSizes, nbSamples, coverParams); | |
| if (ZDICT_isError(coverDictSize)) goto _output_error; | |
| hash = XXH64(coverDict, coverDictSize, 0); | |
| if (hash != expectedHash[dIdx]) { | |
| DISPLAYLEVEL(1, "error : COVER dictionary for d=%u hashes to %08X%08X, expected %08X%08X \n", | |
| coverParams.d, | |
| (unsigned)(hash >> 32), (unsigned)hash, | |
| (unsigned)(expectedHash[dIdx] >> 32), (unsigned)expectedHash[dIdx]); | |
| goto _output_error; | |
| } | |
| } | |
| free(coverDict); | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : COVER_selectDict shrinkDict with content < ZDICT_DICTSIZE_MIN : ", testNb++); | |
| /* The shrink loop tries suffixes of the content, starting at ZDICT_DICTSIZE_MIN. | |
| * With smaller content no suffix fits, so the full dictionary must be returned. | |
| * The content is allocated at its exact size, so under ASan any read before it | |
| * is reported as a heap-buffer-overflow. */ | |
| { size_t offsets[8]; | |
| unsigned const nbSelectSamples = (unsigned)(sizeof(offsets) / sizeof(offsets[0])); | |
| size_t const contentSize = ZDICT_DICTSIZE_MIN - 16; | |
| size_t const selectCapacity = 4 KB; | |
| BYTE* const content = (BYTE*)malloc(contentSize); | |
| BYTE* const expected = (BYTE*)malloc(selectCapacity); | |
| ZDICT_cover_params_t coverParams; | |
| COVER_dictSelection_t selection; | |
| size_t expectedSize; | |
| unsigned u; | |
| if (content == NULL || expected == NULL) { | |
| free(content); | |
| free(expected); | |
| goto _output_error; | |
| } | |
| memcpy(content, CNBuffer, contentSize); | |
| for (u = 0; u < nbSelectSamples; u++) offsets[u] = u * sampleUnitSize; | |
| memset(&coverParams, 0, sizeof(coverParams)); | |
| coverParams.splitPoint = 1.0; | |
| coverParams.shrinkDict = 1; | |
| coverParams.shrinkDictMaxRegression = 100000; /* accept any candidate */ | |
| expectedSize = ZDICT_finalizeDictionary(expected, selectCapacity, | |
| content, contentSize, | |
| CNBuffer, samplesSizes, nbSelectSamples, | |
| coverParams.zParams); | |
| selection = COVER_selectDict(content, selectCapacity, contentSize, | |
| (const BYTE*)CNBuffer, samplesSizes, nbSelectSamples, | |
| nbSelectSamples, nbSelectSamples, coverParams, offsets, 0); | |
| { int const ok = !ZDICT_isError(expectedSize) | |
| && !COVER_dictSelectionIsError(selection) | |
| && selection.dictSize == expectedSize | |
| && !memcmp(selection.dictContent, expected, expectedSize); | |
| COVER_dictSelectionFree(selection); | |
| free(content); | |
| free(expected); | |
| if (!ok) goto _output_error; | |
| } | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : check dictID : ", testNb++); | |
| dictID = ZDICT_getDictID(dictBuffer, dictSize); | |
| if (dictID==0) goto _output_error; | |
| DISPLAYLEVEL(3, "OK : %u \n", (unsigned)dictID); | |
| DISPLAYLEVEL(3, "test%3i : check dict header size no error : ", testNb++); | |
| dictHeaderSize = ZDICT_getDictHeaderSize(dictBuffer, dictSize); | |
| if (dictHeaderSize==0) goto _output_error; | |
| DISPLAYLEVEL(3, "OK : %u \n", (unsigned)dictHeaderSize); | |
| DISPLAYLEVEL(3, "test%3i : check dict header size correctness : ", testNb++); | |
| { dictHeaderSize = ZDICT_getDictHeaderSize(dictBufferFixed, dictBufferFixedSize); | |
| if (dictHeaderSize != 115) goto _output_error; | |
| } | |
| DISPLAYLEVEL(3, "OK : %u \n", (unsigned)dictHeaderSize); | |
| DISPLAYLEVEL(3, "test%3i : Dictionary creation errors : ", testNb++); | |
| test_dictionaryLoadErrors(dictBuffer, dictSize); | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : Large dictionary repcodes : ", testNb++); | |
| test_largeDictionaryRepcodes(dictBuffer, dictSize); | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : compress with dictionary : ", testNb++); | |
| cSize = ZSTD_compress_usingDict(cctx, compressedBuffer, compressedBufferSize, | |
| CNBuffer, CNBuffSize, | |
| dictBuffer, dictSize, 4); | |
| if (ZSTD_isError(cSize)) goto _output_error; | |
| DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100); | |
| DISPLAYLEVEL(3, "test%3i : retrieve dictID from dictionary : ", testNb++); | |
| { U32 const did = ZSTD_getDictID_fromDict(dictBuffer, dictSize); | |
| if (did != dictID) goto _output_error; /* non-conformant (content-only) dictionary */ | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : retrieve dictID from frame : ", testNb++); | |
| { U32 const did = ZSTD_getDictID_fromFrame(compressedBuffer, cSize); | |
| if (did != dictID) goto _output_error; /* non-conformant (content-only) dictionary */ | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : frame built with dictionary should be decompressible : ", testNb++); | |
| { ZSTD_DCtx* const dctx = ZSTD_createDCtx(); assert(dctx != NULL); | |
| CHECKPLUS(r, ZSTD_decompress_usingDict(dctx, | |
| decodedBuffer, CNBuffSize, | |
| compressedBuffer, cSize, | |
| dictBuffer, dictSize), | |
| if (r != CNBuffSize) goto _output_error); | |
| ZSTD_freeDCtx(dctx); | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : dictionary IDs in the encoded error range (32-bit size_t) : ", testNb++); | |
| /* On 32-bit, (size_t)-120 == 4294967176 : | |
| * dictIDs above must not be confused with an error code while loading the dictionary. */ | |
| { U32 const dictIDs[] = { 1U, 0x7FFFFFFFU, 0x80000000U, | |
| 0xFFFFFF88U /* last non-error value */, | |
| 0xFFFFFF89U /* first value in the error range */, | |
| 0xFFFFFFFFU }; | |
| size_t const nbDictIDs = sizeof(dictIDs) / sizeof(dictIDs[0]); | |
| size_t const srcSize = MIN(CNBuffSize, (size_t)(16 KB)); | |
| void* const dictCopy = malloc(dictSize); | |
| ZSTD_DCtx* const dctx = ZSTD_createDCtx(); | |
| size_t n; | |
| CHECK(dictCopy != NULL); | |
| CHECK(dctx != NULL); | |
| memcpy(dictCopy, dictBuffer, dictSize); | |
| for (n = 0; n < nbDictIDs; n++) { | |
| int mode; | |
| MEM_writeLE32((char*)dictCopy + 4, dictIDs[n]); /* after the magic number */ | |
| CHECK_EQ(ZSTD_getDictID_fromDict(dictCopy, dictSize), dictIDs[n]); | |
| for (mode = 0; mode < 3; mode++) { | |
| size_t r; | |
| if (mode == 0) { /* ZSTD_compress_usingDict() */ | |
| cSize = ZSTD_compress_usingDict(cctx, compressedBuffer, compressedBufferSize, | |
| CNBuffer, srcSize, dictCopy, dictSize, 3); | |
| } else if (mode == 1) { /* ZSTD_CDict */ | |
| ZSTD_CDict* const cdict = ZSTD_createCDict(dictCopy, dictSize, 3); | |
| CHECK(cdict != NULL); | |
| CHECK_EQ(ZSTD_getDictID_fromCDict(cdict), dictIDs[n]); | |
| cSize = ZSTD_compress_usingCDict(cctx, compressedBuffer, compressedBufferSize, | |
| CNBuffer, srcSize, cdict); | |
| ZSTD_freeCDict(cdict); | |
| } else { /* ZSTD_CCtx_loadDictionary() */ | |
| CHECK_Z(ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters)); | |
| CHECK_Z(ZSTD_CCtx_loadDictionary(cctx, dictCopy, dictSize)); | |
| cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, | |
| CNBuffer, srcSize); | |
| } | |
| CHECK_Z(cSize); | |
| CHECK_EQ(ZSTD_getDictID_fromFrame(compressedBuffer, cSize), dictIDs[n]); | |
| r = ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, | |
| compressedBuffer, cSize, dictCopy, dictSize); | |
| CHECK_Z(r); | |
| CHECK_EQ(r, srcSize); | |
| CHECK(!memcmp(decodedBuffer, CNBuffer, srcSize)); | |
| } | |
| } | |
| ZSTD_freeDCtx(dctx); | |
| free(dictCopy); | |
| } | |
| CHECK_Z(ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters)); | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : estimate CDict size : ", testNb++); | |
| { ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize); | |
| size_t const estimatedSize = ZSTD_estimateCDictSize_advanced(dictSize, cParams, ZSTD_dlm_byRef); | |
| DISPLAYLEVEL(3, "OK : %u \n", (unsigned)estimatedSize); | |
| } | |
| DISPLAYLEVEL(3, "test%3i : compress with CDict ", testNb++); | |
| { ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize); | |
| ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize, | |
| ZSTD_dlm_byRef, ZSTD_dct_auto, | |
| cParams, ZSTD_defaultCMem); | |
| assert(cdict != NULL); | |
| DISPLAYLEVEL(3, "(size : %u) : ", (unsigned)ZSTD_sizeof_CDict(cdict)); | |
| assert(ZSTD_getDictID_fromDict(dictBuffer, dictSize) == ZSTD_getDictID_fromCDict(cdict)); | |
| cSize = ZSTD_compress_usingCDict(cctx, compressedBuffer, compressedBufferSize, | |
| CNBuffer, CNBuffSize, cdict); | |
| ZSTD_freeCDict(cdict); | |
| if (ZSTD_isError(cSize)) goto _output_error; | |
| } | |
| DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100); | |
| DISPLAYLEVEL(3, "test%3i : retrieve dictID from frame : ", testNb++); | |
| { U32 const did = ZSTD_getDictID_fromFrame(compressedBuffer, cSize); | |
| if (did != dictID) goto _output_error; /* non-conformant (content-only) dictionary */ | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : frame built with dictionary should be decompressible : ", testNb++); | |
| { ZSTD_DCtx* const dctx = ZSTD_createDCtx(); assert(dctx != NULL); | |
| CHECKPLUS(r, ZSTD_decompress_usingDict(dctx, | |
| decodedBuffer, CNBuffSize, | |
| compressedBuffer, cSize, | |
| dictBuffer, dictSize), | |
| if (r != CNBuffSize) goto _output_error); | |
| ZSTD_freeDCtx(dctx); | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : compress with static CDict : ", testNb++); | |
| { int const maxLevel = ZSTD_maxCLevel(); | |
| int level; | |
| for (level = 1; level <= maxLevel; ++level) { | |
| ZSTD_compressionParameters const cParams = ZSTD_getCParams(level, CNBuffSize, dictSize); | |
| size_t const cdictSize = ZSTD_estimateCDictSize_advanced(dictSize, cParams, ZSTD_dlm_byCopy); | |
| void* const cdictBuffer = malloc(cdictSize); | |
| if (cdictBuffer==NULL) goto _output_error; | |
| { const ZSTD_CDict* const cdict = ZSTD_initStaticCDict( | |
| cdictBuffer, cdictSize, | |
| dictBuffer, dictSize, | |
| ZSTD_dlm_byCopy, ZSTD_dct_auto, | |
| cParams); | |
| if (cdict == NULL) { | |
| DISPLAY("ZSTD_initStaticCDict failed "); | |
| goto _output_error; | |
| } | |
| cSize = ZSTD_compress_usingCDict(cctx, | |
| compressedBuffer, compressedBufferSize, | |
| CNBuffer, MIN(10 KB, CNBuffSize), cdict); | |
| if (ZSTD_isError(cSize)) { | |
| DISPLAY("ZSTD_compress_usingCDict failed "); | |
| goto _output_error; | |
| } } | |
| free(cdictBuffer); | |
| } } | |
| DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100); | |
| DISPLAYLEVEL(3, "test%3i : ZSTD_compress_usingCDict_advanced, no contentSize, no dictID : ", testNb++); | |
| { ZSTD_frameParameters const fParams = { 0 /* frameSize */, 1 /* checksum */, 1 /* noDictID*/ }; | |
| ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize); | |
| ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto, cParams, ZSTD_defaultCMem); | |
| assert(cdict != NULL); | |
| cSize = ZSTD_compress_usingCDict_advanced(cctx, | |
| compressedBuffer, compressedBufferSize, | |
| CNBuffer, CNBuffSize, | |
| cdict, fParams); | |
| ZSTD_freeCDict(cdict); | |
| if (ZSTD_isError(cSize)) goto _output_error; | |
| } | |
| DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100); | |
| DISPLAYLEVEL(3, "test%3i : try retrieving contentSize from frame : ", testNb++); | |
| { U64 const contentSize = ZSTD_getFrameContentSize(compressedBuffer, cSize); | |
| if (contentSize != ZSTD_CONTENTSIZE_UNKNOWN) goto _output_error; | |
| } | |
| DISPLAYLEVEL(3, "OK (unknown)\n"); | |
| DISPLAYLEVEL(3, "test%3i : frame built without dictID should be decompressible : ", testNb++); | |
| { ZSTD_DCtx* const dctx = ZSTD_createDCtx(); | |
| assert(dctx != NULL); | |
| CHECKPLUS(r, ZSTD_decompress_usingDict(dctx, | |
| decodedBuffer, CNBuffSize, | |
| compressedBuffer, cSize, | |
| dictBuffer, dictSize), | |
| if (r != CNBuffSize) goto _output_error); | |
| ZSTD_freeDCtx(dctx); | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : ZSTD_compress_advanced, no dictID : ", testNb++); | |
| { ZSTD_parameters p = ZSTD_getParams(3, CNBuffSize, dictSize); | |
| p.fParams.noDictIDFlag = 1; | |
| cSize = ZSTD_compress_advanced(cctx, compressedBuffer, compressedBufferSize, | |
| CNBuffer, CNBuffSize, | |
| dictBuffer, dictSize, p); | |
| if (ZSTD_isError(cSize)) goto _output_error; | |
| } | |
| DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100); | |
| DISPLAYLEVEL(3, "test%3i : frame built without dictID should be decompressible : ", testNb++); | |
| { ZSTD_DCtx* const dctx = ZSTD_createDCtx(); assert(dctx != NULL); | |
| CHECKPLUS(r, ZSTD_decompress_usingDict(dctx, | |
| decodedBuffer, CNBuffSize, | |
| compressedBuffer, cSize, | |
| dictBuffer, dictSize), | |
| if (r != CNBuffSize) goto _output_error); | |
| ZSTD_freeDCtx(dctx); | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : dictionary containing only header should return error : ", testNb++); | |
| { ZSTD_DCtx* const dctx = ZSTD_createDCtx(); | |
| assert(dctx != NULL); | |
| { const size_t ret = ZSTD_decompress_usingDict( | |
| dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, | |
| "\x37\xa4\x30\xec\x11\x22\x33\x44", 8); | |
| if (ZSTD_getErrorCode(ret) != ZSTD_error_dictionary_corrupted) | |
| goto _output_error; | |
| } | |
| ZSTD_freeDCtx(dctx); | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : loading a dictionary containing only header should return dictionary_corrupted : ", testNb++); | |
| { ZSTD_DCtx* const dctx = ZSTD_createDCtx(); | |
| const char* const headerOnly = "\x37\xa4\x30\xec\x11\x22\x33\x44"; | |
| assert(dctx != NULL); | |
| CHECK_EQ(ZSTD_getErrorCode(ZSTD_DCtx_loadDictionary(dctx, headerOnly, 8)), ZSTD_error_dictionary_corrupted); | |
| CHECK_EQ(ZSTD_getErrorCode(ZSTD_DCtx_loadDictionary_byReference(dctx, headerOnly, 8)), ZSTD_error_dictionary_corrupted); | |
| /* the context remains usable */ | |
| CHECK_Z(ZSTD_DCtx_loadDictionary(dctx, dictBuffer, dictSize)); | |
| CHECK_EQ(ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize), CNBuffSize); | |
| ZSTD_freeDCtx(dctx); | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3d : bufferless api with cdict : ", testNb++); | |
| { ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer, dictSize, 1); | |
| ZSTD_DCtx* const dctx = ZSTD_createDCtx(); | |
| ZSTD_frameParameters const fParams = { 0, 1, 0 }; | |
| size_t cBlockSize; | |
| cSize = 0; | |
| CHECK_Z(ZSTD_compressBegin_usingCDict_advanced(cctx, cdict, fParams, ZSTD_CONTENTSIZE_UNKNOWN)); | |
| cBlockSize = ZSTD_compressContinue(cctx, (char*)compressedBuffer + cSize, compressedBufferSize - cSize, CNBuffer, 1000); | |
| CHECK_Z(cBlockSize); | |
| cSize += cBlockSize; | |
| cBlockSize = ZSTD_compressEnd(cctx, (char*)compressedBuffer + cSize, compressedBufferSize - cSize, (char const*)CNBuffer + 2000, 1000); | |
| CHECK_Z(cBlockSize); | |
| cSize += cBlockSize; | |
| CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dictBuffer, dictSize)); | |
| ZSTD_freeCDict(cdict); | |
| ZSTD_freeDCtx(dctx); | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : Building cdict w/ ZSTD_dct_fullDict on a good dictionary : ", testNb++); | |
| { ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize); | |
| ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_fullDict, cParams, ZSTD_defaultCMem); | |
| if (cdict==NULL) goto _output_error; | |
| ZSTD_freeCDict(cdict); | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : Building cdict w/ ZSTD_dct_fullDict on a rawContent (must fail) : ", testNb++); | |
| { ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize); | |
| ZSTD_CDict* const cdict = ZSTD_createCDict_advanced((const char*)dictBuffer+1, dictSize-1, ZSTD_dlm_byRef, ZSTD_dct_fullDict, cParams, ZSTD_defaultCMem); | |
| if (cdict!=NULL) goto _output_error; | |
| ZSTD_freeCDict(cdict); | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| { char* rawDictBuffer = (char*)malloc(dictSize); | |
| assert(rawDictBuffer); | |
| memcpy(rawDictBuffer, (char*)dictBuffer + 2, dictSize - 2); | |
| memset(rawDictBuffer + dictSize - 2, 0, 2); | |
| MEM_writeLE32((char*)rawDictBuffer, ZSTD_MAGIC_DICTIONARY); | |
| DISPLAYLEVEL(3, "test%3i : Loading rawContent starting with dict header w/ ZSTD_dct_auto should fail : ", testNb++); | |
| { | |
| size_t ret; | |
| /* Either operation is allowed to fail, but one must fail. */ | |
| ret = ZSTD_CCtx_loadDictionary_advanced( | |
| cctx, (const char*)rawDictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto); | |
| if (!ZSTD_isError(ret)) { | |
| ret = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100)); | |
| if (!ZSTD_isError(ret)) goto _output_error; | |
| } | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : Loading rawContent starting with dict header w/ ZSTD_dct_rawContent should pass : ", testNb++); | |
| { | |
| size_t ret; | |
| ret = ZSTD_CCtx_loadDictionary_advanced( | |
| cctx, (const char*)rawDictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_rawContent); | |
| if (ZSTD_isError(ret)) goto _output_error; | |
| ret = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100)); | |
| if (ZSTD_isError(ret)) goto _output_error; | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : Testing non-attached CDict with ZSTD_dct_rawContent : ", testNb++); | |
| { size_t const srcSize = MIN(CNBuffSize, 100); | |
| ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters); | |
| /* Force the dictionary to be reloaded in raw content mode */ | |
| CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_forceAttachDict, ZSTD_dictForceLoad)); | |
| CHECK_Z(ZSTD_CCtx_loadDictionary_advanced(cctx, rawDictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_rawContent)); | |
| cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, srcSize); | |
| CHECK_Z(cSize); | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| free(rawDictBuffer); | |
| } | |
| DISPLAYLEVEL(3, "test%3i : ZSTD_CCtx_refCDict() then set parameters : ", testNb++); | |
| { ZSTD_CDict* const cdict = ZSTD_createCDict(CNBuffer, dictSize, 1); | |
| ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters); | |
| CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 1) ); | |
| CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_hashLog, 12 )); | |
| CHECK_Z( ZSTD_CCtx_refCDict(cctx, cdict) ); | |
| CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 1) ); | |
| CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_hashLog, 12 )); | |
| ZSTD_freeCDict(cdict); | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : Loading dictionary before setting parameters is the same as loading after : ", testNb++); | |
| { | |
| size_t size1, size2; | |
| ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters); | |
| CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 7) ); | |
| CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, CNBuffer, MIN(CNBuffSize, 10 KB)) ); | |
| size1 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100 KB)); | |
| if (ZSTD_isError(size1)) goto _output_error; | |
| ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters); | |
| CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, CNBuffer, MIN(CNBuffSize, 10 KB)) ); | |
| CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 7) ); | |
| size2 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100 KB)); | |
| if (ZSTD_isError(size2)) goto _output_error; | |
| if (size1 != size2) goto _output_error; | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : Loading a dictionary clears the prefix : ", testNb++); | |
| { | |
| CHECK_Z( ZSTD_CCtx_refPrefix(cctx, (const char*)dictBuffer, dictSize) ); | |
| CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, (const char*)dictBuffer, dictSize) ); | |
| CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100)) ); | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : Loading a dictionary clears the cdict : ", testNb++); | |
| { | |
| ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer, dictSize, 1); | |
| CHECK_Z( ZSTD_CCtx_refCDict(cctx, cdict) ); | |
| CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, (const char*)dictBuffer, dictSize) ); | |
| CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100)) ); | |
| ZSTD_freeCDict(cdict); | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : Loading a cdict clears the prefix : ", testNb++); | |
| { | |
| ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer, dictSize, 1); | |
| CHECK_Z( ZSTD_CCtx_refPrefix(cctx, (const char*)dictBuffer, dictSize) ); | |
| CHECK_Z( ZSTD_CCtx_refCDict(cctx, cdict) ); | |
| CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100)) ); | |
| ZSTD_freeCDict(cdict); | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : Loading a cdict clears the dictionary : ", testNb++); | |
| { | |
| ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer, dictSize, 1); | |
| CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, (const char*)dictBuffer, dictSize) ); | |
| CHECK_Z( ZSTD_CCtx_refCDict(cctx, cdict) ); | |
| CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100)) ); | |
| ZSTD_freeCDict(cdict); | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : Loading a prefix clears the dictionary : ", testNb++); | |
| { | |
| CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, (const char*)dictBuffer, dictSize) ); | |
| CHECK_Z( ZSTD_CCtx_refPrefix(cctx, (const char*)dictBuffer, dictSize) ); | |
| CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100)) ); | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : Loading a prefix clears the cdict : ", testNb++); | |
| { | |
| ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer, dictSize, 1); | |
| CHECK_Z( ZSTD_CCtx_refCDict(cctx, cdict) ); | |
| CHECK_Z( ZSTD_CCtx_refPrefix(cctx, (const char*)dictBuffer, dictSize) ); | |
| CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100)) ); | |
| ZSTD_freeCDict(cdict); | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : Loaded dictionary persists across reset session : ", testNb++); | |
| { | |
| size_t size1, size2; | |
| ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters); | |
| CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, CNBuffer, MIN(CNBuffSize, 10 KB)) ); | |
| size1 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100 KB)); | |
| if (ZSTD_isError(size1)) goto _output_error; | |
| ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only); | |
| size2 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100 KB)); | |
| if (ZSTD_isError(size2)) goto _output_error; | |
| if (size1 != size2) goto _output_error; | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : Loaded dictionary is cleared after resetting parameters : ", testNb++); | |
| { | |
| size_t size1, size2; | |
| ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters); | |
| CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, CNBuffer, MIN(CNBuffSize, 10 KB)) ); | |
| size1 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100 KB)); | |
| if (ZSTD_isError(size1)) goto _output_error; | |
| ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters); | |
| size2 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100 KB)); | |
| if (ZSTD_isError(size2)) goto _output_error; | |
| if (size1 == size2) goto _output_error; | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters); | |
| CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, dictBuffer, dictSize) ); | |
| cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100 KB)); | |
| CHECK_Z(cSize); | |
| DISPLAYLEVEL(3, "test%3i : ZSTD_decompressDCtx() with dictionary : ", testNb++); | |
| { | |
| ZSTD_DCtx* dctx = ZSTD_createDCtx(); | |
| size_t ret; | |
| /* We should fail to decompress without a dictionary. */ | |
| ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters); | |
| ret = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize); | |
| if (!ZSTD_isError(ret)) goto _output_error; | |
| /* We should succeed to decompress with the dictionary. */ | |
| ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters); | |
| CHECK_Z( ZSTD_DCtx_loadDictionary(dctx, dictBuffer, dictSize) ); | |
| CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) ); | |
| /* The dictionary should persist across calls. */ | |
| CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) ); | |
| /* When we reset the context the dictionary is cleared. */ | |
| ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters); | |
| ret = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize); | |
| if (!ZSTD_isError(ret)) goto _output_error; | |
| ZSTD_freeDCtx(dctx); | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : ZSTD_decompressDCtx() with ddict : ", testNb++); | |
| { | |
| ZSTD_DCtx* dctx = ZSTD_createDCtx(); | |
| ZSTD_DDict* ddict = ZSTD_createDDict(dictBuffer, dictSize); | |
| size_t ret; | |
| /* We should succeed to decompress with the ddict. */ | |
| ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters); | |
| CHECK_Z( ZSTD_DCtx_refDDict(dctx, ddict) ); | |
| CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) ); | |
| /* The ddict should persist across calls. */ | |
| CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) ); | |
| /* When we reset the context the ddict is cleared. */ | |
| ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters); | |
| ret = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize); | |
| if (!ZSTD_isError(ret)) goto _output_error; | |
| ZSTD_freeDCtx(dctx); | |
| ZSTD_freeDDict(ddict); | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : ZSTD_decompressDCtx() with prefix : ", testNb++); | |
| { | |
| ZSTD_DCtx* dctx = ZSTD_createDCtx(); | |
| size_t ret; | |
| /* We should succeed to decompress with the prefix. */ | |
| ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters); | |
| CHECK_Z( ZSTD_DCtx_refPrefix_advanced(dctx, dictBuffer, dictSize, ZSTD_dct_auto) ); | |
| CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) ); | |
| /* The prefix should be cleared after the first compression. */ | |
| ret = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize); | |
| if (!ZSTD_isError(ret)) goto _output_error; | |
| ZSTD_freeDCtx(dctx); | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : ZSTD_fast attach dictionary with hashLog = 25 and chainLog = 25 : ", testNb++); | |
| { | |
| ZSTD_CCtx_params* cctxParams = ZSTD_createCCtxParams(); | |
| ZSTD_customMem customMem = {NULL, NULL, NULL}; | |
| ZSTD_DCtx* dctx = ZSTD_createDCtx(); | |
| ZSTD_CDict* cdict; | |
| CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_strategy, ZSTD_fast)); | |
| /* Set windowLog to 25 so hash/chain logs don't get sized down */ | |
| CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_windowLog, 25)); | |
| CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_hashLog, 25)); | |
| CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_chainLog, 25)); | |
| /* Set srcSizeHint to 2^25 so hash/chain logs don't get sized down */ | |
| CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_srcSizeHint, 1u << 25)); | |
| cdict = ZSTD_createCDict_advanced2(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto, cctxParams, customMem); | |
| CHECK_Z(ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters)); | |
| CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_forceAttachDict, ZSTD_dictForceAttach)); | |
| CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1)); | |
| CHECK_Z(ZSTD_CCtx_refCDict(cctx, cdict)); | |
| cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize); | |
| CHECK_Z(cSize); | |
| CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dictBuffer, dictSize)); | |
| ZSTD_freeCDict(cdict); | |
| ZSTD_freeDCtx(dctx); | |
| ZSTD_freeCCtxParams(cctxParams); | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : ZSTD_dfast attach dictionary with hashLog = 25 and chainLog = 25 : ", testNb++); | |
| { | |
| ZSTD_CCtx_params* cctxParams = ZSTD_createCCtxParams(); | |
| ZSTD_customMem customMem = {NULL, NULL, NULL}; | |
| ZSTD_DCtx* dctx = ZSTD_createDCtx(); | |
| ZSTD_CDict* cdict; | |
| CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_strategy, ZSTD_dfast)); | |
| /* Set windowLog to 25 so hash/chain logs don't get sized down */ | |
| CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_windowLog, 25)); | |
| CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_hashLog, 25)); | |
| CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_chainLog, 25)); | |
| /* Set srcSizeHint to 2^25 so hash/chain logs don't get sized down */ | |
| CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_srcSizeHint, 1u << 25)); | |
| cdict = ZSTD_createCDict_advanced2(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto, cctxParams, customMem); | |
| CHECK_Z(ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters)); | |
| CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_forceAttachDict, ZSTD_dictForceAttach)); | |
| CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1)); | |
| CHECK_Z(ZSTD_CCtx_refCDict(cctx, cdict)); | |
| cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize); | |
| CHECK_Z(cSize); | |
| CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dictBuffer, dictSize)); | |
| ZSTD_freeCDict(cdict); | |
| ZSTD_freeDCtx(dctx); | |
| ZSTD_freeCCtxParams(cctxParams); | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : ZSTD_lazy attach dictionary with hashLog = 29 and searchLog = 4 : ", testNb++); | |
| if (MEM_64bits()) { | |
| ZSTD_CCtx_params* cctxParams = ZSTD_createCCtxParams(); | |
| ZSTD_customMem customMem = {NULL, NULL, NULL}; | |
| ZSTD_DCtx* dctx = ZSTD_createDCtx(); | |
| ZSTD_CDict* cdict; | |
| CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_strategy, ZSTD_lazy)); | |
| /* Force enable row based match finder, and disable dedicated dict search. */ | |
| CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_useRowMatchFinder, ZSTD_ps_enable)); | |
| CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_enableDedicatedDictSearch, 0)); | |
| CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_searchLog, 4)); | |
| /* Set windowLog to 29 so hash/chain logs don't get sized down */ | |
| CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_windowLog, 29)); | |
| CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_hashLog, 29)); | |
| /* Set srcSizeHint to 2^29 so hash/chain logs don't get sized down */ | |
| CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_srcSizeHint, 1u << 29)); | |
| cdict = ZSTD_createCDict_advanced2(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto, cctxParams, customMem); | |
| CHECK_Z(ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters)); | |
| CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_forceAttachDict, ZSTD_dictForceAttach)); | |
| CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1)); | |
| CHECK_Z(ZSTD_CCtx_refCDict(cctx, cdict)); | |
| cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize); | |
| CHECK_Z(cSize); | |
| CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dictBuffer, dictSize)); | |
| ZSTD_freeCDict(cdict); | |
| ZSTD_freeDCtx(dctx); | |
| ZSTD_freeCCtxParams(cctxParams); | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : Dictionary with non-default repcodes : ", testNb++); | |
| { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; } | |
| dictSize = ZDICT_trainFromBuffer(dictBuffer, dictSize, | |
| CNBuffer, samplesSizes, nbSamples); | |
| if (ZDICT_isError(dictSize)) goto _output_error; | |
| /* Set all the repcodes to non-default */ | |
| { | |
| BYTE* dictPtr = (BYTE*)dictBuffer; | |
| BYTE* dictLimit = dictPtr + dictSize - 12; | |
| /* Find the repcodes */ | |
| while (dictPtr < dictLimit && | |
| (MEM_readLE32(dictPtr) != 1 || MEM_readLE32(dictPtr + 4) != 4 || | |
| MEM_readLE32(dictPtr + 8) != 8)) { | |
| ++dictPtr; | |
| } | |
| if (dictPtr >= dictLimit) goto _output_error; | |
| MEM_writeLE32(dictPtr + 0, 10); | |
| MEM_writeLE32(dictPtr + 4, 10); | |
| MEM_writeLE32(dictPtr + 8, 10); | |
| /* Set the last 8 bytes to 'x' */ | |
| memset((BYTE*)dictBuffer + dictSize - 8, 'x', 8); | |
| } | |
| /* The optimal parser checks all the repcodes. | |
| * Make sure at least one is a match >= targetLength so that it is | |
| * immediately chosen. This will make sure that the compressor and | |
| * decompressor agree on at least one of the repcodes. | |
| */ | |
| { size_t dSize; | |
| BYTE data[1024]; | |
| ZSTD_DCtx* const dctx = ZSTD_createDCtx(); | |
| ZSTD_compressionParameters const cParams = ZSTD_getCParams(19, CNBuffSize, dictSize); | |
| ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize, | |
| ZSTD_dlm_byRef, ZSTD_dct_auto, | |
| cParams, ZSTD_defaultCMem); | |
| assert(dctx != NULL); assert(cdict != NULL); | |
| memset(data, 'x', sizeof(data)); | |
| cSize = ZSTD_compress_usingCDict(cctx, compressedBuffer, compressedBufferSize, | |
| data, sizeof(data), cdict); | |
| ZSTD_freeCDict(cdict); | |
| if (ZSTD_isError(cSize)) { DISPLAYLEVEL(5, "Compression error %s : ", ZSTD_getErrorName(cSize)); goto _output_error; } | |
| dSize = ZSTD_decompress_usingDict(dctx, decodedBuffer, sizeof(data), compressedBuffer, cSize, dictBuffer, dictSize); | |
| if (ZSTD_isError(dSize)) { DISPLAYLEVEL(5, "Decompression error %s : ", ZSTD_getErrorName(dSize)); goto _output_error; } | |
| if (memcmp(data, decodedBuffer, sizeof(data))) { DISPLAYLEVEL(5, "Data corruption : "); goto _output_error; } | |
| ZSTD_freeDCtx(dctx); | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| testNb = test_sessionReset(testNb); | |
| testNb = test_multipleDDicts(testNb); | |
| DISPLAYLEVEL(3, "test%3i : ZSTD_decompressDCtx() with multiple ddicts : ", testNb++); | |
| { | |
| const size_t numDicts = 128; | |
| const size_t numFrames = 4; | |
| size_t i; | |
| ZSTD_DCtx* dctx = ZSTD_createDCtx(); | |
| ZSTD_DDict** ddictTable = (ZSTD_DDict**)malloc(sizeof(ZSTD_DDict*)*numDicts); | |
| ZSTD_CDict** cdictTable = (ZSTD_CDict**)malloc(sizeof(ZSTD_CDict*)*numDicts); | |
| U32 dictIDSeed = seed; | |
| /* Create new compressed buffer that will hold frames with differing dictIDs */ | |
| char* dictBufferMulti = (char*)malloc(sizeof(char) * dictBufferFixedSize); /* Modifiable copy of fixed full dict buffer */ | |
| ZSTD_memcpy(dictBufferMulti, dictBufferFixed, dictBufferFixedSize); | |
| /* Create a bunch of DDicts with random dict IDs */ | |
| for (i = 0; i < numDicts; ++i) { | |
| U32 currDictID = FUZ_rand(&dictIDSeed); | |
| MEM_writeLE32(dictBufferMulti+ZSTD_FRAMEIDSIZE, currDictID); | |
| ddictTable[i] = ZSTD_createDDict(dictBufferMulti, dictBufferFixedSize); | |
| cdictTable[i] = ZSTD_createCDict(dictBufferMulti, dictBufferFixedSize, 3); | |
| if (!ddictTable[i] || !cdictTable[i] || ZSTD_getDictID_fromCDict(cdictTable[i]) != ZSTD_getDictID_fromDDict(ddictTable[i])) { | |
| goto _output_error; | |
| } | |
| } | |
| /* Compress a few frames using random CDicts */ | |
| { | |
| size_t off = 0; | |
| /* only use the first half so we don't push against size limit of compressedBuffer */ | |
| size_t const segSize = (CNBuffSize / 2) / numFrames; | |
| for (i = 0; i < numFrames; i++) { | |
| size_t dictIdx = FUZ_rand(&dictIDSeed) % numDicts; | |
| ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters); | |
| { CHECK_NEWV(r, ZSTD_compress_usingCDict(cctx, | |
| (BYTE*)compressedBuffer + off, CNBuffSize - off, | |
| (BYTE*)CNBuffer + segSize * (size_t)i, segSize, | |
| cdictTable[dictIdx])); | |
| off += r; | |
| } | |
| } | |
| cSize = off; | |
| } | |
| /* We should succeed to decompression even though different dicts were used on different frames */ | |
| ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters); | |
| ZSTD_DCtx_setParameter(dctx, ZSTD_d_refMultipleDDicts, ZSTD_rmd_refMultipleDDicts); | |
| /* Reference every single ddict we made */ | |
| for (i = 0; i < numDicts; ++i) { | |
| CHECK_Z( ZSTD_DCtx_refDDict(dctx, ddictTable[i])); | |
| } | |
| CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) ); | |
| /* Streaming decompression should also work */ | |
| { | |
| ZSTD_inBuffer in = {compressedBuffer, cSize, 0}; | |
| ZSTD_outBuffer out = {decodedBuffer, CNBuffSize, 0}; | |
| while (in.pos < in.size) { | |
| CHECK_Z(ZSTD_decompressStream(dctx, &out, &in)); | |
| } | |
| } | |
| ZSTD_freeDCtx(dctx); | |
| for (i = 0; i < numDicts; ++i) { | |
| ZSTD_freeCDict(cdictTable[i]); | |
| ZSTD_freeDDict(ddictTable[i]); | |
| } | |
| free(dictBufferMulti); | |
| free(ddictTable); | |
| free(cdictTable); | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : DDict hashset OOB T283341343 (ASAN) : ", testNb++); | |
| { | |
| /* Simplified repro for T283341343 - relies on ASAN to catch OOB read. | |
| * Original bug: probe does idx &= mask; idx++ -> idx can become 64 OOB. | |
| * IDs 3 and 47 both hash XXH64(id) & 63 == 63 (last slot). | |
| * Setup: register dict 3 (occupies slot 63), then decompress frame | |
| * whose header dictID is patched to 47 (collides to same slot). | |
| * Buggy getDDict reads slot 64 OOB. With ASAN=1 this is reported as | |
| * heap-buffer-overflow even though glibc would see 0 and return | |
| * "dictionary mismatch". No custom allocator needed. | |
| * See P2447503582 mode 0. | |
| */ | |
| U32 victimID; | |
| U32 attackerID; | |
| size_t victimIdx; | |
| size_t attackerIdx; | |
| ZSTD_DCtx* dctx2; | |
| ZSTD_CCtx* cctx2; | |
| char* dictBufVictim; | |
| ZSTD_DDict* victimDDict; | |
| size_t srcSize; | |
| BYTE* frame; | |
| BYTE fhd; | |
| int dictIDSizeCode; | |
| int nb; | |
| int singleSegment; | |
| int pos; | |
| int i; | |
| victimID = 3; | |
| attackerID = 47; | |
| victimIdx = XXH64(&victimID, sizeof(victimID), 0) & 63; | |
| attackerIdx = XXH64(&attackerID, sizeof(attackerID), 0) & 63; | |
| if (victimIdx != 63 || attackerIdx != 63) { | |
| DISPLAY("hash assumption broken\n"); | |
| goto _output_error; | |
| } | |
| dctx2 = ZSTD_createDCtx(); | |
| cctx2 = ZSTD_createCCtx(); | |
| dictBufVictim = (char*)malloc(dictBufferFixedSize); | |
| victimDDict = NULL; | |
| if (!dctx2 || !cctx2 || !dictBufVictim) { | |
| DISPLAY("alloc failed\n"); | |
| goto _oob_error2; | |
| } | |
| ZSTD_memcpy(dictBufVictim, dictBufferFixed, dictBufferFixedSize); | |
| MEM_writeLE32(dictBufVictim + ZSTD_FRAMEIDSIZE, victimID); | |
| victimDDict = ZSTD_createDDict(dictBufVictim, dictBufferFixedSize); | |
| if (!victimDDict) { | |
| DISPLAY("DDict creation failed\n"); | |
| goto _oob_error2; | |
| } | |
| CHECK_Z( ZSTD_DCtx_setParameter(dctx2, ZSTD_d_refMultipleDDicts, ZSTD_rmd_refMultipleDDicts) ); | |
| CHECK_Z( ZSTD_DCtx_refDDict(dctx2, victimDDict) ); | |
| ZSTD_CCtx_reset(cctx2, ZSTD_reset_session_and_parameters); | |
| srcSize = MIN(CNBuffSize, 1 KB); | |
| cSize = ZSTD_compress_usingDict(cctx2, compressedBuffer, compressedBufferSize, | |
| CNBuffer, srcSize, | |
| dictBufVictim, dictBufferFixedSize, 3); | |
| if (ZSTD_isError(cSize)) { | |
| DISPLAY("compress_usingDict failed %s\n", ZSTD_getErrorName(cSize)); | |
| goto _oob_error2; | |
| } | |
| /* Patch dictID in frame header to attackerID (47) */ | |
| frame = (BYTE*)compressedBuffer; | |
| fhd = frame[4]; | |
| dictIDSizeCode = fhd & 3; | |
| nb = (dictIDSizeCode == 3) ? 4 : dictIDSizeCode; | |
| singleSegment = (fhd >> 5) & 1; | |
| pos = 4 + 1 + (singleSegment ? 0 : 1); | |
| if (nb == 0) { | |
| DISPLAY("frame has no dictID field\n"); | |
| goto _oob_error2; | |
| } | |
| for (i = 0; i < nb; i++) frame[pos + i] = (BYTE)(attackerID >> (8 * i)); | |
| /* With ASAN=1 buggy code triggers heap-buffer-overflow here. | |
| * Without ASAN it just returns dictionary_wrong, which is OK for fixed code. | |
| * So this test always passes without ASAN, but fails under ASAN if bug present. | |
| */ | |
| { | |
| size_t const ret = ZSTD_decompressDCtx(dctx2, decodedBuffer, CNBuffSize, compressedBuffer, cSize); | |
| if (!ZSTD_isError(ret)) { | |
| DISPLAY("unexpected success %zu (should be dict mismatch)\n", ret); | |
| goto _oob_error2; | |
| } | |
| /* Expected: dictionary_wrong / mismatch -> OK means OOB was not taken as success, | |
| * but ASAN would have already aborted if OOB read happened. */ | |
| } | |
| ZSTD_freeDDict(victimDDict); | |
| ZSTD_freeDCtx(dctx2); | |
| ZSTD_freeCCtx(cctx2); | |
| free(dictBufVictim); | |
| goto _oob_skip2; | |
| _oob_error2: | |
| if (victimDDict) ZSTD_freeDDict(victimDDict); | |
| if (dctx2) ZSTD_freeDCtx(dctx2); | |
| if (cctx2) ZSTD_freeCCtx(cctx2); | |
| if (dictBufVictim) free(dictBufVictim); | |
| goto _output_error; | |
| _oob_skip2: ; | |
| } | |
| DISPLAYLEVEL(3, "OK (ASAN would report OOB read if vulnerable)\n"); | |
| DISPLAYLEVEL(3, "test%3i : DDict hashset OOB write T283341343 (ASAN) : ", testNb++); | |
| { | |
| /* Write path version of above: same colliding IDs 3 and 47 → slot 63. | |
| * Insertion does while(table[idx]!=NULL) { idx&=mask; idx++; } | |
| * Second insertion with colliding ID should wrap to slot 0, but buggy code | |
| * writes to slot 64 OOB (8-byte write). With ASAN this is heap-buffer-overflow | |
| * on write. Without ASAN glibc slack hides it. | |
| */ | |
| U32 victimID2; | |
| U32 attackerID2; | |
| size_t vIdx2; | |
| size_t aIdx2; | |
| ZSTD_DCtx* dctx3; | |
| char* dictBufV2; | |
| char* dictBufA2; | |
| ZSTD_DDict* victimDDict2; | |
| ZSTD_DDict* attackerDDict2; | |
| victimID2 = 3; | |
| attackerID2 = 47; | |
| vIdx2 = XXH64(&victimID2, sizeof(victimID2), 0) & 63; | |
| aIdx2 = XXH64(&attackerID2, sizeof(attackerID2), 0) & 63; | |
| if (vIdx2 != 63 || aIdx2 != 63) { | |
| DISPLAY("hash assumption broken for write test\n"); | |
| goto _output_error; | |
| } | |
| dctx3 = ZSTD_createDCtx(); | |
| dictBufV2 = (char*)malloc(dictBufferFixedSize); | |
| dictBufA2 = (char*)malloc(dictBufferFixedSize); | |
| victimDDict2 = NULL; | |
| attackerDDict2 = NULL; | |
| if (!dctx3 || !dictBufV2 || !dictBufA2) { | |
| DISPLAY("alloc failed for OOB write test\n"); | |
| goto _oob_write_error2; | |
| } | |
| ZSTD_memcpy(dictBufV2, dictBufferFixed, dictBufferFixedSize); | |
| ZSTD_memcpy(dictBufA2, dictBufferFixed, dictBufferFixedSize); | |
| MEM_writeLE32(dictBufV2 + ZSTD_FRAMEIDSIZE, victimID2); | |
| MEM_writeLE32(dictBufA2 + ZSTD_FRAMEIDSIZE, attackerID2); | |
| victimDDict2 = ZSTD_createDDict(dictBufV2, dictBufferFixedSize); | |
| attackerDDict2 = ZSTD_createDDict(dictBufA2, dictBufferFixedSize); | |
| if (!victimDDict2 || !attackerDDict2) { | |
| DISPLAY("DDict creation failed for write test\n"); | |
| goto _oob_write_error2; | |
| } | |
| CHECK_Z( ZSTD_DCtx_setParameter(dctx3, ZSTD_d_refMultipleDDicts, ZSTD_rmd_refMultipleDDicts) ); | |
| CHECK_Z( ZSTD_DCtx_refDDict(dctx3, victimDDict2) ); | |
| /* Second colliding insertion - triggers OOB write in buggy emplaceDDict */ | |
| CHECK_Z( ZSTD_DCtx_refDDict(dctx3, attackerDDict2) ); | |
| ZSTD_freeDDict(victimDDict2); | |
| ZSTD_freeDDict(attackerDDict2); | |
| ZSTD_freeDCtx(dctx3); | |
| free(dictBufV2); | |
| free(dictBufA2); | |
| goto _oob_write_skip2; | |
| _oob_write_error2: | |
| if (victimDDict2) ZSTD_freeDDict(victimDDict2); | |
| if (attackerDDict2) ZSTD_freeDDict(attackerDDict2); | |
| if (dctx3) ZSTD_freeDCtx(dctx3); | |
| if (dictBufV2) free(dictBufV2); | |
| if (dictBufA2) free(dictBufA2); | |
| goto _output_error; | |
| _oob_write_skip2: ; | |
| } | |
| DISPLAYLEVEL(3, "OK (ASAN would report OOB write if vulnerable)\n"); | |
| DISPLAYLEVEL(3, "test%3i : DDict hashset is dropped by reset and refDDict(NULL) : ", testNb++); | |
| { /* After a parameter reset or refDDict(NULL), a frame must not select | |
| * a DDict referenced before, which the caller may have freed since */ | |
| char* const dictBufA = (char*)malloc(dictBufferFixedSize); | |
| ZSTD_DDict* const ddictB = ZSTD_createDDict(dictBufferFixed, dictBufferFixedSize); | |
| ZSTD_DCtx* const dctx = ZSTD_createDCtx(); | |
| size_t frameSize; | |
| int n; | |
| if (!dictBufA || !ddictB || !dctx) goto _output_error; | |
| ZSTD_memcpy(dictBufA, dictBufferFixed, dictBufferFixedSize); | |
| MEM_writeLE32(dictBufA + ZSTD_FRAMEIDSIZE, ZSTD_getDictID_fromDDict(ddictB) + 1); | |
| frameSize = ZSTD_compress_usingDict(cctx, compressedBuffer, compressedBufferSize, | |
| CNBuffer, 100, dictBufA, dictBufferFixedSize, 1); | |
| CHECK_Z(frameSize); | |
| for (n = 0; n < 2; n++) { | |
| ZSTD_DDict* const ddictA = ZSTD_createDDict(dictBufA, dictBufferFixedSize); | |
| if (!ddictA) goto _output_error; | |
| CHECK_Z( ZSTD_DCtx_setParameter(dctx, ZSTD_d_refMultipleDDicts, ZSTD_rmd_refMultipleDDicts) ); | |
| CHECK_Z( ZSTD_DCtx_refDDict(dctx, ddictA) ); | |
| if (n == 0) { | |
| CHECK_Z( ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters) ); | |
| } else { | |
| CHECK_Z( ZSTD_DCtx_refDDict(dctx, NULL) ); | |
| } | |
| ZSTD_freeDDict(ddictA); | |
| /* only ddictB is referenced now : the frame needs ddictA, so it must fail */ | |
| CHECK_Z( ZSTD_DCtx_setParameter(dctx, ZSTD_d_refMultipleDDicts, ZSTD_rmd_refMultipleDDicts) ); | |
| CHECK_Z( ZSTD_DCtx_refDDict(dctx, ddictB) ); | |
| if (!ZSTD_isError(ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, frameSize))) | |
| goto _output_error; | |
| } | |
| ZSTD_freeDCtx(dctx); | |
| ZSTD_freeDDict(ddictB); | |
| free(dictBufA); | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| ZSTD_freeCCtx(cctx); | |
| free(dictBuffer); | |
| free(samplesSizes); | |
| } | |
| /* COVER dictionary builder tests */ | |
| { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); | |
| size_t dictSize = 16 KB; | |
| size_t optDictSize = dictSize; | |
| void* dictBuffer = malloc(dictSize); | |
| size_t const totalSampleSize = 1 MB; | |
| size_t const sampleUnitSize = 8 KB; | |
| U32 const nbSamples = (U32)(totalSampleSize / sampleUnitSize); | |
| size_t* const samplesSizes = (size_t*) malloc(nbSamples * sizeof(size_t)); | |
| U32 seed32 = seed; | |
| ZDICT_cover_params_t params; | |
| U32 dictID; | |
| if (dictBuffer==NULL || samplesSizes==NULL) { | |
| free(dictBuffer); | |
| free(samplesSizes); | |
| goto _output_error; | |
| } | |
| DISPLAYLEVEL(3, "test%3i : ZDICT_trainFromBuffer_cover : ", testNb++); | |
| { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; } | |
| memset(¶ms, 0, sizeof(params)); | |
| params.d = 1 + (FUZ_rand(&seed32) % 16); | |
| params.k = params.d + (FUZ_rand(&seed32) % 256); | |
| dictSize = ZDICT_trainFromBuffer_cover(dictBuffer, dictSize, | |
| CNBuffer, samplesSizes, nbSamples, | |
| params); | |
| if (ZDICT_isError(dictSize)) goto _output_error; | |
| DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)dictSize); | |
| DISPLAYLEVEL(3, "test%3i : check dictID : ", testNb++); | |
| dictID = ZDICT_getDictID(dictBuffer, dictSize); | |
| if (dictID==0) goto _output_error; | |
| DISPLAYLEVEL(3, "OK : %u \n", (unsigned)dictID); | |
| DISPLAYLEVEL(3, "test%3i : ZDICT_optimizeTrainFromBuffer_cover : ", testNb++); | |
| memset(¶ms, 0, sizeof(params)); | |
| params.steps = 4; | |
| optDictSize = ZDICT_optimizeTrainFromBuffer_cover(dictBuffer, optDictSize, | |
| CNBuffer, samplesSizes, | |
| nbSamples / 4, ¶ms); | |
| if (ZDICT_isError(optDictSize)) goto _output_error; | |
| DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)optDictSize); | |
| DISPLAYLEVEL(3, "test%3i : check dictID : ", testNb++); | |
| dictID = ZDICT_getDictID(dictBuffer, optDictSize); | |
| if (dictID==0) goto _output_error; | |
| DISPLAYLEVEL(3, "OK : %u \n", (unsigned)dictID); | |
| DISPLAYLEVEL(3, "test%3i : ZDICT_optimizeTrainFromBuffer_cover with tiny training set : ", testNb++); | |
| { /* With splitPoint < 1.0 the training set is a subset of the corpus. | |
| * If that subset is smaller than MAX(d, sizeof(U64)) the partial | |
| * suffix array size used to underflow size_t (issue #4682). The | |
| * builder must reject it cleanly instead. The first few samples are | |
| * tiny so the 0.75 training split stays below the threshold, while | |
| * the total corpus is comfortably large. */ | |
| size_t tinySizes[8] = { 1, 1, 1, 1, 1, 1, 100, 100 }; | |
| ZDICT_cover_params_t tinyParams; | |
| size_t tinyResult; | |
| memset(&tinyParams, 0, sizeof(tinyParams)); | |
| tinyParams.steps = 4; | |
| tinyParams.splitPoint = 0.75; | |
| tinyResult = ZDICT_optimizeTrainFromBuffer_cover( | |
| dictBuffer, optDictSize, CNBuffer, tinySizes, 8, &tinyParams); | |
| if (!ZDICT_isError(tinyResult)) goto _output_error; | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| ZSTD_freeCCtx(cctx); | |
| free(dictBuffer); | |
| free(samplesSizes); | |
| } | |
| /* Decompression defense tests */ | |
| DISPLAYLEVEL(3, "test%3i : Check input length for magic number : ", testNb++); | |
| { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, CNBuffer, 3); /* too small input */ | |
| if (!ZSTD_isError(r)) goto _output_error; | |
| if (ZSTD_getErrorCode(r) != ZSTD_error_srcSize_wrong) goto _output_error; } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : Check magic Number : ", testNb++); | |
| ((char*)(CNBuffer))[0] = 1; | |
| { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, CNBuffer, 4); | |
| if (!ZSTD_isError(r)) goto _output_error; } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| /* content size verification test */ | |
| DISPLAYLEVEL(3, "test%3i : Content size verification : ", testNb++); | |
| { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); | |
| size_t const srcSize = 5000; | |
| size_t const wrongSrcSize = (srcSize + 1000); | |
| ZSTD_parameters params = ZSTD_getParams(1, wrongSrcSize, 0); | |
| params.fParams.contentSizeFlag = 1; | |
| CHECK_Z( ZSTD_compressBegin_advanced(cctx, NULL, 0, params, wrongSrcSize) ); | |
| { size_t const result = ZSTD_compressEnd(cctx, decodedBuffer, CNBuffSize, CNBuffer, srcSize); | |
| if (!ZSTD_isError(result)) goto _output_error; | |
| if (ZSTD_getErrorCode(result) != ZSTD_error_srcSize_wrong) goto _output_error; | |
| DISPLAYLEVEL(3, "OK : %s \n", ZSTD_getErrorName(result)); | |
| } | |
| ZSTD_freeCCtx(cctx); | |
| } | |
| /* negative compression level test : ensure simple API and advanced API produce same result */ | |
| DISPLAYLEVEL(3, "test%3i : negative compression level : ", testNb++); | |
| { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); | |
| size_t const srcSize = CNBuffSize / 5; | |
| int const compressionLevel = -1; | |
| assert(cctx != NULL); | |
| { size_t const cSize_1pass = ZSTD_compress(compressedBuffer, compressedBufferSize, | |
| CNBuffer, srcSize, compressionLevel); | |
| if (ZSTD_isError(cSize_1pass)) goto _output_error; | |
| CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, compressionLevel) ); | |
| { size_t const compressionResult = ZSTD_compress2(cctx, | |
| compressedBuffer, compressedBufferSize, | |
| CNBuffer, srcSize); | |
| DISPLAYLEVEL(5, "simple=%u vs %u=advanced : ", (unsigned)cSize_1pass, (unsigned)compressionResult); | |
| if (ZSTD_isError(compressionResult)) goto _output_error; | |
| if (compressionResult != cSize_1pass) goto _output_error; | |
| } } | |
| ZSTD_freeCCtx(cctx); | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| /* parameters order test */ | |
| { size_t const inputSize = CNBuffSize / 2; | |
| U64 xxh64; | |
| { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); | |
| DISPLAYLEVEL(3, "test%3i : parameters in order : ", testNb++); | |
| assert(cctx != NULL); | |
| CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 2) ); | |
| CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable) ); | |
| CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, 18) ); | |
| { size_t const compressedSize = ZSTD_compress2(cctx, | |
| compressedBuffer, ZSTD_compressBound(inputSize), | |
| CNBuffer, inputSize); | |
| CHECK_Z(compressedSize); | |
| cSize = compressedSize; | |
| xxh64 = XXH64(compressedBuffer, compressedSize, 0); | |
| } | |
| DISPLAYLEVEL(3, "OK (compress : %u -> %u bytes)\n", (unsigned)inputSize, (unsigned)cSize); | |
| ZSTD_freeCCtx(cctx); | |
| } | |
| { ZSTD_CCtx* cctx = ZSTD_createCCtx(); | |
| DISPLAYLEVEL(3, "test%3i : parameters disordered : ", testNb++); | |
| CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, 18) ); | |
| CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable) ); | |
| CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 2) ); | |
| { size_t const result = ZSTD_compress2(cctx, | |
| compressedBuffer, ZSTD_compressBound(inputSize), | |
| CNBuffer, inputSize); | |
| CHECK_Z(result); | |
| if (result != cSize) goto _output_error; /* must result in same compressed result, hence same size */ | |
| if (XXH64(compressedBuffer, result, 0) != xxh64) goto _output_error; /* must result in exactly same content, hence same hash */ | |
| DISPLAYLEVEL(3, "OK (compress : %u -> %u bytes)\n", (unsigned)inputSize, (unsigned)result); | |
| } | |
| ZSTD_freeCCtx(cctx); | |
| } | |
| } | |
| DISPLAYLEVEL(3, "test%3i : oversized round buffer must fail cleanly on 32-bit : ", testNb++); | |
| /* On 32-bit builds the round buffer size can exceed 4 GiB, which | |
| * wraps in 32-bit size_t. Without the overflow check in | |
| * ZSTDMT_initCStream_internal(), init succeeds with a buffer that | |
| * is too small for the requested configuration (and if it wraps | |
| * to 0, the first streaming copy overruns it). With the check, | |
| * the first ZSTD_compressStream2() call fails cleanly with | |
| * memory_allocation and nothing is copied. Both cases are | |
| * unreachable on 64-bit builds, where they get a real buffer. | |
| * - case 0: windowLog 30 with btultra2 (level 22) and LDM disabled | |
| * makes the overlap prefix 1 GiB, which bumps the job size to | |
| * 1 GiB; 1 worker needs 4 GiB, which wraps to 0. | |
| * - case 1: 16 workers with 256 MB jobs need 4.75 GiB, which | |
| * wraps to 768 MB. */ | |
| if (MEM_32bits()) { | |
| #ifdef ZSTD_MULTITHREAD | |
| struct { int nbWorkers; int windowLog; int jobSize; } const cases[2] = { | |
| { 1, 30, 0 }, | |
| { 16, 27, 256 MB }, | |
| }; | |
| size_t caseNb; | |
| for (caseNb = 0; caseNb < 2; caseNb++) { | |
| ZSTD_CCtx* const cctx = ZSTD_createCCtx(); | |
| ZSTD_outBuffer out = { compressedBuffer, compressedBufferSize, 0 }; | |
| ZSTD_inBuffer in = { CNBuffer, CNBuffSize, 0 }; | |
| size_t r; | |
| CHECK(cctx != NULL); | |
| CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, cases[caseNb].nbWorkers) ); | |
| CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, ZSTD_maxCLevel()) ); | |
| CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, cases[caseNb].windowLog) ); | |
| CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_jobSize, cases[caseNb].jobSize) ); | |
| CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_disable) ); | |
| r = ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_continue); | |
| ZSTD_freeCCtx(cctx); | |
| if (ZSTD_getErrorCode(r) != ZSTD_error_memory_allocation) goto _output_error; | |
| } | |
| #endif | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| /* advanced parameters for decompression */ | |
| { ZSTD_DCtx* const dctx = ZSTD_createDCtx(); | |
| assert(dctx != NULL); | |
| DISPLAYLEVEL(3, "test%3i : get dParameter bounds ", testNb++); | |
| { ZSTD_bounds const bounds = ZSTD_dParam_getBounds(ZSTD_d_windowLogMax); | |
| CHECK_Z(bounds.error); | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : wrong dParameter : ", testNb++); | |
| { size_t const sr = ZSTD_DCtx_setParameter(dctx, (ZSTD_dParameter)999999, 0); | |
| if (!ZSTD_isError(sr)) goto _output_error; | |
| } | |
| { ZSTD_bounds const bounds = ZSTD_dParam_getBounds((ZSTD_dParameter)999998); | |
| if (!ZSTD_isError(bounds.error)) goto _output_error; | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : out of bound dParameter : ", testNb++); | |
| { size_t const sr = ZSTD_DCtx_setParameter(dctx, ZSTD_d_windowLogMax, 9999); | |
| if (!ZSTD_isError(sr)) goto _output_error; | |
| } | |
| { size_t const sr = ZSTD_DCtx_setParameter(dctx, ZSTD_d_format, (ZSTD_format_e)888); | |
| if (!ZSTD_isError(sr)) goto _output_error; | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| ZSTD_freeDCtx(dctx); | |
| } | |
| /* custom formats tests */ | |
| { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); | |
| ZSTD_DCtx* const dctx = ZSTD_createDCtx(); | |
| size_t const inputSize = CNBuffSize / 2; /* won't cause pb with small dict size */ | |
| assert(dctx != NULL); assert(cctx != NULL); | |
| /* basic block compression */ | |
| DISPLAYLEVEL(3, "test%3i : magic-less format test : ", testNb++); | |
| CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_format, ZSTD_f_zstd1_magicless) ); | |
| { ZSTD_inBuffer in = { CNBuffer, inputSize, 0 }; | |
| ZSTD_outBuffer out = { compressedBuffer, ZSTD_compressBound(inputSize), 0 }; | |
| size_t const result = ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end); | |
| if (result != 0) goto _output_error; | |
| if (in.pos != in.size) goto _output_error; | |
| cSize = out.pos; | |
| } | |
| DISPLAYLEVEL(3, "OK (compress : %u -> %u bytes)\n", (unsigned)inputSize, (unsigned)cSize); | |
| DISPLAYLEVEL(3, "test%3i : decompress normally (should fail) : ", testNb++); | |
| { size_t const decodeResult = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize); | |
| if (ZSTD_getErrorCode(decodeResult) != ZSTD_error_prefix_unknown) goto _output_error; | |
| DISPLAYLEVEL(3, "OK : %s \n", ZSTD_getErrorName(decodeResult)); | |
| } | |
| DISPLAYLEVEL(3, "test%3i : decompress of magic-less frame : ", testNb++); | |
| ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters); | |
| CHECK_Z( ZSTD_DCtx_setParameter(dctx, ZSTD_d_format, ZSTD_f_zstd1_magicless) ); | |
| { ZSTD_FrameHeader zfh; | |
| size_t const zfhrt = ZSTD_getFrameHeader_advanced(&zfh, compressedBuffer, cSize, ZSTD_f_zstd1_magicless); | |
| if (zfhrt != 0) goto _output_error; | |
| } | |
| /* one shot */ | |
| { size_t const result = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize); | |
| if (result != inputSize) goto _output_error; | |
| DISPLAYLEVEL(3, "one-shot OK, "); | |
| } | |
| /* streaming */ | |
| { ZSTD_inBuffer in = { compressedBuffer, cSize, 0 }; | |
| ZSTD_outBuffer out = { decodedBuffer, CNBuffSize, 0 }; | |
| size_t const result = ZSTD_decompressStream(dctx, &out, &in); | |
| if (result != 0) goto _output_error; | |
| if (in.pos != in.size) goto _output_error; | |
| if (out.pos != inputSize) goto _output_error; | |
| DISPLAYLEVEL(3, "streaming OK : regenerated %u bytes \n", (unsigned)out.pos); | |
| } | |
| /* basic block compression */ | |
| DISPLAYLEVEL(3, "test%3i : empty magic-less format test : ", testNb++); | |
| CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_format, ZSTD_f_zstd1_magicless) ); | |
| { ZSTD_inBuffer in = { CNBuffer, 0, 0 }; | |
| ZSTD_outBuffer out = { compressedBuffer, ZSTD_compressBound(0), 0 }; | |
| size_t const result = ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end); | |
| if (result != 0) goto _output_error; | |
| if (in.pos != in.size) goto _output_error; | |
| cSize = out.pos; | |
| } | |
| DISPLAYLEVEL(3, "OK (compress : %u -> %u bytes)\n", (unsigned)0, (unsigned)cSize); | |
| DISPLAYLEVEL(3, "test%3i : decompress of empty magic-less frame : ", testNb++); | |
| ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters); | |
| CHECK_Z( ZSTD_DCtx_setParameter(dctx, ZSTD_d_format, ZSTD_f_zstd1_magicless) ); | |
| /* one shot */ | |
| { size_t const result = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize); | |
| if (result != 0) goto _output_error; | |
| DISPLAYLEVEL(3, "one-shot OK, "); | |
| } | |
| /* streaming */ | |
| { ZSTD_inBuffer in = { compressedBuffer, cSize, 0 }; | |
| ZSTD_outBuffer out = { decodedBuffer, CNBuffSize, 0 }; | |
| size_t const result = ZSTD_decompressStream(dctx, &out, &in); | |
| if (result != 0) goto _output_error; | |
| if (in.pos != in.size) goto _output_error; | |
| if (out.pos != 0) goto _output_error; | |
| DISPLAYLEVEL(3, "streaming OK : regenerated %u bytes \n", (unsigned)out.pos); | |
| } | |
| ZSTD_freeCCtx(cctx); | |
| ZSTD_freeDCtx(dctx); | |
| } | |
| DISPLAYLEVEL(3, "test%3i : Decompression parameter reset test : ", testNb++); | |
| { | |
| ZSTD_DCtx* const dctx = ZSTD_createDCtx(); | |
| /* Attempt to future proof this to new parameters. */ | |
| int const maxParam = 2000; | |
| int param; | |
| if (ZSTD_d_experimentalParam3 > maxParam) goto _output_error; | |
| for (param = 0; param < maxParam; ++param) { | |
| ZSTD_dParameter dParam = (ZSTD_dParameter)param; | |
| ZSTD_bounds bounds = ZSTD_dParam_getBounds(dParam); | |
| int value1; | |
| int value2; | |
| int check; | |
| if (ZSTD_isError(bounds.error)) | |
| continue; | |
| CHECK_Z(ZSTD_DCtx_getParameter(dctx, dParam, &value1)); | |
| value2 = (value1 != bounds.lowerBound) ? bounds.lowerBound : bounds.upperBound; | |
| CHECK_Z(ZSTD_DCtx_setParameter(dctx, dParam, value2)); | |
| CHECK_Z(ZSTD_DCtx_getParameter(dctx, dParam, &check)); | |
| if (check != value2) goto _output_error; | |
| CHECK_Z(ZSTD_DCtx_reset(dctx, ZSTD_reset_parameters)); | |
| CHECK_Z(ZSTD_DCtx_getParameter(dctx, dParam, &check)); | |
| if (check != value1) goto _output_error; | |
| } | |
| ZSTD_freeDCtx(dctx); | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| /* block API tests */ | |
| { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); | |
| ZSTD_DCtx* const dctx = ZSTD_createDCtx(); | |
| static const size_t dictSize = 65 KB; | |
| static const size_t blockSize = 100 KB; /* won't cause pb with small dict size */ | |
| size_t cSize2; | |
| assert(cctx != NULL); assert(dctx != NULL); | |
| /* basic block compression */ | |
| DISPLAYLEVEL(3, "test%3i : Block compression test : ", testNb++); | |
| CHECK_Z( ZSTD_compressBegin(cctx, 5) ); | |
| CHECK_Z( ZSTD_getBlockSize(cctx) >= blockSize); | |
| CHECK_VAR(cSize, ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), CNBuffer, blockSize) ); | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : Block decompression test : ", testNb++); | |
| CHECK_Z( ZSTD_decompressBegin(dctx) ); | |
| { CHECK_NEWV(r, ZSTD_decompressBlock(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) ); | |
| if (r != blockSize) goto _output_error; } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| /* very long stream of block compression */ | |
| DISPLAYLEVEL(3, "test%3i : Huge block streaming compression test : ", testNb++); | |
| CHECK_Z( ZSTD_compressBegin(cctx, -199) ); /* we just want to quickly overflow internal U32 index */ | |
| CHECK_Z( ZSTD_getBlockSize(cctx) >= blockSize); | |
| { U64 const toCompress = 5000000000ULL; /* > 4 GB */ | |
| U64 compressed = 0; | |
| while (compressed < toCompress) { | |
| size_t const blockCSize = ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), CNBuffer, blockSize); | |
| assert(blockCSize != 0); | |
| if (ZSTD_isError(blockCSize)) goto _output_error; | |
| compressed += blockCSize; | |
| } } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| /* dictionary block compression */ | |
| DISPLAYLEVEL(3, "test%3i : Dictionary Block compression test : ", testNb++); | |
| CHECK_Z( ZSTD_compressBegin_usingDict(cctx, CNBuffer, dictSize, 5) ); | |
| CHECK_VAR(cSize, ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), (char*)CNBuffer+dictSize, blockSize)); | |
| RDG_genBuffer((char*)CNBuffer+dictSize+blockSize, blockSize, 0.0, 0.0, seed); /* create a non-compressible second block */ | |
| { CHECK_NEWV(r, ZSTD_compressBlock(cctx, (char*)compressedBuffer+cSize, ZSTD_compressBound(blockSize), (char*)CNBuffer+dictSize+blockSize, blockSize) ); /* for cctx history consistency */ | |
| assert(r == 0); /* non-compressible block */ } | |
| memcpy((char*)compressedBuffer+cSize, (char*)CNBuffer+dictSize+blockSize, blockSize); /* send non-compressed block (without header) */ | |
| CHECK_VAR(cSize2, ZSTD_compressBlock(cctx, (char*)compressedBuffer+cSize+blockSize, ZSTD_compressBound(blockSize), | |
| (char*)CNBuffer+dictSize+2*blockSize, blockSize)); | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : Dictionary Block decompression test : ", testNb++); | |
| CHECK_Z( ZSTD_decompressBegin_usingDict(dctx, CNBuffer, dictSize) ); | |
| { CHECK_NEWV( r, ZSTD_decompressBlock(dctx, decodedBuffer, blockSize, compressedBuffer, cSize) ); | |
| if (r != blockSize) { | |
| DISPLAYLEVEL(1, "ZSTD_decompressBlock() with _usingDict() fails : %u, instead of %u expected \n", (unsigned)r, (unsigned)blockSize); | |
| goto _output_error; | |
| } } | |
| memcpy((char*)decodedBuffer+blockSize, (char*)compressedBuffer+cSize, blockSize); | |
| ZSTD_insertBlock(dctx, (char*)decodedBuffer+blockSize, blockSize); /* insert non-compressed block into dctx history */ | |
| { CHECK_NEWV( r, ZSTD_decompressBlock(dctx, (char*)decodedBuffer+2*blockSize, blockSize, (char*)compressedBuffer+cSize+blockSize, cSize2) ); | |
| if (r != blockSize) { | |
| DISPLAYLEVEL(1, "ZSTD_decompressBlock() with _usingDict() and after insertBlock() fails : %u, instead of %u expected \n", (unsigned)r, (unsigned)blockSize); | |
| goto _output_error; | |
| } } | |
| assert(memcpy((char*)CNBuffer+dictSize, decodedBuffer, blockSize*3)); /* ensure regenerated content is identical to origin */ | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : Block compression with CDict : ", testNb++); | |
| { ZSTD_CDict* const cdict = ZSTD_createCDict(CNBuffer, dictSize, 3); | |
| if (cdict==NULL) goto _output_error; | |
| CHECK_Z( ZSTD_compressBegin_usingCDict(cctx, cdict) ); | |
| CHECK_Z( ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), (char*)CNBuffer+dictSize, blockSize) ); | |
| ZSTD_freeCDict(cdict); | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| ZSTD_freeCCtx(cctx); | |
| ZSTD_freeDCtx(dctx); | |
| } | |
| /* rle detection test: must compress better blocks with a single identical byte repeated */ | |
| { size_t sampleSize = 0; | |
| size_t maxCompressedSize = 46; /* block 1, 2: compressed, block 3: RLE, zstd 1.4.4 */ | |
| DISPLAYLEVEL(3, "test%3i : RLE detection test : ", testNb++); | |
| memset((char*)CNBuffer+sampleSize, 'B', 256 KB - 2); | |
| sampleSize += 256 KB - 2; | |
| memset((char*)CNBuffer+sampleSize, 'A', 100 KB); | |
| sampleSize += 100 KB; | |
| cSize = ZSTD_compress(compressedBuffer, ZSTD_compressBound(sampleSize), CNBuffer, sampleSize, 1); | |
| if (ZSTD_isError(cSize) || cSize > maxCompressedSize) { | |
| DISPLAYLEVEL(4, "error: cSize %u > %u expected ! \n", (unsigned)cSize, (unsigned)maxCompressedSize); | |
| goto _output_error; | |
| } | |
| { CHECK_NEWV(regenSize, ZSTD_decompress(decodedBuffer, sampleSize, compressedBuffer, cSize)); | |
| if (regenSize!=sampleSize) goto _output_error; } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| } | |
| DISPLAYLEVEL(3, "test%3i : ZSTD_generateSequences decode from sequences test : ", testNb++); | |
| { | |
| size_t srcSize = 150 KB; | |
| BYTE* src = (BYTE*)CNBuffer; | |
| BYTE* decoded = (BYTE*)compressedBuffer; | |
| ZSTD_CCtx* cctx = ZSTD_createCCtx(); | |
| ZSTD_Sequence* seqs = (ZSTD_Sequence*)malloc(srcSize * sizeof(ZSTD_Sequence)); | |
| size_t seqsSize; | |
| if (seqs == NULL) goto _output_error; | |
| assert(cctx != NULL); | |
| ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 19); | |
| /* Populate src with random data */ | |
| RDG_genBuffer(CNBuffer, srcSize, compressibility, 0.5, seed); | |
| /* Test with block delimiters roundtrip */ | |
| seqsSize = ZSTD_generateSequences(cctx, seqs, srcSize, src, srcSize); | |
| CHECK_Z(seqsSize); | |
| FUZ_decodeSequences(decoded, seqs, seqsSize, src, srcSize, ZSTD_sf_explicitBlockDelimiters); | |
| assert(!memcmp(CNBuffer, compressedBuffer, srcSize)); | |
| /* Test no block delimiters roundtrip */ | |
| seqsSize = ZSTD_mergeBlockDelimiters(seqs, seqsSize); | |
| CHECK_Z(seqsSize); | |
| FUZ_decodeSequences(decoded, seqs, seqsSize, src, srcSize, ZSTD_sf_noBlockDelimiters); | |
| assert(!memcmp(CNBuffer, compressedBuffer, srcSize)); | |
| ZSTD_freeCCtx(cctx); | |
| free(seqs); | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : ZSTD_generateSequences too small output buffer : ", testNb++); | |
| { | |
| const size_t seqsCapacity = 10; | |
| const size_t srcSize = 150 KB; | |
| const BYTE* src = (BYTE*)CNBuffer; | |
| ZSTD_CCtx* const cctx = ZSTD_createCCtx(); | |
| ZSTD_Sequence* const seqs = (ZSTD_Sequence*)malloc(seqsCapacity * sizeof(ZSTD_Sequence)); | |
| if (seqs == NULL) goto _output_error; | |
| if (cctx == NULL) goto _output_error; | |
| /* Populate src with random data */ | |
| RDG_genBuffer(CNBuffer, srcSize, compressibility, 0.5, seed); | |
| /* Test with block delimiters roundtrip */ | |
| { | |
| size_t const seqsSize = ZSTD_generateSequences(cctx, seqs, seqsCapacity, src, srcSize); | |
| if (!ZSTD_isError(seqsSize)) goto _output_error; | |
| } | |
| ZSTD_freeCCtx(cctx); | |
| free(seqs); | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : ZSTD_getSequences followed by ZSTD_compressSequences : ", testNb++); | |
| { | |
| const size_t srcSize = 500 KB; | |
| const BYTE* const src = (BYTE*)CNBuffer; | |
| BYTE* const dst = (BYTE*)compressedBuffer; | |
| const size_t dstCapacity = ZSTD_compressBound(srcSize); | |
| const size_t decompressSize = srcSize; | |
| char* const decompressBuffer = (char*)malloc(decompressSize); | |
| size_t compressedSize; | |
| ZSTD_CCtx* const cctx = ZSTD_createCCtx(); | |
| ZSTD_Sequence* const seqs = (ZSTD_Sequence*)malloc(srcSize * sizeof(ZSTD_Sequence)); | |
| size_t nbSeqs; | |
| if (seqs == NULL) goto _output_error; | |
| assert(cctx != NULL); | |
| /* Populate src with compressible random data */ | |
| RDG_genBuffer(CNBuffer, srcSize, compressibility, 0., seed); | |
| /* Roundtrip Test with block delimiters generated by ZSTD_generateSequences() */ | |
| nbSeqs = ZSTD_generateSequences(cctx, seqs, srcSize, src, srcSize); | |
| ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters); | |
| ZSTD_CCtx_setParameter(cctx, ZSTD_c_blockDelimiters, ZSTD_sf_explicitBlockDelimiters); | |
| compressedSize = ZSTD_compressSequences(cctx, dst, dstCapacity, seqs, nbSeqs, src, srcSize); | |
| if (ZSTD_isError(compressedSize)) { | |
| DISPLAY("Error in sequence compression with block delims\n"); | |
| goto _output_error; | |
| } | |
| { size_t const dSize = ZSTD_decompress(decompressBuffer, decompressSize, dst, compressedSize); | |
| if (ZSTD_isError(dSize)) { | |
| DISPLAY("Error in sequence compression roundtrip with block delims\n"); | |
| goto _output_error; | |
| } } | |
| assert(!memcmp(decompressBuffer, src, srcSize)); | |
| /* Roundtrip Test with no block delimiters */ | |
| { size_t const nbSeqsAfterMerge = ZSTD_mergeBlockDelimiters(seqs, nbSeqs); | |
| ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters); | |
| ZSTD_CCtx_setParameter(cctx, ZSTD_c_blockDelimiters, ZSTD_sf_noBlockDelimiters); | |
| compressedSize = ZSTD_compressSequences(cctx, dst, dstCapacity, seqs, nbSeqsAfterMerge, src, srcSize); | |
| } | |
| if (ZSTD_isError(compressedSize)) { | |
| DISPLAY("Error in sequence compression with no block delims\n"); | |
| goto _output_error; | |
| } | |
| { size_t const dSize = ZSTD_decompress(decompressBuffer, decompressSize, dst, compressedSize); | |
| if (ZSTD_isError(dSize)) { | |
| DISPLAY("Error in sequence compression roundtrip with no block delims\n"); | |
| goto _output_error; | |
| } } | |
| assert(!memcmp(decompressBuffer, src, srcSize)); | |
| ZSTD_freeCCtx(cctx); | |
| free(decompressBuffer); | |
| free(seqs); | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| testNb = test_convertSequences_noRepcodes(seed, testNb); | |
| testNb = test_get1BlockSummary(testNb); | |
| testNb = test_blockSplitLimit(testNb); | |
| DISPLAYLEVEL(3, "test%3i : ZSTD_compressSequencesAndLiterals : ", testNb++); | |
| { | |
| const size_t srcSize = 497000; | |
| const BYTE* const src = (BYTE*)CNBuffer; | |
| BYTE* const dst = (BYTE*)compressedBuffer; | |
| const size_t dstCapacity = ZSTD_compressBound(srcSize); | |
| const size_t decompressSize = srcSize; | |
| char* const decompressBuffer = (char*)malloc(decompressSize); | |
| char* const litBuffer = (char*)malloc(decompressSize); | |
| size_t compressedSize; | |
| ZSTD_CCtx* const cctx = ZSTD_createCCtx(); | |
| ZSTD_Sequence* const seqs = (ZSTD_Sequence*)malloc(srcSize * sizeof(ZSTD_Sequence)); | |
| size_t nbSeqs; | |
| if (litBuffer == NULL) goto _output_error; | |
| if (decompressBuffer == NULL) goto _output_error; | |
| if (seqs == NULL) goto _output_error; | |
| assert(cctx != NULL); | |
| /* Populate src with compressible random data */ | |
| RDG_genBuffer(CNBuffer, srcSize, compressibility, 0., seed); | |
| /* Roundtrip Test using the AndLiterals() variant */ | |
| nbSeqs = ZSTD_generateSequences(cctx, seqs, srcSize, src, srcSize); | |
| ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters); | |
| ZSTD_CCtx_setParameter(cctx, ZSTD_c_blockDelimiters, ZSTD_sf_explicitBlockDelimiters); | |
| { size_t const litSize = FUZ_getLitSize(seqs, nbSeqs); | |
| FUZ_transferLiterals(litBuffer, decompressSize, CNBuffer, srcSize, seqs, nbSeqs); | |
| /* literal buffer capacity without 8 bytes of margin: must fail */ | |
| compressedSize = ZSTD_compressSequencesAndLiterals(cctx, dst, dstCapacity, seqs, nbSeqs, litBuffer, litSize, litSize+7, srcSize); | |
| if (ZSTD_getErrorCode(compressedSize) != ZSTD_error_workSpace_tooSmall) { | |
| DISPLAY("ZSTD_compressSequencesAndLiterals() should have failed: literals buffer capacity is too small\n"); | |
| goto _output_error; | |
| } | |
| /* not enough literals: must fail */ | |
| compressedSize = ZSTD_compressSequencesAndLiterals(cctx, dst, dstCapacity, seqs, nbSeqs, src, litSize-1, decompressSize, srcSize); | |
| if (!ZSTD_isError(compressedSize)) { | |
| DISPLAY("ZSTD_compressSequencesAndLiterals() should have failed: not enough literals provided\n"); | |
| goto _output_error; | |
| } | |
| /* too many literals: must fail */ | |
| compressedSize = ZSTD_compressSequencesAndLiterals(cctx, dst, dstCapacity, seqs, nbSeqs, src, litSize+1, decompressSize, srcSize); | |
| if (!ZSTD_isError(compressedSize)) { | |
| DISPLAY("ZSTD_compressSequencesAndLiterals() should have failed: too many literals provided\n"); | |
| goto _output_error; | |
| } | |
| /* srcSize too large: must fail */ | |
| compressedSize = ZSTD_compressSequencesAndLiterals(cctx, dst, dstCapacity, seqs, nbSeqs, litBuffer, litSize, decompressSize, srcSize+1); | |
| if (!ZSTD_isError(compressedSize)) { | |
| DISPLAY("ZSTD_compressSequencesAndLiterals() should have failed: srcSize is too large\n"); | |
| goto _output_error; | |
| } | |
| /* srcSize too small: must fail */ | |
| compressedSize = ZSTD_compressSequencesAndLiterals(cctx, dst, dstCapacity, seqs, nbSeqs, litBuffer, litSize, decompressSize, srcSize-1); | |
| if (!ZSTD_isError(compressedSize)) { | |
| DISPLAY("ZSTD_compressSequencesAndLiterals() should have failed: srcSize is too small\n"); | |
| goto _output_error; | |
| } | |
| /* correct amount of literals: should compress successfully */ | |
| compressedSize = ZSTD_compressSequencesAndLiterals(cctx, dst, dstCapacity, seqs, nbSeqs, litBuffer, litSize, litSize+8, srcSize); | |
| if (ZSTD_isError(compressedSize)) { | |
| DISPLAY("Error in ZSTD_compressSequencesAndLiterals()\n"); | |
| goto _output_error; | |
| } | |
| } | |
| { ZSTD_FrameHeader zfh; | |
| size_t const zfhStatus = ZSTD_getFrameHeader(&zfh, dst, compressedSize); | |
| if (zfhStatus != 0) { | |
| DISPLAY("Error reading frame header\n"); | |
| goto _output_error; | |
| } | |
| if (zfh.frameContentSize != srcSize) { | |
| DISPLAY("Error: ZSTD_compressSequencesAndLiterals() did not report srcSize in the frame header\n"); | |
| goto _output_error; | |
| } | |
| if (zfh.windowSize > srcSize) { | |
| DISPLAY("Error: ZSTD_compressSequencesAndLiterals() did not resized window size to smaller contentSize\n"); | |
| goto _output_error; | |
| } | |
| } | |
| { size_t const dSize = ZSTD_decompress(decompressBuffer, decompressSize, dst, compressedSize); | |
| if (ZSTD_isError(dSize)) { | |
| DISPLAY("Error during decompression of frame produced by ZSTD_compressSequencesAndLiterals()\n"); | |
| goto _output_error; | |
| } | |
| if (dSize != srcSize) { | |
| DISPLAY("Error: decompression of frame produced by ZSTD_compressSequencesAndLiterals() has different size\n"); | |
| goto _output_error; | |
| } | |
| if (memcmp(decompressBuffer, src, srcSize)) { | |
| DISPLAY("Error: decompression of frame produced by ZSTD_compressSequencesAndLiterals() produces a different content (of same size)\n"); | |
| goto _output_error; | |
| } | |
| } | |
| ZSTD_freeCCtx(cctx); | |
| free(litBuffer); | |
| free(decompressBuffer); | |
| free(seqs); | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : ZSTD_compressSequencesAndLiterals : reject a too large block : ", testNb++); | |
| { /* one block of ZSTD_BLOCKSIZE_MAX + 1 bytes : 16 literals and a match, then the block delimiter */ | |
| ZSTD_Sequence const seqs[2] = { { 1, 16, ZSTD_BLOCKSIZE_MAX + 1 - 16, 0 }, { 0, 0, 0, 0 } }; | |
| ZSTD_CCtx* const cctx = ZSTD_createCCtx(); | |
| CHECK(cctx != NULL); | |
| CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_blockDelimiters, ZSTD_sf_explicitBlockDelimiters) ); | |
| CHECK( ZSTD_isError(ZSTD_compressSequencesAndLiterals(cctx, compressedBuffer, compressedBufferSize, | |
| seqs, 2, CNBuffer, 16, CNBuffSize, ZSTD_BLOCKSIZE_MAX + 1)) ); | |
| ZSTD_freeCCtx(cctx); | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| /* Multiple blocks of zeros test */ | |
| #define LONGZEROSLENGTH 1000000 /* 1MB of zeros */ | |
| DISPLAYLEVEL(3, "test%3i : compress %u zeroes : ", testNb++, LONGZEROSLENGTH); | |
| memset(CNBuffer, 0, LONGZEROSLENGTH); | |
| CHECK_VAR(cSize, ZSTD_compress(compressedBuffer, ZSTD_compressBound(LONGZEROSLENGTH), CNBuffer, LONGZEROSLENGTH, 1) ); | |
| DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/LONGZEROSLENGTH*100); | |
| DISPLAYLEVEL(3, "test%3i : decompress %u zeroes : ", testNb++, LONGZEROSLENGTH); | |
| { CHECK_NEWV(r, ZSTD_decompress(decodedBuffer, LONGZEROSLENGTH, compressedBuffer, cSize) ); | |
| if (r != LONGZEROSLENGTH) goto _output_error; } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| /* All zeroes test (test bug #137) */ | |
| #define ZEROESLENGTH 100 | |
| DISPLAYLEVEL(3, "test%3i : compress %u zeroes : ", testNb++, ZEROESLENGTH); | |
| memset(CNBuffer, 0, ZEROESLENGTH); | |
| CHECK_VAR(cSize, ZSTD_compress(compressedBuffer, ZSTD_compressBound(ZEROESLENGTH), CNBuffer, ZEROESLENGTH, 1) ); | |
| DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/ZEROESLENGTH*100); | |
| DISPLAYLEVEL(3, "test%3i : decompress %u zeroes : ", testNb++, ZEROESLENGTH); | |
| { CHECK_NEWV(r, ZSTD_decompress(decodedBuffer, ZEROESLENGTH, compressedBuffer, cSize) ); | |
| if (r != ZEROESLENGTH) goto _output_error; } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| /* nbSeq limit test */ | |
| #define _3BYTESTESTLENGTH 131000 | |
| #define NB3BYTESSEQLOG 9 | |
| #define NB3BYTESSEQ (1 << NB3BYTESSEQLOG) | |
| #define NB3BYTESSEQMASK (NB3BYTESSEQ-1) | |
| /* creates a buffer full of 3-bytes sequences */ | |
| { BYTE _3BytesSeqs[NB3BYTESSEQ][3]; | |
| U32 rSeed = 1; | |
| /* create batch of 3-bytes sequences */ | |
| { int i; | |
| for (i=0; i < NB3BYTESSEQ; i++) { | |
| _3BytesSeqs[i][0] = (BYTE)(FUZ_rand(&rSeed) & 255); | |
| _3BytesSeqs[i][1] = (BYTE)(FUZ_rand(&rSeed) & 255); | |
| _3BytesSeqs[i][2] = (BYTE)(FUZ_rand(&rSeed) & 255); | |
| } } | |
| /* randomly fills CNBuffer with prepared 3-bytes sequences */ | |
| { int i; | |
| for (i=0; i < _3BYTESTESTLENGTH; i += 3) { /* note : CNBuffer size > _3BYTESTESTLENGTH+3 */ | |
| U32 const id = FUZ_rand(&rSeed) & NB3BYTESSEQMASK; | |
| ((BYTE*)CNBuffer)[i+0] = _3BytesSeqs[id][0]; | |
| ((BYTE*)CNBuffer)[i+1] = _3BytesSeqs[id][1]; | |
| ((BYTE*)CNBuffer)[i+2] = _3BytesSeqs[id][2]; | |
| } } } | |
| DISPLAYLEVEL(3, "test%3i : growing nbSeq : ", testNb++); | |
| { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); | |
| size_t const maxNbSeq = _3BYTESTESTLENGTH / 3; | |
| size_t const bound = ZSTD_compressBound(_3BYTESTESTLENGTH); | |
| size_t nbSeq = 1; | |
| while (nbSeq <= maxNbSeq) { | |
| CHECK_Z(ZSTD_compressCCtx(cctx, compressedBuffer, bound, CNBuffer, nbSeq * 3, 19)); | |
| /* Check every sequence for the first 100, then skip more rapidly. */ | |
| if (nbSeq < 100) { | |
| ++nbSeq; | |
| } else { | |
| nbSeq += (nbSeq >> 2); | |
| } | |
| } | |
| ZSTD_freeCCtx(cctx); | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : compress lots 3-bytes sequences : ", testNb++); | |
| CHECK_VAR(cSize, ZSTD_compress(compressedBuffer, ZSTD_compressBound(_3BYTESTESTLENGTH), | |
| CNBuffer, _3BYTESTESTLENGTH, 19) ); | |
| DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/_3BYTESTESTLENGTH*100); | |
| DISPLAYLEVEL(3, "test%3i : decompress lots 3-bytes sequence : ", testNb++); | |
| { CHECK_NEWV(r, ZSTD_decompress(decodedBuffer, _3BYTESTESTLENGTH, compressedBuffer, cSize) ); | |
| if (r != _3BYTESTESTLENGTH) goto _output_error; } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : growing literals buffer : ", testNb++); | |
| RDG_genBuffer(CNBuffer, CNBuffSize, 0.0, 0.1, seed); | |
| { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); | |
| size_t const bound = ZSTD_compressBound(CNBuffSize); | |
| size_t size = 1; | |
| while (size <= CNBuffSize) { | |
| CHECK_Z(ZSTD_compressCCtx(cctx, compressedBuffer, bound, CNBuffer, size, 3)); | |
| /* Check every size for the first 100, then skip more rapidly. */ | |
| if (size < 100) { | |
| ++size; | |
| } else { | |
| size += (size >> 2); | |
| } | |
| } | |
| ZSTD_freeCCtx(cctx); | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : incompressible data and ill suited dictionary : ", testNb++); | |
| { /* Train a dictionary on low characters */ | |
| size_t dictSize = 16 KB; | |
| void* const dictBuffer = malloc(dictSize); | |
| size_t const totalSampleSize = 1 MB; | |
| size_t const sampleUnitSize = 8 KB; | |
| U32 const nbSamples = (U32)(totalSampleSize / sampleUnitSize); | |
| size_t* const samplesSizes = (size_t*) malloc(nbSamples * sizeof(size_t)); | |
| if (!dictBuffer || !samplesSizes) goto _output_error; | |
| { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; } | |
| dictSize = ZDICT_trainFromBuffer(dictBuffer, dictSize, CNBuffer, samplesSizes, nbSamples); | |
| if (ZDICT_isError(dictSize)) goto _output_error; | |
| /* Reverse the characters to make the dictionary ill suited */ | |
| { U32 u; | |
| for (u = 0; u < CNBuffSize; ++u) { | |
| ((BYTE*)CNBuffer)[u] = 255 - ((BYTE*)CNBuffer)[u]; | |
| } } | |
| { /* Compress the data */ | |
| size_t const inputSize = 500; | |
| size_t const outputSize = ZSTD_compressBound(inputSize); | |
| void* const outputBuffer = malloc(outputSize); | |
| ZSTD_CCtx* const cctx = ZSTD_createCCtx(); | |
| if (!outputBuffer || !cctx) goto _output_error; | |
| CHECK_Z(ZSTD_compress_usingDict(cctx, outputBuffer, outputSize, CNBuffer, inputSize, dictBuffer, dictSize, 1)); | |
| free(outputBuffer); | |
| ZSTD_freeCCtx(cctx); | |
| } | |
| free(dictBuffer); | |
| free(samplesSizes); | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| /* frame operations on skippable frames */ | |
| { const char skippableFrame[] = "\x52\x2a\x4d\x18\x05\x0\x0\0abcde"; | |
| size_t const skippableFrameSize = sizeof(skippableFrame) - 1 /* remove the terminating /0 */; | |
| DISPLAYLEVEL(3, "test%3i : ZSTD_findFrameCompressedSize on skippable frame : ", testNb++); | |
| CHECK(ZSTD_findFrameCompressedSize(skippableFrame, skippableFrameSize) == skippableFrameSize); | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : ZSTD_getFrameContentSize on skippable frame : ", testNb++); | |
| CHECK(ZSTD_getFrameContentSize(skippableFrame, skippableFrameSize) == 0); | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : ZSTD_getFrameHeader on skippable frame : ", testNb++); | |
| { ZSTD_FrameHeader zfh; | |
| size_t const s = ZSTD_getFrameHeader(&zfh, skippableFrame, skippableFrameSize); | |
| CHECK_Z(s); | |
| CHECK(s == 0); /* success */ | |
| CHECK(zfh.frameType == ZSTD_skippableFrame); | |
| CHECK(zfh.headerSize == ZSTD_SKIPPABLEHEADERSIZE); | |
| CHECK(zfh.dictID == 2); /* magic variant */ | |
| assert(skippableFrameSize >= ZSTD_SKIPPABLEHEADERSIZE); | |
| CHECK(zfh.frameContentSize == skippableFrameSize - ZSTD_SKIPPABLEHEADERSIZE); | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| } | |
| /* error string tests */ | |
| DISPLAYLEVEL(3, "test%3i : testing ZSTD error code strings : ", testNb++); | |
| if (strcmp("No error detected", ZSTD_getErrorName((ZSTD_ErrorCode)(0-ZSTD_error_no_error))) != 0) goto _output_error; | |
| if (strcmp("No error detected", ZSTD_getErrorString(ZSTD_error_no_error)) != 0) goto _output_error; | |
| if (strcmp("Unspecified error code", ZSTD_getErrorString((ZSTD_ErrorCode)(0-ZSTD_error_GENERIC))) != 0) goto _output_error; | |
| if (strcmp("Error (generic)", ZSTD_getErrorName((size_t)0-ZSTD_error_GENERIC)) != 0) goto _output_error; | |
| if (strcmp("Error (generic)", ZSTD_getErrorString(ZSTD_error_GENERIC)) != 0) goto _output_error; | |
| if (strcmp("No error detected", ZSTD_getErrorName(ZSTD_error_GENERIC)) != 0) goto _output_error; | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : testing ZSTD dictionary sizes : ", testNb++); | |
| RDG_genBuffer(CNBuffer, CNBuffSize, compressibility, 0., seed); | |
| { | |
| size_t const size = MIN(128 KB, CNBuffSize); | |
| ZSTD_CCtx* const cctx = ZSTD_createCCtx(); | |
| ZSTD_CDict* const lgCDict = ZSTD_createCDict(CNBuffer, size, 1); | |
| ZSTD_CDict* const smCDict = ZSTD_createCDict(CNBuffer, 1 KB, 1); | |
| ZSTD_FrameHeader lgHeader; | |
| ZSTD_FrameHeader smHeader; | |
| CHECK_Z(ZSTD_compress_usingCDict(cctx, compressedBuffer, compressedBufferSize, CNBuffer, size, lgCDict)); | |
| CHECK_Z(ZSTD_getFrameHeader(&lgHeader, compressedBuffer, compressedBufferSize)); | |
| CHECK_Z(ZSTD_compress_usingCDict(cctx, compressedBuffer, compressedBufferSize, CNBuffer, size, smCDict)); | |
| CHECK_Z(ZSTD_getFrameHeader(&smHeader, compressedBuffer, compressedBufferSize)); | |
| if (lgHeader.windowSize != smHeader.windowSize) goto _output_error; | |
| ZSTD_freeCDict(smCDict); | |
| ZSTD_freeCDict(lgCDict); | |
| ZSTD_freeCCtx(cctx); | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : testing FSE_normalizeCount() PR#1255: ", testNb++); | |
| { | |
| short norm[32]; | |
| unsigned count[32]; | |
| unsigned const tableLog = 5; | |
| size_t const nbSeq = 32; | |
| unsigned const maxSymbolValue = 31; | |
| size_t i; | |
| for (i = 0; i < 32; ++i) | |
| count[i] = 1; | |
| /* Calling FSE_normalizeCount() on a uniform distribution should not | |
| * cause a division by zero. | |
| */ | |
| FSE_normalizeCount(norm, tableLog, count, nbSeq, maxSymbolValue, /* useLowProbCount */ 1); | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : testing FSE_writeNCount() PR#2779: ", testNb++); | |
| { | |
| size_t const outBufSize = 9; | |
| short const count[11] = {1, 0, 1, 0, 1, 0, 1, 0, 1, 9, 18}; | |
| unsigned const tableLog = 5; | |
| unsigned const maxSymbolValue = 10; | |
| BYTE* outBuf = (BYTE*)malloc(outBufSize*sizeof(BYTE)); | |
| /* Ensure that this write doesn't write out of bounds, and that | |
| * FSE_writeNCount_generic() is *not* called with writeIsSafe == 1. | |
| */ | |
| FSE_writeNCount(outBuf, outBufSize, count, maxSymbolValue, tableLog); | |
| free(outBuf); | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : testing bitwise intrinsics PR#3045: ", testNb++); | |
| { | |
| U32 seed_copy = seed; /* need non-const seed to avoid compiler warning for FUZ_rand(&seed) */ | |
| U32 rand32 = FUZ_rand(&seed_copy); | |
| U64 rand64 = ((U64)FUZ_rand(&seed_copy) << 32) | FUZ_rand(&seed_copy); | |
| U32 lowbit_only_32 = 1; | |
| U64 lowbit_only_64 = 1; | |
| U32 highbit_only_32 = (U32)1 << 31; | |
| U64 highbit_only_64 = (U64)1 << 63; | |
| U32 i; | |
| if (rand32 == 0) rand32 = 1; /* CLZ and CTZ are undefined on 0 */ | |
| if (rand64 == 0) rand64 = 1; /* CLZ and CTZ are undefined on 0 */ | |
| /* Test ZSTD_countTrailingZeros32 */ | |
| CHECK_EQ(ZSTD_countTrailingZeros32(lowbit_only_32), 0u); | |
| CHECK_EQ(ZSTD_countTrailingZeros32(highbit_only_32), 31u); | |
| CHECK_EQ(ZSTD_countTrailingZeros32(rand32), ZSTD_countTrailingZeros32_fallback(rand32)); | |
| /* Test ZSTD_countLeadingZeros32 */ | |
| CHECK_EQ(ZSTD_countLeadingZeros32(lowbit_only_32), 31u); | |
| CHECK_EQ(ZSTD_countLeadingZeros32(highbit_only_32), 0u); | |
| CHECK_EQ(ZSTD_countLeadingZeros32(rand32), ZSTD_countLeadingZeros32_fallback(rand32)); | |
| /* Test ZSTD_countTrailingZeros64 */ | |
| CHECK_EQ(ZSTD_countTrailingZeros64(lowbit_only_64), 0u); | |
| CHECK_EQ(ZSTD_countTrailingZeros64(highbit_only_64), 63u); | |
| /* Test ZSTD_countLeadingZeros64 */ | |
| CHECK_EQ(ZSTD_countLeadingZeros64(lowbit_only_64), 63u); | |
| CHECK_EQ(ZSTD_countLeadingZeros64(highbit_only_64), 0u); | |
| /* Test ZSTD_highbit32 */ | |
| CHECK_EQ(ZSTD_highbit32(lowbit_only_32), 0u); | |
| CHECK_EQ(ZSTD_highbit32(highbit_only_32), 31u); | |
| /* Test ZSTD_NbCommonBytes */ | |
| if (MEM_isLittleEndian()) { | |
| if (MEM_64bits()) { | |
| CHECK_EQ(ZSTD_NbCommonBytes(lowbit_only_32), 0u); | |
| CHECK_EQ(ZSTD_NbCommonBytes(highbit_only_32), 3u); | |
| } else { | |
| CHECK_EQ(ZSTD_NbCommonBytes(lowbit_only_32), 0u); | |
| CHECK_EQ(ZSTD_NbCommonBytes(highbit_only_32), 3u); | |
| } | |
| } else { | |
| if (MEM_64bits()) { | |
| CHECK_EQ(ZSTD_NbCommonBytes(lowbit_only_32), 7u); | |
| CHECK_EQ(ZSTD_NbCommonBytes(highbit_only_32), 4u); | |
| } else { | |
| CHECK_EQ(ZSTD_NbCommonBytes(lowbit_only_32), 3u); | |
| CHECK_EQ(ZSTD_NbCommonBytes(highbit_only_32), 0u); | |
| } | |
| } | |
| /* Test MEM_ intrinsics */ | |
| CHECK_EQ(MEM_swap32(rand32), MEM_swap32_fallback(rand32)); | |
| CHECK_EQ(MEM_swap64(rand64), MEM_swap64_fallback(rand64)); | |
| /* Test fallbacks vs intrinsics on a range of small integers */ | |
| for (i=1; i <= 1000; i++) { | |
| CHECK_EQ(MEM_swap32(i), MEM_swap32_fallback(i)); | |
| CHECK_EQ(MEM_swap64((U64)i), MEM_swap64_fallback((U64)i)); | |
| CHECK_EQ(ZSTD_countTrailingZeros32(i), ZSTD_countTrailingZeros32_fallback(i)); | |
| CHECK_EQ(ZSTD_countLeadingZeros32(i), ZSTD_countLeadingZeros32_fallback(i)); | |
| } | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| #ifdef ZSTD_MULTITHREAD | |
| DISPLAYLEVEL(3, "test%3i : passing wrong full dict should fail on compressStream2 refPrefix ", testNb++); | |
| { ZSTD_CCtx* cctx = ZSTD_createCCtx(); | |
| size_t const srcSize = 1 MB + 5; /* A little more than ZSTDMT_JOBSIZE_MIN */ | |
| size_t const dstSize = ZSTD_compressBound(srcSize); | |
| void* const src = CNBuffer; | |
| void* const dst = compressedBuffer; | |
| void* dict = (void*)malloc(srcSize); | |
| RDG_genBuffer(src, srcSize, compressibility, 0.5, seed); | |
| RDG_genBuffer(dict, srcSize, compressibility, 0., seed); | |
| /* Make sure there is no ZSTD_MAGIC_NUMBER */ | |
| memset(dict, 0, sizeof(U32)); | |
| /* something more than 1 */ | |
| CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 2)); | |
| /* lie and claim this is a full dict */ | |
| CHECK_Z(ZSTD_CCtx_refPrefix_advanced(cctx, dict, srcSize, ZSTD_dct_fullDict)); | |
| { ZSTD_outBuffer out = {dst, dstSize, 0}; | |
| ZSTD_inBuffer in = {src, srcSize, 0}; | |
| /* should fail because its not a full dict like we said it was */ | |
| assert(ZSTD_isError(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush))); | |
| } | |
| ZSTD_freeCCtx(cctx); | |
| free(dict); | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : small dictionary with multithreading and LDM ", testNb++); | |
| { ZSTD_CCtx* cctx = ZSTD_createCCtx(); | |
| size_t const srcSize = 1 MB + 5; /* A little more than ZSTDMT_JOBSIZE_MIN */ | |
| size_t const dictSize = 10; | |
| size_t const dstSize = ZSTD_compressBound(srcSize); | |
| void* const src = CNBuffer; | |
| void* const dst = compressedBuffer; | |
| void* dict = (void*)malloc(dictSize); | |
| RDG_genBuffer(src, srcSize, compressibility, 0.5, seed); | |
| RDG_genBuffer(dict, dictSize, compressibility, 0., seed); | |
| /* Make sure there is no ZSTD_MAGIC_NUMBER */ | |
| memset(dict, 0, sizeof(U32)); | |
| /* Enable MT, LDM, and use refPrefix() for a small dict */ | |
| CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 2)); | |
| CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable)); | |
| CHECK_Z(ZSTD_CCtx_refPrefix(cctx, dict, dictSize)); | |
| CHECK_Z(ZSTD_compress2(cctx, dst, dstSize, src, srcSize)); | |
| ZSTD_freeCCtx(cctx); | |
| free(dict); | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : ZSTD_getCParams() + dictionary ", testNb++); | |
| { | |
| ZSTD_compressionParameters const medium = ZSTD_getCParams(1, 16*1024-1, 0); | |
| ZSTD_compressionParameters const large = ZSTD_getCParams(1, 128*1024-1, 0); | |
| ZSTD_compressionParameters const smallDict = ZSTD_getCParams(1, 0, 400); | |
| ZSTD_compressionParameters const mediumDict = ZSTD_getCParams(1, 0, 10000); | |
| ZSTD_compressionParameters const largeDict = ZSTD_getCParams(1, 0, 100000); | |
| assert(!memcmp(&smallDict, &mediumDict, sizeof(smallDict))); | |
| assert(!memcmp(&medium, &mediumDict, sizeof(medium))); | |
| assert(!memcmp(&large, &largeDict, sizeof(large))); | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : ZSTD_adjustCParams() + dictionary ", testNb++); | |
| { | |
| ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, 0, 0); | |
| ZSTD_compressionParameters const smallDict = ZSTD_adjustCParams(cParams, 0, 400); | |
| ZSTD_compressionParameters const smallSrcAndDict = ZSTD_adjustCParams(cParams, 500, 400); | |
| assert(smallSrcAndDict.windowLog == 10); | |
| assert(!memcmp(&cParams, &smallDict, sizeof(cParams))); | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : check compression mem usage monotonicity over levels for estimateCCtxSize() : ", testNb++); | |
| { | |
| int level = 1; | |
| size_t prevSize = 0; | |
| for (; level < ZSTD_maxCLevel(); ++level) { | |
| size_t const currSize = ZSTD_estimateCCtxSize(level); | |
| if (prevSize > currSize) { | |
| DISPLAYLEVEL(3, "Error! previous cctx size: %u at level: %d is larger than current cctx size: %u at level: %d", | |
| (unsigned)prevSize, level-1, (unsigned)currSize, level); | |
| goto _output_error; | |
| } | |
| prevSize = currSize; | |
| } | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : check estimateCCtxSize() always larger or equal to ZSTD_estimateCCtxSize_usingCParams() : ", testNb++); | |
| { | |
| size_t const kSizeIncrement = 2 KB; | |
| int level = -3; | |
| for (; level <= ZSTD_maxCLevel(); ++level) { | |
| size_t dictSize = 0; | |
| for (; dictSize <= 256 KB; dictSize += 8 * kSizeIncrement) { | |
| size_t srcSize = 2 KB; | |
| for (; srcSize < 300 KB; srcSize += kSizeIncrement) { | |
| ZSTD_compressionParameters const cParams = ZSTD_getCParams(level, srcSize, dictSize); | |
| size_t const cctxSizeUsingCParams = ZSTD_estimateCCtxSize_usingCParams(cParams); | |
| size_t const cctxSizeUsingLevel = ZSTD_estimateCCtxSize(level); | |
| if (cctxSizeUsingLevel < cctxSizeUsingCParams | |
| || ZSTD_isError(cctxSizeUsingCParams) | |
| || ZSTD_isError(cctxSizeUsingLevel)) { | |
| DISPLAYLEVEL(3, "error! l: %d dict: %u srcSize: %u cctx size cpar: %u, cctx size level: %u\n", | |
| level, (unsigned)dictSize, (unsigned)srcSize, (unsigned)cctxSizeUsingCParams, (unsigned)cctxSizeUsingLevel); | |
| goto _output_error; | |
| } } } } } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "test%3i : thread pool API tests : \n", testNb++) | |
| { | |
| int const threadPoolTestResult = threadPoolTests(); | |
| if (threadPoolTestResult) { | |
| goto _output_error; | |
| } | |
| } | |
| DISPLAYLEVEL(3, "thread pool tests OK \n"); | |
| #endif /* ZSTD_MULTITHREAD */ | |
| _end: | |
| free(CNBuffer); | |
| free(compressedBuffer); | |
| free(decodedBuffer); | |
| return testResult; | |
| _output_error: | |
| testResult = 1; | |
| DISPLAY("Error detected in Unit tests ! \n"); | |
| goto _end; | |
| } | |
| static int longUnitTests(U32 const seed, double compressibility) | |
| { | |
| size_t const CNBuffSize = 5 MB; | |
| void* const CNBuffer = malloc(CNBuffSize); | |
| size_t const compressedBufferSize = ZSTD_compressBound(CNBuffSize); | |
| void* const compressedBuffer = malloc(compressedBufferSize); | |
| void* const decodedBuffer = malloc(CNBuffSize); | |
| int testResult = 0; | |
| unsigned testNb=0; | |
| size_t cSize; | |
| /* Create compressible noise */ | |
| if (!CNBuffer || !compressedBuffer || !decodedBuffer) { | |
| DISPLAY("Not enough memory, aborting\n"); | |
| testResult = 1; | |
| goto _end; | |
| } | |
| RDG_genBuffer(CNBuffer, CNBuffSize, compressibility, 0., seed); | |
| /* note : this test is rather long, it would be great to find a way to speed up its execution */ | |
| DISPLAYLEVEL(3, "longtest%3i : table cleanliness through index reduction : ", testNb++); | |
| { int cLevel; | |
| size_t approxIndex = 0; | |
| size_t maxIndex = ((3U << 29) + (1U << ZSTD_WINDOWLOG_MAX)); /* ZSTD_CURRENT_MAX from zstd_compress_internal.h */ | |
| /* Provision enough space in a static context so that we can do all | |
| * this without ever reallocating, which would reset the indices. */ | |
| size_t const staticCCtxSize = ZSTD_estimateCStreamSize(22); | |
| void* const staticCCtxBuffer = malloc(staticCCtxSize); | |
| ZSTD_CCtx* const cctx = ZSTD_initStaticCCtx(staticCCtxBuffer, staticCCtxSize); | |
| /* bump the indices so the following compressions happen at high | |
| * indices. */ | |
| { ZSTD_outBuffer out = { compressedBuffer, compressedBufferSize, 0 }; | |
| ZSTD_inBuffer in = { CNBuffer, CNBuffSize, 0 }; | |
| ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters); | |
| CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, -500)); | |
| while (approxIndex <= (maxIndex / 4) * 3) { | |
| CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush)); | |
| approxIndex += in.pos; | |
| CHECK_Z(in.pos == in.size); | |
| in.pos = 0; | |
| out.pos = 0; | |
| } | |
| CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end)); | |
| } | |
| /* spew a bunch of stuff into the table area */ | |
| for (cLevel = 1; cLevel <= 22; cLevel++) { | |
| ZSTD_outBuffer out = { compressedBuffer, compressedBufferSize / (unsigned)cLevel, 0 }; | |
| ZSTD_inBuffer in = { CNBuffer, CNBuffSize, 0 }; | |
| ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters); | |
| CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, cLevel)); | |
| CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush)); | |
| CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end)); | |
| approxIndex += in.pos; | |
| } | |
| /* now crank the indices so we overflow */ | |
| { ZSTD_outBuffer out = { compressedBuffer, compressedBufferSize, 0 }; | |
| ZSTD_inBuffer in = { CNBuffer, CNBuffSize, 0 }; | |
| ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters); | |
| CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, -500)); | |
| while (approxIndex <= maxIndex) { | |
| CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush)); | |
| approxIndex += in.pos; | |
| CHECK_Z(in.pos == in.size); | |
| in.pos = 0; | |
| out.pos = 0; | |
| } | |
| CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end)); | |
| } | |
| /* do a bunch of compressions again in low indices and ensure we don't | |
| * hit untracked invalid indices */ | |
| for (cLevel = 1; cLevel <= 22; cLevel++) { | |
| ZSTD_outBuffer out = { compressedBuffer, compressedBufferSize / (unsigned)cLevel, 0 }; | |
| ZSTD_inBuffer in = { CNBuffer, CNBuffSize, 0 }; | |
| ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters); | |
| CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, cLevel)); | |
| CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush)); | |
| CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end)); | |
| approxIndex += in.pos; | |
| } | |
| free(staticCCtxBuffer); | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "longtest%3i : testing ldm no regressions in size for opt parser : ", testNb++); | |
| { size_t cSizeLdm; | |
| size_t cSizeNoLdm; | |
| ZSTD_CCtx* const cctx = ZSTD_createCCtx(); | |
| RDG_genBuffer(CNBuffer, CNBuffSize, 0.5, 0.5, seed); | |
| /* Enable checksum to verify round trip. */ | |
| CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1)); | |
| CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable)); | |
| CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 19)); | |
| /* Round trip once with ldm. */ | |
| cSizeLdm = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize); | |
| CHECK_Z(cSizeLdm); | |
| CHECK_Z(ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSizeLdm)); | |
| ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters); | |
| CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1)); | |
| CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_disable)); | |
| CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 19)); | |
| /* Round trip once without ldm. */ | |
| cSizeNoLdm = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize); | |
| CHECK_Z(cSizeNoLdm); | |
| CHECK_Z(ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSizeNoLdm)); | |
| if (cSizeLdm > cSizeNoLdm) { | |
| DISPLAY("Using long mode should not cause regressions for btopt+\n"); | |
| testResult = 1; | |
| goto _end; | |
| } | |
| ZSTD_freeCCtx(cctx); | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| DISPLAYLEVEL(3, "longtest%3i : testing cdict compression with different attachment strategies : ", testNb++); | |
| { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); | |
| ZSTD_DCtx* const dctx = ZSTD_createDCtx(); | |
| size_t dictSize = CNBuffSize; | |
| void* dict = (void*)malloc(dictSize); | |
| ZSTD_CCtx_params* cctx_params = ZSTD_createCCtxParams(); | |
| ZSTD_dictAttachPref_e const attachPrefs[] = { | |
| ZSTD_dictDefaultAttach, | |
| ZSTD_dictForceAttach, | |
| ZSTD_dictForceCopy, | |
| ZSTD_dictForceLoad, | |
| ZSTD_dictDefaultAttach, | |
| ZSTD_dictForceAttach, | |
| ZSTD_dictForceCopy, | |
| ZSTD_dictForceLoad | |
| }; | |
| int const enableDedicatedDictSearch[] = {0, 0, 0, 0, 1, 1, 1, 1}; | |
| int cLevel; | |
| int i; | |
| RDG_genBuffer(dict, dictSize, 0.5, 0.5, seed); | |
| RDG_genBuffer(CNBuffer, CNBuffSize, 0.6, 0.6, seed); | |
| CHECK_Z(cctx_params != NULL); | |
| for (dictSize = CNBuffSize; dictSize; dictSize = dictSize >> 3) { | |
| DISPLAYLEVEL(3, "\n Testing with dictSize %u ", (U32)dictSize); | |
| for (cLevel = 4; cLevel < 13; cLevel++) { | |
| for (i = 0; i < 8; ++i) { | |
| ZSTD_dictAttachPref_e const attachPref = attachPrefs[i]; | |
| int const enableDDS = enableDedicatedDictSearch[i]; | |
| ZSTD_CDict* cdict; | |
| DISPLAYLEVEL(5, "\n dictSize %u cLevel %d iter %d ", (U32)dictSize, cLevel, i); | |
| ZSTD_CCtxParams_init(cctx_params, cLevel); | |
| CHECK_Z(ZSTD_CCtxParams_setParameter(cctx_params, ZSTD_c_enableDedicatedDictSearch, enableDDS)); | |
| cdict = ZSTD_createCDict_advanced2(dict, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto, cctx_params, ZSTD_defaultCMem); | |
| CHECK(cdict != NULL); | |
| CHECK_Z(ZSTD_CCtx_refCDict(cctx, cdict)); | |
| CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_forceAttachDict, (int)attachPref)); | |
| cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize); | |
| CHECK_Z(cSize); | |
| CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dict, dictSize)); | |
| DISPLAYLEVEL(5, "compressed to %u bytes ", (U32)cSize); | |
| CHECK_Z(ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters)); | |
| ZSTD_freeCDict(cdict); | |
| } } } | |
| ZSTD_freeCCtx(cctx); | |
| ZSTD_freeDCtx(dctx); | |
| ZSTD_freeCCtxParams(cctx_params); | |
| free(dict); | |
| } | |
| DISPLAYLEVEL(3, "OK \n"); | |
| _end: | |
| free(CNBuffer); | |
| free(compressedBuffer); | |
| free(decodedBuffer); | |
| return testResult; | |
| } | |
| static size_t findDiff(const void* buf1, const void* buf2, size_t max) | |
| { | |
| const BYTE* b1 = (const BYTE*)buf1; | |
| const BYTE* b2 = (const BYTE*)buf2; | |
| size_t u; | |
| for (u=0; u<max; u++) { | |
| if (b1[u] != b2[u]) break; | |
| } | |
| return u; | |
| } | |
| static ZSTD_parameters FUZ_makeParams(ZSTD_compressionParameters cParams, ZSTD_frameParameters fParams) | |
| { | |
| ZSTD_parameters params; | |
| params.cParams = cParams; | |
| params.fParams = fParams; | |
| return params; | |
| } | |
| static size_t FUZ_rLogLength(U32* seed, U32 logLength) | |
| { | |
| size_t const lengthMask = ((size_t)1 << logLength) - 1; | |
| return (lengthMask+1) + (FUZ_rand(seed) & lengthMask); | |
| } | |
| static size_t FUZ_randomLength(U32* seed, U32 maxLog) | |
| { | |
| U32 const logLength = FUZ_rand(seed) % maxLog; | |
| return FUZ_rLogLength(seed, logLength); | |
| } | |
| #undef CHECK | |
| #define CHECK(cond, ...) { \ | |
| if (cond) { \ | |
| DISPLAY("Error => "); \ | |
| DISPLAY(__VA_ARGS__); \ | |
| DISPLAY(" (seed %u, test nb %u) \n", (unsigned)seed, testNb); \ | |
| goto _output_error; \ | |
| } } | |
| #undef CHECK_Z | |
| #define CHECK_Z(f) { \ | |
| size_t const err = f; \ | |
| if (ZSTD_isError(err)) { \ | |
| DISPLAY("Error => %s : %s ", \ | |
| #f, ZSTD_getErrorName(err)); \ | |
| DISPLAY(" (seed %u, test nb %u) \n", (unsigned)seed, testNb); \ | |
| goto _output_error; \ | |
| } } | |
| static int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, U32 const maxDurationS, double compressibility, int bigTests) | |
| { | |
| static const U32 maxSrcLog = 23; | |
| static const U32 maxSampleLog = 22; | |
| size_t const srcBufferSize = (size_t)1<<maxSrcLog; | |
| size_t const dstBufferSize = (size_t)1<<maxSampleLog; | |
| size_t const cBufferSize = ZSTD_compressBound(dstBufferSize); | |
| BYTE* cNoiseBuffer[5]; | |
| BYTE* const cBuffer = (BYTE*) malloc (cBufferSize); | |
| BYTE* const dstBuffer = (BYTE*) malloc (dstBufferSize); | |
| BYTE* const mirrorBuffer = (BYTE*) malloc (dstBufferSize); | |
| ZSTD_CCtx* const ctx = ZSTD_createCCtx(); | |
| ZSTD_DCtx* const dctx = ZSTD_createDCtx(); | |
| U32 result = 0; | |
| unsigned testNb = 0; | |
| U32 coreSeed = seed; | |
| UTIL_time_t const startClock = UTIL_getTime(); | |
| U64 const maxClockSpan = maxDurationS * SEC_TO_MICRO; | |
| int const cLevelLimiter = bigTests ? 3 : 2; | |
| /* allocation */ | |
| cNoiseBuffer[0] = (BYTE*)malloc (srcBufferSize); | |
| cNoiseBuffer[1] = (BYTE*)malloc (srcBufferSize); | |
| cNoiseBuffer[2] = (BYTE*)malloc (srcBufferSize); | |
| cNoiseBuffer[3] = (BYTE*)malloc (srcBufferSize); | |
| cNoiseBuffer[4] = (BYTE*)malloc (srcBufferSize); | |
| CHECK (!cNoiseBuffer[0] || !cNoiseBuffer[1] || !cNoiseBuffer[2] || !cNoiseBuffer[3] || !cNoiseBuffer[4] | |
| || !dstBuffer || !mirrorBuffer || !cBuffer || !ctx || !dctx, | |
| "Not enough memory, fuzzer tests cancelled"); | |
| /* Create initial samples */ | |
| RDG_genBuffer(cNoiseBuffer[0], srcBufferSize, 0.00, 0., coreSeed); /* pure noise */ | |
| RDG_genBuffer(cNoiseBuffer[1], srcBufferSize, 0.05, 0., coreSeed); /* barely compressible */ | |
| RDG_genBuffer(cNoiseBuffer[2], srcBufferSize, compressibility, 0., coreSeed); | |
| RDG_genBuffer(cNoiseBuffer[3], srcBufferSize, 0.95, 0., coreSeed); /* highly compressible */ | |
| RDG_genBuffer(cNoiseBuffer[4], srcBufferSize, 1.00, 0., coreSeed); /* sparse content */ | |
| /* catch up testNb */ | |
| for (testNb=1; testNb < startTest; testNb++) FUZ_rand(&coreSeed); | |
| /* main test loop */ | |
| for ( ; (testNb <= nbTests) || (UTIL_clockSpanMicro(startClock) < maxClockSpan); testNb++ ) { | |
| BYTE* srcBuffer; /* jumping pointer */ | |
| U32 lseed; | |
| size_t sampleSize, maxTestSize, totalTestSize; | |
| size_t cSize, totalCSize, totalGenSize; | |
| U64 crcOrig; | |
| BYTE* sampleBuffer; | |
| const BYTE* dict; | |
| size_t dictSize; | |
| /* notification */ | |
| if (nbTests >= testNb) { DISPLAYUPDATE(2, "\r%6u/%6u ", testNb, nbTests); } | |
| else { DISPLAYUPDATE(2, "\r%6u ", testNb); } | |
| FUZ_rand(&coreSeed); | |
| { U32 const prime1 = 2654435761U; lseed = coreSeed ^ prime1; } | |
| /* srcBuffer selection [0-4] */ | |
| { U32 buffNb = FUZ_rand(&lseed) & 0x7F; | |
| if (buffNb & 7) buffNb=2; /* most common : compressible (P) */ | |
| else { | |
| buffNb >>= 3; | |
| if (buffNb & 7) { | |
| const U32 tnb[2] = { 1, 3 }; /* barely/highly compressible */ | |
| buffNb = tnb[buffNb >> 3]; | |
| } else { | |
| const U32 tnb[2] = { 0, 4 }; /* not compressible / sparse */ | |
| buffNb = tnb[buffNb >> 3]; | |
| } } | |
| srcBuffer = cNoiseBuffer[buffNb]; | |
| } | |
| /* select src segment */ | |
| sampleSize = FUZ_randomLength(&lseed, maxSampleLog); | |
| /* create sample buffer (to catch read error with valgrind & sanitizers) */ | |
| sampleBuffer = (BYTE*)malloc(sampleSize); | |
| CHECK(sampleBuffer==NULL, "not enough memory for sample buffer"); | |
| { size_t const sampleStart = FUZ_rand(&lseed) % (srcBufferSize - sampleSize); | |
| memcpy(sampleBuffer, srcBuffer + sampleStart, sampleSize); } | |
| crcOrig = XXH64(sampleBuffer, sampleSize, 0); | |
| /* compression tests */ | |
| { int const cLevelPositive = (int) | |
| ( FUZ_rand(&lseed) % | |
| ((U32)ZSTD_maxCLevel() - (FUZ_highbit32((U32)sampleSize) / (U32)cLevelLimiter)) ) | |
| + 1; | |
| int const cLevel = ((FUZ_rand(&lseed) & 15) == 3) ? | |
| - (int)((FUZ_rand(&lseed) & 7) + 1) : /* test negative cLevel */ | |
| cLevelPositive; | |
| DISPLAYLEVEL(5, "fuzzer t%u: Simple compression test (level %i) \n", testNb, cLevel); | |
| cSize = ZSTD_compressCCtx(ctx, cBuffer, cBufferSize, sampleBuffer, sampleSize, cLevel); | |
| CHECK(ZSTD_isError(cSize), "ZSTD_compressCCtx failed : %s", ZSTD_getErrorName(cSize)); | |
| /* compression failure test : too small dest buffer */ | |
| assert(cSize > 3); | |
| { const size_t missing = (FUZ_rand(&lseed) % (cSize-2)) + 1; | |
| const size_t tooSmallSize = cSize - missing; | |
| const unsigned endMark = 0x4DC2B1A9; | |
| memcpy(dstBuffer+tooSmallSize, &endMark, sizeof(endMark)); | |
| DISPLAYLEVEL(5, "fuzzer t%u: compress into too small buffer of size %u (missing %u bytes) \n", | |
| testNb, (unsigned)tooSmallSize, (unsigned)missing); | |
| { size_t const errorCode = ZSTD_compressCCtx(ctx, dstBuffer, tooSmallSize, sampleBuffer, sampleSize, cLevel); | |
| CHECK(ZSTD_getErrorCode(errorCode) != ZSTD_error_dstSize_tooSmall, "ZSTD_compressCCtx should have failed ! (buffer too small : %u < %u)", (unsigned)tooSmallSize, (unsigned)cSize); } | |
| { unsigned endCheck; memcpy(&endCheck, dstBuffer+tooSmallSize, sizeof(endCheck)); | |
| CHECK(endCheck != endMark, "ZSTD_compressCCtx : dst buffer overflow (check.%08X != %08X.mark)", endCheck, endMark); } | |
| } } | |
| /* frame header decompression test */ | |
| { ZSTD_FrameHeader zfh; | |
| CHECK_Z( ZSTD_getFrameHeader(&zfh, cBuffer, cSize) ); | |
| CHECK(zfh.frameContentSize != sampleSize, "Frame content size incorrect"); | |
| } | |
| /* Decompressed size test */ | |
| { unsigned long long const rSize = ZSTD_findDecompressedSize(cBuffer, cSize); | |
| CHECK(rSize != sampleSize, "decompressed size incorrect"); | |
| } | |
| /* successful decompression test */ | |
| DISPLAYLEVEL(5, "fuzzer t%u: simple decompression test \n", testNb); | |
| { size_t const margin = (FUZ_rand(&lseed) & 1) ? 0 : (FUZ_rand(&lseed) & 31) + 1; | |
| size_t const dSize = ZSTD_decompress(dstBuffer, sampleSize + margin, cBuffer, cSize); | |
| CHECK(dSize != sampleSize, "ZSTD_decompress failed (%s) (srcSize : %u ; cSize : %u)", ZSTD_getErrorName(dSize), (unsigned)sampleSize, (unsigned)cSize); | |
| { U64 const crcDest = XXH64(dstBuffer, sampleSize, 0); | |
| CHECK(crcOrig != crcDest, "decompression result corrupted (pos %u / %u)", (unsigned)findDiff(sampleBuffer, dstBuffer, sampleSize), (unsigned)sampleSize); | |
| } } | |
| free(sampleBuffer); /* no longer useful after this point */ | |
| /* truncated src decompression test */ | |
| DISPLAYLEVEL(5, "fuzzer t%u: decompression of truncated source \n", testNb); | |
| { size_t const missing = (FUZ_rand(&lseed) % (cSize-2)) + 1; /* no problem, as cSize > 4 (frameHeaderSizer) */ | |
| size_t const tooSmallSize = cSize - missing; | |
| void* cBufferTooSmall = malloc(tooSmallSize); /* valgrind will catch read overflows */ | |
| CHECK(cBufferTooSmall == NULL, "not enough memory !"); | |
| memcpy(cBufferTooSmall, cBuffer, tooSmallSize); | |
| { size_t const errorCode = ZSTD_decompress(dstBuffer, dstBufferSize, cBufferTooSmall, tooSmallSize); | |
| CHECK(!ZSTD_isError(errorCode), "ZSTD_decompress should have failed ! (truncated src buffer)"); } | |
| free(cBufferTooSmall); | |
| } | |
| /* too small dst decompression test */ | |
| DISPLAYLEVEL(5, "fuzzer t%u: decompress into too small dst buffer \n", testNb); | |
| if (sampleSize > 3) { | |
| size_t const missing = (FUZ_rand(&lseed) % (sampleSize-2)) + 1; /* no problem, as cSize > 4 (frameHeaderSizer) */ | |
| size_t const tooSmallSize = sampleSize - missing; | |
| static const BYTE token = 0xA9; | |
| dstBuffer[tooSmallSize] = token; | |
| { size_t const errorCode = ZSTD_decompress(dstBuffer, tooSmallSize, cBuffer, cSize); | |
| CHECK(ZSTD_getErrorCode(errorCode) != ZSTD_error_dstSize_tooSmall, "ZSTD_decompress should have failed : %u > %u (dst buffer too small)", (unsigned)errorCode, (unsigned)tooSmallSize); } | |
| CHECK(dstBuffer[tooSmallSize] != token, "ZSTD_decompress : dst buffer overflow"); | |
| } | |
| /* noisy src decompression test */ | |
| if (cSize > 6) { | |
| /* insert noise into src */ | |
| { U32 const maxNbBits = FUZ_highbit32((U32)(cSize-4)); | |
| size_t pos = 4; /* preserve magic number (too easy to detect) */ | |
| for (;;) { | |
| /* keep some original src */ | |
| { U32 const nbBits = FUZ_rand(&lseed) % maxNbBits; | |
| size_t const mask = (1<<nbBits) - 1; | |
| size_t const skipLength = FUZ_rand(&lseed) & mask; | |
| pos += skipLength; | |
| } | |
| if (pos >= cSize) break; | |
| /* add noise */ | |
| { U32 const nbBitsCodes = FUZ_rand(&lseed) % maxNbBits; | |
| U32 const nbBits = nbBitsCodes ? nbBitsCodes-1 : 0; | |
| size_t const mask = (1<<nbBits) - 1; | |
| size_t const rNoiseLength = (FUZ_rand(&lseed) & mask) + 1; | |
| size_t const noiseLength = MIN(rNoiseLength, cSize-pos); | |
| size_t const noiseStart = FUZ_rand(&lseed) % (srcBufferSize - noiseLength); | |
| memcpy(cBuffer + pos, srcBuffer + noiseStart, noiseLength); | |
| pos += noiseLength; | |
| } } } | |
| /* decompress noisy source */ | |
| DISPLAYLEVEL(5, "fuzzer t%u: decompress noisy source \n", testNb); | |
| { U32 const endMark = 0xA9B1C3D6; | |
| memcpy(dstBuffer+sampleSize, &endMark, 4); | |
| { size_t const decompressResult = ZSTD_decompress(dstBuffer, sampleSize, cBuffer, cSize); | |
| /* result *may* be an unlikely success, but even then, it must strictly respect dst buffer boundaries */ | |
| CHECK((!ZSTD_isError(decompressResult)) && (decompressResult>sampleSize), | |
| "ZSTD_decompress on noisy src : result is too large : %u > %u (dst buffer)", (unsigned)decompressResult, (unsigned)sampleSize); | |
| } | |
| { U32 endCheck; memcpy(&endCheck, dstBuffer+sampleSize, 4); | |
| CHECK(endMark!=endCheck, "ZSTD_decompress on noisy src : dst buffer overflow"); | |
| } } } /* noisy src decompression test */ | |
| /*===== Bufferless streaming compression test, scattered segments and dictionary =====*/ | |
| DISPLAYLEVEL(5, "fuzzer t%u: Bufferless streaming compression test \n", testNb); | |
| { U32 const testLog = FUZ_rand(&lseed) % maxSrcLog; | |
| U32 const dictLog = FUZ_rand(&lseed) % maxSrcLog; | |
| int const cLevel = (int)(FUZ_rand(&lseed) % | |
| ((U32)ZSTD_maxCLevel() - | |
| (MAX(testLog, dictLog) / (U32)cLevelLimiter))) + | |
| 1; | |
| maxTestSize = FUZ_rLogLength(&lseed, testLog); | |
| if (maxTestSize >= dstBufferSize) maxTestSize = dstBufferSize-1; | |
| dictSize = FUZ_rLogLength(&lseed, dictLog); /* needed also for decompression */ | |
| dict = srcBuffer + (FUZ_rand(&lseed) % (srcBufferSize - dictSize)); | |
| DISPLAYLEVEL(6, "fuzzer t%u: Compressing up to <=%u bytes at level %i with dictionary size %u \n", | |
| testNb, (unsigned)maxTestSize, cLevel, (unsigned)dictSize); | |
| if (FUZ_rand(&lseed) & 0xF) { | |
| CHECK_Z ( ZSTD_compressBegin_usingDict(ctx, dict, dictSize, cLevel) ); | |
| } else { | |
| ZSTD_compressionParameters const cPar = ZSTD_getCParams(cLevel, ZSTD_CONTENTSIZE_UNKNOWN, dictSize); | |
| ZSTD_frameParameters const fPar = { FUZ_rand(&lseed)&1 /* contentSizeFlag */, | |
| !(FUZ_rand(&lseed)&3) /* contentChecksumFlag*/, | |
| 0 /*NodictID*/ }; /* note : since dictionary is fake, dictIDflag has no impact */ | |
| ZSTD_parameters const p = FUZ_makeParams(cPar, fPar); | |
| CHECK_Z ( ZSTD_compressBegin_advanced(ctx, dict, dictSize, p, ZSTD_CONTENTSIZE_UNKNOWN) ); | |
| } | |
| } | |
| { U32 const nbChunks = (FUZ_rand(&lseed) & 127) + 2; | |
| U32 n; | |
| XXH64_state_t xxhState; | |
| XXH64_reset(&xxhState, 0); | |
| for (totalTestSize=0, cSize=0, n=0 ; n<nbChunks ; n++) { | |
| size_t const segmentSize = FUZ_randomLength(&lseed, maxSampleLog); | |
| size_t const segmentStart = FUZ_rand(&lseed) % (srcBufferSize - segmentSize); | |
| if (cBufferSize-cSize < ZSTD_compressBound(segmentSize)) break; /* avoid invalid dstBufferTooSmall */ | |
| if (totalTestSize+segmentSize > maxTestSize) break; | |
| { size_t const compressResult = ZSTD_compressContinue(ctx, cBuffer+cSize, cBufferSize-cSize, srcBuffer+segmentStart, segmentSize); | |
| CHECK (ZSTD_isError(compressResult), "multi-segments compression error : %s", ZSTD_getErrorName(compressResult)); | |
| cSize += compressResult; | |
| } | |
| XXH64_update(&xxhState, srcBuffer+segmentStart, segmentSize); | |
| memcpy(mirrorBuffer + totalTestSize, srcBuffer+segmentStart, segmentSize); | |
| totalTestSize += segmentSize; | |
| } | |
| { size_t const flushResult = ZSTD_compressEnd(ctx, cBuffer+cSize, cBufferSize-cSize, NULL, 0); | |
| CHECK (ZSTD_isError(flushResult), "multi-segments epilogue error : %s", ZSTD_getErrorName(flushResult)); | |
| cSize += flushResult; | |
| } | |
| crcOrig = XXH64_digest(&xxhState); | |
| } | |
| /* streaming decompression test */ | |
| DISPLAYLEVEL(5, "fuzzer t%u: Bufferless streaming decompression test \n", testNb); | |
| /* ensure memory requirement is good enough (should always be true) */ | |
| { ZSTD_FrameHeader zfh; | |
| CHECK( ZSTD_getFrameHeader(&zfh, cBuffer, ZSTD_FRAMEHEADERSIZE_MAX), | |
| "ZSTD_getFrameHeader(): error retrieving frame information"); | |
| { size_t const roundBuffSize = ZSTD_decodingBufferSize_min(zfh.windowSize, zfh.frameContentSize); | |
| CHECK_Z(roundBuffSize); | |
| CHECK((roundBuffSize > totalTestSize) && (zfh.frameContentSize!=ZSTD_CONTENTSIZE_UNKNOWN), | |
| "ZSTD_decodingBufferSize_min() requires more memory (%u) than necessary (%u)", | |
| (unsigned)roundBuffSize, (unsigned)totalTestSize ); | |
| } } | |
| if (dictSize<8) dictSize=0, dict=NULL; /* disable dictionary */ | |
| CHECK_Z( ZSTD_decompressBegin_usingDict(dctx, dict, dictSize) ); | |
| totalCSize = 0; | |
| totalGenSize = 0; | |
| while (totalCSize < cSize) { | |
| size_t const inSize = ZSTD_nextSrcSizeToDecompress(dctx); | |
| size_t const genSize = ZSTD_decompressContinue(dctx, dstBuffer+totalGenSize, dstBufferSize-totalGenSize, cBuffer+totalCSize, inSize); | |
| CHECK (ZSTD_isError(genSize), "ZSTD_decompressContinue error : %s", ZSTD_getErrorName(genSize)); | |
| totalGenSize += genSize; | |
| totalCSize += inSize; | |
| } | |
| CHECK (ZSTD_nextSrcSizeToDecompress(dctx) != 0, "frame not fully decoded"); | |
| CHECK (totalGenSize != totalTestSize, "streaming decompressed data : wrong size") | |
| CHECK (totalCSize != cSize, "compressed data should be fully read") | |
| { U64 const crcDest = XXH64(dstBuffer, totalTestSize, 0); | |
| CHECK(crcOrig != crcDest, "streaming decompressed data corrupted (pos %u / %u)", | |
| (unsigned)findDiff(mirrorBuffer, dstBuffer, totalTestSize), (unsigned)totalTestSize); | |
| } | |
| } /* for ( ; (testNb <= nbTests) */ | |
| DISPLAY("\r%u fuzzer tests completed \n", testNb-1); | |
| _cleanup: | |
| ZSTD_freeCCtx(ctx); | |
| ZSTD_freeDCtx(dctx); | |
| free(cNoiseBuffer[0]); | |
| free(cNoiseBuffer[1]); | |
| free(cNoiseBuffer[2]); | |
| free(cNoiseBuffer[3]); | |
| free(cNoiseBuffer[4]); | |
| free(cBuffer); | |
| free(dstBuffer); | |
| free(mirrorBuffer); | |
| return (int)result; | |
| _output_error: | |
| result = 1; | |
| goto _cleanup; | |
| } | |
| /*_******************************************************* | |
| * Command line | |
| *********************************************************/ | |
| static int FUZ_usage(const char* programName) | |
| { | |
| DISPLAY( "Usage :\n"); | |
| DISPLAY( " %s [args]\n", programName); | |
| DISPLAY( "\n"); | |
| DISPLAY( "Arguments :\n"); | |
| DISPLAY( " -i# : Number of tests (default:%i)\n", nbTestsDefault); | |
| DISPLAY( " -T# : Max duration to run for. Overrides number of tests. (e.g. -T1m or -T60s for one minute)\n"); | |
| DISPLAY( " -s# : Select seed (default:prompt user)\n"); | |
| DISPLAY( " -t# : Select starting test number (default:0)\n"); | |
| DISPLAY( " -P# : Select compressibility in %% (default:%i%%)\n", FUZ_compressibility_default); | |
| DISPLAY( " -v : verbose\n"); | |
| DISPLAY( " -p : pause at the end\n"); | |
| DISPLAY( " -h : display help and exit\n"); | |
| return 0; | |
| } | |
| /*! readU32FromChar() : | |
| @return : unsigned integer value read from input in `char` format | |
| allows and interprets K, KB, KiB, M, MB and MiB suffix. | |
| Will also modify `*stringPtr`, advancing it to position where it stopped reading. | |
| Note : function result can overflow if digit string > MAX_UINT */ | |
| static unsigned readU32FromChar(const char** stringPtr) | |
| { | |
| unsigned result = 0; | |
| while ((**stringPtr >='0') && (**stringPtr <='9')) | |
| result *= 10, result += (unsigned)(**stringPtr - '0'), (*stringPtr)++ ; | |
| if ((**stringPtr=='K') || (**stringPtr=='M')) { | |
| result <<= 10; | |
| if (**stringPtr=='M') result <<= 10; | |
| (*stringPtr)++ ; | |
| if (**stringPtr=='i') (*stringPtr)++; | |
| if (**stringPtr=='B') (*stringPtr)++; | |
| } | |
| return result; | |
| } | |
| /** longCommandWArg() : | |
| * check if *stringPtr is the same as longCommand. | |
| * If yes, @return 1 and advances *stringPtr to the position which immediately follows longCommand. | |
| * @return 0 and doesn't modify *stringPtr otherwise. | |
| */ | |
| static int longCommandWArg(const char** stringPtr, const char* longCommand) | |
| { | |
| size_t const comSize = strlen(longCommand); | |
| int const result = !strncmp(*stringPtr, longCommand, comSize); | |
| if (result) *stringPtr += comSize; | |
| return result; | |
| } | |
| int main(int argc, const char** argv) | |
| { | |
| U32 seed = 0; | |
| int seedset = 0; | |
| int argNb; | |
| int nbTests = nbTestsDefault; | |
| int testNb = 0; | |
| int proba = FUZ_compressibility_default; | |
| double probfloat; | |
| int result = 0; | |
| U32 mainPause = 0; | |
| U32 maxDuration = 0; | |
| int bigTests = 1; | |
| int longTests = 0; | |
| U32 memTestsOnly = 0; | |
| const char* const programName = argv[0]; | |
| /* Check command line */ | |
| for (argNb=1; argNb<argc; argNb++) { | |
| const char* argument = argv[argNb]; | |
| if(!argument) continue; /* Protection if argument empty */ | |
| /* Handle commands. Aggregated commands are allowed */ | |
| if (argument[0]=='-') { | |
| if (longCommandWArg(&argument, "--memtest=")) { memTestsOnly = readU32FromChar(&argument); continue; } | |
| if (!strcmp(argument, "--memtest")) { memTestsOnly=1; continue; } | |
| if (!strcmp(argument, "--no-big-tests")) { bigTests=0; continue; } | |
| if (!strcmp(argument, "--long-tests")) { longTests=1; continue; } | |
| if (!strcmp(argument, "--no-long-tests")) { longTests=0; continue; } | |
| argument++; | |
| while (*argument!=0) { | |
| switch(*argument) | |
| { | |
| case 'h': | |
| return FUZ_usage(programName); | |
| case 'v': | |
| argument++; | |
| g_displayLevel++; | |
| break; | |
| case 'q': | |
| argument++; | |
| g_displayLevel--; | |
| break; | |
| case 'p': /* pause at the end */ | |
| argument++; | |
| mainPause = 1; | |
| break; | |
| case 'i': | |
| argument++; maxDuration = 0; | |
| nbTests = (int)readU32FromChar(&argument); | |
| break; | |
| case 'T': | |
| argument++; | |
| nbTests = 0; | |
| maxDuration = readU32FromChar(&argument); | |
| if (*argument=='s') argument++; /* seconds */ | |
| if (*argument=='m') maxDuration *= 60, argument++; /* minutes */ | |
| if (*argument=='n') argument++; | |
| break; | |
| case 's': | |
| argument++; | |
| seedset = 1; | |
| seed = readU32FromChar(&argument); | |
| break; | |
| case 't': | |
| argument++; | |
| testNb = (int)readU32FromChar(&argument); | |
| break; | |
| case 'P': /* compressibility % */ | |
| argument++; | |
| proba = (int)readU32FromChar(&argument); | |
| if (proba>100) proba = 100; | |
| break; | |
| default: | |
| return (FUZ_usage(programName), 1); | |
| } } } } /* for (argNb=1; argNb<argc; argNb++) */ | |
| /* Get Seed */ | |
| DISPLAY("Starting zstd tester (%i-bits, %s)\n", (int)(sizeof(size_t)*8), ZSTD_VERSION_STRING); | |
| if (!seedset) { | |
| time_t const t = time(NULL); | |
| U32 const h = XXH32(&t, sizeof(t), 1); | |
| seed = h % 10000; | |
| } | |
| DISPLAY("Seed = %u\n", (unsigned)seed); | |
| if (proba!=FUZ_compressibility_default) DISPLAY("Compressibility : %i%%\n", proba); | |
| probfloat = ((double)proba) / 100; | |
| if (memTestsOnly) { | |
| g_displayLevel = MAX(3, g_displayLevel); | |
| return FUZ_mallocTests(seed, probfloat, memTestsOnly); | |
| } | |
| if (nbTests < testNb) nbTests = testNb; | |
| if (testNb==0) { | |
| result = basicUnitTests(0, probfloat); /* constant seed for predictability */ | |
| if (!result && longTests) { | |
| result = longUnitTests(0, probfloat); | |
| } | |
| } | |
| if (!result) | |
| result = fuzzerTests(seed, (unsigned)nbTests, (unsigned)testNb, maxDuration, ((double)proba) / 100, bigTests); | |
| if (mainPause) { | |
| int unused; | |
| DISPLAY("Press Enter \n"); | |
| unused = getchar(); | |
| (void)unused; | |
| } | |
| return result; | |
| } |