Merge pull request #1361 from lz4/fix_s571
Fix seed 571 test
GitOrigin-RevId: f34ecf37176d0dfa8322b33535aace0a883bbf97
Change-Id: I9f6d2ae22c626c6676c5c02aae95b4651910c11d
diff --git a/lib/lz4.c b/lib/lz4.c
index 0a72759..42b03c7 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -1579,8 +1579,11 @@
#endif
+typedef enum { _ld_fast, _ld_slow } LoadDict_mode_e;
#define HASH_UNIT sizeof(reg_t)
-int LZ4_loadDict (LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize)
+int LZ4_loadDict_internal(LZ4_stream_t* LZ4_dict,
+ const char* dictionary, int dictSize,
+ LoadDict_mode_e _ld)
{
LZ4_stream_t_internal* const dict = &LZ4_dict->internal_donotuse;
const tableType_t tableType = byU32;
@@ -1616,13 +1619,39 @@
while (p <= dictEnd-HASH_UNIT) {
U32 const h = LZ4_hashPosition(p, tableType);
+ /* Note: overwriting => favors positions end of dictionary */
LZ4_putIndexOnHash(idx32, h, dict->hashTable, tableType);
p+=3; idx32+=3;
}
+ if (_ld == _ld_slow) {
+ /* Fill hash table with additional references, to improve compression capability */
+ p = dict->dictionary;
+ idx32 = dict->currentOffset - dict->dictSize;
+ while (p <= dictEnd-HASH_UNIT) {
+ U32 const h = LZ4_hashPosition(p, tableType);
+ U32 const limit = dict->currentOffset - 64 KB;
+ if (LZ4_getIndexOnHash(h, dict->hashTable, tableType) <= limit) {
+ /* Note: not overwriting => favors positions beginning of dictionary */
+ LZ4_putIndexOnHash(idx32, h, dict->hashTable, tableType);
+ }
+ p++; idx32++;
+ }
+ }
+
return (int)dict->dictSize;
}
+int LZ4_loadDict(LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize)
+{
+ return LZ4_loadDict_internal(LZ4_dict, dictionary, dictSize, _ld_fast);
+}
+
+int LZ4_loadDictSlow(LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize)
+{
+ return LZ4_loadDict_internal(LZ4_dict, dictionary, dictSize, _ld_slow);
+}
+
void LZ4_attach_dictionary(LZ4_stream_t* workingStream, const LZ4_stream_t* dictionaryStream)
{
const LZ4_stream_t_internal* dictCtx = (dictionaryStream == NULL) ? NULL :
diff --git a/lib/lz4.h b/lib/lz4.h
index 7a2dbfd..5c79972 100644
--- a/lib/lz4.h
+++ b/lib/lz4.h
@@ -148,6 +148,7 @@
**************************************/
/*!
* LZ4_MEMORY_USAGE :
+ * Can be selected at compile time, by setting LZ4_MEMORY_USAGE.
* Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB)
* Increasing memory usage improves compression ratio, generally at the cost of speed.
* Reduced memory usage may improve speed at the cost of ratio, thanks to better cache locality.
@@ -157,6 +158,7 @@
# define LZ4_MEMORY_USAGE LZ4_MEMORY_USAGE_DEFAULT
#endif
+/* These are absolute limits, they should not be changed by users */
#define LZ4_MEMORY_USAGE_MIN 10
#define LZ4_MEMORY_USAGE_DEFAULT 14
#define LZ4_MEMORY_USAGE_MAX 20
@@ -368,6 +370,15 @@
*/
LZ4LIB_API int LZ4_loadDict (LZ4_stream_t* streamPtr, const char* dictionary, int dictSize);
+/*! LZ4_loadDictSlow() : v1.9.5+
+ * Same as LZ4_loadDict(),
+ * but uses a bit more cpu to reference the dictionary content more thoroughly.
+ * This is expected to slightly improve compression ratio.
+ * The extra-cpu cost is likely worth it if the dictionary is re-used across multiple sessions.
+ * @return : loaded dictionary size, in bytes (note: only the last 64 KB are loaded)
+ */
+LZ4LIB_API int LZ4_loadDictSlow(LZ4_stream_t* streamPtr, const char* dictionary, int dictSize);
+
/*! LZ4_compress_fast_continue() :
* Compress 'src' content using data from previously compressed blocks, for better compression ratio.
* 'dst' buffer must be already allocated.
diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index 5a42c0c..4eb9713 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -436,7 +436,7 @@
BYTE* dstPtr = dstStart;
BYTE* const dstEnd = dstStart + dstCapacity;
- DEBUGLOG(4, "LZ4F_compressFrame_usingCDict (srcSize=%u)", srcSize);
+ DEBUGLOG(4, "LZ4F_compressFrame_usingCDict (srcSize=%u)", (unsigned)srcSize);
if (preferencesPtr!=NULL)
prefs = *preferencesPtr;
else
@@ -552,13 +552,13 @@
LZ4_initStream(cdict->fastCtx, sizeof(LZ4_stream_t));
cdict->HCCtx = (LZ4_streamHC_t*)LZ4F_malloc(sizeof(LZ4_streamHC_t), cmem);
if (cdict->HCCtx)
- LZ4_initStream(cdict->HCCtx, sizeof(LZ4_streamHC_t));
+ LZ4_initStreamHC(cdict->HCCtx, sizeof(LZ4_streamHC_t));
if (!cdict->dictContent || !cdict->fastCtx || !cdict->HCCtx) {
LZ4F_freeCDict(cdict);
return NULL;
}
memcpy(cdict->dictContent, dictStart, dictSize);
- LZ4_loadDict (cdict->fastCtx, (const char*)cdict->dictContent, (int)dictSize);
+ LZ4_loadDictSlow(cdict->fastCtx, (const char*)cdict->dictContent, (int)dictSize);
LZ4_setCompressionLevel(cdict->HCCtx, LZ4HC_CLEVEL_DEFAULT);
LZ4_loadDictHC(cdict->HCCtx, (const char*)cdict->dictContent, (int)dictSize);
return cdict;
