Merge pull request #1138 from lz4/dev
stage v1.9.4
diff --git a/.circleci/config.yml b/.circleci/config.yml
index 7f03d1a..ae5aa39 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -44,7 +44,7 @@
# This would typically be a build job when using workflows, possibly combined with build
# This is based on your 1.0 configuration file or project settings
- run: CFLAGS= make clangtest && make clean
- - run: g++ -v; make gpptest && make clean
+ - run: g++ -v; make cxxtest && make clean
- run: gcc -v; g++ -v; make ctocpptest && make clean
- run: gcc-5 -v; CC=gcc-5 CFLAGS="-O2 -Werror" make check && make clean
- run: gcc-5 -v; CC=gcc-5 CFLAGS="-O2 -m32 -Werror" CPPFLAGS=-I/usr/include/x86_64-linux-gnu make check && make clean
diff --git a/.cirrus.yml b/.cirrus.yml
index 0c0e7a7..e423538 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -1,5 +1,5 @@
freebsd_instance:
- image_family: freebsd-12-1
+ image_family: freebsd-12-2
task:
script: pkg install -y gmake && gmake test
diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md
index 86b7696..e47afe7 100644
--- a/.github/ISSUE_TEMPLATE/bug_report.md
+++ b/.github/ISSUE_TEMPLATE/bug_report.md
@@ -26,7 +26,7 @@
- Version [e.g. 22]
- Compiler [e.g. gcc]
- Build System [e.g. Makefile]
- - Other hardware specs [e.g Core 2 duo...]
+ - Other hardware specs [e.g. Core 2 duo...]
**Additional context**
Add any other context about the problem here.
diff --git a/.github/workflows/README.md b/.github/workflows/README.md
new file mode 100644
index 0000000..306d875
--- /dev/null
+++ b/.github/workflows/README.md
@@ -0,0 +1,61 @@
+This directory contains [GitHub Actions](https://github.com/features/actions) workflow files.
+
+# Known issues
+
+## USAN, ASAN (`lz4-ubsan-x64`, `lz4-ubsan-x86`, `lz4-asan-x64`)
+
+For now, `lz4-ubsan-*` ignores the exit code of `make usan` and `make usan32`.
+Because there are several issues which may take relatively long time to resolve.
+
+We'll fully enable it when we ensure `make usan` is ready for all commits and PRs.
+
+See [#983](https://github.com/lz4/lz4/pull/983) for details.
+
+
+## C Compilers (`lz4-c-compilers`)
+
+- Our test doesn't use `gcc-4.5` due to installation issue of its package. (`apt-get install gcc-4.5` fails on GH-Actions VM)
+
+- Currently, the following 32bit executable tests fail with all versions of `clang`.
+ - `CC=clang-X CFLAGS='-O3' make V=1 -C tests clean test-lz4c32`
+ - `CC=clang-X CFLAGS='-O3 -mx32' make V=1 -C tests clean test-lz4c32`
+ - See [#991](https://github.com/lz4/lz4/issues/991) for details.
+
+- Currently, the following 32bit executable tests fail with `gcc-11`
+ - `CC=gcc-11 CFLAGS='-O3' make V=1 -C tests clean test-lz4c32`
+ - `CC=gcc-11 CFLAGS='-O3 -mx32' make V=1 -C tests clean test-lz4c32`
+ - See [#991](https://github.com/lz4/lz4/issues/991) for details.
+
+
+## cppcheck (`lz4-cppcheck`)
+
+This test script ignores the exit code of `make cppcheck`.
+Because this project doesn't 100% follow their recommendation.
+Also sometimes it reports false positives.
+
+
+
+# Notes
+
+- You can investigate various information at the right pane of GitHub
+ Actions report page.
+
+| Item | Section in the right pane |
+| ------------------------- | ------------------------------------- |
+| OS, VM | Set up job |
+| git repo, commit hash | Run actions/checkout@v2 |
+| Version of tools | Environment info |
+
+
+
+# Difference with `.travis.yml`
+
+The following tests are not included yet.
+
+- name: Compile OSS-Fuzz targets
+
+The following tests will not be included due to limitation of GH-Actions.
+
+- name: aarch64 real-hw tests
+- name: PPC64LE real-hw tests
+- name: IBM s390x real-hw tests
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
new file mode 100644
index 0000000..c490bc7
--- /dev/null
+++ b/.github/workflows/ci.yml
@@ -0,0 +1,800 @@
+# For details, see README.md in this directory.
+
+###############################################################
+# C compilers
+#
+# - gcc
+# - clang
+#
+# Known Issue
+# - All test cases which described as 'fail' must be fixed and replaced with 'true'.
+# - gcc-11 (x32, x86) : "../lib/lz4hc.c:148: LZ4HC_countBack: Assertion `(size_t)(match - mMin) < (1U<<31)' failed."
+# - all clangs (x32, x86) : "../lib/lz4hc.c:282: int LZ4HC_InsertAndGetWiderMatch(...): Assertion `matchPtr >= lowPrefixPtr' failed."
+#
+name: lz4 CI
+on: [push, pull_request]
+permissions:
+ contents: read
+
+jobs:
+ lz4-c-compilers:
+ name: CC=${{ matrix.cc }}, ${{ matrix.os }}
+ strategy:
+ fail-fast: false # 'false' means Don't stop matrix workflows even if some matrix failed.
+ matrix:
+ include: [
+ # You can access the following values via ${{ matrix.??? }}
+ #
+ # pkgs : apt-get package names. It can include multiple package names which are delimited by space.
+ # cc : C compiler executable.
+ # cxx : C++ compiler executable for `make ctocpptest`.
+ # x32 : Set 'true' if compiler supports x32. Otherwise, set 'false'.
+ # Set 'fail' if it supports x32 but fails for now. 'fail' cases must be removed.
+ # x86 : Set 'true' if compiler supports x86 (-m32). Otherwise, set 'false'.
+ # Set 'fail' if it supports x86 but fails for now. 'fail' cases must be removed.
+ # cxxtest : Set 'true' if it can be compiled as C++ code. Otherwise, set 'false'.
+ # freestanding : Set 'true' if it can be compiled and execute freestanding code. Otherwise, set 'false'.
+ # Usually, it requires Linux, x86_64 and gcc/g++.
+ # os : GitHub Actions YAML workflow label. See https://github.com/actions/virtual-environments#available-environments
+
+ # cc
+ { pkgs: '', cc: cc, cxx: c++, x32: 'true', x86: 'true', cxxtest: 'true', freestanding: 'true', os: ubuntu-latest, },
+
+ # gcc
+ { pkgs: '', cc: gcc, cxx: g++, x32: 'true', x86: 'true', cxxtest: 'true', freestanding: 'true', os: ubuntu-latest, },
+ { pkgs: 'gcc-12 g++-12 lib32gcc-12-dev libx32gcc-12-dev', cc: gcc-12, cxx: g++-12, x32: 'true', x86: 'true', cxxtest: 'true', freestanding: 'true', os: ubuntu-22.04, },
+ { pkgs: 'gcc-11 g++-11 lib32gcc-11-dev libx32gcc-11-dev', cc: gcc-11, cxx: g++-11, x32: 'true', x86: 'true', cxxtest: 'true', freestanding: 'true', os: ubuntu-22.04, },
+ { pkgs: 'gcc-10 g++-10 lib32gcc-10-dev libx32gcc-10-dev', cc: gcc-10, cxx: g++-10, x32: 'true', x86: 'true', cxxtest: 'true', freestanding: 'true', os: ubuntu-22.04, },
+ { pkgs: 'gcc-9 g++-9 lib32gcc-9-dev libx32gcc-9-dev', cc: gcc-9, cxx: g++-9, x32: 'true', x86: 'true', cxxtest: 'true', freestanding: 'true', os: ubuntu-22.04, },
+ { pkgs: 'gcc-8 g++-8 lib32gcc-8-dev libx32gcc-8-dev', cc: gcc-8, cxx: g++-8, x32: 'true', x86: 'true', cxxtest: 'true', freestanding: 'true', os: ubuntu-20.04, },
+ { pkgs: 'gcc-7 g++-7 lib32gcc-7-dev libx32gcc-7-dev', cc: gcc-7, cxx: g++-7, x32: 'true', x86: 'true', cxxtest: 'true', freestanding: 'true', os: ubuntu-20.04, },
+ { pkgs: 'gcc-6 g++-6 lib32gcc-6-dev libx32gcc-6-dev', cc: gcc-6, cxx: g++-6, x32: 'true', x86: 'true', cxxtest: 'true', freestanding: 'true', os: ubuntu-18.04, },
+ { pkgs: 'gcc-5 g++-5 lib32gcc-5-dev libx32gcc-5-dev', cc: gcc-5, cxx: g++-5, x32: 'true', x86: 'true', cxxtest: 'true', freestanding: 'true', os: ubuntu-18.04, },
+ { pkgs: 'gcc-4.8 g++-4.8 lib32gcc-4.8-dev libx32gcc-4.8-dev', cc: gcc-4.8, cxx: g++-4.8, x32: 'true', x86: 'true', cxxtest: 'true', freestanding: 'true', os: ubuntu-18.04, },
+
+ # clang
+ { pkgs: 'lib32gcc-11-dev libx32gcc-11-dev', cc: clang, cxx: clang++, x32: 'true', x86: 'true', cxxtest: 'true', freestanding: 'false', os: ubuntu-latest, },
+ { pkgs: 'clang-14 lib32gcc-11-dev libx32gcc-11-dev', cc: clang-14, cxx: clang++-14, x32: 'true', x86: 'true', cxxtest: 'true', freestanding: 'false', os: ubuntu-22.04, },
+ { pkgs: 'clang-13 lib32gcc-11-dev libx32gcc-11-dev', cc: clang-13, cxx: clang++-13, x32: 'true', x86: 'true', cxxtest: 'true', freestanding: 'false', os: ubuntu-22.04, },
+ { pkgs: 'clang-12 lib32gcc-11-dev libx32gcc-11-dev', cc: clang-12, cxx: clang++-12, x32: 'true', x86: 'true', cxxtest: 'true', freestanding: 'false', os: ubuntu-22.04, },
+ { pkgs: 'clang-11 lib32gcc-11-dev libx32gcc-11-dev', cc: clang-11, cxx: clang++-11, x32: 'true', x86: 'true', cxxtest: 'true', freestanding: 'false', os: ubuntu-22.04, },
+ { pkgs: 'clang-10 lib32gcc-11-dev libx32gcc-11-dev', cc: clang-10, cxx: clang++-10, x32: 'true', x86: 'true', cxxtest: 'true', freestanding: 'false', os: ubuntu-20.04, },
+ { pkgs: 'clang-9 lib32gcc-11-dev libx32gcc-11-dev', cc: clang-9, cxx: clang++-9, x32: 'true', x86: 'true', cxxtest: 'true', freestanding: 'false', os: ubuntu-20.04, },
+ { pkgs: 'clang-8 lib32gcc-11-dev libx32gcc-11-dev', cc: clang-8, cxx: clang++-8, x32: 'true', x86: 'true', cxxtest: 'true', freestanding: 'false', os: ubuntu-20.04, },
+ { pkgs: 'clang-7 lib32gcc-7-dev libx32gcc-7-dev', cc: clang-7, cxx: clang++-7, x32: 'true', x86: 'true', cxxtest: 'true', freestanding: 'false', os: ubuntu-20.04, },
+ { pkgs: 'clang-6.0 lib32gcc-11-dev libx32gcc-11-dev', cc: clang-6.0, cxx: clang++-6.0, x32: 'true', x86: 'true', cxxtest: 'true', freestanding: 'false', os: ubuntu-20.04, },
+ { pkgs: 'clang-5.0 lib32gcc-11-dev libx32gcc-11-dev', cc: clang-5.0, cxx: clang++-5.0, x32: 'true', x86: 'true', cxxtest: 'true', freestanding: 'false', os: ubuntu-18.04, },
+ { pkgs: 'clang-4.0 lib32gcc-11-dev libx32gcc-11-dev', cc: clang-4.0, cxx: clang++-4.0, x32: 'true', x86: 'true', cxxtest: 'true', freestanding: 'false', os: ubuntu-18.04, },
+ { pkgs: 'clang-3.9', cc: clang-3.9, cxx: clang++-3.9, x32: 'fail', x86: 'fail', cxxtest: 'false', freestanding: 'false', os: ubuntu-18.04, },
+ ]
+
+ runs-on: ${{ matrix.os }}
+ env: # Set environment variables
+ # We globally set CC and CXX to improve compatibility with .travis.yml
+ CC: ${{ matrix.cc }}
+ CXX: ${{ matrix.cxx }}
+ FIXME__LZ4_CI_IGNORE : ' echo Error. But we ignore it for now.'
+ steps:
+ - uses: actions/checkout@v2 # https://github.com/actions/checkout
+
+ - name: apt-get install
+ run: |
+ sudo apt-get update
+ sudo apt-get install gcc-multilib
+ sudo apt-get install ${{ matrix.pkgs }}
+
+ - name: Environment info
+ run: |
+ echo && type $CC && which $CC && $CC --version
+ echo && type $CXX && which $CXX && $CXX --version
+
+ - name: make
+ if: always()
+ run: make V=1
+
+ - name: install test
+ if: always()
+ run: make clean; make V=1 -C tests test-install
+
+ - name: make all
+ if: always()
+ run: make V=1 clean all
+
+ - name: make c_standards (C90)
+ if: always()
+ run: make V=1 clean c_standards_c90
+
+ - name: make c_standards (C11)
+ if: always()
+ run: make V=1 clean c_standards_c11
+
+ - name: make c-to-c++
+ if: always()
+ run: make V=1 clean ctocpptest
+
+ - name: make cxxtest
+ if: ${{ matrix.cxxtest == 'true' }}
+ run: make V=1 clean cxxtest
+
+ - name: make test-freestanding
+ if: ${{ matrix.freestanding == 'true' }}
+ run: make V=1 clean test-freestanding
+
+ - name: make -C programs default
+ if: always()
+ run: make V=1 -C programs clean default
+
+ - name: make -C programs default -D_FORTIFY_SOURCE=2
+ if: always()
+ run: CFLAGS='-fPIC' LDFLAGS='-pie -fPIE -D_FORTIFY_SOURCE=2' make V=1 -C programs clean default
+
+ - name: make -C tests test-lz4
+ if: always()
+ run: make clean; MOREFLAGS='-Werror' make -j V=1 -C tests test-lz4
+
+ - name: make clangtest (clang only)
+ if: ${{ startsWith( matrix.cc , 'clang' ) }}
+ run: make V=1 clean clangtest
+
+ - name: make -C tests test MOREFLAGS='-mx32'
+ if: ${{ matrix.x32 == 'true' }}
+ run: make clean; LDFLAGS='-Wl,--verbose' MOREFLAGS='-mx32' make -j V=1 -C tests test
+
+ - name: make -C tests test-lz4c32
+ if: ${{ matrix.x86 == 'true' }}
+ run: LDFLAGS='-Wl,--verbose' MOREFLAGS='-Werror' make V=1 -C tests clean test-lz4c32
+
+
+ ###############################################################
+ # #
+ # Remove this block when we stabilize the tests. #
+ # #
+
+ - name: make -C tests test MOREFLAGS='-mx32' || echo Ignore failure for now.
+ if: ${{ matrix.x32 == 'fail' }}
+ run: make clean; LDFLAGS='-Wl,--verbose' MOREFLAGS='-mx32' make -j V=1 -C tests test || $FIXME__LZ4_CI_IGNORE
+
+ - name: make -C tests test-lz4c32 || echo Ignore failure for now.
+ if: ${{ matrix.x86 == 'fail' }}
+ run: make clean; LDFLAGS='-Wl,--verbose' MOREFLAGS='-Werror' make V=1 -C tests test-lz4c32 || $FIXME__LZ4_CI_IGNORE
+
+ # #
+ ###############################################################
+
+
+
+###############################################################
+# LZ4 self tests
+#
+# - Benchmark
+# - Fuzzer
+# - LZ4 Frame
+# - LZ4 versions
+# - Custom LZ4_DISTANCE_MAX
+#
+ lz4-benchmark:
+ name: Benchmark
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v2 # https://github.com/actions/checkout
+
+ - name: apt-get install
+ run: |
+ sudo apt-get update
+ sudo apt-get install gcc-multilib
+
+ - name: benchmark (-C tests test-lz4)
+ run: make -j V=1 -C tests test-lz4
+
+ - name: benchmark (-C tests test-lz4c)
+ run: make -j V=1 -C tests test-lz4c
+
+ - name: benchmark (-C tests test-lz4c32)
+ run: make V=1 -C tests test-lz4c32
+
+ - name: benchmark (-C tests test-fullbench)
+ run: make V=1 -C tests test-fullbench
+
+ - name: benchmark (-C tests test-fullbench32)
+ run: make V=1 -C tests test-fullbench32
+
+
+ lz4-fuzzer:
+ name: Fuzzer test
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v2 # https://github.com/actions/checkout
+
+ - name: apt-get install
+ run: |
+ sudo apt-get update
+ sudo apt-get install gcc-multilib
+
+ - name: setup
+ run: sudo sysctl -w vm.mmap_min_addr=4096
+
+ - name: fuzzer
+ run: make V=1 -C tests test-fuzzer
+
+ - name: fuzzer32
+ run: make V=1 -C tests test-fuzzer32
+
+
+ lz4-standard-makefile-variables:
+ name: LZ4 Makefile - support for standard variables
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v2 # https://github.com/actions/checkout
+
+ - name: make standard_variables
+ run: make V=1 standard_variables
+
+
+ lz4-versions:
+ name: LZ4 versions test
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v2 # https://github.com/actions/checkout
+
+ - name: apt-get install
+ run: |
+ sudo apt-get update
+ sudo apt-get install gcc-multilib
+
+ - name: make -C tests versionsTest
+ run: make V=1 -C tests versionsTest
+
+
+ lz4-abi:
+ name: LZ4 inter-versions ABI test
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v2 # https://github.com/actions/checkout
+
+ - name: apt-get install
+ run: |
+ sudo apt-get update
+ sudo apt-get install gcc-multilib
+
+ - name: make -C tests abiTests
+ run: make V=1 -C tests abiTests
+
+
+ lz4-frame:
+ name: LZ4 frame test
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v2 # https://github.com/actions/checkout
+
+ - name: apt-get install
+ run: |
+ sudo apt-get update
+ sudo apt-get install gcc-multilib
+
+ - name: LZ4 frame test
+ run: make V=1 -C tests test-frametest
+
+ - name: LZ4 frame test (32-bit)
+ run: make V=1 -C tests test-frametest32
+
+ lz4-memory-usage:
+ name: test different values of LZ4_MEMORY_USAGE
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v2 # https://github.com/actions/checkout
+ - name: LZ4_MEMORY_USAGE
+ run: make V=1 -C tests test-compile-with-lz4-memory-usage
+
+ # Custom LZ4_DISTANCE_MAX ; lz4-wlib (CLI linked to dynamic library); LZ4_USER_MEMORY_FUNCTIONS
+ lz4-custom-distance:
+ name: Custom LZ4_DISTANCE_MAX
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v2 # https://github.com/actions/checkout
+ - name: custom LZ4_DISTANCE_MAX; test LZ4_USER_MEMORY_FUNCTIONS
+ run: |
+ MOREFLAGS='-DLZ4_DISTANCE_MAX=8000' make V=1 check
+ make V=1 clean
+ make V=1 -C programs lz4-wlib
+ make V=1 clean
+ make V=1 -C tests fullbench-wmalloc # test LZ4_USER_MEMORY_FUNCTIONS
+ make V=1 clean
+ CC="c++ -Wno-deprecated" make V=1 -C tests fullbench-wmalloc # stricter function signature check
+
+ # test block device compression #1086
+ lz4cli-block-device:
+ name: Test lz4 compression on a block device
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v2 # https://github.com/actions/checkout
+ - name: create a block device, compress it with lz4 # alternative : blindly use /dev/loop0, seems to always exist
+ run: |
+ make lz4
+ dd if=/dev/zero of=full0.img bs=2M count=1
+ BLOCK_DEVICE=$(sudo losetup --show -fP full0.img)
+ sudo chmod 666 $BLOCK_DEVICE
+ ./lz4 -v $BLOCK_DEVICE -c > /dev/null
+ sudo losetup -d $BLOCK_DEVICE
+ rm full0.img
+
+
+###############################################################
+# Check tools
+#
+# - cppcheck
+# - scan-build
+# - valgrind
+# - ubsan
+# - asan
+# - unicode-lint
+# - build examples
+#
+ lz4-cppcheck:
+ name: make cppcheck
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v2 # https://github.com/actions/checkout
+ - name: apt-get install
+ run: |
+ sudo apt-get update
+ sudo apt-get install cppcheck
+
+ - name: Environment info
+ run: echo && type cppcheck && which cppcheck && cppcheck --version
+
+ - name: cppcheck
+ # This test script ignores the exit code of cppcheck.
+ # See known issues in README.md.
+ run: make V=1 clean cppcheck || echo There are some cppcheck reports but we ignore it.
+
+
+ lz4-scan-build:
+ name: make staticAnalyze
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v2 # https://github.com/actions/checkout
+ - name: apt-get install
+ run: |
+ sudo apt-get update
+ sudo apt-get install clang-tools
+
+ - name: Environment info
+ run: |
+ echo && type gcc && which gcc && gcc --version
+ echo && type clang && which clang && clang --version
+ echo && type scan-build && which scan-build # scan-build doesn't have any --version equivalent option
+ echo && type make && which make && make -v
+ echo && cat /proc/cpuinfo || echo /proc/cpuinfo is not present
+
+ - name: make staticAnalyze
+ run: make V=1 clean staticAnalyze
+
+
+ lz4-valgrind:
+ name: valgrind
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v2 # https://github.com/actions/checkout
+ - name: apt-get install
+ run: |
+ sudo apt-get update
+ sudo apt-get install valgrind
+
+ - name: Environment info
+ run: |
+ echo && type cc && which cc && cc --version
+ echo && type valgrind && which valgrind && valgrind --version
+
+ - name: valgrind
+ run: make V=1 -C tests test-mem
+
+
+ lz4-ubsan-x64:
+ name: Linux x64 ubsan
+ runs-on: ubuntu-latest
+ env: # Set environment variables
+ FIXME__LZ4_CI_IGNORE : ' echo Error. But we ignore it for now.'
+ steps:
+ - uses: actions/checkout@v2 # https://github.com/actions/checkout
+
+ - name: ubsan
+ #########################################################
+ # For now, we ignore the exit code of `make usan`.
+ # See "Known issues / lz4-ubsan-x64" in README.md
+ # When we'll resolve this issue, remove "|| $FIXME__LZ4_CI_IGNORE"
+ #########################################################
+ run: make V=1 clean usan MOREFLAGS='-Wcomma -Werror' || $FIXME__LZ4_CI_IGNORE
+
+
+ lz4-ubsan-x86:
+ name: Linux x86 ubsan
+ runs-on: ubuntu-latest
+ env: # Set environment variables
+ FIXME__LZ4_CI_IGNORE : ' echo Error. But we ignore it for now.'
+ steps:
+ - uses: actions/checkout@v2 # https://github.com/actions/checkout
+
+ - name: apt-get install
+ run: |
+ sudo apt-get update
+ sudo apt-get install gcc-multilib
+ sudo apt-get install lib32gcc-11-dev
+
+ - name: ubsan32
+ #########################################################
+ # For now, we ignore the exit code of `make usan32`.
+ # See "Known issues / lz4-ubsaan-x86" in README.md.
+ # When we'll resolve this issue, remove "|| $FIXME__LZ4_CI_IGNORE"
+ #########################################################
+ run: CC=clang make V=1 clean usan32 MOREFLAGS='-Wcomma -Werror' || $FIXME__LZ4_CI_IGNORE
+
+
+ lz4-asan-x64:
+ name: Linux x64 ASAN
+ runs-on: ubuntu-latest
+ env: # Set environment variables
+ FIXME__LZ4_CI_IGNORE : ' echo Error. But we ignore it for now.'
+ steps:
+ - uses: actions/checkout@v2 # https://github.com/actions/checkout
+
+ - name: setup
+ run: sudo sysctl -w vm.mmap_min_addr=4096
+
+ - name: frametest
+ run: CC=clang MOREFLAGS=-fsanitize=address make V=1 -C tests clean test-frametest
+
+ - name: fuzzer
+ run: CC=clang MOREFLAGS=-fsanitize=address make V=1 -C tests clean test-fuzzer
+
+ unicode-lint:
+ name: lint unicode in ./lib/, ./tests/ and ./programs/
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v2 # https://github.com/actions/checkout
+ - name: unicode lint
+ run: bash ./tests/unicode_lint.sh
+
+ lz4-examples:
+ name: make examples
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v2 # https://github.com/actions/checkout
+ - name: apt-get install
+ run: |
+ sudo apt-get update
+
+ - name: Environment info
+ run: |
+ echo && type cc && which cc && cc --version
+ echo && type c++ && which c++ && c++ --version
+
+ - name: examples
+ run: make V=1 clean examples
+
+ - name: examples (compile as C++ code)
+ run: make V=1 -C examples clean cxxtest
+
+ # lasts ~20mn
+ oss-fuzz:
+ runs-on: ubuntu-latest
+ strategy:
+ fail-fast: false
+ matrix:
+ sanitizer: [address, undefined, memory]
+ steps:
+ - name: Build Fuzzers (${{ matrix.sanitizer }})
+ id: build
+ uses: google/oss-fuzz/infra/cifuzz/actions/build_fuzzers@master
+ with:
+ oss-fuzz-project-name: 'lz4'
+ dry-run: false
+ sanitizer: ${{ matrix.sanitizer }}
+ - name: Run Fuzzers (${{ matrix.sanitizer }})
+ uses: google/oss-fuzz/infra/cifuzz/actions/run_fuzzers@master
+ with:
+ oss-fuzz-project-name: 'lz4'
+ fuzz-seconds: 600
+ dry-run: false
+ sanitizer: ${{ matrix.sanitizer }}
+ - name: Upload Crash
+ uses: actions/upload-artifact@v1
+ if: failure() && steps.build.outcome == 'success'
+ with:
+ name: ${{ matrix.sanitizer }}-artifacts
+ path: ./out/artifacts
+
+
+
+###############################################################
+# Platforms
+#
+# - QEMU (ARM, ARM64, PPC, PPC64LE, S390X)
+# - macOS
+#
+
+ # QEMU
+ # All tests use QEMU (static) and gcc cross compiler.
+ #
+ # note:
+ # We don't employ completely matrix method which provides `MOREFLAGS`
+ # etc in the matrix. Because some platform may need its special
+ # compiler options and test.
+ # For example, xxHash already has tests for scalar and SIMD version of
+ # it. But compiler options are quite different between platforms.
+ #
+ # So, please keep them simple and independent.
+ #
+ lz4-qemu-platforms:
+ name: QEMU ${{ matrix.type }}
+ strategy:
+ fail-fast: false # 'false' means Don't stop matrix workflows even if some matrix instance failed.
+ matrix:
+ include: [
+ # You can access the following values via ${{ matrix.??? }}
+ # type : Architecture type for `if:` statement.
+ # pkgs : apt-get package names. You can include multiple packages which are delimited by space.
+ # xcc : gcc cross C compiler executable.
+ # xemu : QEMU static emulator executable.
+ # os : GitHub Actions YAML workflow label. See https://github.com/actions/virtual-environments#available-environments
+
+ { type: ARM, pkgs: 'qemu-system-arm gcc-arm-linux-gnueabi', xcc: arm-linux-gnueabi-gcc, xemu: qemu-arm-static, os: ubuntu-latest, },
+ { type: ARM64, pkgs: 'qemu-system-arm gcc-aarch64-linux-gnu', xcc: aarch64-linux-gnu-gcc, xemu: qemu-aarch64-static, os: ubuntu-latest, },
+ { type: PPC, pkgs: 'qemu-system-ppc gcc-powerpc-linux-gnu', xcc: powerpc-linux-gnu-gcc, xemu: qemu-ppc-static, os: ubuntu-latest, },
+ { type: PPC64LE, pkgs: 'qemu-system-ppc gcc-powerpc64le-linux-gnu', xcc: powerpc64le-linux-gnu-gcc, xemu: qemu-ppc64le-static, os: ubuntu-latest, },
+ { type: S390X, pkgs: 'qemu-system-s390x gcc-s390x-linux-gnu', xcc: s390x-linux-gnu-gcc, xemu: qemu-s390x-static, os: ubuntu-latest, },
+ ]
+
+ runs-on: ${{ matrix.os }}
+ env: # Set environment variables
+ XCC: ${{ matrix.xcc }}
+ XEMU: ${{ matrix.xemu }}
+ steps:
+ - uses: actions/checkout@v2 # https://github.com/actions/checkout
+
+ - name: apt-get install
+ run: |
+ sudo apt-get update
+ sudo apt-get install gcc-multilib
+ sudo apt-get install qemu-utils qemu-user-static
+ sudo apt-get install ${{ matrix.pkgs }}
+
+ - name: Environment info
+ run: |
+ echo && type $XCC && which $XCC && $XCC --version
+ echo && $XCC -v # Show built-in specs
+ echo && type $XEMU && which $XEMU && $XEMU --version
+
+ - name: ARM64
+ if: ${{ matrix.type == 'ARM64' }}
+ run: make V=1 platformTest CC=$XCC QEMU_SYS=$XEMU
+
+ - name: ARM
+ if: ${{ matrix.type == 'ARM' }}
+ run: make V=1 platformTest CC=$XCC QEMU_SYS=$XEMU
+
+ - name: PPC
+ if: ${{ matrix.type == 'PPC' }}
+ run: make V=1 platformTest CC=$XCC QEMU_SYS=$XEMU
+
+ - name: PPC64LE
+ if: ${{ matrix.type == 'PPC64LE' }}
+ run: make V=1 platformTest CC=$XCC QEMU_SYS=$XEMU MOREFLAGS=-m64
+
+ - name: S390X
+ if: ${{ matrix.type == 'S390X' }}
+ run: make V=1 platformTest CC=$XCC QEMU_SYS=$XEMU
+
+
+ # macOS
+ lz4-platform-macos-latest:
+ name: macOS
+ runs-on: macos-latest
+ steps:
+ - uses: actions/checkout@v2
+
+ - name: Environment info
+ run: |
+ echo && type cc && which cc && cc --version
+ echo && type make && which make && make -v
+ echo && sysctl -a | grep machdep.cpu # cpuinfo
+
+ - name: make default
+ run: CFLAGS="-Werror" make V=1 clean default
+
+ - name: make test
+ run: make clean; make -j V=1 test MOREFLAGS='-Werror -Wconversion -Wno-sign-conversion'
+
+ - name: Ensure `make test` doesn't depend on the status of the console
+ # see issue #990 for detailed explanations
+ run: make -j test > /dev/null
+
+
+###############################################################
+# Build systems
+#
+# - make
+# - cmake
+# - meson
+#
+
+ # make
+ lz4-build-make:
+ name: make
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v2 # https://github.com/actions/checkout
+
+ - name: Environment info
+ run: |
+ echo && type cc && which cc && cc --version
+ echo && type make && which make && make -v
+
+ - name: make
+ run: make V=1
+
+
+ lz4-build-make-travis-install:
+ name: make travis-install
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v2 # https://github.com/actions/checkout
+
+ - name: travis-install
+ run: make V=1 clean travis-install
+
+ - name: travis-install result
+ run: |
+ echo && echo Installed files
+ ( cd ~/install_test_dir; find .; )
+
+
+ # cmake
+ lz4-build-cmake:
+ name: cmake
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v2 # https://github.com/actions/checkout
+
+ - name: Environment info
+ run: |
+ echo && type cmake && which cmake && cmake --version
+ echo && type make && which make && make -v
+
+ - name: cmake
+ run: |
+ cd build/cmake
+ mkdir build
+ cd build
+ cmake ..
+ CFLAGS=-Werror make VERBOSE=1
+
+
+ # Invoke cmake via Makefile
+ lz4-build-make-cmake:
+ name: make cmake
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v2 # https://github.com/actions/checkout
+ - name: make cmake
+ # V=1 for lz4 Makefile, VERBOSE=1 for cmake Makefile.
+ run: make V=1 VERBOSE=1 clean cmake
+
+
+ # Meson
+ lz4-build-meson:
+ name: Meson + Ninja
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v2 # https://github.com/actions/checkout
+ - uses: actions/setup-python@v2 # https://github.com/actions/setup-python
+ with:
+ python-version: '3.x'
+
+ - name: Install
+ run: |
+ sudo apt-get update
+ sudo apt-get install tree ninja-build
+ python -m pip install --upgrade pip
+ pip3 install --user meson
+
+ - name: Environment info
+ run: |
+ echo && type clang && which clang && clang --version
+ echo && type python && which python && python --version
+ echo && type meson && which meson && meson --version
+
+ - name: meson
+ # 'run: >' replaces all newlines in the following block with spaces
+ run: >
+ meson setup
+ --buildtype=debug
+ -Db_lundef=false
+ -Dauto_features=enabled
+ -Dprograms=true
+ -Dcontrib=true
+ -Dtests=true
+ -Dexamples=true
+ contrib/meson build
+
+ - name: staging
+ run: |
+ cd build
+ DESTDIR=./staging ninja install
+ tree ./staging
+
+
+
+############################################################
+# Check git tag for LZ4 releases
+#
+ lz4-check-tag:
+ name: git version tag checking for release
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v2
+ - name: make -C tests checkTag
+ if: startsWith(github.ref, 'refs/tags/v') # If git tag name starts with 'v'
+ run: |
+ echo "tag=${GITHUB_REF#refs/*/}"
+ make -C tests checkTag
+ tests/checkTag ${GITHUB_REF#refs/*/}
+
+
+
+############################################################
+# Gather CI environment information.
+#
+ lz4-env-info:
+ name: GH-Actions Virtual Env Info (${{ matrix.os }})
+ strategy:
+ matrix:
+ include: [
+ { os: ubuntu-latest, }, # https://github.com/actions/virtual-environments/
+ { os: ubuntu-22.04, }, # https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu2204-Readme.md
+ { os: ubuntu-20.04, }, # https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu2004-Readme.md
+ { os: ubuntu-18.04, }, # https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu1804-Readme.md
+ ]
+
+ runs-on: ${{ matrix.os }}
+ steps:
+ - uses: actions/checkout@v2
+
+ - name: init
+ run: |
+ sudo apt-get update
+
+ - name: cc --version
+ run: echo && type cc && which cc && cc --version
+
+ - name: gcc --version
+ run: echo && type gcc && which gcc && gcc --version
+
+ - name: clang --version
+ run: echo && type clang && which clang && clang --version
+
+ - name: make -v
+ run: echo && type make && which make && make -v
+
+ - name: g++ --version
+ run: echo && type g++ && which g++ && g++ --version
+
+ - name: git --version
+ run: echo && type git && which git && git --version
+
+ - name: gcc packages (apt-cache)
+ run: apt-cache search gcc | grep "^gcc-[0-9\.]* " | sort
+
+ - name: lib32gcc packages for i386 (apt-cache)
+ run: apt-cache search lib32gcc | grep "^lib32gcc-" | sort
+
+ - name: libx32gcc packages for x32 (apt-cache)
+ run: apt-cache search libx32gcc | grep "^libx32gcc-" | sort
+
+ - name: gcc multilib packages (apt-cache)
+ run: apt-cache search multilib | grep "gcc-" | sort
+
+ - name: clang packages (apt-cache)
+ run: apt-cache search clang | grep "^clang-[0-9\.]* " | sort
+
+ - name: QEMU packages (apt-cache)
+ run: apt-cache search qemu | grep "^qemu-system-.*QEMU full system" | sort
diff --git a/.gitignore b/.gitignore
index d7ba96e..ed02057 100644
--- a/.gitignore
+++ b/.gitignore
@@ -25,6 +25,7 @@
_codelite_lz4/
bin/
*.zip
+*.swp
# analyzers
infer-out
@@ -37,5 +38,6 @@
nul
ld.exe*
-# test files
+# test artifacts
*.lz4
+tmp*
diff --git a/.travis.yml b/.travis.yml
index f201d52..0aeea6e 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -3,125 +3,6 @@
matrix:
fast_finish: true
include:
- # OS X Mavericks
- - name: (macOS) General Test
- os: osx
- compiler: clang
- script:
- - make # test library build
- - make clean
- - make test MOREFLAGS='-Werror -Wconversion -Wno-sign-conversion' | tee # test scenario where `stdout` is not the console
-
- # Container-based 12.04 LTS Server Edition 64 bit (doesn't support 32-bit includes)
- - name: (Precise) benchmark test
- dist: precise
- script:
- - make -C tests test-lz4 test-lz4c test-fullbench
-
- - name: (Precise) frame and fuzzer test
- dist: precise
- install:
- - sudo sysctl -w vm.mmap_min_addr=4096
- script:
- - make -C tests test-frametest test-fuzzer
-
- - name: ASAN tests with fuzzer and frametest
- install:
- - sudo sysctl -w vm.mmap_min_addr=4096
- script:
- - CC=clang MOREFLAGS=-fsanitize=address make -C tests test-frametest test-fuzzer
-
- - name: Custom LZ4_DISTANCE_MAX ; lz4-wlib (CLI linked to dynamic library); LZ4_USER_MEMORY_FUNCTIONS
- script:
- - MOREFLAGS=-DLZ4_DISTANCE_MAX=8000 make check
- - make clean
- - make -C programs lz4-wlib
- - make clean
- - make -C tests fullbench-wmalloc # test LZ4_USER_MEMORY_FUNCTIONS
- - make clean
- - CC="c++ -Wno-deprecated" make -C tests fullbench-wmalloc # stricter function signature check
-
- - name: (Precise) g++ and clang CMake test
- dist: precise
- script:
- - make gpptest
- - make clean
- - make examples
- - make clean cmake
- - make clean travis-install
- - make clean clangtest
-
- - name: x32 compatibility test
- addons:
- apt:
- packages:
- - gcc-multilib
- script:
- - make -C tests test MOREFLAGS=-mx32
-
- # 14.04 LTS Server Edition 64 bit
- # presume clang >= v3.9.0
- - name: (Trusty) USan test
- dist: trusty
- compiler: clang
- script:
- - make usan MOREFLAGS=-Wcomma -Werror
-
- - name: (Trusty) valgrind test
- dist: trusty
- install:
- - sudo apt-get install -qq valgrind
- script:
- - make c_standards
- - make -C tests test-lz4 test-mem
-
- - name: (Trusty) c-to-c++ test
- dist: trusty
- script:
- - make ctocpptest
-
- - name: (Trusty) i386 benchmark + version test
- dist: trusty
- install:
- - sudo apt-get install -qq python3 libc6-dev-i386 gcc-multilib
- script:
- - make -C tests test-lz4c32 test-fullbench32 versionsTest
-
- - name: (Trusty) i386 frame + fuzzer test
- dist: trusty
- install:
- - sudo apt-get install -qq libc6-dev-i386 gcc-multilib
- - sudo sysctl -w vm.mmap_min_addr=4096
- script:
- - make -C tests test-frametest32 test-fuzzer32
-
- - name: (Trusty) gcc-6 standard C compilation
- dist: trusty
- addons:
- apt:
- sources:
- - ubuntu-toolchain-r-test
- packages:
- - gcc-6
- env:
- - CC=gcc-6
- script:
- - make c_standards
- - make -C tests test-lz4 MOREFLAGS=-Werror
-
- - name: (Trusty) arm + aarch64 compilation
- dist: trusty
- install:
- - sudo apt-get install -qq
- qemu-system-arm
- qemu-user-static
- gcc-arm-linux-gnueabi
- libc6-dev-armel-cross
- gcc-aarch64-linux-gnu
- libc6-dev-arm64-cross
- script:
- - make platformTest CC=arm-linux-gnueabi-gcc QEMU_SYS=qemu-arm-static
- - make platformTest CC=aarch64-linux-gnu-gcc QEMU_SYS=qemu-aarch64-static
- name: aarch64 real-hw tests
arch: arm64
@@ -138,57 +19,6 @@
script:
- make test
- - name: (Xenial) gcc-5 compilation
- dist: xenial
- install:
- - sudo apt-get install -qq libc6-dev-i386 gcc-multilib
- script:
- - make -C tests test-lz4 clean test-lz4c32 MOREFLAGS=-Werror
-
- - name: (Trusty) clang-3.8 compilation
- dist: trusty
- addons:
- apt:
- sources:
- - ubuntu-toolchain-r-test
- - llvm-toolchain-precise-3.8
- packages:
- - clang-3.8
- script:
- - make -C tests test-lz4 CC=clang-3.8
-
- - name: (Trusty) PowerPC + PPC64 compilation
- dist: trusty
- install:
- - sudo apt-get install -qq qemu-system-ppc qemu-user-static gcc-powerpc-linux-gnu
- script:
- - make platformTest CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc-static
- - make platformTest CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc64-static MOREFLAGS=-m64
-
- - name: (Trusty) scan-build + cppcheck
- dist: trusty
- compiler: clang
- install:
- - sudo apt-get install -qq cppcheck
- script:
- - make staticAnalyze
- - make cppcheck
-
- - name: (Trusty) gcc-4.4 compilation
- dist: trusty
- addons:
- apt:
- sources:
- - ubuntu-toolchain-r-test
- packages:
- - libc6-dev-i386
- - gcc-multilib
- - gcc-4.4
- script:
- - make clean all CC=gcc-4.4 MOREFLAGS=-Werror
- - make clean
- - CFLAGS=-fPIC LDFLAGS='-pie -fPIE -D_FORTIFY_SOURCE=2' make -C programs
-
# tag-specific test
- name: tag build
if: tag =~ ^v[0-9]\.[0-9]
@@ -197,40 +27,16 @@
- make -C tests checkTag
- tests/checkTag "$TRAVIS_BRANCH"
- - name: (Xenial) Meson + clang build
- #env: ALLOW_FAILURES=true
- dist: xenial
- language: cpp
- compiler: clang
- install:
- - sudo apt-get install -qq python3 tree
- - |
- travis_retry curl -o ~/ninja.zip -L 'https://github.com/ninja-build/ninja/releases/download/v1.9.0/ninja-linux.zip' &&
- unzip ~/ninja.zip -d ~/.local/bin
- - |
- travis_retry curl -o ~/get-pip.py 'https://bootstrap.pypa.io/get-pip.py' &&
- python3 ~/get-pip.py --user &&
- pip3 install --user meson
- script:
- - |
- meson setup \
- --buildtype=debug \
- -Db_lundef=false \
- -Dauto_features=enabled \
- -Ddefault_library=both \
- -Dbin_programs=true \
- -Dbin_contrib=true \
- -Dbin_tests=true \
- -Dbin_examples=true \
- contrib/meson build
- - pushd build
- - DESTDIR=./staging ninja install
- - tree ./staging
-
# oss-fuzz compilation test
- name: Compile OSS-Fuzz targets
script:
- ./ossfuzz/travisoss.sh
+ # Unicode lint
+ # See https://github.com/lz4/lz4/issues/1018
+ - name: Run Unicode lint
+ script:
+ - ./tests/unicode_lint.sh
+
allow_failures:
- env: ALLOW_FAILURES=true
diff --git a/LICENSE b/LICENSE
index c221aeb..1b84cc3 100644
--- a/LICENSE
+++ b/LICENSE
@@ -8,4 +8,5 @@
This model is selected to emphasize that
files in the `lib` directory are designed to be included into 3rd party applications,
while all other files, in `programs`, `tests` or `examples`,
-receive more limited attention and support for such scenario.
+are intended to be used "as is", as part of their intended scenarios,
+with no intention to support 3rd party integration use cases.
diff --git a/Makefile b/Makefile
index 744005f..e70c3db 100644
--- a/Makefile
+++ b/Makefile
@@ -1,6 +1,6 @@
# ################################################################
# LZ4 - Makefile
-# Copyright (C) Yann Collet 2011-present
+# Copyright (C) Yann Collet 2011-2020
# All rights reserved.
#
# BSD license
@@ -38,9 +38,13 @@
include Makefile.inc
+
.PHONY: default
default: lib-release lz4-release
+# silent mode by default; verbose can be triggered by V=1 or VERBOSE=1
+$(V)$(VERBOSE).SILENT:
+
.PHONY: all
all: allmost examples manuals build_tests
@@ -50,14 +54,14 @@
.PHONY: lib lib-release liblz4.a
lib: liblz4.a
lib lib-release liblz4.a:
- @$(MAKE) -C $(LZ4DIR) $@
+ $(MAKE) -C $(LZ4DIR) $@
.PHONY: lz4 lz4-release
lz4 : liblz4.a
lz4-release : lib-release
lz4 lz4-release :
- @$(MAKE) -C $(PRGDIR) $@
- @cp $(PRGDIR)/lz4$(EXT) .
+ $(MAKE) -C $(PRGDIR) $@
+ cp $(PRGDIR)/lz4$(EXT) .
.PHONY: examples
examples: liblz4.a
@@ -65,58 +69,64 @@
.PHONY: manuals
manuals:
- @$(MAKE) -C contrib/gen_manual $@
+ $(MAKE) -C contrib/gen_manual $@
.PHONY: build_tests
build_tests:
- @$(MAKE) -C $(TESTDIR) all
+ $(MAKE) -C $(TESTDIR) all
.PHONY: clean
clean:
- @$(MAKE) -C $(LZ4DIR) $@ > $(VOID)
- @$(MAKE) -C $(PRGDIR) $@ > $(VOID)
- @$(MAKE) -C $(TESTDIR) $@ > $(VOID)
- @$(MAKE) -C $(EXDIR) $@ > $(VOID)
- @$(MAKE) -C $(FUZZDIR) $@ > $(VOID)
- @$(MAKE) -C contrib/gen_manual $@ > $(VOID)
- @$(RM) lz4$(EXT)
+ $(MAKE) -C $(LZ4DIR) $@ > $(VOID)
+ $(MAKE) -C $(PRGDIR) $@ > $(VOID)
+ $(MAKE) -C $(TESTDIR) $@ > $(VOID)
+ $(MAKE) -C $(EXDIR) $@ > $(VOID)
+ $(MAKE) -C $(FUZZDIR) $@ > $(VOID)
+ $(MAKE) -C contrib/gen_manual $@ > $(VOID)
+ $(RM) lz4$(EXT)
+ $(RM) -r $(CMAKE_BUILD_DIR)
@echo Cleaning completed
#-----------------------------------------------------------------------------
-# make install is validated only for Linux, OSX, BSD, Hurd and Solaris targets
+# make install is validated only for Posix environments
#-----------------------------------------------------------------------------
ifeq ($(POSIX_ENV),Yes)
HOST_OS = POSIX
.PHONY: install uninstall
install uninstall:
- @$(MAKE) -C $(LZ4DIR) $@
- @$(MAKE) -C $(PRGDIR) $@
+ $(MAKE) -C $(LZ4DIR) $@
+ $(MAKE) -C $(PRGDIR) $@
+.PHONY: travis-install
travis-install:
$(MAKE) -j1 install DESTDIR=~/install_test_dir
-cmake:
- @cd build/cmake; cmake $(CMAKE_PARAMS) CMakeLists.txt; $(MAKE)
-
-endif
+endif # POSIX_ENV
-ifneq (,$(filter MSYS%,$(shell uname)))
+CMAKE ?= cmake
+CMAKE_BUILD_DIR ?= build/cmake/build
+ifneq (,$(filter MSYS%,$(shell $(UNAME))))
HOST_OS = MSYS
CMAKE_PARAMS = -G"MSYS Makefiles"
endif
+.PHONY: cmake
+cmake:
+ mkdir -p $(CMAKE_BUILD_DIR)
+ cd $(CMAKE_BUILD_DIR); $(CMAKE) $(CMAKE_PARAMS) ..; $(CMAKE) --build .
+
#------------------------------------------------------------------------
-#make tests validated only for MSYS, Linux, OSX, kFreeBSD and Hurd targets
+# make tests validated only for MSYS and Posix environments
#------------------------------------------------------------------------
ifneq (,$(filter $(HOST_OS),MSYS POSIX))
.PHONY: list
list:
- @$(MAKE) -pRrq -f $(lastword $(MAKEFILE_LIST)) : 2>/dev/null | awk -v RS= -F: '/^# File/,/^# Finished Make data base/ {if ($$1 !~ "^[#.]") {print $$1}}' | sort | egrep -v -e '^[^[:alnum:]]' -e '^$@$$' | xargs
+ $(MAKE) -pRrq -f $(lastword $(MAKEFILE_LIST)) : 2>/dev/null | awk -v RS= -F: '/^# File/,/^# Finished Make data base/ {if ($$1 !~ "^[#.]") {print $$1}}' | sort | egrep -v -e '^[^[:alnum:]]' -e '^$@$$' | xargs
.PHONY: check
check:
@@ -124,32 +134,38 @@
.PHONY: test
test:
- CFLAGS="$(CFLAGS)" LDFLAGS="$(LDFLAGS)" $(MAKE) -C $(TESTDIR) $@
- CFLAGS="$(CFLAGS)" LDFLAGS="$(LDFLAGS)" $(MAKE) -C $(EXDIR) $@
+ $(MAKE) -C $(TESTDIR) $@
+ $(MAKE) -C $(EXDIR) $@
-clangtest: CFLAGS ?= -O3
+.PHONY: clangtest
clangtest: CFLAGS += -Werror -Wconversion -Wno-sign-conversion
clangtest: CC = clang
clangtest: clean
$(CC) -v
- @CFLAGS="$(CFLAGS)" $(MAKE) -C $(LZ4DIR) all CC=$(CC)
- @CFLAGS="$(CFLAGS)" $(MAKE) -C $(PRGDIR) all CC=$(CC)
- @CFLAGS="$(CFLAGS)" $(MAKE) -C $(TESTDIR) all CC=$(CC)
+ $(MAKE) -C $(LZ4DIR) all CC=$(CC)
+ $(MAKE) -C $(PRGDIR) all CC=$(CC)
+ $(MAKE) -C $(TESTDIR) all CC=$(CC)
+.PHONY: clangtest-native
+clangtest-native: CFLAGS = -O3 -Werror -Wconversion -Wno-sign-conversion
clangtest-native: clean
clang -v
- @CFLAGS="-O3 -Werror -Wconversion -Wno-sign-conversion" $(MAKE) -C $(LZ4DIR) all CC=clang
- @CFLAGS="-O3 -Werror -Wconversion -Wno-sign-conversion" $(MAKE) -C $(PRGDIR) native CC=clang
- @CFLAGS="-O3 -Werror -Wconversion -Wno-sign-conversion" $(MAKE) -C $(TESTDIR) native CC=clang
+ $(MAKE) -C $(LZ4DIR) all CC=clang
+ $(MAKE) -C $(PRGDIR) native CC=clang
+ $(MAKE) -C $(TESTDIR) native CC=clang
+.PHONY: usan
usan: CC = clang
usan: CFLAGS = -O3 -g -fsanitize=undefined -fno-sanitize-recover=undefined -fsanitize-recover=pointer-overflow
usan: LDFLAGS = $(CFLAGS)
usan: clean
- CFLAGS="$(CFLAGS)" LDFLAGS="$(LDFLAGS)" $(MAKE) test FUZZER_TIME="-T30s" NB_LOOPS=-i1
+ CC=$(CC) CFLAGS='$(CFLAGS)' LDFLAGS='$(LDFLAGS)' $(MAKE) test FUZZER_TIME="-T30s" NB_LOOPS=-i1
+.PHONY: usan32
+usan32: CFLAGS = -m32 -O3 -g -fsanitize=undefined
+usan32: LDFLAGS = $(CFLAGS)
usan32: clean
- CFLAGS="-m32 -O3 -g -fsanitize=undefined" $(MAKE) test FUZZER_TIME="-T30s" NB_LOOPS=-i1
+ $(MAKE) test FUZZER_TIME="-T30s" NB_LOOPS=-i1
SCANBUILD ?= scan-build
SCANBUILD_FLAGS += --status-bugs -v --force-analyze-debug-code
@@ -161,9 +177,10 @@
cppcheck:
cppcheck . --force --enable=warning,portability,performance,style --error-exitcode=1 > /dev/null
+.PHONY: platformTest
platformTest: clean
@echo "\n ---- test lz4 with $(CC) compiler ----"
- @$(CC) -v
+ $(CC) -v
CFLAGS="-O3 -Werror" $(MAKE) -C $(LZ4DIR) all
CFLAGS="-O3 -Werror -static" $(MAKE) -C $(PRGDIR) all
CFLAGS="-O3 -Werror -static" $(MAKE) -C $(TESTDIR) all
@@ -173,15 +190,21 @@
versionsTest: clean
$(MAKE) -C $(TESTDIR) $@
-gpptest gpptest32: CC = "$(CXX) -Wno-deprecated"
-gpptest gpptest32: CFLAGS = -O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror
-gpptest32: CFLAGS += -m32
-gpptest gpptest32: clean
+.PHONY: test-freestanding
+test-freestanding:
+ $(MAKE) -C $(TESTDIR) clean $@
+
+.PHONY: cxxtest cxx32test
+cxxtest cxx32test: CC := "$(CXX) -Wno-deprecated"
+cxxtest cxx32test: CFLAGS = -O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror
+cxx32test: CFLAGS += -m32
+cxxtest cxx32test: clean
$(CXX) -v
CC=$(CC) $(MAKE) -C $(LZ4DIR) all CFLAGS="$(CFLAGS)"
CC=$(CC) $(MAKE) -C $(PRGDIR) all CFLAGS="$(CFLAGS)"
CC=$(CC) $(MAKE) -C $(TESTDIR) all CFLAGS="$(CFLAGS)"
+.PHONY: cxx17build
cxx17build : CC = "$(CXX) -Wno-deprecated"
cxx17build : CFLAGS = -std=c++17 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror -pedantic
cxx17build : clean
@@ -190,19 +213,67 @@
CC=$(CC) $(MAKE) -C $(PRGDIR) all CFLAGS="$(CFLAGS)"
CC=$(CC) $(MAKE) -C $(TESTDIR) all CFLAGS="$(CFLAGS)"
+.PHONY: ctocpptest
ctocpptest: LIBCC="$(CC)"
ctocpptest: TESTCC="$(CXX)"
-ctocpptest: CFLAGS=""
+ctocpptest: CFLAGS=
ctocpptest: clean
CC=$(LIBCC) $(MAKE) -C $(LZ4DIR) CFLAGS="$(CFLAGS)" all
CC=$(LIBCC) $(MAKE) -C $(TESTDIR) CFLAGS="$(CFLAGS)" lz4.o lz4hc.o lz4frame.o
CC=$(TESTCC) $(MAKE) -C $(TESTDIR) CFLAGS="$(CFLAGS)" all
-c_standards: clean
+.PHONY: c_standards
+c_standards: clean c_standards_c11 c_standards_c99 c_standards_c90
+
+.PHONY: c_standards_c90
+c_standards_c90: clean
$(MAKE) clean; CFLAGS="-std=c90 -Werror -pedantic -Wno-long-long -Wno-variadic-macros" $(MAKE) allmost
$(MAKE) clean; CFLAGS="-std=gnu90 -Werror -pedantic -Wno-long-long -Wno-variadic-macros" $(MAKE) allmost
+
+.PHONY: c_standards_c99
+c_standards_c99: clean
$(MAKE) clean; CFLAGS="-std=c99 -Werror -pedantic" $(MAKE) all
$(MAKE) clean; CFLAGS="-std=gnu99 -Werror -pedantic" $(MAKE) all
- $(MAKE) clean; CFLAGS="-std=c11 -Werror" $(MAKE) all
-endif
+.PHONY: c_standards_c11
+c_standards_c11: clean
+ $(MAKE) clean; CFLAGS="-std=c11 -Werror" $(MAKE) all
+
+# The following test ensures that standard Makefile variables set through environment
+# are correctly transmitted at compilation stage.
+# This test is meant to detect issues like https://github.com/lz4/lz4/issues/958
+.PHONY: standard_variables
+standard_variables: clean
+ @echo =================
+ @echo Check support of Makefile Standard variables through environment
+ @echo note : this test requires V=1 to work properly
+ @echo =================
+ CC="cc -DCC_TEST" \
+ CFLAGS=-DCFLAGS_TEST \
+ CPPFLAGS=-DCPPFLAGS_TEST \
+ LDFLAGS=-DLDFLAGS_TEST \
+ LDLIBS=-DLDLIBS_TEST \
+ $(MAKE) V=1 > tmpsv
+ # Note: just checking the presence of custom flags
+ # would not detect situations where custom flags are
+ # supported in some part of the Makefile, and missed in others.
+ # So the test checks if they are present the _right nb of times_.
+ # However, checking static quantities makes this test brittle,
+ # because quantities (7, 2 and 1) can still evolve in future,
+ # for example when source directories or Makefile evolve.
+ if [ $$(grep CC_TEST tmpsv | wc -l) -ne 7 ]; then \
+ echo "CC environment variable missed" && False; fi
+ if [ $$(grep CFLAGS_TEST tmpsv | wc -l) -ne 7 ]; then \
+ echo "CFLAGS environment variable missed" && False; fi
+ if [ $$(grep CPPFLAGS_TEST tmpsv | wc -l) -ne 7 ]; then \
+ echo "CPPFLAGS environment variable missed" && False; fi
+ if [ $$(grep LDFLAGS_TEST tmpsv | wc -l) -ne 2 ]; then \
+ echo "LDFLAGS environment variable missed" && False; fi
+ if [ $$(grep LDLIBS_TEST tmpsv | wc -l) -ne 1 ]; then \
+ echo "LDLIBS environment variable missed" && False; fi
+ @echo =================
+ @echo all custom variables detected
+ @echo =================
+ $(RM) tmpsv
+
+endif # MSYS POSIX
diff --git a/Makefile.inc b/Makefile.inc
index 2d64405..e78298c 100644
--- a/Makefile.inc
+++ b/Makefile.inc
@@ -1,10 +1,38 @@
-ifeq ($(V), 1)
-Q =
-else
-Q = @
-endif
+# ################################################################
+# LZ4 - Makefile common definitions
+# Copyright (C) Yann Collet 2020
+# All rights reserved.
+#
+# BSD license
+# Redistribution and use in source and binary forms, with or without modification,
+# are permitted provided that the following conditions are met:
+#
+# * Redistributions of source code must retain the above copyright notice, this
+# list of conditions and the following disclaimer.
+#
+# * Redistributions in binary form must reproduce the above copyright notice, this
+# list of conditions and the following disclaimer in the documentation and/or
+# other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+# You can contact the author at :
+# - LZ4 source repository : https://github.com/lz4/lz4
+# - LZ4 forum froup : https://groups.google.com/forum/#!forum/lz4c
+# ################################################################
-TARGET_OS ?= $(shell uname)
+UNAME ?= uname
+
+TARGET_OS ?= $(shell $(UNAME))
ifeq ($(TARGET_OS),)
TARGET_OS ?= $(OS)
endif
@@ -41,7 +69,7 @@
endif
#determine if dev/nul based on host environment
-ifneq (,$(filter MINGW% MSYS% CYGWIN%,$(shell uname)))
+ifneq (,$(filter MINGW% MSYS% CYGWIN%,$(shell $(UNAME))))
VOID := /dev/null
else
ifneq (,$(filter Windows%,$(OS)))
@@ -51,32 +79,28 @@
endif
endif
-ifneq (,$(filter Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS Haiku MidnightBSD MINGW% CYGWIN% MSYS%,$(shell uname)))
+ifneq (,$(filter Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS Haiku MidnightBSD MINGW% CYGWIN% MSYS%,$(shell $(UNAME))))
POSIX_ENV = Yes
else
POSIX_ENV = No
endif
-# Avoid symlinks when targetting Windows or building on a Windows host
+# Avoid symlinks when targeting Windows or building on a Windows host
ifeq ($(WINBASED),yes)
-LN_S = cp -p
LN_SF = cp -p
else
- ifneq (,$(filter MINGW% MSYS% CYGWIN%,$(shell uname)))
-LN_S = cp -p
+ ifneq (,$(filter MINGW% MSYS% CYGWIN%,$(shell $(UNAME))))
LN_SF = cp -p
else
ifneq (,$(filter Windows%,$(OS)))
-LN_S = cp -p
LN_SF = cp -p
else
-LN_S = ln -s
LN_SF = ln -sf
endif
endif
endif
-ifneq (,$(filter $(shell uname),SunOS))
+ifneq (,$(filter $(shell $(UNAME)),SunOS))
INSTALL ?= ginstall
else
INSTALL ?= install
diff --git a/NEWS b/NEWS
index 401931e..0a56992 100644
--- a/NEWS
+++ b/NEWS
@@ -1,3 +1,24 @@
+v1.9.4
+perf : faster decoding speed (~+20%) on aarch64 platforms
+perf : faster decoding speed (~+70%) for -BD4 setting in CLI
+api : new function `LZ4_decompress_safe_partial_usingDict()` by @yawqi
+api : lz4frame: ability to provide custom allocators at state creation
+api : can skip checksum validation for improved decoding speed
+api : new experimental unit `lz4file` for file i/o API, by @anjiahao1
+api : new experimental function `LZ4F_uncompressedUpdate()`, by @alexmohr
+cli : `--list` works on `stdin` input, by @Low-power
+cli : `--no-crc` does not produce (compression) nor check (decompression) checksums
+cli : fix: `--test` and `--list` produce an error code when parsing invalid input
+cli : fix: support skippable frames when passed via `stdin`, reported by @davidmankin
+build: fix: Makefile respects CFLAGS directives passed via environment variable
+build: `LZ4_FREESTANDING`, new build macro for freestanding environments, by @t-mat
+build: `make` and `make test` are compatible with `-j` parallel run
+build: AS/400 compatibility, by @jonrumsey
+build: Solaris 10 compatibility, by @pekdon
+build: MSVC 2022 support, by @t-mat
+build: improved meson script, by @eli-schwartz
+doc : Updated LZ4 block format, provide an "implementation notes" section
+
v1.9.3
perf: highly improved speed in kernel space, by @terrelln
perf: faster speed with Visual Studio, thanks to @wolfpld and @remittor
diff --git a/build/VS2010/liblz4-dll/liblz4-dll.rc b/build/VS2010/liblz4-dll/liblz4-dll.rc
index b1871fe..e089c24 100644
--- a/build/VS2010/liblz4-dll/liblz4-dll.rc
+++ b/build/VS2010/liblz4-dll/liblz4-dll.rc
@@ -36,7 +36,7 @@
VALUE "FileDescription", "Extremely fast compression"
VALUE "FileVersion", LZ4_VERSION_STRING
VALUE "InternalName", "lz4.dll"
- VALUE "LegalCopyright", "Copyright (C) 2013-2016, Yann Collet"
+ VALUE "LegalCopyright", "Copyright (C) 2013-2020, Yann Collet"
VALUE "OriginalFilename", "lz4.dll"
VALUE "ProductName", "LZ4"
VALUE "ProductVersion", LZ4_VERSION_STRING
diff --git a/build/VS2010/lz4/lz4.rc b/build/VS2010/lz4/lz4.rc
index c593edf..5eec36b 100644
--- a/build/VS2010/lz4/lz4.rc
+++ b/build/VS2010/lz4/lz4.rc
@@ -36,7 +36,7 @@
VALUE "FileDescription", "Extremely fast compression"
VALUE "FileVersion", LZ4_VERSION_STRING
VALUE "InternalName", "lz4.exe"
- VALUE "LegalCopyright", "Copyright (C) 2013-2016, Yann Collet"
+ VALUE "LegalCopyright", "Copyright (C) 2013-2020, Yann Collet"
VALUE "OriginalFilename", "lz4.exe"
VALUE "ProductName", "LZ4"
VALUE "ProductVersion", LZ4_VERSION_STRING
diff --git a/build/VS2017/liblz4-dll/liblz4-dll.rc b/build/VS2017/liblz4-dll/liblz4-dll.rc
index b1871fe..e089c24 100644
--- a/build/VS2017/liblz4-dll/liblz4-dll.rc
+++ b/build/VS2017/liblz4-dll/liblz4-dll.rc
@@ -36,7 +36,7 @@
VALUE "FileDescription", "Extremely fast compression"
VALUE "FileVersion", LZ4_VERSION_STRING
VALUE "InternalName", "lz4.dll"
- VALUE "LegalCopyright", "Copyright (C) 2013-2016, Yann Collet"
+ VALUE "LegalCopyright", "Copyright (C) 2013-2020, Yann Collet"
VALUE "OriginalFilename", "lz4.dll"
VALUE "ProductName", "LZ4"
VALUE "ProductVersion", LZ4_VERSION_STRING
diff --git a/build/VS2017/lz4/lz4.rc b/build/VS2017/lz4/lz4.rc
index c593edf..5eec36b 100644
--- a/build/VS2017/lz4/lz4.rc
+++ b/build/VS2017/lz4/lz4.rc
@@ -36,7 +36,7 @@
VALUE "FileDescription", "Extremely fast compression"
VALUE "FileVersion", LZ4_VERSION_STRING
VALUE "InternalName", "lz4.exe"
- VALUE "LegalCopyright", "Copyright (C) 2013-2016, Yann Collet"
+ VALUE "LegalCopyright", "Copyright (C) 2013-2020, Yann Collet"
VALUE "OriginalFilename", "lz4.exe"
VALUE "ProductName", "LZ4"
VALUE "ProductVersion", LZ4_VERSION_STRING
diff --git a/build/VS2017/lz4/lz4.vcxproj b/build/VS2017/lz4/lz4.vcxproj
index b4fed24..f16c1ec 100644
--- a/build/VS2017/lz4/lz4.vcxproj
+++ b/build/VS2017/lz4/lz4.vcxproj
@@ -80,6 +80,17 @@
<IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
<LinkIncremental>false</LinkIncremental>
</PropertyGroup>
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
+ <OutDir>$(SolutionDir)bin\$(Platform)_$(Configuration)\</OutDir>
+ <IntDir>$(SolutionDir)bin\obj\$(RootNamespace)_$(Platform)_$(Configuration)\</IntDir>
+ <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+ </PropertyGroup>
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+ <OutDir>$(SolutionDir)bin\$(Platform)_$(Configuration)\</OutDir>
+ <IntDir>$(SolutionDir)bin\obj\$(RootNamespace)_$(Platform)_$(Configuration)\</IntDir>
+ <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+ <LinkIncremental>false</LinkIncremental>
+ </PropertyGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<ClCompile>
<WarningLevel>Level4</WarningLevel>
@@ -161,4 +172,4 @@
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
<ImportGroup Label="ExtensionTargets">
</ImportGroup>
-</Project>
\ No newline at end of file
+</Project>
diff --git a/build/VS2022/datagen/datagen.vcxproj b/build/VS2022/datagen/datagen.vcxproj
new file mode 100644
index 0000000..69034d4
--- /dev/null
+++ b/build/VS2022/datagen/datagen.vcxproj
@@ -0,0 +1,173 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project DefaultTargets="Build" ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+ <ItemGroup Label="ProjectConfigurations">
+ <ProjectConfiguration Include="Debug|Win32">
+ <Configuration>Debug</Configuration>
+ <Platform>Win32</Platform>
+ </ProjectConfiguration>
+ <ProjectConfiguration Include="Debug|x64">
+ <Configuration>Debug</Configuration>
+ <Platform>x64</Platform>
+ </ProjectConfiguration>
+ <ProjectConfiguration Include="Release|Win32">
+ <Configuration>Release</Configuration>
+ <Platform>Win32</Platform>
+ </ProjectConfiguration>
+ <ProjectConfiguration Include="Release|x64">
+ <Configuration>Release</Configuration>
+ <Platform>x64</Platform>
+ </ProjectConfiguration>
+ </ItemGroup>
+ <PropertyGroup Label="Globals">
+ <ProjectGuid>{D745AE2F-596A-403A-9B91-81A8C6779243}</ProjectGuid>
+ <Keyword>Win32Proj</Keyword>
+ <RootNamespace>datagen</RootNamespace>
+ <OutDir>$(SolutionDir)bin\$(Platform)_$(Configuration)\</OutDir>
+ <IntDir>$(SolutionDir)bin\obj\$(RootNamespace)_$(Platform)_$(Configuration)\</IntDir>
+ </PropertyGroup>
+ <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
+ <ConfigurationType>Application</ConfigurationType>
+ <UseDebugLibraries>true</UseDebugLibraries>
+ <CharacterSet>Unicode</CharacterSet>
+ <PlatformToolset>v143</PlatformToolset>
+ </PropertyGroup>
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
+ <ConfigurationType>Application</ConfigurationType>
+ <UseDebugLibraries>true</UseDebugLibraries>
+ <CharacterSet>Unicode</CharacterSet>
+ <PlatformToolset>v143</PlatformToolset>
+ </PropertyGroup>
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
+ <ConfigurationType>Application</ConfigurationType>
+ <UseDebugLibraries>false</UseDebugLibraries>
+ <CharacterSet>Unicode</CharacterSet>
+ <WholeProgramOptimization Condition="'$(EnableWholeProgramOptimization)'=='true'">true</WholeProgramOptimization>
+ <PlatformToolset>v143</PlatformToolset>
+ </PropertyGroup>
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
+ <ConfigurationType>Application</ConfigurationType>
+ <UseDebugLibraries>false</UseDebugLibraries>
+ <CharacterSet>Unicode</CharacterSet>
+ <WholeProgramOptimization Condition="'$(EnableWholeProgramOptimization)'=='true'">true</WholeProgramOptimization>
+ <PlatformToolset>v143</PlatformToolset>
+ </PropertyGroup>
+ <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+ <ImportGroup Label="ExtensionSettings">
+ </ImportGroup>
+ <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+ <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+ </ImportGroup>
+ <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="PropertySheets">
+ <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+ </ImportGroup>
+ <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+ <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+ </ImportGroup>
+ <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="PropertySheets">
+ <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+ </ImportGroup>
+ <PropertyGroup Label="UserMacros" />
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+ <LinkIncremental>true</LinkIncremental>
+ <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\programs;$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+ </PropertyGroup>
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
+ <LinkIncremental>true</LinkIncremental>
+ <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\programs;$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+ <RunCodeAnalysis>true</RunCodeAnalysis>
+ </PropertyGroup>
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+ <LinkIncremental>false</LinkIncremental>
+ <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\programs;$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+ </PropertyGroup>
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+ <LinkIncremental>false</LinkIncremental>
+ <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\programs;$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+ <RunCodeAnalysis>true</RunCodeAnalysis>
+ </PropertyGroup>
+ <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+ <ClCompile>
+ <PrecompiledHeader>
+ </PrecompiledHeader>
+ <WarningLevel>Level4</WarningLevel>
+ <Optimization>Disabled</Optimization>
+ <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ <TreatWarningAsError>true</TreatWarningAsError>
+ <EnablePREfast>false</EnablePREfast>
+ <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreadedDebug</RuntimeLibrary>
+ </ClCompile>
+ <Link>
+ <SubSystem>Console</SubSystem>
+ <GenerateDebugInformation>true</GenerateDebugInformation>
+ </Link>
+ </ItemDefinitionGroup>
+ <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
+ <ClCompile>
+ <PrecompiledHeader>
+ </PrecompiledHeader>
+ <WarningLevel>Level4</WarningLevel>
+ <Optimization>Disabled</Optimization>
+ <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ <TreatWarningAsError>true</TreatWarningAsError>
+ <EnablePREfast>true</EnablePREfast>
+ <AdditionalOptions>/analyze:stacksize295252 %(AdditionalOptions)</AdditionalOptions>
+ <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreadedDebug</RuntimeLibrary>
+ </ClCompile>
+ <Link>
+ <SubSystem>Console</SubSystem>
+ <GenerateDebugInformation>true</GenerateDebugInformation>
+ </Link>
+ </ItemDefinitionGroup>
+ <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+ <ClCompile>
+ <WarningLevel>Level4</WarningLevel>
+ <PrecompiledHeader>
+ </PrecompiledHeader>
+ <Optimization>MaxSpeed</Optimization>
+ <FunctionLevelLinking>true</FunctionLevelLinking>
+ <IntrinsicFunctions>true</IntrinsicFunctions>
+ <PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ <TreatWarningAsError>false</TreatWarningAsError>
+ <EnablePREfast>false</EnablePREfast>
+ <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreaded</RuntimeLibrary>
+ </ClCompile>
+ <Link>
+ <SubSystem>Console</SubSystem>
+ <GenerateDebugInformation>true</GenerateDebugInformation>
+ <EnableCOMDATFolding>true</EnableCOMDATFolding>
+ <OptimizeReferences>true</OptimizeReferences>
+ </Link>
+ </ItemDefinitionGroup>
+ <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+ <ClCompile>
+ <WarningLevel>Level4</WarningLevel>
+ <PrecompiledHeader>
+ </PrecompiledHeader>
+ <Optimization>MaxSpeed</Optimization>
+ <FunctionLevelLinking>true</FunctionLevelLinking>
+ <IntrinsicFunctions>true</IntrinsicFunctions>
+ <PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ <TreatWarningAsError>false</TreatWarningAsError>
+ <EnablePREfast>true</EnablePREfast>
+ <AdditionalOptions>/analyze:stacksize295252 %(AdditionalOptions)</AdditionalOptions>
+ <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreaded</RuntimeLibrary>
+ </ClCompile>
+ <Link>
+ <SubSystem>Console</SubSystem>
+ <GenerateDebugInformation>true</GenerateDebugInformation>
+ <EnableCOMDATFolding>true</EnableCOMDATFolding>
+ <OptimizeReferences>true</OptimizeReferences>
+ </Link>
+ </ItemDefinitionGroup>
+ <ItemGroup>
+ <ClCompile Include="..\..\..\programs\datagen.c" />
+ <ClCompile Include="..\..\..\tests\datagencli.c" />
+ </ItemGroup>
+ <ItemGroup>
+ <ClInclude Include="..\..\..\programs\datagen.h" />
+ </ItemGroup>
+ <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
+ <ImportGroup Label="ExtensionTargets">
+ </ImportGroup>
+</Project>
\ No newline at end of file
diff --git a/build/VS2022/frametest/frametest.vcxproj b/build/VS2022/frametest/frametest.vcxproj
new file mode 100644
index 0000000..6b7ff75
--- /dev/null
+++ b/build/VS2022/frametest/frametest.vcxproj
@@ -0,0 +1,180 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project DefaultTargets="Build" ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+ <ItemGroup Label="ProjectConfigurations">
+ <ProjectConfiguration Include="Debug|Win32">
+ <Configuration>Debug</Configuration>
+ <Platform>Win32</Platform>
+ </ProjectConfiguration>
+ <ProjectConfiguration Include="Debug|x64">
+ <Configuration>Debug</Configuration>
+ <Platform>x64</Platform>
+ </ProjectConfiguration>
+ <ProjectConfiguration Include="Release|Win32">
+ <Configuration>Release</Configuration>
+ <Platform>Win32</Platform>
+ </ProjectConfiguration>
+ <ProjectConfiguration Include="Release|x64">
+ <Configuration>Release</Configuration>
+ <Platform>x64</Platform>
+ </ProjectConfiguration>
+ </ItemGroup>
+ <PropertyGroup Label="Globals">
+ <ProjectGuid>{39AD6ECC-8BAD-4368-95E4-A1AA2F077BB7}</ProjectGuid>
+ <Keyword>Win32Proj</Keyword>
+ <RootNamespace>frametest</RootNamespace>
+ <OutDir>$(SolutionDir)bin\$(Platform)_$(Configuration)\</OutDir>
+ <IntDir>$(SolutionDir)bin\obj\$(RootNamespace)_$(Platform)_$(Configuration)\</IntDir>
+ </PropertyGroup>
+ <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
+ <ConfigurationType>Application</ConfigurationType>
+ <UseDebugLibraries>true</UseDebugLibraries>
+ <CharacterSet>Unicode</CharacterSet>
+ <PlatformToolset>v143</PlatformToolset>
+ </PropertyGroup>
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
+ <ConfigurationType>Application</ConfigurationType>
+ <UseDebugLibraries>true</UseDebugLibraries>
+ <CharacterSet>Unicode</CharacterSet>
+ <PlatformToolset>v143</PlatformToolset>
+ </PropertyGroup>
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
+ <ConfigurationType>Application</ConfigurationType>
+ <UseDebugLibraries>false</UseDebugLibraries>
+ <CharacterSet>Unicode</CharacterSet>
+ <WholeProgramOptimization Condition="'$(EnableWholeProgramOptimization)'=='true'">true</WholeProgramOptimization>
+ <PlatformToolset>v143</PlatformToolset>
+ </PropertyGroup>
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
+ <ConfigurationType>Application</ConfigurationType>
+ <UseDebugLibraries>false</UseDebugLibraries>
+ <CharacterSet>Unicode</CharacterSet>
+ <WholeProgramOptimization Condition="'$(EnableWholeProgramOptimization)'=='true'">true</WholeProgramOptimization>
+ <PlatformToolset>v143</PlatformToolset>
+ </PropertyGroup>
+ <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+ <ImportGroup Label="ExtensionSettings">
+ </ImportGroup>
+ <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+ <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+ </ImportGroup>
+ <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="PropertySheets">
+ <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+ </ImportGroup>
+ <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+ <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+ </ImportGroup>
+ <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="PropertySheets">
+ <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+ </ImportGroup>
+ <PropertyGroup Label="UserMacros" />
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+ <LinkIncremental>true</LinkIncremental>
+ <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+ </PropertyGroup>
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
+ <LinkIncremental>true</LinkIncremental>
+ <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+ <RunCodeAnalysis>true</RunCodeAnalysis>
+ </PropertyGroup>
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+ <LinkIncremental>false</LinkIncremental>
+ <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+ </PropertyGroup>
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+ <LinkIncremental>false</LinkIncremental>
+ <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+ <RunCodeAnalysis>true</RunCodeAnalysis>
+ </PropertyGroup>
+ <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+ <ClCompile>
+ <PrecompiledHeader>
+ </PrecompiledHeader>
+ <WarningLevel>Level4</WarningLevel>
+ <Optimization>Disabled</Optimization>
+ <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ <TreatWarningAsError>true</TreatWarningAsError>
+ <EnablePREfast>false</EnablePREfast>
+ <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreadedDebug</RuntimeLibrary>
+ </ClCompile>
+ <Link>
+ <SubSystem>Console</SubSystem>
+ <GenerateDebugInformation>true</GenerateDebugInformation>
+ </Link>
+ </ItemDefinitionGroup>
+ <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
+ <ClCompile>
+ <PrecompiledHeader>
+ </PrecompiledHeader>
+ <WarningLevel>Level4</WarningLevel>
+ <Optimization>Disabled</Optimization>
+ <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ <TreatWarningAsError>true</TreatWarningAsError>
+ <EnablePREfast>true</EnablePREfast>
+ <AdditionalOptions>/analyze:stacksize295252 %(AdditionalOptions)</AdditionalOptions>
+ <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreadedDebug</RuntimeLibrary>
+ </ClCompile>
+ <Link>
+ <SubSystem>Console</SubSystem>
+ <GenerateDebugInformation>true</GenerateDebugInformation>
+ </Link>
+ </ItemDefinitionGroup>
+ <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+ <ClCompile>
+ <WarningLevel>Level4</WarningLevel>
+ <PrecompiledHeader>
+ </PrecompiledHeader>
+ <Optimization>MaxSpeed</Optimization>
+ <FunctionLevelLinking>true</FunctionLevelLinking>
+ <IntrinsicFunctions>true</IntrinsicFunctions>
+ <PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ <TreatWarningAsError>false</TreatWarningAsError>
+ <EnablePREfast>false</EnablePREfast>
+ <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreaded</RuntimeLibrary>
+ </ClCompile>
+ <Link>
+ <SubSystem>Console</SubSystem>
+ <GenerateDebugInformation>true</GenerateDebugInformation>
+ <EnableCOMDATFolding>true</EnableCOMDATFolding>
+ <OptimizeReferences>true</OptimizeReferences>
+ </Link>
+ </ItemDefinitionGroup>
+ <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+ <ClCompile>
+ <WarningLevel>Level4</WarningLevel>
+ <PrecompiledHeader>
+ </PrecompiledHeader>
+ <Optimization>MaxSpeed</Optimization>
+ <FunctionLevelLinking>true</FunctionLevelLinking>
+ <IntrinsicFunctions>true</IntrinsicFunctions>
+ <PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ <TreatWarningAsError>false</TreatWarningAsError>
+ <EnablePREfast>true</EnablePREfast>
+ <AdditionalOptions>/analyze:stacksize295252 %(AdditionalOptions)</AdditionalOptions>
+ <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreaded</RuntimeLibrary>
+ </ClCompile>
+ <Link>
+ <SubSystem>Console</SubSystem>
+ <GenerateDebugInformation>true</GenerateDebugInformation>
+ <EnableCOMDATFolding>true</EnableCOMDATFolding>
+ <OptimizeReferences>true</OptimizeReferences>
+ </Link>
+ </ItemDefinitionGroup>
+ <ItemGroup>
+ <ClCompile Include="..\..\..\lib\lz4.c" />
+ <ClCompile Include="..\..\..\lib\lz4frame.c" />
+ <ClCompile Include="..\..\..\lib\lz4hc.c" />
+ <ClCompile Include="..\..\..\lib\xxhash.c" />
+ <ClCompile Include="..\..\..\tests\frametest.c" />
+ </ItemGroup>
+ <ItemGroup>
+ <ClInclude Include="..\..\..\lib\lz4.h" />
+ <ClInclude Include="..\..\..\lib\lz4frame.h" />
+ <ClInclude Include="..\..\..\lib\lz4frame_static.h" />
+ <ClInclude Include="..\..\..\lib\lz4hc.h" />
+ <ClInclude Include="..\..\..\lib\xxhash.h" />
+ </ItemGroup>
+ <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
+ <ImportGroup Label="ExtensionTargets">
+ </ImportGroup>
+</Project>
\ No newline at end of file
diff --git a/build/VS2022/fullbench-dll/fullbench-dll.vcxproj b/build/VS2022/fullbench-dll/fullbench-dll.vcxproj
new file mode 100644
index 0000000..143dc06
--- /dev/null
+++ b/build/VS2022/fullbench-dll/fullbench-dll.vcxproj
@@ -0,0 +1,184 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project DefaultTargets="Build" ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+ <ItemGroup Label="ProjectConfigurations">
+ <ProjectConfiguration Include="Debug|Win32">
+ <Configuration>Debug</Configuration>
+ <Platform>Win32</Platform>
+ </ProjectConfiguration>
+ <ProjectConfiguration Include="Debug|x64">
+ <Configuration>Debug</Configuration>
+ <Platform>x64</Platform>
+ </ProjectConfiguration>
+ <ProjectConfiguration Include="Release|Win32">
+ <Configuration>Release</Configuration>
+ <Platform>Win32</Platform>
+ </ProjectConfiguration>
+ <ProjectConfiguration Include="Release|x64">
+ <Configuration>Release</Configuration>
+ <Platform>x64</Platform>
+ </ProjectConfiguration>
+ </ItemGroup>
+ <PropertyGroup Label="Globals">
+ <ProjectGuid>{13992FD2-077E-4954-B065-A428198201A9}</ProjectGuid>
+ <Keyword>Win32Proj</Keyword>
+ <RootNamespace>fullbench-dll</RootNamespace>
+ <OutDir>$(SolutionDir)bin\$(Platform)_$(Configuration)\</OutDir>
+ <IntDir>$(SolutionDir)bin\obj\$(RootNamespace)_$(Platform)_$(Configuration)\</IntDir>
+ </PropertyGroup>
+ <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
+ <ConfigurationType>Application</ConfigurationType>
+ <UseDebugLibraries>true</UseDebugLibraries>
+ <CharacterSet>Unicode</CharacterSet>
+ <PlatformToolset>v143</PlatformToolset>
+ </PropertyGroup>
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
+ <ConfigurationType>Application</ConfigurationType>
+ <UseDebugLibraries>true</UseDebugLibraries>
+ <CharacterSet>Unicode</CharacterSet>
+ <PlatformToolset>v143</PlatformToolset>
+ </PropertyGroup>
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
+ <ConfigurationType>Application</ConfigurationType>
+ <UseDebugLibraries>false</UseDebugLibraries>
+ <CharacterSet>Unicode</CharacterSet>
+ <WholeProgramOptimization Condition="'$(EnableWholeProgramOptimization)'=='true'">true</WholeProgramOptimization>
+ <PlatformToolset>v143</PlatformToolset>
+ </PropertyGroup>
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
+ <ConfigurationType>Application</ConfigurationType>
+ <UseDebugLibraries>false</UseDebugLibraries>
+ <CharacterSet>Unicode</CharacterSet>
+ <WholeProgramOptimization Condition="'$(EnableWholeProgramOptimization)'=='true'">true</WholeProgramOptimization>
+ <PlatformToolset>v143</PlatformToolset>
+ </PropertyGroup>
+ <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+ <ImportGroup Label="ExtensionSettings">
+ </ImportGroup>
+ <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+ <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+ </ImportGroup>
+ <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="PropertySheets">
+ <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+ </ImportGroup>
+ <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+ <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+ </ImportGroup>
+ <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="PropertySheets">
+ <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+ </ImportGroup>
+ <PropertyGroup Label="UserMacros" />
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+ <LinkIncremental>true</LinkIncremental>
+ <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+ </PropertyGroup>
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
+ <LinkIncremental>true</LinkIncremental>
+ <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+ <RunCodeAnalysis>true</RunCodeAnalysis>
+ </PropertyGroup>
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+ <LinkIncremental>false</LinkIncremental>
+ <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+ </PropertyGroup>
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+ <LinkIncremental>false</LinkIncremental>
+ <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+ <RunCodeAnalysis>true</RunCodeAnalysis>
+ </PropertyGroup>
+ <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+ <ClCompile>
+ <PrecompiledHeader>
+ </PrecompiledHeader>
+ <WarningLevel>Level4</WarningLevel>
+ <Optimization>Disabled</Optimization>
+ <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;LZ4_DLL_IMPORT=1;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ <TreatWarningAsError>true</TreatWarningAsError>
+ <EnablePREfast>false</EnablePREfast>
+ <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreadedDebug</RuntimeLibrary>
+ </ClCompile>
+ <Link>
+ <SubSystem>Console</SubSystem>
+ <GenerateDebugInformation>true</GenerateDebugInformation>
+ <AdditionalLibraryDirectories>$(SolutionDir)bin\$(Platform)_$(Configuration);%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>
+ <AdditionalDependencies>liblz4.lib;%(AdditionalDependencies)</AdditionalDependencies>
+ </Link>
+ </ItemDefinitionGroup>
+ <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
+ <ClCompile>
+ <PrecompiledHeader>
+ </PrecompiledHeader>
+ <WarningLevel>Level4</WarningLevel>
+ <Optimization>Disabled</Optimization>
+ <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;LZ4_DLL_IMPORT=1;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ <TreatWarningAsError>true</TreatWarningAsError>
+ <EnablePREfast>true</EnablePREfast>
+ <AdditionalOptions>/analyze:stacksize295252 %(AdditionalOptions)</AdditionalOptions>
+ <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreadedDebug</RuntimeLibrary>
+ </ClCompile>
+ <Link>
+ <SubSystem>Console</SubSystem>
+ <GenerateDebugInformation>true</GenerateDebugInformation>
+ <AdditionalLibraryDirectories>$(SolutionDir)bin\$(Platform)_$(Configuration);%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>
+ <AdditionalDependencies>liblz4.lib;%(AdditionalDependencies)</AdditionalDependencies>
+ </Link>
+ </ItemDefinitionGroup>
+ <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+ <ClCompile>
+ <WarningLevel>Level4</WarningLevel>
+ <PrecompiledHeader>
+ </PrecompiledHeader>
+ <Optimization>MaxSpeed</Optimization>
+ <FunctionLevelLinking>true</FunctionLevelLinking>
+ <IntrinsicFunctions>true</IntrinsicFunctions>
+ <PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;LZ4_DLL_IMPORT=1;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ <TreatWarningAsError>false</TreatWarningAsError>
+ <EnablePREfast>false</EnablePREfast>
+ <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreaded</RuntimeLibrary>
+ </ClCompile>
+ <Link>
+ <SubSystem>Console</SubSystem>
+ <GenerateDebugInformation>true</GenerateDebugInformation>
+ <EnableCOMDATFolding>true</EnableCOMDATFolding>
+ <OptimizeReferences>true</OptimizeReferences>
+ <AdditionalLibraryDirectories>$(SolutionDir)bin\$(Platform)_$(Configuration);%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>
+ <AdditionalDependencies>liblz4.lib;%(AdditionalDependencies)</AdditionalDependencies>
+ </Link>
+ </ItemDefinitionGroup>
+ <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+ <ClCompile>
+ <WarningLevel>Level4</WarningLevel>
+ <PrecompiledHeader>
+ </PrecompiledHeader>
+ <Optimization>MaxSpeed</Optimization>
+ <FunctionLevelLinking>true</FunctionLevelLinking>
+ <IntrinsicFunctions>true</IntrinsicFunctions>
+ <PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;LZ4_DLL_IMPORT=1;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ <TreatWarningAsError>false</TreatWarningAsError>
+ <EnablePREfast>true</EnablePREfast>
+ <AdditionalOptions>/analyze:stacksize295252 %(AdditionalOptions)</AdditionalOptions>
+ <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreaded</RuntimeLibrary>
+ </ClCompile>
+ <Link>
+ <SubSystem>Console</SubSystem>
+ <GenerateDebugInformation>true</GenerateDebugInformation>
+ <EnableCOMDATFolding>true</EnableCOMDATFolding>
+ <OptimizeReferences>true</OptimizeReferences>
+ <AdditionalLibraryDirectories>$(SolutionDir)bin\$(Platform)_$(Configuration);%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>
+ <AdditionalDependencies>liblz4.lib;%(AdditionalDependencies)</AdditionalDependencies>
+ </Link>
+ </ItemDefinitionGroup>
+ <ItemGroup>
+ <ClCompile Include="..\..\..\lib\xxhash.c" />
+ <ClCompile Include="..\..\..\tests\fullbench.c" />
+ </ItemGroup>
+ <ItemGroup>
+ <ClInclude Include="..\..\..\lib\lz4.h" />
+ <ClInclude Include="..\..\..\lib\lz4frame.h" />
+ <ClInclude Include="..\..\..\lib\lz4hc.h" />
+ <ClInclude Include="..\..\..\lib\xxhash.h" />
+ </ItemGroup>
+ <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
+ <ImportGroup Label="ExtensionTargets">
+ </ImportGroup>
+</Project>
\ No newline at end of file
diff --git a/build/VS2022/fullbench/fullbench.vcxproj b/build/VS2022/fullbench/fullbench.vcxproj
new file mode 100644
index 0000000..57f4b5a
--- /dev/null
+++ b/build/VS2022/fullbench/fullbench.vcxproj
@@ -0,0 +1,180 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project DefaultTargets="Build" ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+ <ItemGroup Label="ProjectConfigurations">
+ <ProjectConfiguration Include="Debug|Win32">
+ <Configuration>Debug</Configuration>
+ <Platform>Win32</Platform>
+ </ProjectConfiguration>
+ <ProjectConfiguration Include="Debug|x64">
+ <Configuration>Debug</Configuration>
+ <Platform>x64</Platform>
+ </ProjectConfiguration>
+ <ProjectConfiguration Include="Release|Win32">
+ <Configuration>Release</Configuration>
+ <Platform>Win32</Platform>
+ </ProjectConfiguration>
+ <ProjectConfiguration Include="Release|x64">
+ <Configuration>Release</Configuration>
+ <Platform>x64</Platform>
+ </ProjectConfiguration>
+ </ItemGroup>
+ <PropertyGroup Label="Globals">
+ <ProjectGuid>{6A4DF4EF-C77F-43C6-8901-DDCD20879E4E}</ProjectGuid>
+ <Keyword>Win32Proj</Keyword>
+ <RootNamespace>fullbench</RootNamespace>
+ <OutDir>$(SolutionDir)bin\$(Platform)_$(Configuration)\</OutDir>
+ <IntDir>$(SolutionDir)bin\obj\$(RootNamespace)_$(Platform)_$(Configuration)\</IntDir>
+ </PropertyGroup>
+ <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
+ <ConfigurationType>Application</ConfigurationType>
+ <UseDebugLibraries>true</UseDebugLibraries>
+ <CharacterSet>Unicode</CharacterSet>
+ <PlatformToolset>v143</PlatformToolset>
+ </PropertyGroup>
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
+ <ConfigurationType>Application</ConfigurationType>
+ <UseDebugLibraries>true</UseDebugLibraries>
+ <CharacterSet>Unicode</CharacterSet>
+ <PlatformToolset>v143</PlatformToolset>
+ </PropertyGroup>
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
+ <ConfigurationType>Application</ConfigurationType>
+ <UseDebugLibraries>false</UseDebugLibraries>
+ <CharacterSet>Unicode</CharacterSet>
+ <WholeProgramOptimization Condition="'$(EnableWholeProgramOptimization)'=='true'">true</WholeProgramOptimization>
+ <PlatformToolset>v143</PlatformToolset>
+ </PropertyGroup>
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
+ <ConfigurationType>Application</ConfigurationType>
+ <UseDebugLibraries>false</UseDebugLibraries>
+ <CharacterSet>Unicode</CharacterSet>
+ <WholeProgramOptimization Condition="'$(EnableWholeProgramOptimization)'=='true'">true</WholeProgramOptimization>
+ <PlatformToolset>v143</PlatformToolset>
+ </PropertyGroup>
+ <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+ <ImportGroup Label="ExtensionSettings">
+ </ImportGroup>
+ <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+ <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+ </ImportGroup>
+ <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="PropertySheets">
+ <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+ </ImportGroup>
+ <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+ <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+ </ImportGroup>
+ <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="PropertySheets">
+ <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+ </ImportGroup>
+ <PropertyGroup Label="UserMacros" />
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+ <LinkIncremental>true</LinkIncremental>
+ <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+ </PropertyGroup>
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
+ <LinkIncremental>true</LinkIncremental>
+ <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+ <RunCodeAnalysis>true</RunCodeAnalysis>
+ </PropertyGroup>
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+ <LinkIncremental>false</LinkIncremental>
+ <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+ </PropertyGroup>
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+ <LinkIncremental>false</LinkIncremental>
+ <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+ <RunCodeAnalysis>true</RunCodeAnalysis>
+ </PropertyGroup>
+ <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+ <ClCompile>
+ <PrecompiledHeader>
+ </PrecompiledHeader>
+ <WarningLevel>Level4</WarningLevel>
+ <Optimization>Disabled</Optimization>
+ <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ <TreatWarningAsError>true</TreatWarningAsError>
+ <EnablePREfast>false</EnablePREfast>
+ <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreadedDebug</RuntimeLibrary>
+ </ClCompile>
+ <Link>
+ <SubSystem>Console</SubSystem>
+ <GenerateDebugInformation>true</GenerateDebugInformation>
+ </Link>
+ </ItemDefinitionGroup>
+ <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
+ <ClCompile>
+ <PrecompiledHeader>
+ </PrecompiledHeader>
+ <WarningLevel>Level4</WarningLevel>
+ <Optimization>Disabled</Optimization>
+ <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ <TreatWarningAsError>true</TreatWarningAsError>
+ <EnablePREfast>true</EnablePREfast>
+ <AdditionalOptions>/analyze:stacksize295252 %(AdditionalOptions)</AdditionalOptions>
+ <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreadedDebug</RuntimeLibrary>
+ </ClCompile>
+ <Link>
+ <SubSystem>Console</SubSystem>
+ <GenerateDebugInformation>true</GenerateDebugInformation>
+ </Link>
+ </ItemDefinitionGroup>
+ <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+ <ClCompile>
+ <WarningLevel>Level4</WarningLevel>
+ <PrecompiledHeader>
+ </PrecompiledHeader>
+ <Optimization>MaxSpeed</Optimization>
+ <FunctionLevelLinking>true</FunctionLevelLinking>
+ <IntrinsicFunctions>true</IntrinsicFunctions>
+ <PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ <TreatWarningAsError>false</TreatWarningAsError>
+ <EnablePREfast>false</EnablePREfast>
+ <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreaded</RuntimeLibrary>
+ </ClCompile>
+ <Link>
+ <SubSystem>Console</SubSystem>
+ <GenerateDebugInformation>true</GenerateDebugInformation>
+ <EnableCOMDATFolding>true</EnableCOMDATFolding>
+ <OptimizeReferences>true</OptimizeReferences>
+ </Link>
+ </ItemDefinitionGroup>
+ <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+ <ClCompile>
+ <WarningLevel>Level4</WarningLevel>
+ <PrecompiledHeader>
+ </PrecompiledHeader>
+ <Optimization>MaxSpeed</Optimization>
+ <FunctionLevelLinking>true</FunctionLevelLinking>
+ <IntrinsicFunctions>true</IntrinsicFunctions>
+ <PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ <TreatWarningAsError>false</TreatWarningAsError>
+ <EnablePREfast>true</EnablePREfast>
+ <AdditionalOptions>/analyze:stacksize295252 %(AdditionalOptions)</AdditionalOptions>
+ <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreaded</RuntimeLibrary>
+ </ClCompile>
+ <Link>
+ <SubSystem>Console</SubSystem>
+ <GenerateDebugInformation>true</GenerateDebugInformation>
+ <EnableCOMDATFolding>true</EnableCOMDATFolding>
+ <OptimizeReferences>true</OptimizeReferences>
+ </Link>
+ </ItemDefinitionGroup>
+ <ItemGroup>
+ <ClCompile Include="..\..\..\lib\lz4.c" />
+ <ClCompile Include="..\..\..\lib\lz4frame.c" />
+ <ClCompile Include="..\..\..\lib\lz4hc.c" />
+ <ClCompile Include="..\..\..\lib\xxhash.c" />
+ <ClCompile Include="..\..\..\tests\fullbench.c" />
+ </ItemGroup>
+ <ItemGroup>
+ <ClInclude Include="..\..\..\lib\lz4.h" />
+ <ClInclude Include="..\..\..\lib\lz4frame.h" />
+ <ClInclude Include="..\..\..\lib\lz4frame_static.h" />
+ <ClInclude Include="..\..\..\lib\lz4hc.h" />
+ <ClInclude Include="..\..\..\lib\xxhash.h" />
+ </ItemGroup>
+ <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
+ <ImportGroup Label="ExtensionTargets">
+ </ImportGroup>
+</Project>
\ No newline at end of file
diff --git a/build/VS2022/fuzzer/fuzzer.vcxproj b/build/VS2022/fuzzer/fuzzer.vcxproj
new file mode 100644
index 0000000..83482c2
--- /dev/null
+++ b/build/VS2022/fuzzer/fuzzer.vcxproj
@@ -0,0 +1,177 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project DefaultTargets="Build" ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+ <ItemGroup Label="ProjectConfigurations">
+ <ProjectConfiguration Include="Debug|Win32">
+ <Configuration>Debug</Configuration>
+ <Platform>Win32</Platform>
+ </ProjectConfiguration>
+ <ProjectConfiguration Include="Debug|x64">
+ <Configuration>Debug</Configuration>
+ <Platform>x64</Platform>
+ </ProjectConfiguration>
+ <ProjectConfiguration Include="Release|Win32">
+ <Configuration>Release</Configuration>
+ <Platform>Win32</Platform>
+ </ProjectConfiguration>
+ <ProjectConfiguration Include="Release|x64">
+ <Configuration>Release</Configuration>
+ <Platform>x64</Platform>
+ </ProjectConfiguration>
+ </ItemGroup>
+ <PropertyGroup Label="Globals">
+ <ProjectGuid>{18B9F1A7-9C66-4352-898B-30804DADE0FD}</ProjectGuid>
+ <Keyword>Win32Proj</Keyword>
+ <RootNamespace>fuzzer</RootNamespace>
+ <OutDir>$(SolutionDir)bin\$(Platform)_$(Configuration)\</OutDir>
+ <IntDir>$(SolutionDir)bin\obj\$(RootNamespace)_$(Platform)_$(Configuration)\</IntDir>
+ </PropertyGroup>
+ <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
+ <ConfigurationType>Application</ConfigurationType>
+ <UseDebugLibraries>true</UseDebugLibraries>
+ <CharacterSet>Unicode</CharacterSet>
+ <PlatformToolset>v143</PlatformToolset>
+ </PropertyGroup>
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
+ <ConfigurationType>Application</ConfigurationType>
+ <UseDebugLibraries>true</UseDebugLibraries>
+ <CharacterSet>Unicode</CharacterSet>
+ <PlatformToolset>v143</PlatformToolset>
+ </PropertyGroup>
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
+ <ConfigurationType>Application</ConfigurationType>
+ <UseDebugLibraries>false</UseDebugLibraries>
+ <CharacterSet>Unicode</CharacterSet>
+ <WholeProgramOptimization Condition="'$(EnableWholeProgramOptimization)'=='true'">true</WholeProgramOptimization>
+ <PlatformToolset>v143</PlatformToolset>
+ </PropertyGroup>
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
+ <ConfigurationType>Application</ConfigurationType>
+ <UseDebugLibraries>false</UseDebugLibraries>
+ <CharacterSet>Unicode</CharacterSet>
+ <WholeProgramOptimization Condition="'$(EnableWholeProgramOptimization)'=='true'">true</WholeProgramOptimization>
+ <PlatformToolset>v143</PlatformToolset>
+ </PropertyGroup>
+ <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+ <ImportGroup Label="ExtensionSettings">
+ </ImportGroup>
+ <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+ <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+ </ImportGroup>
+ <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="PropertySheets">
+ <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+ </ImportGroup>
+ <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+ <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+ </ImportGroup>
+ <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="PropertySheets">
+ <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+ </ImportGroup>
+ <PropertyGroup Label="UserMacros" />
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+ <LinkIncremental>true</LinkIncremental>
+ <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+ </PropertyGroup>
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
+ <LinkIncremental>true</LinkIncremental>
+ <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+ <RunCodeAnalysis>true</RunCodeAnalysis>
+ </PropertyGroup>
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+ <LinkIncremental>false</LinkIncremental>
+ <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+ </PropertyGroup>
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+ <LinkIncremental>false</LinkIncremental>
+ <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+ <RunCodeAnalysis>true</RunCodeAnalysis>
+ </PropertyGroup>
+ <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+ <ClCompile>
+ <PrecompiledHeader>
+ </PrecompiledHeader>
+ <WarningLevel>Level4</WarningLevel>
+ <Optimization>Disabled</Optimization>
+ <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ <TreatWarningAsError>true</TreatWarningAsError>
+ <EnablePREfast>false</EnablePREfast>
+ <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreadedDebug</RuntimeLibrary>
+ </ClCompile>
+ <Link>
+ <SubSystem>Console</SubSystem>
+ <GenerateDebugInformation>true</GenerateDebugInformation>
+ </Link>
+ </ItemDefinitionGroup>
+ <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
+ <ClCompile>
+ <PrecompiledHeader>
+ </PrecompiledHeader>
+ <WarningLevel>Level4</WarningLevel>
+ <Optimization>Disabled</Optimization>
+ <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ <TreatWarningAsError>true</TreatWarningAsError>
+ <EnablePREfast>true</EnablePREfast>
+ <AdditionalOptions>/analyze:stacksize295252 %(AdditionalOptions)</AdditionalOptions>
+ <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreadedDebug</RuntimeLibrary>
+ </ClCompile>
+ <Link>
+ <SubSystem>Console</SubSystem>
+ <GenerateDebugInformation>true</GenerateDebugInformation>
+ </Link>
+ </ItemDefinitionGroup>
+ <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+ <ClCompile>
+ <WarningLevel>Level4</WarningLevel>
+ <PrecompiledHeader>
+ </PrecompiledHeader>
+ <Optimization>MaxSpeed</Optimization>
+ <FunctionLevelLinking>true</FunctionLevelLinking>
+ <IntrinsicFunctions>true</IntrinsicFunctions>
+ <PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ <TreatWarningAsError>false</TreatWarningAsError>
+ <EnablePREfast>false</EnablePREfast>
+ <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreaded</RuntimeLibrary>
+ </ClCompile>
+ <Link>
+ <SubSystem>Console</SubSystem>
+ <GenerateDebugInformation>true</GenerateDebugInformation>
+ <EnableCOMDATFolding>true</EnableCOMDATFolding>
+ <OptimizeReferences>true</OptimizeReferences>
+ </Link>
+ </ItemDefinitionGroup>
+ <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+ <ClCompile>
+ <WarningLevel>Level4</WarningLevel>
+ <PrecompiledHeader>
+ </PrecompiledHeader>
+ <Optimization>MaxSpeed</Optimization>
+ <FunctionLevelLinking>true</FunctionLevelLinking>
+ <IntrinsicFunctions>true</IntrinsicFunctions>
+ <PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ <TreatWarningAsError>false</TreatWarningAsError>
+ <EnablePREfast>true</EnablePREfast>
+ <AdditionalOptions>/analyze:stacksize295252 %(AdditionalOptions)</AdditionalOptions>
+ <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreaded</RuntimeLibrary>
+ </ClCompile>
+ <Link>
+ <SubSystem>Console</SubSystem>
+ <GenerateDebugInformation>true</GenerateDebugInformation>
+ <EnableCOMDATFolding>true</EnableCOMDATFolding>
+ <OptimizeReferences>true</OptimizeReferences>
+ </Link>
+ </ItemDefinitionGroup>
+ <ItemGroup>
+ <ClCompile Include="..\..\..\lib\lz4.c" />
+ <ClCompile Include="..\..\..\lib\lz4hc.c" />
+ <ClCompile Include="..\..\..\lib\xxhash.c" />
+ <ClCompile Include="..\..\..\tests\fuzzer.c" />
+ </ItemGroup>
+ <ItemGroup>
+ <ClInclude Include="..\..\..\lib\lz4.h" />
+ <ClInclude Include="..\..\..\lib\lz4hc.h" />
+ <ClInclude Include="..\..\..\lib\xxhash.h" />
+ </ItemGroup>
+ <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
+ <ImportGroup Label="ExtensionTargets">
+ </ImportGroup>
+</Project>
\ No newline at end of file
diff --git a/build/VS2022/liblz4-dll/liblz4-dll.rc b/build/VS2022/liblz4-dll/liblz4-dll.rc
new file mode 100644
index 0000000..e089c24
--- /dev/null
+++ b/build/VS2022/liblz4-dll/liblz4-dll.rc
@@ -0,0 +1,51 @@
+// Microsoft Visual C++ generated resource script.
+//
+
+#include "lz4.h" /* LZ4_VERSION_STRING */
+#define APSTUDIO_READONLY_SYMBOLS
+#include "verrsrc.h"
+#undef APSTUDIO_READONLY_SYMBOLS
+
+
+#if !defined(AFX_RESOURCE_DLL) || defined(AFX_TARG_ENU)
+LANGUAGE 9, 1
+
+/////////////////////////////////////////////////////////////////////////////
+//
+// Version
+//
+
+VS_VERSION_INFO VERSIONINFO
+ FILEVERSION LZ4_VERSION_MAJOR,LZ4_VERSION_MINOR,LZ4_VERSION_RELEASE,0
+ PRODUCTVERSION LZ4_VERSION_MAJOR,LZ4_VERSION_MINOR,LZ4_VERSION_RELEASE,0
+ FILEFLAGSMASK VS_FFI_FILEFLAGSMASK
+#ifdef _DEBUG
+ FILEFLAGS VS_FF_DEBUG
+#else
+ FILEFLAGS 0x0L
+#endif
+ FILEOS VOS_NT_WINDOWS32
+ FILETYPE VFT_DLL
+ FILESUBTYPE VFT2_UNKNOWN
+BEGIN
+ BLOCK "StringFileInfo"
+ BEGIN
+ BLOCK "040904B0"
+ BEGIN
+ VALUE "CompanyName", "Yann Collet"
+ VALUE "FileDescription", "Extremely fast compression"
+ VALUE "FileVersion", LZ4_VERSION_STRING
+ VALUE "InternalName", "lz4.dll"
+ VALUE "LegalCopyright", "Copyright (C) 2013-2020, Yann Collet"
+ VALUE "OriginalFilename", "lz4.dll"
+ VALUE "ProductName", "LZ4"
+ VALUE "ProductVersion", LZ4_VERSION_STRING
+ END
+ END
+ BLOCK "VarFileInfo"
+ BEGIN
+ VALUE "Translation", 0x0409, 1200
+ END
+END
+
+#endif
diff --git a/build/VS2022/liblz4-dll/liblz4-dll.vcxproj b/build/VS2022/liblz4-dll/liblz4-dll.vcxproj
new file mode 100644
index 0000000..532ac75
--- /dev/null
+++ b/build/VS2022/liblz4-dll/liblz4-dll.vcxproj
@@ -0,0 +1,183 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project DefaultTargets="Build" ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+ <ItemGroup Label="ProjectConfigurations">
+ <ProjectConfiguration Include="Debug|Win32">
+ <Configuration>Debug</Configuration>
+ <Platform>Win32</Platform>
+ </ProjectConfiguration>
+ <ProjectConfiguration Include="Debug|x64">
+ <Configuration>Debug</Configuration>
+ <Platform>x64</Platform>
+ </ProjectConfiguration>
+ <ProjectConfiguration Include="Release|Win32">
+ <Configuration>Release</Configuration>
+ <Platform>Win32</Platform>
+ </ProjectConfiguration>
+ <ProjectConfiguration Include="Release|x64">
+ <Configuration>Release</Configuration>
+ <Platform>x64</Platform>
+ </ProjectConfiguration>
+ </ItemGroup>
+ <PropertyGroup Label="Globals">
+ <ProjectGuid>{9800039D-4AAA-43A4-BB78-FEF6F4836927}</ProjectGuid>
+ <Keyword>Win32Proj</Keyword>
+ <RootNamespace>liblz4-dll</RootNamespace>
+ <OutDir>$(SolutionDir)bin\$(Platform)_$(Configuration)\</OutDir>
+ <IntDir>$(SolutionDir)bin\obj\$(RootNamespace)_$(Platform)_$(Configuration)\</IntDir>
+ <ProjectName>liblz4-dll</ProjectName>
+ </PropertyGroup>
+ <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
+ <ConfigurationType>DynamicLibrary</ConfigurationType>
+ <UseDebugLibraries>true</UseDebugLibraries>
+ <CharacterSet>Unicode</CharacterSet>
+ <PlatformToolset>v143</PlatformToolset>
+ </PropertyGroup>
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
+ <ConfigurationType>DynamicLibrary</ConfigurationType>
+ <UseDebugLibraries>true</UseDebugLibraries>
+ <CharacterSet>Unicode</CharacterSet>
+ <PlatformToolset>v143</PlatformToolset>
+ </PropertyGroup>
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
+ <ConfigurationType>DynamicLibrary</ConfigurationType>
+ <UseDebugLibraries>false</UseDebugLibraries>
+ <CharacterSet>Unicode</CharacterSet>
+ <WholeProgramOptimization Condition="'$(EnableWholeProgramOptimization)'=='true'">true</WholeProgramOptimization>
+ <PlatformToolset>v143</PlatformToolset>
+ </PropertyGroup>
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
+ <ConfigurationType>DynamicLibrary</ConfigurationType>
+ <UseDebugLibraries>false</UseDebugLibraries>
+ <CharacterSet>Unicode</CharacterSet>
+ <WholeProgramOptimization Condition="'$(EnableWholeProgramOptimization)'=='true'">true</WholeProgramOptimization>
+ <PlatformToolset>v143</PlatformToolset>
+ </PropertyGroup>
+ <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+ <ImportGroup Label="ExtensionSettings">
+ </ImportGroup>
+ <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+ <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+ </ImportGroup>
+ <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="PropertySheets">
+ <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+ </ImportGroup>
+ <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+ <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+ </ImportGroup>
+ <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="PropertySheets">
+ <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+ </ImportGroup>
+ <PropertyGroup Label="UserMacros" />
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+ <LinkIncremental>true</LinkIncremental>
+ <TargetName>liblz4</TargetName>
+ <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+ </PropertyGroup>
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
+ <LinkIncremental>true</LinkIncremental>
+ <TargetName>liblz4</TargetName>
+ <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+ <RunCodeAnalysis>true</RunCodeAnalysis>
+ </PropertyGroup>
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+ <LinkIncremental>false</LinkIncremental>
+ <TargetName>liblz4</TargetName>
+ <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+ </PropertyGroup>
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+ <LinkIncremental>false</LinkIncremental>
+ <TargetName>liblz4</TargetName>
+ <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+ <RunCodeAnalysis>true</RunCodeAnalysis>
+ </PropertyGroup>
+ <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+ <ClCompile>
+ <PrecompiledHeader>
+ </PrecompiledHeader>
+ <WarningLevel>Level4</WarningLevel>
+ <Optimization>Disabled</Optimization>
+ <PreprocessorDefinitions>WIN32;_DEBUG;LZ4_DLL_EXPORT=1;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ <TreatWarningAsError>true</TreatWarningAsError>
+ <EnablePREfast>false</EnablePREfast>
+ <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreadedDebug</RuntimeLibrary>
+ </ClCompile>
+ <Link>
+ <GenerateDebugInformation>true</GenerateDebugInformation>
+ </Link>
+ </ItemDefinitionGroup>
+ <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
+ <ClCompile>
+ <PrecompiledHeader>
+ </PrecompiledHeader>
+ <WarningLevel>Level4</WarningLevel>
+ <Optimization>Disabled</Optimization>
+ <PreprocessorDefinitions>WIN32;_DEBUG;LZ4_DLL_EXPORT=1;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ <TreatWarningAsError>true</TreatWarningAsError>
+ <EnablePREfast>true</EnablePREfast>
+ <AdditionalOptions>/analyze:stacksize295252 %(AdditionalOptions)</AdditionalOptions>
+ <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreadedDebug</RuntimeLibrary>
+ </ClCompile>
+ <Link>
+ <GenerateDebugInformation>true</GenerateDebugInformation>
+ </Link>
+ </ItemDefinitionGroup>
+ <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+ <ClCompile>
+ <WarningLevel>Level4</WarningLevel>
+ <PrecompiledHeader>
+ </PrecompiledHeader>
+ <Optimization>MaxSpeed</Optimization>
+ <FunctionLevelLinking>true</FunctionLevelLinking>
+ <IntrinsicFunctions>true</IntrinsicFunctions>
+ <PreprocessorDefinitions>WIN32;NDEBUG;LZ4_DLL_EXPORT=1;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ <TreatWarningAsError>false</TreatWarningAsError>
+ <EnablePREfast>false</EnablePREfast>
+ <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreaded</RuntimeLibrary>
+ </ClCompile>
+ <Link>
+ <GenerateDebugInformation>true</GenerateDebugInformation>
+ <EnableCOMDATFolding>true</EnableCOMDATFolding>
+ <OptimizeReferences>true</OptimizeReferences>
+ </Link>
+ </ItemDefinitionGroup>
+ <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+ <ClCompile>
+ <WarningLevel>Level4</WarningLevel>
+ <PrecompiledHeader>
+ </PrecompiledHeader>
+ <Optimization>MaxSpeed</Optimization>
+ <FunctionLevelLinking>true</FunctionLevelLinking>
+ <IntrinsicFunctions>true</IntrinsicFunctions>
+ <PreprocessorDefinitions>WIN32;NDEBUG;LZ4_DLL_EXPORT=1;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ <TreatWarningAsError>false</TreatWarningAsError>
+ <EnablePREfast>true</EnablePREfast>
+ <AdditionalOptions>/analyze:stacksize295252 %(AdditionalOptions)</AdditionalOptions>
+ <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreaded</RuntimeLibrary>
+ </ClCompile>
+ <Link>
+ <GenerateDebugInformation>true</GenerateDebugInformation>
+ <EnableCOMDATFolding>true</EnableCOMDATFolding>
+ <OptimizeReferences>true</OptimizeReferences>
+ </Link>
+ </ItemDefinitionGroup>
+ <ItemGroup>
+ <ClInclude Include="..\..\..\lib\lz4.h" />
+ <ClInclude Include="..\..\..\lib\lz4frame.h" />
+ <ClInclude Include="..\..\..\lib\lz4frame_static.h" />
+ <ClInclude Include="..\..\..\lib\lz4hc.h" />
+ <ClInclude Include="..\..\..\lib\xxhash.h" />
+ </ItemGroup>
+ <ItemGroup>
+ <ClCompile Include="..\..\..\lib\lz4.c" />
+ <ClCompile Include="..\..\..\lib\lz4frame.c" />
+ <ClCompile Include="..\..\..\lib\lz4hc.c" />
+ <ClCompile Include="..\..\..\lib\xxhash.c" />
+ </ItemGroup>
+ <ItemGroup>
+ <ResourceCompile Include="liblz4-dll.rc" />
+ </ItemGroup>
+ <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
+ <ImportGroup Label="ExtensionTargets">
+ </ImportGroup>
+</Project>
\ No newline at end of file
diff --git a/build/VS2022/liblz4/liblz4.vcxproj b/build/VS2022/liblz4/liblz4.vcxproj
new file mode 100644
index 0000000..fdddaaa
--- /dev/null
+++ b/build/VS2022/liblz4/liblz4.vcxproj
@@ -0,0 +1,179 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project DefaultTargets="Build" ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+ <ItemGroup Label="ProjectConfigurations">
+ <ProjectConfiguration Include="Debug|Win32">
+ <Configuration>Debug</Configuration>
+ <Platform>Win32</Platform>
+ </ProjectConfiguration>
+ <ProjectConfiguration Include="Debug|x64">
+ <Configuration>Debug</Configuration>
+ <Platform>x64</Platform>
+ </ProjectConfiguration>
+ <ProjectConfiguration Include="Release|Win32">
+ <Configuration>Release</Configuration>
+ <Platform>Win32</Platform>
+ </ProjectConfiguration>
+ <ProjectConfiguration Include="Release|x64">
+ <Configuration>Release</Configuration>
+ <Platform>x64</Platform>
+ </ProjectConfiguration>
+ </ItemGroup>
+ <PropertyGroup Label="Globals">
+ <ProjectGuid>{9092C5CC-3E71-41B3-BF68-4A7BDD8A5476}</ProjectGuid>
+ <Keyword>Win32Proj</Keyword>
+ <RootNamespace>liblz4</RootNamespace>
+ <OutDir>$(SolutionDir)bin\$(Platform)_$(Configuration)\</OutDir>
+ <IntDir>$(SolutionDir)bin\obj\$(RootNamespace)_$(Platform)_$(Configuration)\</IntDir>
+ </PropertyGroup>
+ <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
+ <ConfigurationType>StaticLibrary</ConfigurationType>
+ <UseDebugLibraries>true</UseDebugLibraries>
+ <CharacterSet>Unicode</CharacterSet>
+ <PlatformToolset>v143</PlatformToolset>
+ </PropertyGroup>
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
+ <ConfigurationType>StaticLibrary</ConfigurationType>
+ <UseDebugLibraries>true</UseDebugLibraries>
+ <CharacterSet>Unicode</CharacterSet>
+ <PlatformToolset>v143</PlatformToolset>
+ </PropertyGroup>
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
+ <ConfigurationType>StaticLibrary</ConfigurationType>
+ <UseDebugLibraries>false</UseDebugLibraries>
+ <CharacterSet>Unicode</CharacterSet>
+ <WholeProgramOptimization Condition="'$(EnableWholeProgramOptimization)'=='true'">true</WholeProgramOptimization>
+ <PlatformToolset>v143</PlatformToolset>
+ </PropertyGroup>
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
+ <ConfigurationType>StaticLibrary</ConfigurationType>
+ <UseDebugLibraries>false</UseDebugLibraries>
+ <CharacterSet>Unicode</CharacterSet>
+ <WholeProgramOptimization Condition="'$(EnableWholeProgramOptimization)'=='true'">true</WholeProgramOptimization>
+ <PlatformToolset>v143</PlatformToolset>
+ </PropertyGroup>
+ <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+ <ImportGroup Label="ExtensionSettings">
+ </ImportGroup>
+ <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+ <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+ </ImportGroup>
+ <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="PropertySheets">
+ <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+ </ImportGroup>
+ <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+ <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+ </ImportGroup>
+ <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="PropertySheets">
+ <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+ </ImportGroup>
+ <PropertyGroup Label="UserMacros" />
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+ <LinkIncremental>true</LinkIncremental>
+ <TargetName>liblz4_static</TargetName>
+ <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+ </PropertyGroup>
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
+ <LinkIncremental>true</LinkIncremental>
+ <TargetName>liblz4_static</TargetName>
+ <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+ <RunCodeAnalysis>true</RunCodeAnalysis>
+ </PropertyGroup>
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+ <LinkIncremental>false</LinkIncremental>
+ <TargetName>liblz4_static</TargetName>
+ <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+ </PropertyGroup>
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+ <LinkIncremental>false</LinkIncremental>
+ <TargetName>liblz4_static</TargetName>
+ <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+ <RunCodeAnalysis>true</RunCodeAnalysis>
+ </PropertyGroup>
+ <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+ <ClCompile>
+ <PrecompiledHeader>
+ </PrecompiledHeader>
+ <WarningLevel>Level4</WarningLevel>
+ <Optimization>Disabled</Optimization>
+ <PreprocessorDefinitions>WIN32;_DEBUG;LZ4_DLL_EXPORT=1;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ <TreatWarningAsError>true</TreatWarningAsError>
+ <EnablePREfast>false</EnablePREfast>
+ <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreadedDebug</RuntimeLibrary>
+ </ClCompile>
+ <Link>
+ <GenerateDebugInformation>true</GenerateDebugInformation>
+ </Link>
+ </ItemDefinitionGroup>
+ <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
+ <ClCompile>
+ <PrecompiledHeader>
+ </PrecompiledHeader>
+ <WarningLevel>Level4</WarningLevel>
+ <Optimization>Disabled</Optimization>
+ <PreprocessorDefinitions>WIN32;_DEBUG;LZ4_DLL_EXPORT=1;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ <TreatWarningAsError>true</TreatWarningAsError>
+ <EnablePREfast>true</EnablePREfast>
+ <AdditionalOptions>/analyze:stacksize295252 %(AdditionalOptions)</AdditionalOptions>
+ <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreadedDebug</RuntimeLibrary>
+ </ClCompile>
+ <Link>
+ <GenerateDebugInformation>true</GenerateDebugInformation>
+ </Link>
+ </ItemDefinitionGroup>
+ <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+ <ClCompile>
+ <WarningLevel>Level4</WarningLevel>
+ <PrecompiledHeader>
+ </PrecompiledHeader>
+ <Optimization>MaxSpeed</Optimization>
+ <FunctionLevelLinking>true</FunctionLevelLinking>
+ <IntrinsicFunctions>true</IntrinsicFunctions>
+ <PreprocessorDefinitions>WIN32;NDEBUG;LZ4_DLL_EXPORT=1;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ <TreatWarningAsError>false</TreatWarningAsError>
+ <EnablePREfast>false</EnablePREfast>
+ <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreaded</RuntimeLibrary>
+ </ClCompile>
+ <Link>
+ <GenerateDebugInformation>true</GenerateDebugInformation>
+ <EnableCOMDATFolding>true</EnableCOMDATFolding>
+ <OptimizeReferences>true</OptimizeReferences>
+ </Link>
+ </ItemDefinitionGroup>
+ <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+ <ClCompile>
+ <WarningLevel>Level4</WarningLevel>
+ <PrecompiledHeader>
+ </PrecompiledHeader>
+ <Optimization>MaxSpeed</Optimization>
+ <FunctionLevelLinking>true</FunctionLevelLinking>
+ <IntrinsicFunctions>true</IntrinsicFunctions>
+ <PreprocessorDefinitions>WIN32;NDEBUG;LZ4_DLL_EXPORT=1;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ <TreatWarningAsError>false</TreatWarningAsError>
+ <EnablePREfast>true</EnablePREfast>
+ <AdditionalOptions>/analyze:stacksize295252 %(AdditionalOptions)</AdditionalOptions>
+ <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreaded</RuntimeLibrary>
+ </ClCompile>
+ <Link>
+ <GenerateDebugInformation>true</GenerateDebugInformation>
+ <EnableCOMDATFolding>true</EnableCOMDATFolding>
+ <OptimizeReferences>true</OptimizeReferences>
+ </Link>
+ </ItemDefinitionGroup>
+ <ItemGroup>
+ <ClInclude Include="..\..\..\lib\lz4.h" />
+ <ClInclude Include="..\..\..\lib\lz4frame.h" />
+ <ClInclude Include="..\..\..\lib\lz4frame_static.h" />
+ <ClInclude Include="..\..\..\lib\lz4hc.h" />
+ <ClInclude Include="..\..\..\lib\xxhash.h" />
+ </ItemGroup>
+ <ItemGroup>
+ <ClCompile Include="..\..\..\lib\lz4.c" />
+ <ClCompile Include="..\..\..\lib\lz4frame.c" />
+ <ClCompile Include="..\..\..\lib\lz4hc.c" />
+ <ClCompile Include="..\..\..\lib\xxhash.c" />
+ </ItemGroup>
+ <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
+ <ImportGroup Label="ExtensionTargets">
+ </ImportGroup>
+</Project>
\ No newline at end of file
diff --git a/build/VS2022/lz4.sln b/build/VS2022/lz4.sln
new file mode 100644
index 0000000..6a2779f
--- /dev/null
+++ b/build/VS2022/lz4.sln
@@ -0,0 +1,103 @@
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio 15
+VisualStudioVersion = 15.0.28307.271
+MinimumVisualStudioVersion = 10.0.40219.1
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "liblz4-dll", "liblz4-dll\liblz4-dll.vcxproj", "{9800039D-4AAA-43A4-BB78-FEF6F4836927}"
+EndProject
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "liblz4", "liblz4\liblz4.vcxproj", "{9092C5CC-3E71-41B3-BF68-4A7BDD8A5476}"
+EndProject
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "fuzzer", "fuzzer\fuzzer.vcxproj", "{18B9F1A7-9C66-4352-898B-30804DADE0FD}"
+EndProject
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "fullbench", "fullbench\fullbench.vcxproj", "{6A4DF4EF-C77F-43C6-8901-DDCD20879E4E}"
+EndProject
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "frametest", "frametest\frametest.vcxproj", "{39AD6ECC-8BAD-4368-95E4-A1AA2F077BB7}"
+EndProject
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "datagen", "datagen\datagen.vcxproj", "{D745AE2F-596A-403A-9B91-81A8C6779243}"
+EndProject
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "fullbench-dll", "fullbench-dll\fullbench-dll.vcxproj", "{13992FD2-077E-4954-B065-A428198201A9}"
+ ProjectSection(ProjectDependencies) = postProject
+ {9800039D-4AAA-43A4-BB78-FEF6F4836927} = {9800039D-4AAA-43A4-BB78-FEF6F4836927}
+ EndProjectSection
+EndProject
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "lz4", "lz4\lz4.vcxproj", "{60A3115E-B988-41EE-8815-F4D4F253D866}"
+EndProject
+Global
+ GlobalSection(SolutionConfigurationPlatforms) = preSolution
+ Debug|Win32 = Debug|Win32
+ Debug|x64 = Debug|x64
+ Release|Win32 = Release|Win32
+ Release|x64 = Release|x64
+ EndGlobalSection
+ GlobalSection(ProjectConfigurationPlatforms) = postSolution
+ {9800039D-4AAA-43A4-BB78-FEF6F4836927}.Debug|Win32.ActiveCfg = Debug|Win32
+ {9800039D-4AAA-43A4-BB78-FEF6F4836927}.Debug|Win32.Build.0 = Debug|Win32
+ {9800039D-4AAA-43A4-BB78-FEF6F4836927}.Debug|x64.ActiveCfg = Debug|x64
+ {9800039D-4AAA-43A4-BB78-FEF6F4836927}.Debug|x64.Build.0 = Debug|x64
+ {9800039D-4AAA-43A4-BB78-FEF6F4836927}.Release|Win32.ActiveCfg = Release|Win32
+ {9800039D-4AAA-43A4-BB78-FEF6F4836927}.Release|Win32.Build.0 = Release|Win32
+ {9800039D-4AAA-43A4-BB78-FEF6F4836927}.Release|x64.ActiveCfg = Release|x64
+ {9800039D-4AAA-43A4-BB78-FEF6F4836927}.Release|x64.Build.0 = Release|x64
+ {9092C5CC-3E71-41B3-BF68-4A7BDD8A5476}.Debug|Win32.ActiveCfg = Debug|Win32
+ {9092C5CC-3E71-41B3-BF68-4A7BDD8A5476}.Debug|Win32.Build.0 = Debug|Win32
+ {9092C5CC-3E71-41B3-BF68-4A7BDD8A5476}.Debug|x64.ActiveCfg = Debug|x64
+ {9092C5CC-3E71-41B3-BF68-4A7BDD8A5476}.Debug|x64.Build.0 = Debug|x64
+ {9092C5CC-3E71-41B3-BF68-4A7BDD8A5476}.Release|Win32.ActiveCfg = Release|Win32
+ {9092C5CC-3E71-41B3-BF68-4A7BDD8A5476}.Release|Win32.Build.0 = Release|Win32
+ {9092C5CC-3E71-41B3-BF68-4A7BDD8A5476}.Release|x64.ActiveCfg = Release|x64
+ {9092C5CC-3E71-41B3-BF68-4A7BDD8A5476}.Release|x64.Build.0 = Release|x64
+ {18B9F1A7-9C66-4352-898B-30804DADE0FD}.Debug|Win32.ActiveCfg = Debug|Win32
+ {18B9F1A7-9C66-4352-898B-30804DADE0FD}.Debug|Win32.Build.0 = Debug|Win32
+ {18B9F1A7-9C66-4352-898B-30804DADE0FD}.Debug|x64.ActiveCfg = Debug|x64
+ {18B9F1A7-9C66-4352-898B-30804DADE0FD}.Debug|x64.Build.0 = Debug|x64
+ {18B9F1A7-9C66-4352-898B-30804DADE0FD}.Release|Win32.ActiveCfg = Release|Win32
+ {18B9F1A7-9C66-4352-898B-30804DADE0FD}.Release|Win32.Build.0 = Release|Win32
+ {18B9F1A7-9C66-4352-898B-30804DADE0FD}.Release|x64.ActiveCfg = Release|x64
+ {18B9F1A7-9C66-4352-898B-30804DADE0FD}.Release|x64.Build.0 = Release|x64
+ {6A4DF4EF-C77F-43C6-8901-DDCD20879E4E}.Debug|Win32.ActiveCfg = Debug|Win32
+ {6A4DF4EF-C77F-43C6-8901-DDCD20879E4E}.Debug|Win32.Build.0 = Debug|Win32
+ {6A4DF4EF-C77F-43C6-8901-DDCD20879E4E}.Debug|x64.ActiveCfg = Debug|x64
+ {6A4DF4EF-C77F-43C6-8901-DDCD20879E4E}.Debug|x64.Build.0 = Debug|x64
+ {6A4DF4EF-C77F-43C6-8901-DDCD20879E4E}.Release|Win32.ActiveCfg = Release|Win32
+ {6A4DF4EF-C77F-43C6-8901-DDCD20879E4E}.Release|Win32.Build.0 = Release|Win32
+ {6A4DF4EF-C77F-43C6-8901-DDCD20879E4E}.Release|x64.ActiveCfg = Release|x64
+ {6A4DF4EF-C77F-43C6-8901-DDCD20879E4E}.Release|x64.Build.0 = Release|x64
+ {39AD6ECC-8BAD-4368-95E4-A1AA2F077BB7}.Debug|Win32.ActiveCfg = Debug|Win32
+ {39AD6ECC-8BAD-4368-95E4-A1AA2F077BB7}.Debug|Win32.Build.0 = Debug|Win32
+ {39AD6ECC-8BAD-4368-95E4-A1AA2F077BB7}.Debug|x64.ActiveCfg = Debug|x64
+ {39AD6ECC-8BAD-4368-95E4-A1AA2F077BB7}.Debug|x64.Build.0 = Debug|x64
+ {39AD6ECC-8BAD-4368-95E4-A1AA2F077BB7}.Release|Win32.ActiveCfg = Release|Win32
+ {39AD6ECC-8BAD-4368-95E4-A1AA2F077BB7}.Release|Win32.Build.0 = Release|Win32
+ {39AD6ECC-8BAD-4368-95E4-A1AA2F077BB7}.Release|x64.ActiveCfg = Release|x64
+ {39AD6ECC-8BAD-4368-95E4-A1AA2F077BB7}.Release|x64.Build.0 = Release|x64
+ {D745AE2F-596A-403A-9B91-81A8C6779243}.Debug|Win32.ActiveCfg = Debug|Win32
+ {D745AE2F-596A-403A-9B91-81A8C6779243}.Debug|Win32.Build.0 = Debug|Win32
+ {D745AE2F-596A-403A-9B91-81A8C6779243}.Debug|x64.ActiveCfg = Debug|x64
+ {D745AE2F-596A-403A-9B91-81A8C6779243}.Debug|x64.Build.0 = Debug|x64
+ {D745AE2F-596A-403A-9B91-81A8C6779243}.Release|Win32.ActiveCfg = Release|Win32
+ {D745AE2F-596A-403A-9B91-81A8C6779243}.Release|Win32.Build.0 = Release|Win32
+ {D745AE2F-596A-403A-9B91-81A8C6779243}.Release|x64.ActiveCfg = Release|x64
+ {D745AE2F-596A-403A-9B91-81A8C6779243}.Release|x64.Build.0 = Release|x64
+ {13992FD2-077E-4954-B065-A428198201A9}.Debug|Win32.ActiveCfg = Debug|Win32
+ {13992FD2-077E-4954-B065-A428198201A9}.Debug|Win32.Build.0 = Debug|Win32
+ {13992FD2-077E-4954-B065-A428198201A9}.Debug|x64.ActiveCfg = Debug|x64
+ {13992FD2-077E-4954-B065-A428198201A9}.Debug|x64.Build.0 = Debug|x64
+ {13992FD2-077E-4954-B065-A428198201A9}.Release|Win32.ActiveCfg = Release|Win32
+ {13992FD2-077E-4954-B065-A428198201A9}.Release|Win32.Build.0 = Release|Win32
+ {13992FD2-077E-4954-B065-A428198201A9}.Release|x64.ActiveCfg = Release|x64
+ {13992FD2-077E-4954-B065-A428198201A9}.Release|x64.Build.0 = Release|x64
+ {60A3115E-B988-41EE-8815-F4D4F253D866}.Debug|Win32.ActiveCfg = Debug|Win32
+ {60A3115E-B988-41EE-8815-F4D4F253D866}.Debug|Win32.Build.0 = Debug|Win32
+ {60A3115E-B988-41EE-8815-F4D4F253D866}.Debug|x64.ActiveCfg = Debug|x64
+ {60A3115E-B988-41EE-8815-F4D4F253D866}.Debug|x64.Build.0 = Debug|x64
+ {60A3115E-B988-41EE-8815-F4D4F253D866}.Release|Win32.ActiveCfg = Release|Win32
+ {60A3115E-B988-41EE-8815-F4D4F253D866}.Release|Win32.Build.0 = Release|Win32
+ {60A3115E-B988-41EE-8815-F4D4F253D866}.Release|x64.ActiveCfg = Release|x64
+ {60A3115E-B988-41EE-8815-F4D4F253D866}.Release|x64.Build.0 = Release|x64
+ EndGlobalSection
+ GlobalSection(SolutionProperties) = preSolution
+ HideSolutionNode = FALSE
+ EndGlobalSection
+ GlobalSection(ExtensibilityGlobals) = postSolution
+ SolutionGuid = {BBC259B2-BABF-47CD-8A6A-7B8318A803AC}
+ EndGlobalSection
+EndGlobal
diff --git a/build/cmake/.gitignore b/build/cmake/.gitignore
index d39505d..0ad8240 100644
--- a/build/cmake/.gitignore
+++ b/build/cmake/.gitignore
@@ -1,4 +1,4 @@
-# cmake artefact
+# cmake build artefact
CMakeCache.txt
CMakeFiles
@@ -7,3 +7,4 @@
liblz4.pc
lz4c
install_manifest.txt
+build
diff --git a/build/cmake/CMakeLists.txt b/build/cmake/CMakeLists.txt
index 57501ee..eb7007b 100644
--- a/build/cmake/CMakeLists.txt
+++ b/build/cmake/CMakeLists.txt
@@ -10,10 +10,9 @@
# LZ4's CMake support is maintained by Evan Nemerson; when filing
# bugs please mention @nemequ to make sure I see it.
-set(LZ4_TOP_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../..")
+cmake_minimum_required(VERSION 2.8.12)
-option(LZ4_BUILD_CLI "Build lz4 program" ON)
-option(LZ4_BUILD_LEGACY_LZ4C "Build lz4c progam with legacy argument support" ON)
+set(LZ4_TOP_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../..")
# Parse version information
file(STRINGS "${LZ4_TOP_SOURCE_DIR}/lib/lz4.h" LZ4_VERSION_MAJOR REGEX "^#define LZ4_VERSION_MAJOR +([0-9]+) +.*$")
@@ -34,7 +33,8 @@
LANGUAGES C)
endif()
-cmake_minimum_required (VERSION 2.8.6)
+option(LZ4_BUILD_CLI "Build lz4 program" ON)
+option(LZ4_BUILD_LEGACY_LZ4C "Build lz4c program with legacy argument support" ON)
# If LZ4 is being bundled in another project, we don't want to
# install anything. However, we want to let people override this, so
@@ -103,6 +103,9 @@
set(LZ4_LIBRARIES_BUILT)
if(BUILD_SHARED_LIBS)
add_library(lz4_shared SHARED ${LZ4_SOURCES})
+ target_include_directories(lz4_shared
+ PUBLIC $<BUILD_INTERFACE:${LZ4_LIB_SOURCE_DIR}>
+ INTERFACE $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>)
set_target_properties(lz4_shared PROPERTIES
OUTPUT_NAME lz4
SOVERSION "${LZ4_VERSION_MAJOR}"
@@ -114,18 +117,24 @@
list(APPEND LZ4_LIBRARIES_BUILT lz4_shared)
endif()
if(BUILD_STATIC_LIBS)
+ set(STATIC_LIB_NAME lz4)
+ if (MSVC AND BUILD_SHARED_LIBS)
+ set(STATIC_LIB_NAME lz4_static)
+ endif()
add_library(lz4_static STATIC ${LZ4_SOURCES})
+ target_include_directories(lz4_static
+ PUBLIC $<BUILD_INTERFACE:${LZ4_LIB_SOURCE_DIR}>
+ INTERFACE $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>)
set_target_properties(lz4_static PROPERTIES
- OUTPUT_NAME lz4
+ OUTPUT_NAME ${STATIC_LIB_NAME}
POSITION_INDEPENDENT_CODE ${LZ4_POSITION_INDEPENDENT_LIB})
list(APPEND LZ4_LIBRARIES_BUILT lz4_static)
endif()
-# link to shared whenever possible, to static otherwise
-if(BUILD_SHARED_LIBS)
- set(LZ4_LINK_LIBRARY lz4_shared)
-else()
+if(BUILD_STATIC_LIBS)
set(LZ4_LINK_LIBRARY lz4_static)
+else()
+ list(APPEND LZ4_CLI_SOURCES ${LZ4_SOURCES})
endif()
# lz4
@@ -133,7 +142,9 @@
set(LZ4_PROGRAMS_BUILT lz4cli)
add_executable(lz4cli ${LZ4_CLI_SOURCES})
set_target_properties(lz4cli PROPERTIES OUTPUT_NAME lz4)
- target_link_libraries(lz4cli ${LZ4_LINK_LIBRARY})
+ if (BUILD_STATIC_LIBS)
+ target_link_libraries(lz4cli ${LZ4_LINK_LIBRARY})
+ endif()
endif()
# lz4c
@@ -141,7 +152,9 @@
list(APPEND LZ4_PROGRAMS_BUILT lz4c)
add_executable(lz4c ${LZ4_CLI_SOURCES})
set_target_properties(lz4c PROPERTIES COMPILE_DEFINITIONS "ENABLE_LZ4C_LEGACY_OPTIONS")
- target_link_libraries(lz4c ${LZ4_LINK_LIBRARY})
+ if (BUILD_STATIC_LIBS)
+ target_link_libraries(lz4c ${LZ4_LINK_LIBRARY})
+ endif()
endif()
# Extra warning flags
@@ -182,6 +195,7 @@
BUNDLE DESTINATION "${CMAKE_INSTALL_BINDIR}"
RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}")
install(TARGETS ${LZ4_LIBRARIES_BUILT}
+ EXPORT lz4Targets
LIBRARY DESTINATION "${CMAKE_INSTALL_LIBDIR}"
ARCHIVE DESTINATION "${CMAKE_INSTALL_LIBDIR}"
RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}")
@@ -195,6 +209,30 @@
install(FILES "${CMAKE_CURRENT_BINARY_DIR}/liblz4.pc"
DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig")
+ include(CMakePackageConfigHelpers)
+ write_basic_package_version_file(
+ "${CMAKE_CURRENT_BINARY_DIR}/lz4ConfigVersion.cmake"
+ VERSION ${LZ4_VERSION_STRING}
+ COMPATIBILITY SameMajorVersion)
+
+ set(LZ4_PKG_INSTALLDIR "${CMAKE_INSTALL_LIBDIR}/cmake/lz4")
+ configure_package_config_file(
+ "${CMAKE_CURRENT_LIST_DIR}/lz4Config.cmake.in"
+ "${CMAKE_CURRENT_BINARY_DIR}/lz4Config.cmake"
+ INSTALL_DESTINATION ${LZ4_PKG_INSTALLDIR})
+ export(EXPORT lz4Targets
+ FILE ${CMAKE_CURRENT_BINARY_DIR}/lz4Targets.cmake
+ NAMESPACE LZ4::)
+
+ install(EXPORT lz4Targets
+ FILE lz4Targets.cmake
+ NAMESPACE LZ4::
+ DESTINATION ${LZ4_PKG_INSTALLDIR})
+ install(FILES
+ ${CMAKE_CURRENT_BINARY_DIR}/lz4Config.cmake
+ ${CMAKE_CURRENT_BINARY_DIR}/lz4ConfigVersion.cmake
+ DESTINATION ${LZ4_PKG_INSTALLDIR})
+
# install lz4cat and unlz4 symlinks on *nix
if(UNIX AND LZ4_BUILD_CLI)
install(CODE "
diff --git a/build/cmake/lz4Config.cmake.in b/build/cmake/lz4Config.cmake.in
new file mode 100644
index 0000000..e9c9473
--- /dev/null
+++ b/build/cmake/lz4Config.cmake.in
@@ -0,0 +1,2 @@
+@PACKAGE_INIT@
+include( "${CMAKE_CURRENT_LIST_DIR}/lz4Targets.cmake" )
\ No newline at end of file
diff --git a/contrib/gen_manual/Makefile b/contrib/gen_manual/Makefile
index 95abe2e..262c80d 100644
--- a/contrib/gen_manual/Makefile
+++ b/contrib/gen_manual/Makefile
@@ -30,10 +30,10 @@
# ################################################################
-CXXFLAGS ?= -O3
+CXXFLAGS ?= -O2
CXXFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 -Wswitch-enum -Wno-comment
-CXXFLAGS += $(MOREFLAGS)
-FLAGS = $(CPPFLAGS) $(CXXFLAGS) $(LDFLAGS)
+CPPFLAGS += $(MOREFLAGS)
+FLAGS = $(CXXFLAGS) $(CPPFLAGS) $(LDFLAGS)
LZ4API = ../../lib/lz4.h
LZ4MANUAL = ../../doc/lz4_manual.html
@@ -68,7 +68,7 @@
./gen_manual $(LZ4VER) $(LZ4FAPI) $@
.PHONY: manuals
-manuals: gen_manual $(LZ4MANUAL) $(LZ4FMANUAL)
+manuals: $(LZ4MANUAL) $(LZ4FMANUAL)
.PHONY: clean
clean:
diff --git a/contrib/meson/README.md b/contrib/meson/README.md
index a44850a..1dc1bd9 100644
--- a/contrib/meson/README.md
+++ b/contrib/meson/README.md
@@ -13,7 +13,7 @@
`cd` to this meson directory (`contrib/meson`)
```sh
-meson setup --buildtype=release -Ddefault_library=shared -Dbin_programs=true builddir
+meson setup --buildtype=release -Ddefault_library=shared -Dprograms=true builddir
cd builddir
ninja # to build
ninja install # to install
diff --git a/contrib/meson/meson.build b/contrib/meson/meson.build
index d1e97d9..39672c8 100644
--- a/contrib/meson/meson.build
+++ b/contrib/meson/meson.build
@@ -11,11 +11,17 @@
# The intention is that it can be easily moved to the root of the project
# (together with meson_options.txt) and packaged for wrapdb.
-project('lz4', ['c'],
- license: ['BSD', 'GPLv2'],
- default_options : ['c_std=c99',
- 'buildtype=release'],
+project(
+ 'lz4',
+ ['c'],
+ license: 'BSD-2-Clause-Patent AND GPL-2.0-or-later',
+ default_options: [
+ 'c_std=c99',
+ 'buildtype=release',
+ 'warning_level=3'
+ ],
version: 'DUMMY',
- meson_version: '>=0.47.0')
+ meson_version: '>=0.49.0'
+)
subdir('meson')
diff --git a/contrib/meson/meson/InstallSymlink.py b/contrib/meson/meson/InstallSymlink.py
deleted file mode 100644
index 3f2998c..0000000
--- a/contrib/meson/meson/InstallSymlink.py
+++ /dev/null
@@ -1,55 +0,0 @@
-#!/usr/bin/env python3
-# #############################################################################
-# Copyright (c) 2018-present lzutao <taolzu(at)gmail.com>
-# All rights reserved.
-#
-# This source code is licensed under both the BSD-style license (found in the
-# LICENSE file in the root directory of this source tree) and the GPLv2 (found
-# in the COPYING file in the root directory of this source tree).
-# #############################################################################
-# This file should be synced with https://github.com/lzutao/meson-symlink
-
-import os
-import pathlib # since Python 3.4
-
-
-def install_symlink(src, dst, install_dir, dst_is_dir=False, dir_mode=0o777):
- if not install_dir.exists():
- install_dir.mkdir(mode=dir_mode, parents=True, exist_ok=True)
- if not install_dir.is_dir():
- raise NotADirectoryError(install_dir)
-
- new_dst = install_dir.joinpath(dst)
- if new_dst.is_symlink() and os.readlink(new_dst) == src:
- print('File exists: {!r} -> {!r}'.format(new_dst, src))
- return
- print('Installing symlink {!r} -> {!r}'.format(new_dst, src))
- new_dst.symlink_to(src, target_is_directory=dst_is_dir)
-
-
-def main():
- import argparse
- parser = argparse.ArgumentParser(description='Install a symlink',
- usage='{0} [-h] [-d] [-m MODE] source dest install_dir\n\n'
- 'example:\n'
- ' {0} dash sh /bin'.format(pathlib.Path(__file__).name))
- parser.add_argument('source', help='target to link')
- parser.add_argument('dest', help='link name')
- parser.add_argument('install_dir', help='installation directory')
- parser.add_argument('-d', '--isdir',
- action='store_true',
- help='dest is a directory')
- parser.add_argument('-m', '--mode',
- help='directory mode on creating if not exist',
- default='0o755')
- args = parser.parse_args()
-
- dir_mode = int(args.mode, 8)
-
- meson_destdir = os.environ.get('MESON_INSTALL_DESTDIR_PREFIX', default='')
- install_dir = pathlib.Path(meson_destdir, args.install_dir)
- install_symlink(args.source, args.dest, install_dir, args.isdir, dir_mode)
-
-
-if __name__ == '__main__':
- main()
diff --git a/contrib/meson/meson/contrib/gen_manual/meson.build b/contrib/meson/meson/contrib/gen_manual/meson.build
index a872bd6..84a95a9 100644
--- a/contrib/meson/meson/contrib/gen_manual/meson.build
+++ b/contrib/meson/meson/contrib/gen_manual/meson.build
@@ -1,5 +1,6 @@
# #############################################################################
-# Copyright (c) 2018-present lzutao <taolzu(at)gmail.com>
+# Copyright (c) 2018-present lzutao <taolzu(at)gmail.com>
+# Copyright (c) 2022-present Tristan Partin <tristan(at)partin.io>
# All rights reserved.
#
# This source code is licensed under both the BSD-style license (found in the
@@ -7,37 +8,35 @@
# in the COPYING file in the root directory of this source tree).
# #############################################################################
-lz4_root_dir = '../../../../..'
+lz4_source_root = '../../../../..'
add_languages('cpp')
-cxx = meson.get_compiler('cpp')
-gen_manual_includes = include_directories(join_paths(lz4_root_dir, 'contrib/gen_manual'))
+sources = files(
+ lz4_source_root / 'contrib/gen_manual/gen_manual.cpp'
+)
-gen_manual_cppflags = cxx.get_supported_arguments(['-Wextra', '-Wcast-qual',
- '-Wcast-align', '-Wshadow', '-Wstrict-aliasing=1', '-Wswitch-enum',
- '-Wno-comment'])
-
-gen_manual = executable('gen_manual',
- join_paths(lz4_root_dir, 'contrib/gen_manual/gen_manual.cpp'),
- cpp_args: gen_manual_cppflags,
- include_directories: gen_manual_includes,
+gen_manual = executable(
+ 'gen_manual',
+ sources,
native: true,
- install: false)
+ install: false
+)
-# Update lz4 manual
-lz4_manual_html = custom_target('lz4_manual.html',
- output : 'lz4_manual.html',
- command : [gen_manual,
- lz4_version,
- join_paths(meson.current_source_dir(), lz4_root_dir, 'lib/lz4.h'),
- '@OUTPUT@'],
- install : false)
-# Update lz4frame manual
-lz4_manual_html = custom_target('lz4frame_manual.html',
- output : 'lz4frame_manual.html',
- command : [gen_manual,
- lz4_version,
- join_paths(meson.current_source_dir(), lz4_root_dir, 'lib/lz4frame.h'),
- '@OUTPUT@'],
- install : false)
+manual_pages = ['lz4', 'lz4frame']
+
+foreach mp : manual_pages
+ custom_target(
+ '@0@_manual.html'.format(mp),
+ build_by_default: true,
+ input: lz4_source_root / 'lib/@0@.h'.format(mp),
+ output: '@0@_manual.html'.format(mp),
+ command: [
+ gen_manual,
+ lz4_version,
+ '@INPUT@',
+ '@OUTPUT@',
+ ],
+ install: false
+ )
+endforeach
diff --git a/contrib/meson/meson/contrib/meson.build b/contrib/meson/meson/contrib/meson.build
index 5249a4c..ef780fb 100644
--- a/contrib/meson/meson/contrib/meson.build
+++ b/contrib/meson/meson/contrib/meson.build
@@ -1,5 +1,6 @@
# #############################################################################
# Copyright (c) 2018-present lzutao <taolzu(at)gmail.com>
+# Copyright (c) 2022-present Tristan Partin <tristan(at)partin.io>
# All rights reserved.
#
# This source code is licensed under both the BSD-style license (found in the
diff --git a/contrib/meson/meson/examples/meson.build b/contrib/meson/meson/examples/meson.build
index 493049d..65f54ca 100644
--- a/contrib/meson/meson/examples/meson.build
+++ b/contrib/meson/meson/examples/meson.build
@@ -1,5 +1,6 @@
# #############################################################################
-# Copyright (c) 2018-present lzutao <taolzu(at)gmail.com>
+# Copyright (c) 2018-present lzutao <taolzu(at)gmail.com>
+# Copyright (c) 2022-present Tristan Partin <tristan(at)partin.io>
# All rights reserved.
#
# This source code is licensed under both the BSD-style license (found in the
@@ -7,43 +8,25 @@
# in the COPYING file in the root directory of this source tree).
# #############################################################################
-lz4_root_dir = '../../../..'
+lz4_source_root = '../../../..'
-#examples_c_args = ['-Wextra', '-Wundef', '-Wshadow', '-Wcast-align', '-Wstrict-prototypes']
+examples = {
+ 'printVersion': 'printVersion.c',
+ 'doubleBuffer': 'blockStreaming_doubleBuffer.c',
+ 'dictionaryRandomAccess': 'dictionaryRandomAccess.c',
+ 'ringBuffer': 'blockStreaming_ringBuffer.c',
+ 'ringBufferHC': 'HCStreaming_ringBuffer.c',
+ 'lineCompress': 'blockStreaming_lineByLine.c',
+ 'frameCompress': 'frameCompress.c',
+ 'compressFunctions': 'compress_functions.c',
+ 'simpleBuffer': 'simple_buffer.c',
+}
-printVersion = executable('printVersion',
- join_paths(lz4_root_dir, 'examples/printVersion.c'),
- dependencies: liblz4_dep,
- install: false)
-doubleBuffer = executable('doubleBuffer',
- join_paths(lz4_root_dir, 'examples/blockStreaming_doubleBuffer.c'),
- dependencies: liblz4_dep,
- install: false)
-dictionaryRandomAccess = executable('dictionaryRandomAccess',
- join_paths(lz4_root_dir, 'examples/dictionaryRandomAccess.c'),
- dependencies: liblz4_dep,
- install: false)
-ringBuffer = executable('ringBuffer',
- join_paths(lz4_root_dir, 'examples/blockStreaming_ringBuffer.c'),
- dependencies: liblz4_dep,
- install: false)
-ringBufferHC = executable('ringBufferHC',
- join_paths(lz4_root_dir, 'examples/HCStreaming_ringBuffer.c'),
- dependencies: liblz4_dep,
- install: false)
-lineCompress = executable('lineCompress',
- join_paths(lz4_root_dir, 'examples/blockStreaming_lineByLine.c'),
- dependencies: liblz4_dep,
- install: false)
-frameCompress = executable('frameCompress',
- join_paths(lz4_root_dir, 'examples/frameCompress.c'),
- dependencies: liblz4_dep,
- install: false)
-compressFunctions = executable('compressFunctions',
- join_paths(lz4_root_dir, 'examples/compress_functions.c'),
- dependencies: liblz4_dep,
- install: false)
-simpleBuffer = executable('simpleBuffer',
- join_paths(lz4_root_dir, 'examples/simple_buffer.c'),
- dependencies: liblz4_dep,
- install: false)
+foreach e, src : examples
+ executable(
+ e,
+ lz4_source_root / 'examples' / src,
+ dependencies: [liblz4_internal_dep],
+ install: false
+ )
+endforeach
diff --git a/contrib/meson/meson/lib/meson.build b/contrib/meson/meson/lib/meson.build
index 131edcb..469cd09 100644
--- a/contrib/meson/meson/lib/meson.build
+++ b/contrib/meson/meson/lib/meson.build
@@ -1,5 +1,6 @@
# #############################################################################
-# Copyright (c) 2018-present lzutao <taolzu(at)gmail.com>
+# Copyright (c) 2018-present lzutao <taolzu(at)gmail.com>
+# Copyright (c) 2022-present Tristan Partin <tristan(at)partin.io>
# All rights reserved.
#
# This source code is licensed under both the BSD-style license (found in the
@@ -7,51 +8,69 @@
# in the COPYING file in the root directory of this source tree).
# #############################################################################
-lz4_root_dir = '../../../..'
+lz4_source_root = '../../../..'
-liblz4_includes = [include_directories(join_paths(lz4_root_dir, 'lib'))]
-liblz4_sources = [join_paths(lz4_root_dir, 'lib/lz4.c'),
- join_paths(lz4_root_dir, 'lib/lz4frame.c'),
- join_paths(lz4_root_dir, 'lib/lz4hc.c'),
- join_paths(lz4_root_dir, 'lib/xxhash.c')]
-liblz4_c_args = []
+sources = files(
+ lz4_source_root / 'lib/lz4.c',
+ lz4_source_root / 'lib/lz4frame.c',
+ lz4_source_root / 'lib/lz4hc.c',
+ lz4_source_root / 'lib/xxhash.c'
+)
-liblz4_debug_cflags = []
-if use_debug
- liblz4_c_args += '-DLZ4_DEBUG=@0@'.format(debug_level)
- if [compiler_gcc, compiler_clang].contains(cc_id)
- liblz4_debug_cflags = ['-Wextra', '-Wcast-qual', '-Wcast-align', '-Wshadow',
- '-Wswitch-enum', '-Wdeclaration-after-statement', '-Wstrict-prototypes',
- '-Wundef', '-Wpointer-arith', '-Wstrict-aliasing=1']
+c_args = []
+
+if host_machine.system() == 'windows' and get_option('default_library') != 'static'
+ c_args += '-DLZ4_DLL_EXPORT=1'
+endif
+
+if get_option('unstable')
+ compile_args += '-DLZ4_STATIC_LINKING_ONLY'
+ if get_option('default_library') != 'static'
+ c_args += '-DLZ4_PUBLISH_STATIC_FUNCTIONS'
endif
endif
-liblz4_c_args += cc.get_supported_arguments(liblz4_debug_cflags)
-if host_machine_os == os_windows and default_library != 'static'
- liblz4_c_args += '-DLZ4_DLL_EXPORT=1'
+liblz4 = library(
+ 'lz4',
+ sources,
+ install: true,
+ version: lz4_version,
+ gnu_symbol_visibility: 'hidden'
+)
+
+liblz4_dep = declare_dependency(
+ link_with: liblz4,
+ include_directories: include_directories(lz4_source_root / 'lib')
+)
+
+if get_option('tests') or get_option('programs') or get_option('examples')
+ liblz4_internal = static_library(
+ 'lz4-internal',
+ objects: liblz4.extract_all_objects(recursive: true),
+ gnu_symbol_visibility: 'hidden'
+ )
+
+ liblz4_internal_dep = declare_dependency(
+ link_with: liblz4_internal,
+ include_directories: include_directories(lz4_source_root / 'lib')
+ )
endif
-liblz4 = library('lz4',
- liblz4_sources,
- include_directories: liblz4_includes,
- c_args: liblz4_c_args,
- install: true,
- version: lz4_libversion)
-
-liblz4_dep = declare_dependency(link_with: liblz4,
- include_directories: liblz4_includes)
-
-pkgconfig.generate(liblz4,
+pkgconfig.generate(
+ liblz4,
name: 'lz4',
filebase: 'liblz4',
description: 'extremely fast lossless compression algorithm library',
- version: lz4_libversion,
- url: 'http://www.lz4.org/')
+ version: lz4_version,
+ url: 'http://www.lz4.org/'
+)
-install_headers(join_paths(lz4_root_dir, 'lib/lz4.h'),
- join_paths(lz4_root_dir, 'lib/lz4hc.h'),
- join_paths(lz4_root_dir, 'lib/lz4frame.h'))
+install_headers(
+ lz4_source_root / 'lib/lz4.h',
+ lz4_source_root / 'lib/lz4hc.h',
+ lz4_source_root / 'lib/lz4frame.h'
+)
-if default_library != 'shared'
- install_headers(join_paths(lz4_root_dir, 'lib/lz4frame_static.h'))
+if get_option('default_library') != 'shared'
+ install_headers(lz4_source_root / 'lib/lz4frame_static.h')
endif
diff --git a/contrib/meson/meson/meson.build b/contrib/meson/meson/meson.build
index b278b7c..9e8b8c6 100644
--- a/contrib/meson/meson/meson.build
+++ b/contrib/meson/meson/meson.build
@@ -1,5 +1,6 @@
# #############################################################################
-# Copyright (c) 2018-present lzutao <taolzu(at)gmail.com>
+# Copyright (c) 2018-present lzutao <taolzu(at)gmail.com>
+# Copyright (c) 2022-present Tristan Partin <tristan(at)partin.io>
# All rights reserved.
#
# This source code is licensed under both the BSD-style license (found in the
@@ -8,110 +9,59 @@
# #############################################################################
cc = meson.get_compiler('c')
+
pkgconfig = import('pkgconfig')
-c_std = get_option('c_std')
-default_library = get_option('default_library')
-host_machine_os = host_machine.system()
-os_windows = 'windows'
-os_linux = 'linux'
-os_darwin = 'darwin'
-os_freebsd = 'freebsd'
-os_sun = 'sunos'
-
-cc_id = cc.get_id()
-compiler_gcc = 'gcc'
-compiler_clang = 'clang'
-compiler_msvc = 'msvc'
+lz4_source_root = '../../..'
lz4_version = meson.project_version()
-lz4_h_file = join_paths(meson.current_source_dir(), '../../../lib/lz4.h')
-GetLz4LibraryVersion_py = find_program('GetLz4LibraryVersion.py', native : true)
-r = run_command(GetLz4LibraryVersion_py, lz4_h_file)
-if r.returncode() == 0
- lz4_version = r.stdout().strip()
- message('Project version is now: @0@'.format(lz4_version))
-else
- error('Cannot find project version in @0@'.format(lz4_h_file))
+lz4_h_file = lz4_source_root / 'lib/lz4.h'
+GetLz4LibraryVersion_py = find_program('GetLz4LibraryVersion.py')
+lz4_version = run_command(GetLz4LibraryVersion_py, lz4_h_file, check: true).stdout().strip()
+message('Project version is now: @0@'.format(lz4_version))
+
+add_project_arguments('-DXXH_NAMESPACE=LZ4_', language: 'c')
+
+if get_option('debug')
+ add_project_arguments(cc.get_supported_arguments([
+ '-Wcast-qual',
+ '-Wcast-align',
+ '-Wshadow',
+ '-Wswitch-enum',
+ '-Wdeclaration-after-statement',
+ '-Wstrict-prototypes',
+ '-Wundef',
+ '-Wpointer-arith',
+ '-Wstrict-aliasing=1',
+ '-DLZ4_DEBUG=@0@'.format(get_option('debug-level')),
+ ]
+ ),
+ language: 'c',
+ )
endif
-lz4_libversion = lz4_version
-
-# =============================================================================
-# Installation directories
-# =============================================================================
-
-lz4_prefix = get_option('prefix')
-lz4_bindir = get_option('bindir')
-lz4_datadir = get_option('datadir')
-lz4_mandir = get_option('mandir')
-lz4_docdir = join_paths(lz4_datadir, 'doc', meson.project_name())
-
-# =============================================================================
-# Project options
-# =============================================================================
-
-buildtype = get_option('buildtype')
-
-# Built-in options
-use_debug = get_option('debug')
-
-# Custom options
-debug_level = get_option('debug_level')
-use_backtrace = get_option('backtrace')
-
-bin_programs = get_option('bin_programs')
-bin_contrib = get_option('bin_contrib')
-bin_tests = get_option('bin_tests')
-bin_examples = get_option('bin_examples')
-#feature_multi_thread = get_option('multi_thread')
-
-# =============================================================================
-# Dependencies
-# =============================================================================
-
-#libm_dep = cc.find_library('m', required: bin_tests)
-#thread_dep = dependency('threads', required: feature_multi_thread)
-#use_multi_thread = thread_dep.found()
-
-# =============================================================================
-# Compiler flags
-# =============================================================================
-
-add_project_arguments(['-DXXH_NAMESPACE=LZ4_'], language: 'c')
-
-if [compiler_gcc, compiler_clang].contains(cc_id)
- common_warning_flags = []
- # Should use Meson's own --werror build option
- #common_warning_flags += ['-Werror']
- if c_std == 'c89' or c_std == 'gnu89'
- common_warning_flags += ['-pedantic', '-Wno-long-long', '-Wno-variadic-macros']
- elif c_std == 'c99' or c_std == 'gnu99'
- common_warning_flags += ['-pedantic']
- endif
- cc_compile_flags = cc.get_supported_arguments(common_warning_flags)
- add_project_arguments(cc_compile_flags, language: 'c')
+if get_option('memory-usage') > 0
+ add_project_arguments(
+ '-DLZ4_MEMORY_USAGE=@0@'.format(get_option('memory-usage')),
+ language: 'c'
+ )
endif
-# =============================================================================
-# Subdirs
-# =============================================================================
-
subdir('lib')
-if bin_programs
+if get_option('programs')
subdir('programs')
endif
-if bin_tests
+if get_option('tests')
subdir('tests')
endif
-if bin_contrib
+if get_option('contrib')
subdir('contrib')
endif
-if bin_examples
+if get_option('examples')
subdir('examples')
endif
diff --git a/contrib/meson/meson/programs/meson.build b/contrib/meson/meson/programs/meson.build
index 705dbf5..f9d5bf1 100644
--- a/contrib/meson/meson/programs/meson.build
+++ b/contrib/meson/meson/programs/meson.build
@@ -1,5 +1,6 @@
# #############################################################################
-# Copyright (c) 2018-present lzutao <taolzu(at)gmail.com>
+# Copyright (c) 2018-present lzutao <taolzu(at)gmail.com>
+# Copyright (c) 2022-present Tristan Partin <tristan(at)partin.io>
# All rights reserved.
#
# This source code is licensed under both the BSD-style license (found in the
@@ -7,46 +8,37 @@
# in the COPYING file in the root directory of this source tree).
# #############################################################################
-lz4_root_dir = '../../../..'
+lz4_source_root = '../../../..'
-lz4_includes = include_directories(join_paths(lz4_root_dir, 'programs'))
-lz4_sources = [join_paths(lz4_root_dir, 'programs/bench.c'),
- join_paths(lz4_root_dir, 'programs/datagen.c'),
- join_paths(lz4_root_dir, 'programs/lz4cli.c'),
- join_paths(lz4_root_dir, 'programs/lz4io.c')]
-lz4_c_args = []
+sources = files(
+ lz4_source_root / 'programs/bench.c',
+ lz4_source_root / 'programs/datagen.c',
+ lz4_source_root / 'programs/lz4cli.c',
+ lz4_source_root / 'programs/lz4io.c',
+)
-export_dynamic_on_windows = false
-# explicit backtrace enable/disable for Linux & Darwin
-if not use_backtrace
- lz4_c_args += '-DBACKTRACE_ENABLE=0'
-elif use_debug and host_machine_os == os_windows # MinGW target
- lz4_c_args += '-DBACKTRACE_ENABLE=1'
- export_dynamic_on_windows = true
+lz4 = executable(
+ 'lz4',
+ sources,
+ include_directories: include_directories(lz4_source_root / 'programs'),
+ dependencies: [liblz4_internal_dep],
+ export_dynamic: get_option('debug') and host_machine.system() == 'windows',
+ install: true
+)
+
+install_man(lz4_source_root / 'programs/lz4.1')
+
+if meson.version().version_compare('>=0.61.0')
+ foreach alias : ['lz4c', 'lz4cat', 'unlz4']
+ install_symlink(
+ alias,
+ install_dir: get_option('bindir'),
+ pointing_to: 'lz4'
+ )
+ install_symlink(
+ '@0@.1'.format(alias),
+ install_dir: get_option('mandir') / 'man1',
+ pointing_to: 'lz4.1'
+ )
+ endforeach
endif
-
-lz4_deps = [ liblz4_dep ]
-
-lz4 = executable('lz4',
- lz4_sources,
- include_directories: lz4_includes,
- c_args: lz4_c_args,
- dependencies: lz4_deps,
- export_dynamic: export_dynamic_on_windows, # Since Meson 0.45.0
- install: true)
-
-# =============================================================================
-# Programs and manpages installing
-# =============================================================================
-
-install_man(join_paths(lz4_root_dir, 'programs/lz4.1'))
-
-InstallSymlink_py = '../InstallSymlink.py'
-lz4_man1_dir = join_paths(lz4_mandir, 'man1')
-bin_EXT = host_machine_os == os_windows ? '.exe' : ''
-man1_EXT = meson.version().version_compare('>=0.49.0') ? '.1' : '.1.gz'
-
-foreach f : ['lz4c', 'lz4cat', 'unlz4']
- meson.add_install_script(InstallSymlink_py, 'lz4' + bin_EXT, f + bin_EXT, lz4_bindir)
- meson.add_install_script(InstallSymlink_py, 'lz4' + man1_EXT, f + man1_EXT, lz4_man1_dir)
-endforeach
diff --git a/contrib/meson/meson/tests/meson.build b/contrib/meson/meson/tests/meson.build
index 7800475..18479e4 100644
--- a/contrib/meson/meson/tests/meson.build
+++ b/contrib/meson/meson/tests/meson.build
@@ -1,5 +1,6 @@
# #############################################################################
-# Copyright (c) 2018-present lzutao <taolzu(at)gmail.com>
+# Copyright (c) 2018-present lzutao <taolzu(at)gmail.com>
+# Copyright (c) 2022-present Tristan Partin <tristan(at)partin.io>
# All rights reserved.
#
# This source code is licensed under both the BSD-style license (found in the
@@ -7,87 +8,45 @@
# in the COPYING file in the root directory of this source tree).
# #############################################################################
-lz4_root_dir = '../../../..'
-programs_dir_inc = include_directories(join_paths(lz4_root_dir, 'programs'))
-lib_dir_inc = include_directories(join_paths(lz4_root_dir, 'lib'))
+lz4_source_root = '../../../..'
-# =============================================================================
-# Test flags
-# =============================================================================
+exes = {
+ 'fullbench': {
+ 'sources': files(lz4_source_root / 'tests/fullbench.c'),
+ 'include_directories': include_directories(lz4_source_root / 'programs'),
+ },
+ 'fuzzer': {
+ 'sources': files(lz4_source_root / 'tests/fuzzer.c'),
+ 'include_directories': include_directories(lz4_source_root / 'programs'),
+ },
+ 'frametest': {
+ 'sources': files(lz4_source_root / 'tests/frametest.c'),
+ 'include_directories': include_directories(lz4_source_root / 'programs'),
+ },
+ 'roundTripTest': {
+ 'sources': files(lz4_source_root / 'tests/roundTripTest.c'),
+ },
+ 'datagen': {
+ 'sources': files(lz4_source_root / 'tests/datagencli.c'),
+ 'objects': lz4.extract_objects(lz4_source_root / 'programs/datagen.c'),
+ 'include_directories': include_directories(lz4_source_root / 'programs'),
+ },
+ 'checkFrame': {
+ 'sources': files(lz4_source_root / 'tests/checkFrame.c'),
+ 'include_directories': include_directories(lz4_source_root / 'programs'),
+ },
+ 'checkTag': {
+ 'sources': files(lz4_source_root / 'tests/checkTag.c'),
+ },
+}
-TEST_FILES = join_paths(meson.current_source_dir(), lz4_root_dir, 'tests/COPYING')
-FUZZER_TIME = '-T90s'
-NB_LOOPS = '-i1'
-
-# =============================================================================
-# Executables
-# =============================================================================
-
-fullbench_sources = [join_paths(lz4_root_dir, 'tests/fullbench.c')]
-fullbench = executable('fullbench',
- fullbench_sources,
- include_directories: programs_dir_inc,
- dependencies: liblz4_dep,
- install: false)
-
-fuzzer_sources = [join_paths(lz4_root_dir, 'tests/fuzzer.c')]
-fuzzer = executable('fuzzer',
- fuzzer_sources,
- c_args: ['-D_DEFAULT_SOURCE', '-D_BSD_SOURCE'], # since glibc 2.19
- include_directories: programs_dir_inc,
- dependencies: liblz4_dep,
- install: false)
-
-frametest_sources = [join_paths(lz4_root_dir, 'tests/frametest.c')]
-frametest = executable('frametest',
- frametest_sources,
- include_directories: programs_dir_inc,
- dependencies: liblz4_dep,
- install: false)
-
-roundTripTest_sources = [join_paths(lz4_root_dir, 'tests/roundTripTest.c')]
-roundTripTest = executable('roundTripTest',
- roundTripTest_sources,
- dependencies: [ liblz4_dep ],
- install: false)
-
-datagen_sources = [join_paths(lz4_root_dir, 'tests/datagencli.c')]
-datagen = executable('datagen',
- datagen_sources,
- objects: lz4.extract_objects(join_paths(lz4_root_dir, 'programs/datagen.c')),
- include_directories: lz4_includes,
- dependencies: [ liblz4_dep ],
- install: false)
-
-checkFrame_sources = [join_paths(lz4_root_dir, 'tests/checkFrame.c')]
-checkFrame = executable('checkFrame',
- checkFrame_sources,
- include_directories: programs_dir_inc,
- dependencies: [ liblz4_dep ],
- install: false)
-
-checkTag_sources = [join_paths(lz4_root_dir, 'tests/checkTag.c')]
-checkTag = executable('checkTag',
- checkTag_sources,
- include_directories: lib_dir_inc,
- install: false)
-
-# =============================================================================
-# Tests (Use "meson test --list" to list all tests)
-# =============================================================================
-
-# XXX: (Need TEST) These timeouts (in seconds) when running on a HDD should be
-# at least six times bigger than on a SSD
-
-test('test-fullbench',
- fullbench,
- args: ['--no-prompt', NB_LOOPS, TEST_FILES],
- timeout: 420) # Should enough when running on HDD
-test('test-fuzzer',
- fuzzer,
- args: [FUZZER_TIME],
- timeout: 100)
-test('test-frametest',
- frametest,
- args: [FUZZER_TIME],
- timeout: 100)
+foreach e, attrs : exes
+ executable(
+ e,
+ attrs.get('sources'),
+ objects: attrs.get('objects', []),
+ dependencies: [liblz4_internal_dep],
+ include_directories: attrs.get('include_directories', []),
+ install: false
+ )
+endforeach
diff --git a/contrib/meson/meson_options.txt b/contrib/meson/meson_options.txt
index a409c2d..ccb32de 100644
--- a/contrib/meson/meson_options.txt
+++ b/contrib/meson/meson_options.txt
@@ -1,5 +1,6 @@
# #############################################################################
# Copyright (c) 2018-present lzutao <taolzu(at)gmail.com>
+# Copyright (c) 2022-present Tristan Partin <tristan(at)partin.io>
# All rights reserved.
#
# This source code is licensed under both the BSD-style license (found in the
@@ -7,18 +8,17 @@
# in the COPYING file in the root directory of this source tree).
# #############################################################################
-# Read guidelines from https://wiki.gnome.org/Initiatives/GnomeGoals/MesonPorting
-
-option('debug_level', type: 'integer', min: 0, max: 7, value: 1,
+option('debug-level', type: 'integer', min: 0, max: 7, value: 1,
description: 'Enable run-time debug. See lib/lz4hc.c')
-option('backtrace', type: 'boolean', value: false,
- description: 'Display a stack backtrace when execution generates a runtime exception')
-
-option('bin_programs', type: 'boolean', value: false,
+option('unstable', type: 'boolean', value: false,
+ description: 'Expose unstable interfaces')
+option('programs', type: 'boolean', value: false,
description: 'Enable programs build')
-option('bin_tests', type: 'boolean', value: false,
+option('tests', type: 'boolean', value: false,
description: 'Enable tests build')
-option('bin_contrib', type: 'boolean', value: false,
+option('contrib', type: 'boolean', value: false,
description: 'Enable contrib build')
-option('bin_examples', type: 'boolean', value: false,
+option('examples', type: 'boolean', value: false,
description: 'Enable examples build')
+option('memory-usage', type: 'integer', min: 0, value: 0,
+ description: 'See LZ4_MEMORY_USAGE. 0 means use the LZ4 default')
diff --git a/contrib/snap/README.md b/contrib/snap/README.md
index 612d6d7..55c97e0 100644
--- a/contrib/snap/README.md
+++ b/contrib/snap/README.md
@@ -6,7 +6,7 @@
build your application from any source and ship it to any Linux
distribution by publishing it to https://snapcraft.io/. A key attribute
of a snap package is that it is (ideally) confined such that it
-executes within a controlled environmenti with all its dependencies
+executes within a controlled environment with all its dependencies
bundled with it and does not share dependencies with of from any other
package on the system (with a couple of minor exceptions).
diff --git a/contrib/snap/snapcraft.yaml b/contrib/snap/snapcraft.yaml
index 2793c0e..04ad3c4 100644
--- a/contrib/snap/snapcraft.yaml
+++ b/contrib/snap/snapcraft.yaml
@@ -1,5 +1,5 @@
name: lz4
-version: 1.8.4
+version: 1.9.3
summary: Extremely Fast Compression algorithm
description: >
LZ4 is lossless compression algorithm, providing compression
diff --git a/doc/lz4_Block_format.md b/doc/lz4_Block_format.md
index 4344e9b..9e80227 100644
--- a/doc/lz4_Block_format.md
+++ b/doc/lz4_Block_format.md
@@ -1,24 +1,22 @@
LZ4 Block Format Description
============================
-Last revised: 2019-03-30.
+Last revised: 2022-07-31 .
Author : Yann Collet
-This specification is intended for developers
-willing to produce LZ4-compatible compressed data blocks
-using any programming language.
+This specification is intended for developers willing to
+produce or read LZ4 compressed data blocks
+using any programming language of their choice.
-LZ4 is an LZ77-type compressor with a fixed, byte-oriented encoding.
+LZ4 is an LZ77-type compressor with a fixed byte-oriented encoding format.
There is no entropy encoder back-end nor framing layer.
The latter is assumed to be handled by other parts of the system
(see [LZ4 Frame format]).
This design is assumed to favor simplicity and speed.
-It helps later on for optimizations, compactness, and features.
-This document describes only the block format,
+This document describes only the Block Format,
not how the compressor nor decompressor actually work.
-The correctness of the decompressor should not depend
-on implementation details of the compressor, and vice versa.
+For more details on such topics, see later section "Implementation Notes".
[LZ4 Frame format]: lz4_Frame_format.md
@@ -28,7 +26,7 @@
-----------------------
An LZ4 compressed block is composed of sequences.
A sequence is a suite of literals (not-compressed bytes),
-followed by a match copy.
+followed by a match copy operation.
Each sequence starts with a `token`.
The `token` is a one byte value, separated into two 4-bits fields.
@@ -38,13 +36,20 @@
The first field uses the 4 high-bits of the token.
It provides the length of literals to follow.
-If the field value is 0, then there is no literal.
-If it is 15, then we need to add some more bytes to indicate the full length.
-Each additional byte then represent a value from 0 to 255,
+If the field value is smaller than 15,
+then it represents the total nb of literals present in the sequence,
+including 0, in which case there is no literal.
+
+The value 15 is a special case: more bytes are required to indicate the full length.
+Each additional byte then represents a value from 0 to 255,
which is added to the previous value to produce a total length.
-When the byte value is 255, another byte is output.
-There can be any number of bytes following `token`. There is no "size limit".
-(Side note : this is why a not-compressible input block is expanded by 0.4%).
+When the byte value is 255, another byte must be read and added, and so on.
+There can be any number of bytes of value `255` following `token`.
+The Block Format does not define any "size limit",
+though real implementations may feature some practical limits
+(see more details in later chapter "Implementation Notes").
+
+Note : this format explains why a non-compressible input block is expanded by 0.4%.
Example 1 : A literal length of 48 will be represented as :
@@ -55,7 +60,7 @@
- 15 : value for the 4-bits High field
- 255 : following byte is maxed, since 280-15 >= 255
- - 10 : (=280 - 15 - 255) ) remaining length to reach 280
+ - 10 : (=280 - 15 - 255) remaining length to reach 280
Example 3 : A literal length of 15 will be represented as :
@@ -63,94 +68,177 @@
- 0 : (=15-15) yes, the zero must be output
Following `token` and optional length bytes, are the literals themselves.
-They are exactly as numerous as previously decoded (length of literals).
-It's possible that there are zero literal.
+They are exactly as numerous as just decoded (length of literals).
+Reminder: it's possible that there are zero literals.
Following the literals is the match copy operation.
-It starts by the `offset`.
+It starts by the `offset` value.
This is a 2 bytes value, in little endian format
(the 1st byte is the "low" byte, the 2nd one is the "high" byte).
-The `offset` represents the position of the match to be copied from.
-1 means "current position - 1 byte".
-The maximum `offset` value is 65535, 65536 cannot be coded.
-Note that 0 is an invalid value, not used.
+The `offset` represents the position of the match to be copied from the past.
+For example, 1 means "current position - 1 byte".
+The maximum `offset` value is 65535. 65536 and beyond cannot be coded.
+Note that 0 is an invalid `offset` value.
+The presence of a 0 `offset` value denotes an invalid (corrupted) block.
-Then we need to extract the `matchlength`.
-For this, we use the second token field, the low 4-bits.
-Value, obviously, ranges from 0 to 15.
-However here, 0 means that the copy operation will be minimal.
+Then the `matchlength` can be extracted.
+For this, we use the second `token` field, the low 4-bits.
+Such a value, obviously, ranges from 0 to 15.
+However here, 0 means that the copy operation is minimal.
The minimum length of a match, called `minmatch`, is 4.
-As a consequence, a 0 value means 4 bytes, and a value of 15 means 19+ bytes.
-Similar to literal length, on reaching the highest possible value (15),
-we output additional bytes, one at a time, with values ranging from 0 to 255.
+As a consequence, a 0 value means 4 bytes.
+Similarly to literal length, any value smaller than 15 represents a length,
+to which 4 (`minmatch`) must be added, thus ranging from 4 to 18.
+A value of 15 is special, meaning 19+ bytes,
+to which one must read additional bytes, one at a time,
+with each byte value ranging from 0 to 255.
They are added to total to provide the final match length.
A 255 value means there is another byte to read and add.
-There is no limit to the number of optional bytes that can be output this way.
-(This points towards a maximum achievable compression ratio of about 250).
+There is no limit to the number of optional `255` bytes that can be present,
+and therefore no limit to representable match length,
+though real-life implementations are likely going to enforce limits for practical reasons (see more details in "Implementation Notes" section below).
+
+Note: this format has a maximum achievable compression ratio of about ~250.
Decoding the `matchlength` reaches the end of current sequence.
-Next byte will be the start of another sequence.
-But before moving to next sequence,
-it's time to use the decoded match position and length.
-The decoder copies `matchlength` bytes from match position to current position.
-
-In some cases, `matchlength` is larger than `offset`.
-Therefore, `match_pos + matchlength > current_pos`,
-which means that later bytes to copy are not yet decoded.
-This is called an "overlap match", and must be handled with special care.
-A common case is an offset of 1,
-meaning the last byte is repeated `matchlength` times.
+Next byte will be the start of another sequence, and therefore a new `token`.
-End of block restrictions
------------------------
-There are specific rules required to terminate a block.
+End of block conditions
+-------------------------
+There are specific restrictions required to terminate an LZ4 block.
1. The last sequence contains only literals.
- The block ends right after them.
+ The block ends right after the literals (no `offset` field).
2. The last 5 bytes of input are always literals.
Therefore, the last sequence contains at least 5 bytes.
- Special : if input is smaller than 5 bytes,
there is only one sequence, it contains the whole input as literals.
- Empty input can be represented with a zero byte,
+ Even empty input can be represented, using a zero byte,
interpreted as a final token without literal and without a match.
3. The last match must start at least 12 bytes before the end of block.
- The last match is part of the penultimate sequence.
- It is followed by the last sequence, which contains only literals.
+ The last match is part of the _penultimate_ sequence.
+ It is followed by the last sequence, which contains _only_ literals.
- Note that, as a consequence,
- an independent block < 13 bytes cannot be compressed,
- because the match must copy "something",
- so it needs at least one prior byte.
- - When a block can reference data from another block,
- it can start immediately with a match and no literal,
- so a block of 12 bytes can be compressed.
+ blocks < 12 bytes cannot be compressed.
+ And as an extension, _independent_ blocks < 13 bytes cannot be compressed,
+ because they must start by at least one literal,
+ that the match can then copy afterwards.
When a block does not respect these end conditions,
a conformant decoder is allowed to reject the block as incorrect.
-These rules are in place to ensure that a conformant decoder
-can be designed for speed, issuing speculatively instructions,
-while never reading nor writing beyond provided I/O buffers.
+These rules are in place to ensure compatibility with
+a wide range of historical decoders
+which rely on these conditions for their speed-oriented design.
-
-Additional notes
+Implementation notes
-----------------------
-If the decoder will decompress data from an external source,
-it is recommended to ensure that the decoder will not be vulnerable to
-buffer overflow manipulations.
+The LZ4 Block Format only defines the compressed format,
+it does not tell how to create a decoder or an encoder,
+which design is left free to the imagination of the implementer.
+
+However, thanks to experience, there are a number of typical topics that
+most implementations will have to consider.
+This section tries to provide a few guidelines.
+
+#### Metadata
+
+An LZ4-compressed Block requires additional metadata for proper decoding.
+Typically, a decoder will require the compressed block's size,
+and an upper bound of decompressed size.
+Other variants exist, such as knowing the decompressed size,
+and having an upper bound of the input size.
+The Block Format does not specify how to transmit such information,
+which is considered an out-of-band information channel.
+That's because in many cases, the information is present in the environment.
+For example, databases must store the size of their compressed block for indexing,
+and know that their decompressed block can't be larger than a certain threshold.
+
+If you need a format which is "self-contained",
+and also transports the necessary metadata for proper decoding on any platform,
+consider employing the [LZ4 Frame format] instead.
+
+#### Large lengths
+
+While the Block Format does not define any maximum value for length fields,
+in practice, most implementations will feature some form of limit,
+since it's expected for such values to be stored into registers of fixed bit width.
+
+If length fields use 64-bit registers,
+then it can be assumed that there is no practical limit,
+as it would require a single continuous block of multiple petabytes to reach it,
+which is unreasonable by today's standard.
+
+If length fields use 32-bit registers, then it can be overflowed,
+but requires a compressed block of size > 16 MB.
+Therefore, implementations that do not deal with compressed blocks > 16 MB are safe.
+However, if such a case is allowed,
+then it's recommended to check that no large length overflows the register.
+
+If length fields use 16-bit registers,
+then it's definitely possible to overflow such register,
+with less than < 300 bytes of compressed data.
+
+A conformant decoder should be able to detect length overflows when it's possible,
+and simply error out when that happens.
+The input block might not be invalid,
+it's just not decodable by the local decoder implementation.
+
+Note that, in order to be compatible with the larger LZ4 ecosystem,
+it's recommended to be able to read and represent lengths of up to 4 MB,
+and to accept blocks of size up to 4 MB.
+Such limits are compatible with 32-bit length registers,
+and prevent overflow of 32-bit registers.
+
+#### Safe decoding
+
+If a decoder receives compressed data from any external source,
+it is recommended to ensure that the decoder is resilient to corrupted input,
+and made safe from buffer overflow manipulations.
Always ensure that read and write operations
remain within the limits of provided buffers.
-Test the decoder with fuzzers
-to ensure it's resilient to improbable combinations.
-The format makes no assumption nor limits to the way the compressor
+Of particular importance, ensure that the nb of bytes instructed to copy
+does not overflow neither the input nor the output buffers.
+Ensure also, when reading an offset value, that the resulting position to copy
+does not reach beyond the beginning of the buffer.
+Such a situation can happen during the first 64 KB of decoded data.
+
+For more safety, test the decoder with fuzzers
+to ensure it's resilient to improbable sequences of conditions.
+Combine them with sanitizers, in order to catch overflows (asan)
+or initialization issues (msan).
+
+Pay some attention to offset 0 scenario, which is invalid,
+and therefore must not be blindly decoded:
+a naive implementation could preserve destination buffer content,
+which could then result in information disclosure
+if such buffer was uninitialized and still containing private data.
+For reference, in such a scenario, the reference LZ4 decoder
+clears the match segment with `0` bytes,
+though other solutions are certainly possible.
+
+Finally, pay attention to the "overlap match" scenario,
+when `matchlength` is larger than `offset`.
+In which case, since `match_pos + matchlength > current_pos`,
+some of the later bytes to copy do not exist yet,
+and will be generated during the early stage of match copy operation.
+Such scenario must be handled with special care.
+A common case is an offset of 1,
+meaning the last byte is repeated `matchlength` times.
+
+#### Compression techniques
+
+The core of a LZ4 compressor is to detect duplicated data across past 64 KB.
+The format makes no assumption nor limits to the way a compressor
searches and selects matches within the source data block.
-Multiple techniques can be considered,
-featuring distinct time / performance trade offs.
-As long as the format is respected,
-the result will be compatible and decodable by any compliant decoder.
-An upper compression limit can be reached,
-using a technique called "full optimal parsing", at high cpu cost.
+For example, an upper compression limit can be reached,
+using a technique called "full optimal parsing", at high cpu and memory cost.
+But multiple other techniques can be considered,
+featuring distinct time / performance trade-offs.
+As long as the specified format is respected,
+the result will be compatible with and decodable by any compliant decoder.
diff --git a/doc/lz4_Frame_format.md b/doc/lz4_Frame_format.md
index 7e08841..97a2cbe 100644
--- a/doc/lz4_Frame_format.md
+++ b/doc/lz4_Frame_format.md
@@ -3,11 +3,11 @@
### Notices
-Copyright (c) 2013-2015 Yann Collet
+Copyright (c) 2013-2020 Yann Collet
Permission is granted to copy and distribute this document
-for any purpose and without charge,
-including translations into other languages
+for any purpose and without charge,
+including translations into other languages
and incorporation into compilations,
provided that the copyright notice and this notice are preserved,
and that any substantive changes or deletions from the original
@@ -47,7 +47,7 @@
Unless otherwise indicated below,
a compliant compressor must produce data sets
that conform to the specifications presented here.
-It doesn’t need to support all options though.
+It doesn't need to support all options though.
A compliant decompressor must be able to decompress
at least one working set of parameters
@@ -244,8 +244,7 @@
The value is the second byte of `xxh32()` : ` (xxh32()>>8) & 0xFF `
using zero as a seed, and the full Frame Descriptor as an input
(including optional fields when they are present).
-A wrong checksum indicates an error in the descriptor.
-Header checksum is informational and can be skipped.
+A wrong checksum indicates that the descriptor is erroneous.
Data Blocks
@@ -385,7 +384,7 @@
End of legacy frame is implicit only.
It must be followed by a standard EOF (End Of File) signal,
-wether it is a file or a stream.
+whether it is a file or a stream.
Alternatively, if the frame is followed by a valid Frame Magic Number,
it is considered completed.
diff --git a/doc/lz4_manual.html b/doc/lz4_manual.html
index 47fe18d..6fafb21 100644
--- a/doc/lz4_manual.html
+++ b/doc/lz4_manual.html
@@ -1,10 +1,10 @@
<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=ISO-8859-1">
-<title>1.9.3 Manual</title>
+<title>1.9.4 Manual</title>
</head>
<body>
-<h1>1.9.3 Manual</h1>
+<h1>1.9.4 Manual</h1>
<hr>
<a name="Contents"></a><h2>Contents</h2>
<ol>
@@ -48,20 +48,49 @@
The `lz4` CLI can only manage frames.
<BR></pre>
+<pre><b>#if defined(LZ4_FREESTANDING) && (LZ4_FREESTANDING == 1)
+# define LZ4_HEAPMODE 0
+# define LZ4HC_HEAPMODE 0
+# define LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION 1
+# if !defined(LZ4_memcpy)
+# error "LZ4_FREESTANDING requires macro 'LZ4_memcpy'."
+# endif
+# if !defined(LZ4_memset)
+# error "LZ4_FREESTANDING requires macro 'LZ4_memset'."
+# endif
+# if !defined(LZ4_memmove)
+# error "LZ4_FREESTANDING requires macro 'LZ4_memmove'."
+# endif
+#elif ! defined(LZ4_FREESTANDING)
+# define LZ4_FREESTANDING 0
+#endif
+</b><p> When this macro is set to 1, it enables "freestanding mode" that is
+ suitable for typical freestanding environment which doesn't support
+ standard C library.
+
+ - LZ4_FREESTANDING is a compile-time switch.
+ - It requires the following macros to be defined:
+ LZ4_memcpy, LZ4_memmove, LZ4_memset.
+ - It only enables LZ4/HC functions which don't use heap.
+ All LZ4F_* functions are not supported.
+ - See tests/freestanding.c to check its basic setup.
+
+</p></pre><BR>
+
<a name="Chapter2"></a><h2>Version</h2><pre></pre>
-<pre><b>int LZ4_versionNumber (void); </b>/**< library version number; useful to check dll version */<b>
+<pre><b>int LZ4_versionNumber (void); </b>/**< library version number; useful to check dll version; requires v1.3.0+ */<b>
</b></pre><BR>
-<pre><b>const char* LZ4_versionString (void); </b>/**< library version string; useful to check dll version */<b>
+<pre><b>const char* LZ4_versionString (void); </b>/**< library version string; useful to check dll version; requires v1.7.5+ */<b>
</b></pre><BR>
<a name="Chapter3"></a><h2>Tuning parameter</h2><pre></pre>
<pre><b>#ifndef LZ4_MEMORY_USAGE
-# define LZ4_MEMORY_USAGE 14
+# define LZ4_MEMORY_USAGE LZ4_MEMORY_USAGE_DEFAULT
#endif
-</b><p> Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.)
- Increasing memory usage improves compression ratio.
- Reduced memory usage may improve speed, thanks to better cache locality.
+</b><p> Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; )
+ Increasing memory usage improves compression ratio, at the cost of speed.
+ Reduced memory usage may improve speed at the cost of ratio, thanks to better cache locality.
Default value is 14, for 16KB, which nicely fits into Intel x86 L1 cache
</p></pre><BR>
@@ -267,8 +296,10 @@
<a name="Chapter7"></a><h2>Streaming Decompression Functions</h2><pre> Bufferless synchronous API
<BR></pre>
-<pre><b>LZ4_streamDecode_t* LZ4_createStreamDecode(void);
+<pre><b>#if !defined(LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION)
+LZ4_streamDecode_t* LZ4_createStreamDecode(void);
int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream);
+#endif </b>/* !defined(LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION) */<b>
</b><p> creation / destruction of streaming decompression tracking context.
A tracking context can be re-used multiple times.
@@ -297,7 +328,10 @@
</p></pre><BR>
-<pre><b>int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* src, char* dst, int srcSize, int dstCapacity);
+<pre><b>int
+LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode,
+ const char* src, char* dst,
+ int srcSize, int dstCapacity);
</b><p> These decoding functions allow decompression of consecutive blocks in "streaming" mode.
A block is an unsplittable entity, it must be presented entirely to a decompression function.
Decompression functions only accepts one block at a time.
@@ -323,7 +357,10 @@
then indicate where this data is saved using LZ4_setStreamDecode(), before decompressing next block.
</p></pre><BR>
-<pre><b>int LZ4_decompress_safe_usingDict (const char* src, char* dst, int srcSize, int dstCapcity, const char* dictStart, int dictSize);
+<pre><b>int
+LZ4_decompress_safe_usingDict(const char* src, char* dst,
+ int srcSize, int dstCapacity,
+ const char* dictStart, int dictSize);
</b><p> These decoding functions work the same as
a combination of LZ4_setStreamDecode() followed by LZ4_decompress_*_continue()
They are stand-alone, and don't need an LZ4_streamDecode_t structure.
@@ -363,7 +400,9 @@
</p></pre><BR>
-<pre><b>LZ4LIB_STATIC_API void LZ4_attach_dictionary(LZ4_stream_t* workingStream, const LZ4_stream_t* dictionaryStream);
+<pre><b>LZ4LIB_STATIC_API void
+LZ4_attach_dictionary(LZ4_stream_t* workingStream,
+ const LZ4_stream_t* dictionaryStream);
</b><p> This is an experimental API that allows
efficient use of a static dictionary many times.
@@ -393,7 +432,7 @@
<pre><b></b><p>
It's possible to have input and output sharing the same buffer,
- for highly contrained memory environments.
+ for highly constrained memory environments.
In both cases, it requires input to lay at the end of the buffer,
and decompression to start at beginning of the buffer.
Buffer size must feature some margin, hence be larger than final size.
@@ -452,28 +491,9 @@
Accessing members will expose user code to API and/or ABI break in future versions of the library.
<BR></pre>
-<pre><b>typedef struct {
- const LZ4_byte* externalDict;
- size_t extDictSize;
- const LZ4_byte* prefixEnd;
- size_t prefixSize;
-} LZ4_streamDecode_t_internal;
-</b></pre><BR>
-<pre><b>#define LZ4_STREAMSIZE 16416 </b>/* static size, for inter-version compatibility */<b>
-#define LZ4_STREAMSIZE_VOIDP (LZ4_STREAMSIZE / sizeof(void*))
-union LZ4_stream_u {
- void* table[LZ4_STREAMSIZE_VOIDP];
- LZ4_stream_t_internal internal_donotuse;
-}; </b>/* previously typedef'd to LZ4_stream_t */<b>
-</b><p> Do not use below internal definitions directly !
- Declare or allocate an LZ4_stream_t instead.
- LZ4_stream_t can also be created using LZ4_createStream(), which is recommended.
- The structure definition can be convenient for static allocation
- (on stack, or as part of larger structure).
- Init this structure with LZ4_initStream() before first use.
- note : only use this definition in association with static linking !
- this definition is not API/ABI safe, and may change in future versions.
-
+<pre><b></b><p> Never ever use below internal definitions directly !
+ These definitions are not API/ABI safe, and may change in future versions.
+ If you need static allocation, declare or allocate an LZ4_stream_t object.
</p></pre><BR>
<pre><b>LZ4_stream_t* LZ4_initStream (void* buffer, size_t size);
@@ -489,21 +509,17 @@
In which case, the function will @return NULL.
Note2: An LZ4_stream_t structure guarantees correct alignment and size.
Note3: Before v1.9.0, use LZ4_resetStream() instead
-
</p></pre><BR>
-<pre><b>#define LZ4_STREAMDECODESIZE_U64 (4 + ((sizeof(void*)==16) ? 2 : 0) </b>/*AS-400*/ )<b>
-#define LZ4_STREAMDECODESIZE (LZ4_STREAMDECODESIZE_U64 * sizeof(unsigned long long))
-union LZ4_streamDecode_u {
- unsigned long long table[LZ4_STREAMDECODESIZE_U64];
- LZ4_streamDecode_t_internal internal_donotuse;
-} ; </b>/* previously typedef'd to LZ4_streamDecode_t */<b>
-</b><p> information structure to track an LZ4 stream during decompression.
- init this structure using LZ4_setStreamDecode() before first use.
- note : only use in association with static linking !
- this definition is not API/ABI safe,
- and may change in a future version !
-
+<pre><b>typedef struct {
+ const LZ4_byte* externalDict;
+ const LZ4_byte* prefixEnd;
+ size_t extDictSize;
+ size_t prefixSize;
+} LZ4_streamDecode_t_internal;
+</b><p> Never ever use below internal definitions directly !
+ These definitions are not API/ABI safe, and may change in future versions.
+ If you need static allocation, declare or allocate an LZ4_streamDecode_t object.
</p></pre><BR>
<a name="Chapter10"></a><h2>Obsolete Functions</h2><pre></pre>
diff --git a/doc/lz4frame_manual.html b/doc/lz4frame_manual.html
index 2758306..cfb437e 100644
--- a/doc/lz4frame_manual.html
+++ b/doc/lz4frame_manual.html
@@ -1,10 +1,10 @@
<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=ISO-8859-1">
-<title>1.9.3 Manual</title>
+<title>1.9.4 Manual</title>
</head>
<body>
-<h1>1.9.3 Manual</h1>
+<h1>1.9.4 Manual</h1>
<hr>
<a name="Contents"></a><h2>Contents</h2>
<ol>
@@ -22,9 +22,9 @@
</ol>
<hr>
<a name="Chapter1"></a><h2>Introduction</h2><pre>
- lz4frame.h implements LZ4 frame specification (doc/lz4_Frame_format.md).
- lz4frame.h provides frame compression functions that take care
- of encoding standard metadata alongside LZ4-compressed blocks.
+ lz4frame.h implements LZ4 frame specification: see doc/lz4_Frame_format.md .
+ LZ4 Frames are compatible with `lz4` CLI,
+ and designed to be interoperable with any system.
<BR></pre>
<a name="Chapter2"></a><h2>Compiler specifics</h2><pre></pre>
@@ -35,7 +35,8 @@
</b></pre><BR>
<pre><b>const char* LZ4F_getErrorName(LZ4F_errorCode_t code); </b>/**< return error code string; for debugging */<b>
</b></pre><BR>
-<a name="Chapter4"></a><h2>Frame compression types</h2><pre></pre>
+<a name="Chapter4"></a><h2>Frame compression types</h2><pre>
+<BR></pre>
<pre><b>typedef enum {
LZ4F_default=0,
@@ -108,7 +109,7 @@
</b><p> Returns the maximum possible compressed size with LZ4F_compressFrame() given srcSize and preferences.
`preferencesPtr` is optional. It can be replaced by NULL, in which case, the function will assume default preferences.
Note : this result is only usable with LZ4F_compressFrame().
- It may also be used with LZ4F_compressUpdate() _if no flush() operation_ is performed.
+ It may also be relevant to LZ4F_compressUpdate() _only if_ no flush() operation is ever performed.
</p></pre><BR>
@@ -134,13 +135,19 @@
<pre><b>LZ4F_errorCode_t LZ4F_createCompressionContext(LZ4F_cctx** cctxPtr, unsigned version);
LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctx);
-</b><p> The first thing to do is to create a compressionContext object, which will be used in all compression operations.
- This is achieved using LZ4F_createCompressionContext(), which takes as argument a version.
- The version provided MUST be LZ4F_VERSION. It is intended to track potential version mismatch, notably when using DLL.
- The function will provide a pointer to a fully allocated LZ4F_cctx object.
- If @return != zero, there was an error during context creation.
- Object can release its memory using LZ4F_freeCompressionContext();
-
+</b><p> The first thing to do is to create a compressionContext object,
+ which will keep track of operation state during streaming compression.
+ This is achieved using LZ4F_createCompressionContext(), which takes as argument a version,
+ and a pointer to LZ4F_cctx*, to write the resulting pointer into.
+ @version provided MUST be LZ4F_VERSION. It is intended to track potential version mismatch, notably when using DLL.
+ The function provides a pointer to a fully allocated LZ4F_cctx object.
+ @cctxPtr MUST be != NULL.
+ If @return != zero, context creation failed.
+ A created compression context can be employed multiple times for consecutive streaming operations.
+ Once all streaming compression jobs are completed,
+ the state object can be released using LZ4F_freeCompressionContext().
+ Note1 : LZ4F_freeCompressionContext() is always successful. Its return value can be ignored.
+ Note2 : LZ4F_freeCompressionContext() works fine with NULL input pointers (do nothing).
</p></pre><BR>
<a name="Chapter8"></a><h2>Compression</h2><pre></pre>
@@ -181,8 +188,9 @@
Important rule: dstCapacity MUST be large enough to ensure operation success even in worst case situations.
This value is provided by LZ4F_compressBound().
If this condition is not respected, LZ4F_compress() will fail (result is an errorCode).
- LZ4F_compressUpdate() doesn't guarantee error recovery.
- When an error occurs, compression context must be freed or resized.
+ After an error, the state is left in a UB state, and must be re-initialized or freed.
+ If previously an uncompressed block was written, buffered data is flushed
+ before appending compressed data is continued.
`cOptPtr` is optional : NULL can be provided, in which case all options are set to default.
@return : number of bytes written into `dstBuffer` (it can be zero, meaning input data was just buffered).
or an error code if it fails (which can be tested using LZ4F_isError())
@@ -219,16 +227,21 @@
<a name="Chapter9"></a><h2>Decompression functions</h2><pre></pre>
<pre><b>typedef struct {
- unsigned stableDst; </b>/* pledges that last 64KB decompressed data will remain available unmodified. This optimization skips storage operations in tmp buffers. */<b>
- unsigned reserved[3]; </b>/* must be set to zero for forward compatibility */<b>
+ unsigned stableDst; /* pledges that last 64KB decompressed data will remain available unmodified between invocations.
+ * This optimization skips storage operations in tmp buffers. */
+ unsigned skipChecksums; /* disable checksum calculation and verification, even when one is present in frame, to save CPU time.
+ * Setting this option to 1 once disables all checksums for the rest of the frame. */
+ unsigned reserved1; </b>/* must be set to zero for forward compatibility */<b>
+ unsigned reserved0; </b>/* idem */<b>
} LZ4F_decompressOptions_t;
</b></pre><BR>
<pre><b>LZ4F_errorCode_t LZ4F_createDecompressionContext(LZ4F_dctx** dctxPtr, unsigned version);
LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* dctx);
</b><p> Create an LZ4F_dctx object, to track all decompression operations.
- The version provided MUST be LZ4F_VERSION.
- The function provides a pointer to an allocated and initialized LZ4F_dctx object.
- The result is an errorCode, which can be tested using LZ4F_isError().
+ @version provided MUST be LZ4F_VERSION.
+ @dctxPtr MUST be valid.
+ The function fills @dctxPtr with the value of a pointer to an allocated and initialized LZ4F_dctx object.
+ The @return is an errorCode, which can be tested using LZ4F_isError().
dctx memory can be released using LZ4F_freeDecompressionContext();
Result of LZ4F_freeDecompressionContext() indicates current state of decompressionContext when being released.
That is, it should be == 0 if decompression has been completed fully and correctly.
@@ -248,11 +261,12 @@
</p></pre><BR>
-<pre><b>size_t LZ4F_getFrameInfo(LZ4F_dctx* dctx,
- LZ4F_frameInfo_t* frameInfoPtr,
- const void* srcBuffer, size_t* srcSizePtr);
+<pre><b>size_t
+LZ4F_getFrameInfo(LZ4F_dctx* dctx,
+ LZ4F_frameInfo_t* frameInfoPtr,
+ const void* srcBuffer, size_t* srcSizePtr);
</b><p> This function extracts frame parameters (max blockSize, dictID, etc.).
- Its usage is optional: user can call LZ4F_decompress() directly.
+ Its usage is optional: user can also invoke LZ4F_decompress() directly.
Extracted information will fill an existing LZ4F_frameInfo_t structure.
This can be useful for allocation and dictionary identification purposes.
@@ -295,10 +309,11 @@
</p></pre><BR>
-<pre><b>size_t LZ4F_decompress(LZ4F_dctx* dctx,
- void* dstBuffer, size_t* dstSizePtr,
- const void* srcBuffer, size_t* srcSizePtr,
- const LZ4F_decompressOptions_t* dOptPtr);
+<pre><b>size_t
+LZ4F_decompress(LZ4F_dctx* dctx,
+ void* dstBuffer, size_t* dstSizePtr,
+ const void* srcBuffer, size_t* srcSizePtr,
+ const LZ4F_decompressOptions_t* dOptPtr);
</b><p> Call this function repetitively to regenerate data compressed in `srcBuffer`.
The function requires a valid dctx state.
@@ -341,6 +356,30 @@
<pre><b>typedef enum { LZ4F_LIST_ERRORS(LZ4F_GENERATE_ENUM)
_LZ4F_dummy_error_enum_for_c89_never_used } LZ4F_errorCodes;
</b></pre><BR>
+<pre><b>LZ4FLIB_STATIC_API size_t LZ4F_getBlockSize(LZ4F_blockSizeID_t blockSizeID);
+</b><p> Return, in scalar format (size_t),
+ the maximum block size associated with blockSizeID.
+</p></pre><BR>
+
+<pre><b>LZ4FLIB_STATIC_API size_t
+LZ4F_uncompressedUpdate(LZ4F_cctx* cctx,
+ void* dstBuffer, size_t dstCapacity,
+ const void* srcBuffer, size_t srcSize,
+ const LZ4F_compressOptions_t* cOptPtr);
+</b><p> LZ4F_uncompressedUpdate() can be called repetitively to add as much data uncompressed data as necessary.
+ Important rule: dstCapacity MUST be large enough to store the entire source buffer as
+ no compression is done for this operation
+ If this condition is not respected, LZ4F_uncompressedUpdate() will fail (result is an errorCode).
+ After an error, the state is left in a UB state, and must be re-initialized or freed.
+ If previously a compressed block was written, buffered data is flushed
+ before appending uncompressed data is continued.
+ This is only supported when LZ4F_blockIndependent is used
+ `cOptPtr` is optional : NULL can be provided, in which case all options are set to default.
+ @return : number of bytes written into `dstBuffer` (it can be zero, meaning input data was just buffered).
+ or an error code if it fails (which can be tested using LZ4F_isError())
+
+</p></pre><BR>
+
<a name="Chapter11"></a><h2>Bulk processing dictionary API</h2><pre></pre>
<pre><b>LZ4FLIB_STATIC_API LZ4F_CDict* LZ4F_createCDict(const void* dictBuffer, size_t dictSize);
@@ -351,12 +390,12 @@
`dictBuffer` can be released after LZ4_CDict creation, since its content is copied within CDict
</p></pre><BR>
-<pre><b>LZ4FLIB_STATIC_API size_t LZ4F_compressFrame_usingCDict(
- LZ4F_cctx* cctx,
- void* dst, size_t dstCapacity,
- const void* src, size_t srcSize,
- const LZ4F_CDict* cdict,
- const LZ4F_preferences_t* preferencesPtr);
+<pre><b>LZ4FLIB_STATIC_API size_t
+LZ4F_compressFrame_usingCDict(LZ4F_cctx* cctx,
+ void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize,
+ const LZ4F_CDict* cdict,
+ const LZ4F_preferences_t* preferencesPtr);
</b><p> Compress an entire srcBuffer into a valid LZ4 frame using a digested Dictionary.
cctx must point to a context created by LZ4F_createCompressionContext().
If cdict==NULL, compress without a dictionary.
@@ -368,11 +407,11 @@
or an error code if it fails (can be tested using LZ4F_isError())
</p></pre><BR>
-<pre><b>LZ4FLIB_STATIC_API size_t LZ4F_compressBegin_usingCDict(
- LZ4F_cctx* cctx,
- void* dstBuffer, size_t dstCapacity,
- const LZ4F_CDict* cdict,
- const LZ4F_preferences_t* prefsPtr);
+<pre><b>LZ4FLIB_STATIC_API size_t
+LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctx,
+ void* dstBuffer, size_t dstCapacity,
+ const LZ4F_CDict* cdict,
+ const LZ4F_preferences_t* prefsPtr);
</b><p> Inits streaming dictionary compression, and writes the frame header into dstBuffer.
dstCapacity must be >= LZ4F_HEADER_SIZE_MAX bytes.
`prefsPtr` is optional : you may provide NULL as argument,
@@ -381,16 +420,36 @@
or an error code (which can be tested using LZ4F_isError())
</p></pre><BR>
-<pre><b>LZ4FLIB_STATIC_API size_t LZ4F_decompress_usingDict(
- LZ4F_dctx* dctxPtr,
- void* dstBuffer, size_t* dstSizePtr,
- const void* srcBuffer, size_t* srcSizePtr,
- const void* dict, size_t dictSize,
- const LZ4F_decompressOptions_t* decompressOptionsPtr);
+<pre><b>LZ4FLIB_STATIC_API size_t
+LZ4F_decompress_usingDict(LZ4F_dctx* dctxPtr,
+ void* dstBuffer, size_t* dstSizePtr,
+ const void* srcBuffer, size_t* srcSizePtr,
+ const void* dict, size_t dictSize,
+ const LZ4F_decompressOptions_t* decompressOptionsPtr);
</b><p> Same as LZ4F_decompress(), using a predefined dictionary.
Dictionary is used "in place", without any preprocessing.
It must remain accessible throughout the entire frame decoding.
</p></pre><BR>
+<pre><b>typedef void* (*LZ4F_AllocFunction) (void* opaqueState, size_t size);
+typedef void* (*LZ4F_CallocFunction) (void* opaqueState, size_t size);
+typedef void (*LZ4F_FreeFunction) (void* opaqueState, void* address);
+typedef struct {
+ LZ4F_AllocFunction customAlloc;
+ LZ4F_CallocFunction customCalloc; </b>/* optional; when not defined, uses customAlloc + memset */<b>
+ LZ4F_FreeFunction customFree;
+ void* opaqueState;
+} LZ4F_CustomMem;
+static
+#ifdef __GNUC__
+__attribute__((__unused__))
+#endif
+LZ4F_CustomMem const LZ4F_defaultCMem = { NULL, NULL, NULL, NULL }; </b>/**< this constant defers to stdlib's functions */<b>
+</b><p> These prototypes make it possible to pass custom allocation/free functions.
+ LZ4F_customMem is provided at state creation time, using LZ4F_create*_advanced() listed below.
+ All allocation/free operations will be completed using these custom variants instead of regular <stdlib.h> ones.
+
+</p></pre><BR>
+
</html>
</body>
diff --git a/examples/.gitignore b/examples/.gitignore
index 5abeef6..ddc8e21 100644
--- a/examples/.gitignore
+++ b/examples/.gitignore
@@ -6,5 +6,6 @@
/ringBufferHC
/lineCompress
/frameCompress
+/fileCompress
/simpleBuffer
/*.exe
diff --git a/examples/Makefile b/examples/Makefile
index 3ec3e21..8be5c81 100644
--- a/examples/Makefile
+++ b/examples/Makefile
@@ -1,6 +1,6 @@
# ##########################################################################
# LZ4 examples - Makefile
-# Copyright (C) Yann Collet 2011-2014
+# Copyright (C) Yann Collet 2011-2020
#
# GPL v2 License
#
@@ -41,7 +41,7 @@
default: all
all: printVersion doubleBuffer dictionaryRandomAccess ringBuffer ringBufferHC \
- lineCompress frameCompress simpleBuffer
+ lineCompress frameCompress fileCompress simpleBuffer
$(LZ4DIR)/liblz4.a: $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c $(LZ4DIR)/lz4frame.c $(LZ4DIR)/lz4.h $(LZ4DIR)/lz4hc.h $(LZ4DIR)/lz4frame.h $(LZ4DIR)/lz4frame_static.h
$(MAKE) -C $(LZ4DIR) liblz4.a
@@ -67,6 +67,9 @@
frameCompress: frameCompress.c $(LZ4DIR)/liblz4.a
$(CC) $(FLAGS) $^ -o $@$(EXT)
+fileCompress: fileCompress.c $(LZ4DIR)/liblz4.a
+ $(CC) $(FLAGS) $^ -o $@$(EXT)
+
compressFunctions: compress_functions.c $(LZ4DIR)/liblz4.a
$(CC) $(FLAGS) $^ -o $@$(EXT) -lrt
@@ -94,10 +97,18 @@
@echo "\n=== Frame compression ==="
./frameCompress$(EXT) $(TESTFILE)
$(LZ4) -vt $(TESTFILE).lz4
+ @echo "\n=== file compression ==="
+ ./fileCompress$(EXT) $(TESTFILE)
+ $(LZ4) -vt $(TESTFILE).lz4
+
+.PHONY: cxxtest
+cxxtest: CFLAGS := -O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror
+cxxtest: clean
+ CC=$(CXX) $(MAKE) -C . all CFLAGS="$(CFLAGS)"
clean:
@rm -f core *.o *.dec *-0 *-9 *-8192 *.lz4s *.lz4 \
printVersion$(EXT) doubleBuffer$(EXT) dictionaryRandomAccess$(EXT) \
ringBuffer$(EXT) ringBufferHC$(EXT) lineCompress$(EXT) frameCompress$(EXT) \
- compressFunctions$(EXT) simpleBuffer$(EXT)
+ fileCompress$(EXT) compressFunctions$(EXT) simpleBuffer$(EXT)
@echo Cleaning completed
diff --git a/examples/blockStreaming_lineByLine.c b/examples/blockStreaming_lineByLine.c
index 19c3345..3047a3a 100644
--- a/examples/blockStreaming_lineByLine.c
+++ b/examples/blockStreaming_lineByLine.c
@@ -65,7 +65,7 @@
{
const int cmpBytes = LZ4_compress_fast_continue(
- lz4Stream, inpPtr, cmpBuf, inpBytes, cmpBufBytes, 1);
+ lz4Stream, inpPtr, cmpBuf, inpBytes, (int) cmpBufBytes, 1);
if (cmpBytes <= 0) break;
write_uint16(outFp, (uint16_t) cmpBytes);
write_bin(outFp, cmpBuf, cmpBytes);
diff --git a/examples/blockStreaming_lineByLine.md b/examples/blockStreaming_lineByLine.md
index 4735f92..7b66883 100644
--- a/examples/blockStreaming_lineByLine.md
+++ b/examples/blockStreaming_lineByLine.md
@@ -1,7 +1,7 @@
# LZ4 Streaming API Example : Line by Line Text Compression
by *Takayuki Matsuoka*
-`blockStreaming_lineByLine.c` is LZ4 Straming API example which implements line by line incremental (de)compression.
+`blockStreaming_lineByLine.c` is LZ4 Streaming API example which implements line by line incremental (de)compression.
Please note the following restrictions :
@@ -107,7 +107,7 @@
In Line#X+2 (see (5)), finally LZ4 forget almost all memories but still remains Line#X+1.
This is the same situation as Line#2.
-Continue these procedure to the end of text file.
+Continue these procedures to the end of text file.
## How the decompression works
@@ -117,6 +117,6 @@
- Read compressed line from the file to buffer.
- Decompress it to the ringbuffer.
- Output decompressed plain text line to the file.
- - Forward ringbuffer offset. If offset exceedes end of the ringbuffer, reset it.
+ - Forward ringbuffer offset. If offset exceeds end of the ringbuffer, reset it.
-Continue these procedure to the end of the compressed file.
+Continue these procedures to the end of the compressed file.
diff --git a/examples/compress_functions.c b/examples/compress_functions.c
index 7fd6775..2a9d124 100644
--- a/examples/compress_functions.c
+++ b/examples/compress_functions.c
@@ -35,7 +35,7 @@
*
* LZ4_decompress_safe
* This is the recommended function for decompressing data. It is considered safe because the caller specifies
- * both the size of the compresssed buffer to read as well as the maximum size of the output (decompressed) buffer
+ * both the size of the compressed buffer to read as well as the maximum size of the output (decompressed) buffer
* instead of just the latter.
* LZ4_decompress_fast
* Again, despite its name it's not a "fast" version of decompression. It simply frees the caller of sending the
@@ -48,7 +48,7 @@
* Special Note About Decompression:
* Using the LZ4_decompress_safe() function protects against malicious (user) input. If you are using data from a
* trusted source, or if your program is the producer (P) as well as its consumer (C) in a PC or MPMC setup, you can
- * safely use the LZ4_decompress_fast function
+ * safely use the LZ4_decompress_fast function.
*/
/* Since lz4 compiles with c99 and not gnu/std99 we need to enable POSIX linking for time.h structs and functions. */
diff --git a/examples/dictionaryRandomAccess.c b/examples/dictionaryRandomAccess.c
index ecb3b2d..3aa4609 100644
--- a/examples/dictionaryRandomAccess.c
+++ b/examples/dictionaryRandomAccess.c
@@ -78,7 +78,7 @@
}
/* Forget previously compressed data and load the dictionary */
- LZ4_loadDict(lz4Stream, dict, dictSize);
+ LZ4_loadDict(lz4Stream, (const char*) dict, dictSize);
{
char cmpBuf[LZ4_COMPRESSBOUND(BLOCK_BYTES)];
const int cmpBytes = LZ4_compress_fast_continue(
@@ -97,7 +97,7 @@
while (ptr != offsetsEnd) {
write_int(outFp, *ptr++);
}
- write_int(outFp, offsetsEnd - offsets);
+ write_int(outFp, (int) (offsetsEnd - offsets));
}
}
@@ -153,7 +153,7 @@
}
/* Load the dictionary */
- LZ4_setStreamDecode(lz4StreamDecode, dict, dictSize);
+ LZ4_setStreamDecode(lz4StreamDecode, (const char*) dict, dictSize);
{
const int decBytes = LZ4_decompress_safe_continue(
lz4StreamDecode, cmpBuf, decBuf, cmpBytes, BLOCK_BYTES);
diff --git a/examples/dictionaryRandomAccess.md b/examples/dictionaryRandomAccess.md
index 53d825d..c6f4388 100644
--- a/examples/dictionaryRandomAccess.md
+++ b/examples/dictionaryRandomAccess.md
@@ -7,7 +7,7 @@
## What's the point of this example ?
- - Dictionary based compression for homogenous files.
+ - Dictionary based compression for homogeneous files.
- Random access to compressed blocks.
@@ -64,4 +64,4 @@
- Read the next block.
- Decompress it and write that page to the file.
-Continue these procedure until all the required data has been read.
+Continue these procedures until all the required data has been read.
diff --git a/examples/fileCompress.c b/examples/fileCompress.c
new file mode 100644
index 0000000..4486ea8
--- /dev/null
+++ b/examples/fileCompress.c
@@ -0,0 +1,232 @@
+/* LZ4file API example : compress a file
+ * Modified from an example code by anjiahao
+ *
+ * This example will demonstrate how
+ * to manipulate lz4 compressed files like
+ * normal files */
+
+#include <assert.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/stat.h>
+
+#include <lz4file.h>
+
+
+#define CHUNK_SIZE (16*1024)
+
+static size_t get_file_size(char *filename)
+{
+ struct stat statbuf;
+
+ if (filename == NULL) {
+ return 0;
+ }
+
+ if(stat(filename,&statbuf)) {
+ return 0;
+ }
+
+ return statbuf.st_size;
+}
+
+static int compress_file(FILE* f_in, FILE* f_out)
+{
+ assert(f_in != NULL); assert(f_out != NULL);
+
+ LZ4F_errorCode_t ret = LZ4F_OK_NoError;
+ size_t len;
+ LZ4_writeFile_t* lz4fWrite;
+ void* const buf = malloc(CHUNK_SIZE);
+ if (!buf) {
+ printf("error: memory allocation failed \n");
+ }
+
+ /* Of course, you can also use prefsPtr to
+ * set the parameters of the compressed file
+ * NULL is use default
+ */
+ ret = LZ4F_writeOpen(&lz4fWrite, f_out, NULL);
+ if (LZ4F_isError(ret)) {
+ printf("LZ4F_writeOpen error: %s\n", LZ4F_getErrorName(ret));
+ free(buf);
+ return 1;
+ }
+
+ while (1) {
+ len = fread(buf, 1, CHUNK_SIZE, f_in);
+
+ if (ferror(f_in)) {
+ printf("fread error\n");
+ goto out;
+ }
+
+ /* nothing to read */
+ if (len == 0) {
+ break;
+ }
+
+ ret = LZ4F_write(lz4fWrite, buf, len);
+ if (LZ4F_isError(ret)) {
+ printf("LZ4F_write: %s\n", LZ4F_getErrorName(ret));
+ goto out;
+ }
+ }
+
+out:
+ free(buf);
+ if (LZ4F_isError(LZ4F_writeClose(lz4fWrite))) {
+ printf("LZ4F_writeClose: %s\n", LZ4F_getErrorName(ret));
+ return 1;
+ }
+
+ return 0;
+}
+
+static int decompress_file(FILE* f_in, FILE* f_out)
+{
+ assert(f_in != NULL); assert(f_out != NULL);
+
+ LZ4F_errorCode_t ret = LZ4F_OK_NoError;
+ LZ4_readFile_t* lz4fRead;
+ void* const buf= malloc(CHUNK_SIZE);
+ if (!buf) {
+ printf("error: memory allocation failed \n");
+ }
+
+ ret = LZ4F_readOpen(&lz4fRead, f_in);
+ if (LZ4F_isError(ret)) {
+ printf("LZ4F_readOpen error: %s\n", LZ4F_getErrorName(ret));
+ free(buf);
+ return 1;
+ }
+
+ while (1) {
+ ret = LZ4F_read(lz4fRead, buf, CHUNK_SIZE);
+ if (LZ4F_isError(ret)) {
+ printf("LZ4F_read error: %s\n", LZ4F_getErrorName(ret));
+ goto out;
+ }
+
+ /* nothing to read */
+ if (ret == 0) {
+ break;
+ }
+
+ if(fwrite(buf, 1, ret, f_out) != ret) {
+ printf("write error!\n");
+ goto out;
+ }
+ }
+
+out:
+ free(buf);
+ if (LZ4F_isError(LZ4F_readClose(lz4fRead))) {
+ printf("LZ4F_readClose: %s\n", LZ4F_getErrorName(ret));
+ return 1;
+ }
+
+ if (ret) {
+ return 1;
+ }
+
+ return 0;
+}
+
+int compareFiles(FILE* fp0, FILE* fp1)
+{
+ int result = 0;
+
+ while (result==0) {
+ char b0[1024];
+ char b1[1024];
+ size_t const r0 = fread(b0, 1, sizeof(b0), fp0);
+ size_t const r1 = fread(b1, 1, sizeof(b1), fp1);
+
+ result = (r0 != r1);
+ if (!r0 || !r1) break;
+ if (!result) result = memcmp(b0, b1, r0);
+ }
+
+ return result;
+}
+
+int main(int argc, const char **argv) {
+ char inpFilename[256] = { 0 };
+ char lz4Filename[256] = { 0 };
+ char decFilename[256] = { 0 };
+
+ if (argc < 2) {
+ printf("Please specify input filename\n");
+ return 0;
+ }
+
+ snprintf(inpFilename, 256, "%s", argv[1]);
+ snprintf(lz4Filename, 256, "%s.lz4", argv[1]);
+ snprintf(decFilename, 256, "%s.lz4.dec", argv[1]);
+
+ printf("inp = [%s]\n", inpFilename);
+ printf("lz4 = [%s]\n", lz4Filename);
+ printf("dec = [%s]\n", decFilename);
+
+ /* compress */
+ { FILE* const inpFp = fopen(inpFilename, "rb");
+ FILE* const outFp = fopen(lz4Filename, "wb");
+ printf("compress : %s -> %s\n", inpFilename, lz4Filename);
+ LZ4F_errorCode_t ret = compress_file(inpFp, outFp);
+ fclose(inpFp);
+ fclose(outFp);
+
+ if (ret) {
+ printf("compression error: %s\n", LZ4F_getErrorName(ret));
+ return 1;
+ }
+
+ printf("%s: %zu → %zu bytes, %.1f%%\n",
+ inpFilename,
+ get_file_size(inpFilename),
+ get_file_size(lz4Filename), /* might overflow is size_t is 32 bits and size_{in,out} > 4 GB */
+ (double)get_file_size(lz4Filename) / get_file_size(inpFilename) * 100);
+
+ printf("compress : done\n");
+ }
+
+ /* decompress */
+ {
+ FILE* const inpFp = fopen(lz4Filename, "rb");
+ FILE* const outFp = fopen(decFilename, "wb");
+
+ printf("decompress : %s -> %s\n", lz4Filename, decFilename);
+ LZ4F_errorCode_t ret = decompress_file(inpFp, outFp);
+
+ fclose(outFp);
+ fclose(inpFp);
+
+ if (ret) {
+ printf("compression error: %s\n", LZ4F_getErrorName(ret));
+ return 1;
+ }
+
+ printf("decompress : done\n");
+ }
+
+ /* verify */
+ { FILE* const inpFp = fopen(inpFilename, "rb");
+ FILE* const decFp = fopen(decFilename, "rb");
+
+ printf("verify : %s <-> %s\n", inpFilename, decFilename);
+ int const cmp = compareFiles(inpFp, decFp);
+
+ fclose(decFp);
+ fclose(inpFp);
+
+ if (cmp) {
+ printf("corruption detected : decompressed file differs from original\n");
+ return cmp;
+ }
+
+ printf("verify : OK\n");
+ }
+
+}
diff --git a/examples/frameCompress.c b/examples/frameCompress.c
index aac4a3b..25ff729 100644
--- a/examples/frameCompress.c
+++ b/examples/frameCompress.c
@@ -11,8 +11,9 @@
#include <errno.h>
#include <assert.h>
+#include <getopt.h>
#include <lz4frame.h>
-
+#include <lz4frame_static.h>
#define IN_CHUNK_SIZE (16*1024)
@@ -57,10 +58,11 @@
compress_file_internal(FILE* f_in, FILE* f_out,
LZ4F_compressionContext_t ctx,
void* inBuff, size_t inChunkSize,
- void* outBuff, size_t outCapacity)
+ void* outBuff, size_t outCapacity,
+ FILE* f_unc, long uncOffset)
{
compressResult_t result = { 1, 0, 0 }; /* result for an error */
- unsigned long long count_in = 0, count_out;
+ long long count_in = 0, count_out, bytesToOffset = -1;
assert(f_in != NULL); assert(f_out != NULL);
assert(ctx != NULL);
@@ -81,22 +83,48 @@
/* stream file */
for (;;) {
- size_t const readSize = fread(inBuff, 1, IN_CHUNK_SIZE, f_in);
+ size_t compressedSize;
+ long long inSize = IN_CHUNK_SIZE;
+ if (uncOffset >= 0) {
+ bytesToOffset = uncOffset - count_in;
+
+ /* read only remaining bytes to offset position */
+ if (bytesToOffset < IN_CHUNK_SIZE && bytesToOffset > 0) {
+ inSize = bytesToOffset;
+ }
+ }
+
+ /* input data is at uncompressed data offset */
+ if (bytesToOffset <= 0 && uncOffset >= 0 && f_unc) {
+ size_t const readSize = fread(inBuff, 1, inSize, f_unc);
+ if (readSize == 0) {
+ uncOffset = -1;
+ continue;
+ }
+ count_in += readSize;
+ compressedSize = LZ4F_uncompressedUpdate(ctx,
+ outBuff, outCapacity,
+ inBuff, readSize,
+ NULL);
+ } else {
+ size_t const readSize = fread(inBuff, 1, inSize, f_in);
if (readSize == 0) break; /* nothing left to read from input file */
count_in += readSize;
-
- size_t const compressedSize = LZ4F_compressUpdate(ctx,
+ compressedSize = LZ4F_compressUpdate(ctx,
outBuff, outCapacity,
inBuff, readSize,
NULL);
- if (LZ4F_isError(compressedSize)) {
- printf("Compression failed: error %u \n", (unsigned)compressedSize);
- return result;
- }
- printf("Writing %u bytes\n", (unsigned)compressedSize);
- safe_fwrite(outBuff, 1, compressedSize, f_out);
- count_out += compressedSize;
+ }
+
+ if (LZ4F_isError(compressedSize)) {
+ printf("Compression failed: error %u \n", (unsigned)compressedSize);
+ return result;
+ }
+
+ printf("Writing %u bytes\n", (unsigned)compressedSize);
+ safe_fwrite(outBuff, 1, compressedSize, f_out);
+ count_out += compressedSize;
}
/* flush whatever remains within internal buffers */
@@ -120,12 +148,13 @@
}
static compressResult_t
-compress_file(FILE* f_in, FILE* f_out)
+compress_file(FILE* f_in, FILE* f_out,
+ FILE* f_unc, int uncOffset)
{
assert(f_in != NULL);
assert(f_out != NULL);
- /* ressource allocation */
+ /* resource allocation */
LZ4F_compressionContext_t ctx;
size_t const ctxCreation = LZ4F_createCompressionContext(&ctx, LZ4F_VERSION);
void* const src = malloc(IN_CHUNK_SIZE);
@@ -137,9 +166,10 @@
result = compress_file_internal(f_in, f_out,
ctx,
src, IN_CHUNK_SIZE,
- outbuff, outbufCapacity);
+ outbuff, outbufCapacity,
+ f_unc, uncOffset);
} else {
- printf("error : ressource allocation failed \n");
+ printf("error : resource allocation failed \n");
}
LZ4F_freeCompressionContext(ctx); /* supports free on NULL */
@@ -286,7 +316,7 @@
{
assert(f_in != NULL); assert(f_out != NULL);
- /* Ressource allocation */
+ /* Resource allocation */
void* const src = malloc(IN_CHUNK_SIZE);
if (!src) { perror("decompress_file(src)"); return 1; }
@@ -305,52 +335,106 @@
}
-int compareFiles(FILE* fp0, FILE* fp1)
+int compareFiles(FILE* fp0, FILE* fp1, FILE* fpUnc, long uncOffset)
{
int result = 0;
+ long bytesRead = 0;
+ long bytesToOffset = -1;
+ long b1Size = 1024;
while (result==0) {
+ char b1[b1Size];
+ size_t r1;
+ size_t bytesToRead = sizeof b1;
+ if (uncOffset >= 0) {
+ bytesToOffset = uncOffset - bytesRead;
+
+ /* read remainder to offset */
+ if (bytesToOffset < b1Size) {
+ bytesToRead = bytesToOffset;
+ }
+ }
+
char b0[1024];
- char b1[1024];
- size_t const r0 = fread(b0, 1, sizeof(b0), fp0);
- size_t const r1 = fread(b1, 1, sizeof(b1), fp1);
+ size_t r0;
+ if (bytesToOffset <= 0 && fpUnc) {
+ bytesToRead = sizeof b1;
+ r0 = fread(b0, 1,bytesToRead, fpUnc);
+ } else {
+ r0 = fread(b0, 1, bytesToRead, fp0);
+ }
+
+ r1 = fread(b1, 1, r0, fp1);
result = (r0 != r1);
if (!r0 || !r1) break;
if (!result) result = memcmp(b0, b1, r0);
+
+ bytesRead += r1;
}
return result;
}
-int main(int argc, const char **argv) {
+int main(int argc, char **argv) {
char inpFilename[256] = { 0 };
char lz4Filename[256] = { 0 };
char decFilename[256] = { 0 };
+ int uncOffset = -1;
+ char uncFilename[256] = { 0 };
+ int opt;
+
if (argc < 2) {
printf("Please specify input filename\n");
- return 0;
+ return EXIT_FAILURE;
}
snprintf(inpFilename, 256, "%s", argv[1]);
snprintf(lz4Filename, 256, "%s.lz4", argv[1]);
snprintf(decFilename, 256, "%s.lz4.dec", argv[1]);
+ while ((opt = getopt(argc, argv, "o:d:")) != -1) {
+ switch (opt) {
+ case 'd':
+ snprintf(uncFilename, 256, "%s", optarg);
+ break;
+ case 'o':
+ uncOffset = atoi(optarg);
+ break;
+ default:
+ printf("usage: %s <input file> [-o <offset> -d <file>]\n", argv[0]);
+ printf("-o uncompressed data offset\n");
+ printf(" inject uncompressed data at this offset into the lz4 file\n");
+ printf("-d uncompressed file\n");
+ printf(" file to inject without compression into the lz4 file\n");
+ return EXIT_FAILURE;
+ }
+ }
+
printf("inp = [%s]\n", inpFilename);
printf("lz4 = [%s]\n", lz4Filename);
printf("dec = [%s]\n", decFilename);
+ if (uncOffset > 0) {
+ printf("unc = [%s]\n", uncFilename);
+ printf("ofs = [%i]\n", uncOffset);
+ }
/* compress */
{ FILE* const inpFp = fopen(inpFilename, "rb");
FILE* const outFp = fopen(lz4Filename, "wb");
+ FILE* const uncFp = fopen(uncFilename, "rb");
printf("compress : %s -> %s\n", inpFilename, lz4Filename);
- compressResult_t const ret = compress_file(inpFp, outFp);
+ compressResult_t const ret = compress_file(
+ inpFp, outFp,
+ uncFp, uncOffset);
fclose(outFp);
fclose(inpFp);
+ if (uncFp)
+ fclose(uncFp);
if (ret.error) {
printf("compress : failed with code %i\n", ret.error);
@@ -383,12 +467,16 @@
/* verify */
{ FILE* const inpFp = fopen(inpFilename, "rb");
FILE* const decFp = fopen(decFilename, "rb");
+ FILE* const uncFp = fopen(uncFilename, "rb");
printf("verify : %s <-> %s\n", inpFilename, decFilename);
- int const cmp = compareFiles(inpFp, decFp);
+ int const cmp = compareFiles(inpFp, decFp,
+ uncFp, uncOffset);
fclose(decFp);
fclose(inpFp);
+ if (uncFp)
+ fclose(uncFp);
if (cmp) {
printf("corruption detected : decompressed file differs from original\n");
diff --git a/examples/simple_buffer.c b/examples/simple_buffer.c
index 6afc62a..f5c6eb2 100644
--- a/examples/simple_buffer.c
+++ b/examples/simple_buffer.c
@@ -44,10 +44,10 @@
// LZ4 provides a function that will tell you the maximum size of compressed output based on input data via LZ4_compressBound().
const int max_dst_size = LZ4_compressBound(src_size);
// We will use that size for our destination boundary when allocating space.
- char* compressed_data = malloc((size_t)max_dst_size);
+ char* compressed_data = (char*)malloc((size_t)max_dst_size);
if (compressed_data == NULL)
run_screaming("Failed to allocate memory for *compressed_data.", 1);
- // That's all the information and preparation LZ4 needs to compress *src into *compressed_data.
+ // That's all the information and preparation LZ4 needs to compress *src into* compressed_data.
// Invoke LZ4_compress_default now with our size values and pointers to our memory locations.
// Save the return value for error checking.
const int compressed_data_size = LZ4_compress_default(src, compressed_data, src_size, max_dst_size);
@@ -73,7 +73,7 @@
// Sometimes, the metadata can be extracted from the local context.
// First, let's create a *new_src location of size src_size since we know that value.
- char* const regen_buffer = malloc(src_size);
+ char* const regen_buffer = (char*)malloc(src_size);
if (regen_buffer == NULL)
run_screaming("Failed to allocate memory for *regen_buffer.", 1);
// The LZ4_decompress_safe function needs to know where the compressed data is, how many bytes long it is,
diff --git a/examples/streaming_api_basics.md b/examples/streaming_api_basics.md
index 1ccc6e3..6f5ae41 100644
--- a/examples/streaming_api_basics.md
+++ b/examples/streaming_api_basics.md
@@ -9,7 +9,7 @@
It guarantees interoperability with other LZ4 framing format compliant tools/libraries
such as LZ4 command line utility, node-lz4, etc.
- "Block" API : This is recommended for simple purpose.
- It compress single raw memory block to LZ4 memory block and vice versa.
+ It compresses single raw memory block to LZ4 memory block and vice versa.
- "Streaming" API : This is designed for complex things.
For example, compress huge stream data in restricted memory environment.
@@ -22,7 +22,7 @@
Block API (de)compresses a single contiguous memory block.
In other words, LZ4 library finds redundancy from a single contiguous memory block.
Streaming API does same thing but (de)compresses multiple adjacent contiguous memory blocks.
-So LZ4 library could find more redundancy than Block API.
+So Streaming API could find more redundancy than Block API.
The following figure shows difference between API and block sizes.
In these figures, the original data is split into 4KiBytes contiguous chunks.
diff --git a/lib/LICENSE b/lib/LICENSE
index 74c2cdd..4884916 100644
--- a/lib/LICENSE
+++ b/lib/LICENSE
@@ -1,5 +1,5 @@
LZ4 Library
-Copyright (c) 2011-2016, Yann Collet
+Copyright (c) 2011-2020, Yann Collet
All rights reserved.
Redistribution and use in source and binary forms, with or without modification,
diff --git a/lib/Makefile b/lib/Makefile
index c12949b..06503cb 100644
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -1,6 +1,6 @@
# ################################################################
# LZ4 library - Makefile
-# Copyright (C) Yann Collet 2011-2016
+# Copyright (C) Yann Collet 2011-2020
# All rights reserved.
#
# This Makefile is validated for Linux, macOS, *BSD, Hurd, Solaris, MSYS2 targets
@@ -31,11 +31,12 @@
# - LZ4 source repository : https://github.com/lz4/lz4
# - LZ4 forum froup : https://groups.google.com/forum/#!forum/lz4c
# ################################################################
+SED = sed
# Version numbers
-LIBVER_MAJOR_SCRIPT:=`sed -n '/define LZ4_VERSION_MAJOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < ./lz4.h`
-LIBVER_MINOR_SCRIPT:=`sed -n '/define LZ4_VERSION_MINOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < ./lz4.h`
-LIBVER_PATCH_SCRIPT:=`sed -n '/define LZ4_VERSION_RELEASE/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < ./lz4.h`
+LIBVER_MAJOR_SCRIPT:=`$(SED) -n '/define LZ4_VERSION_MAJOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < ./lz4.h`
+LIBVER_MINOR_SCRIPT:=`$(SED) -n '/define LZ4_VERSION_MINOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < ./lz4.h`
+LIBVER_PATCH_SCRIPT:=`$(SED) -n '/define LZ4_VERSION_RELEASE/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < ./lz4.h`
LIBVER_SCRIPT:= $(LIBVER_MAJOR_SCRIPT).$(LIBVER_MINOR_SCRIPT).$(LIBVER_PATCH_SCRIPT)
LIBVER_MAJOR := $(shell echo $(LIBVER_MAJOR_SCRIPT))
LIBVER_MINOR := $(shell echo $(LIBVER_MINOR_SCRIPT))
@@ -46,12 +47,13 @@
BUILD_STATIC:=yes
CPPFLAGS+= -DXXH_NAMESPACE=LZ4_
+CPPFLAGS+= $(MOREFLAGS)
CFLAGS ?= -O3
DEBUGFLAGS:= -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
-Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes \
-Wundef -Wpointer-arith -Wstrict-aliasing=1
-CFLAGS += $(DEBUGFLAGS) $(MOREFLAGS)
-FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS)
+CFLAGS += $(DEBUGFLAGS)
+FLAGS = $(CFLAGS) $(CPPFLAGS) $(LDFLAGS)
SRCFILES := $(sort $(wildcard *.c))
@@ -74,27 +76,33 @@
.PHONY: default
default: lib-release
+# silent mode by default; verbose can be triggered by V=1 or VERBOSE=1
+$(V)$(VERBOSE).SILENT:
+
lib-release: DEBUGFLAGS :=
lib-release: lib
+.PHONY: lib
lib: liblz4.a liblz4
+.PHONY: all
all: lib
+.PHONY: all32
all32: CFLAGS+=-m32
all32: all
liblz4.a: $(SRCFILES)
ifeq ($(BUILD_STATIC),yes) # can be disabled on command line
@echo compiling static library
- $(Q)$(CC) $(CPPFLAGS) $(CFLAGS) -c $^
- $(Q)$(AR) rcs $@ *.o
+ $(COMPILE.c) $^
+ $(AR) rcs $@ *.o
endif
ifeq ($(WINBASED),yes)
liblz4-dll.rc: liblz4-dll.rc.in
@echo creating library resource
- $(Q)sed -e 's|@LIBLZ4@|$(LIBLZ4)|' \
+ $(SED) -e 's|@LIBLZ4@|$(LIBLZ4)|' \
-e 's|@LIBVER_MAJOR@|$(LIBVER_MAJOR)|g' \
-e 's|@LIBVER_MINOR@|$(LIBVER_MINOR)|g' \
-e 's|@LIBVER_PATCH@|$(LIBVER_PATCH)|g' \
@@ -104,31 +112,30 @@
$(WINDRES) -i liblz4-dll.rc -o liblz4-dll.o
$(LIBLZ4): $(SRCFILES) liblz4-dll.o
-else
-$(LIBLZ4): $(SRCFILES)
-endif
-ifeq ($(BUILD_SHARED),yes) # can be disabled on command line
@echo compiling dynamic library $(LIBVER)
- ifeq ($(WINBASED),yes)
- $(Q)$(CC) $(FLAGS) -DLZ4_DLL_EXPORT=1 -shared $^ -o dll/$@.dll -Wl,--out-implib,dll/$(LIBLZ4_EXP)
- else
- $(Q)$(CC) $(FLAGS) -shared $^ -fPIC -fvisibility=hidden $(SONAME_FLAGS) -o $@
+ $(CC) $(FLAGS) -DLZ4_DLL_EXPORT=1 -shared $^ -o dll/$@.dll -Wl,--out-implib,dll/$(LIBLZ4_EXP)
+
+else # not windows
+
+$(LIBLZ4): $(SRCFILES)
+ @echo compiling dynamic library $(LIBVER)
+ $(CC) $(FLAGS) -shared $^ -fPIC -fvisibility=hidden $(SONAME_FLAGS) -o $@
@echo creating versioned links
- $(Q)$(LN_SF) $@ liblz4.$(SHARED_EXT_MAJOR)
- $(Q)$(LN_SF) $@ liblz4.$(SHARED_EXT)
- endif
+ $(LN_SF) $@ liblz4.$(SHARED_EXT_MAJOR)
+ $(LN_SF) $@ liblz4.$(SHARED_EXT)
+
endif
-ifeq (,$(filter MINGW%,$(TARGET_OS)))
+.PHONY: liblz4
liblz4: $(LIBLZ4)
-endif
+.PHONY: clean
clean:
ifeq ($(WINBASED),yes)
- $(Q)$(RM) *.rc
+ $(RM) *.rc
endif
- $(Q)$(RM) core *.o liblz4.pc dll/$(LIBLZ4).dll dll/$(LIBLZ4_EXP)
- $(Q)$(RM) *.a *.$(SHARED_EXT) *.$(SHARED_EXT_MAJOR) *.$(SHARED_EXT_VER)
+ $(RM) core *.o liblz4.pc dll/$(LIBLZ4).dll dll/$(LIBLZ4_EXP)
+ $(RM) *.a *.$(SHARED_EXT) *.$(SHARED_EXT_MAJOR) *.$(SHARED_EXT_VER)
@echo Cleaning library completed
#-----------------------------------------------------------------------------
@@ -164,54 +171,55 @@
liblz4.pc: liblz4.pc.in Makefile
@echo creating pkgconfig
- $(Q)sed -e 's|@PREFIX@|$(prefix)|' \
+ $(SED) -e 's|@PREFIX@|$(prefix)|' \
-e 's|@LIBDIR@|$(libdir)|' \
-e 's|@INCLUDEDIR@|$(includedir)|' \
-e 's|@VERSION@|$(LIBVER)|' \
+ -e 's|=${prefix}/|=$${prefix}/|' \
$< >$@
install: lib liblz4.pc
- $(Q)$(INSTALL_DIR) $(DESTDIR)$(pkgconfigdir)/ $(DESTDIR)$(includedir)/ $(DESTDIR)$(libdir)/ $(DESTDIR)$(bindir)/
- $(Q)$(INSTALL_DATA) liblz4.pc $(DESTDIR)$(pkgconfigdir)/
- @echo Installing libraries
+ $(INSTALL_DIR) $(DESTDIR)$(pkgconfigdir)/ $(DESTDIR)$(includedir)/ $(DESTDIR)$(libdir)/ $(DESTDIR)$(bindir)/
+ $(INSTALL_DATA) liblz4.pc $(DESTDIR)$(pkgconfigdir)/
+ @echo Installing libraries in $(DESTDIR)$(libdir)
ifeq ($(BUILD_STATIC),yes)
- $(Q)$(INSTALL_DATA) liblz4.a $(DESTDIR)$(libdir)/liblz4.a
- $(Q)$(INSTALL_DATA) lz4frame_static.h $(DESTDIR)$(includedir)/lz4frame_static.h
+ $(INSTALL_DATA) liblz4.a $(DESTDIR)$(libdir)/liblz4.a
+ $(INSTALL_DATA) lz4frame_static.h $(DESTDIR)$(includedir)/lz4frame_static.h
endif
ifeq ($(BUILD_SHARED),yes)
-# Traditionnally, one installs the DLLs in the bin directory as programs
+# Traditionally, one installs the DLLs in the bin directory as programs
# search them first in their directory. This allows to not pollute system
# directories (like c:/windows/system32), nor modify the PATH variable.
ifeq ($(WINBASED),yes)
- $(Q)$(INSTALL_PROGRAM) dll/$(LIBLZ4).dll $(DESTDIR)$(bindir)
- $(Q)$(INSTALL_PROGRAM) dll/$(LIBLZ4_EXP) $(DESTDIR)$(libdir)
+ $(INSTALL_PROGRAM) dll/$(LIBLZ4).dll $(DESTDIR)$(bindir)
+ $(INSTALL_PROGRAM) dll/$(LIBLZ4_EXP) $(DESTDIR)$(libdir)
else
- $(Q)$(INSTALL_PROGRAM) liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(libdir)
- $(Q)$(LN_SF) liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(libdir)/liblz4.$(SHARED_EXT_MAJOR)
- $(Q)$(LN_SF) liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(libdir)/liblz4.$(SHARED_EXT)
+ $(INSTALL_PROGRAM) liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(libdir)
+ $(LN_SF) liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(libdir)/liblz4.$(SHARED_EXT_MAJOR)
+ $(LN_SF) liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(libdir)/liblz4.$(SHARED_EXT)
endif
endif
- @echo Installing headers in $(includedir)
- $(Q)$(INSTALL_DATA) lz4.h $(DESTDIR)$(includedir)/lz4.h
- $(Q)$(INSTALL_DATA) lz4hc.h $(DESTDIR)$(includedir)/lz4hc.h
- $(Q)$(INSTALL_DATA) lz4frame.h $(DESTDIR)$(includedir)/lz4frame.h
+ @echo Installing headers in $(DESTDIR)$(includedir)
+ $(INSTALL_DATA) lz4.h $(DESTDIR)$(includedir)/lz4.h
+ $(INSTALL_DATA) lz4hc.h $(DESTDIR)$(includedir)/lz4hc.h
+ $(INSTALL_DATA) lz4frame.h $(DESTDIR)$(includedir)/lz4frame.h
@echo lz4 libraries installed
uninstall:
- $(Q)$(RM) $(DESTDIR)$(pkgconfigdir)/liblz4.pc
+ $(RM) $(DESTDIR)$(pkgconfigdir)/liblz4.pc
ifeq (WINBASED,1)
- $(Q)$(RM) $(DESTDIR)$(bindir)/$(LIBLZ4).dll
- $(Q)$(RM) $(DESTDIR)$(libdir)/$(LIBLZ4_EXP)
+ $(RM) $(DESTDIR)$(bindir)/$(LIBLZ4).dll
+ $(RM) $(DESTDIR)$(libdir)/$(LIBLZ4_EXP)
else
- $(Q)$(RM) $(DESTDIR)$(libdir)/liblz4.$(SHARED_EXT)
- $(Q)$(RM) $(DESTDIR)$(libdir)/liblz4.$(SHARED_EXT_MAJOR)
- $(Q)$(RM) $(DESTDIR)$(libdir)/liblz4.$(SHARED_EXT_VER)
+ $(RM) $(DESTDIR)$(libdir)/liblz4.$(SHARED_EXT)
+ $(RM) $(DESTDIR)$(libdir)/liblz4.$(SHARED_EXT_MAJOR)
+ $(RM) $(DESTDIR)$(libdir)/liblz4.$(SHARED_EXT_VER)
endif
- $(Q)$(RM) $(DESTDIR)$(libdir)/liblz4.a
- $(Q)$(RM) $(DESTDIR)$(includedir)/lz4.h
- $(Q)$(RM) $(DESTDIR)$(includedir)/lz4hc.h
- $(Q)$(RM) $(DESTDIR)$(includedir)/lz4frame.h
- $(Q)$(RM) $(DESTDIR)$(includedir)/lz4frame_static.h
+ $(RM) $(DESTDIR)$(libdir)/liblz4.a
+ $(RM) $(DESTDIR)$(includedir)/lz4.h
+ $(RM) $(DESTDIR)$(includedir)/lz4hc.h
+ $(RM) $(DESTDIR)$(includedir)/lz4frame.h
+ $(RM) $(DESTDIR)$(includedir)/lz4frame_static.h
@echo lz4 libraries successfully uninstalled
endif
diff --git a/lib/README.md b/lib/README.md
index e2af868..08d1cef 100644
--- a/lib/README.md
+++ b/lib/README.md
@@ -2,16 +2,20 @@
================================
The `/lib` directory contains many files, but depending on project's objectives,
-not all of them are necessary.
+not all of them are required.
+Limited systems may want to reduce the nb of source files to include
+as a way to reduce binary size and dependencies.
-#### Minimal LZ4 build
+Capabilities are added at the "level" granularity, detailed below.
+
+#### Level 1 : Minimal LZ4 build
The minimum required is **`lz4.c`** and **`lz4.h`**,
which provides the fast compression and decompression algorithms.
They generate and decode data using the [LZ4 block format].
-#### High Compression variant
+#### Level 2 : High Compression variant
For more compression ratio at the cost of compression speed,
the High Compression variant called **lz4hc** is available.
@@ -20,7 +24,7 @@
and depends on regular `lib/lz4.*` source files.
-#### Frame support, for interoperability
+#### Level 3 : Frame support, for interoperability
In order to produce compressed data compatible with `lz4` command line utility,
it's necessary to use the [official interoperable frame format].
@@ -28,14 +32,29 @@
Its public API is described in `lib/lz4frame.h`.
In order to work properly, lz4frame needs all other modules present in `/lib`,
including, lz4 and lz4hc, and also **xxhash**.
-So it's necessary to include all `*.c` and `*.h` files present in `/lib`.
+So it's necessary to also include `xxhash.c` and `xxhash.h`.
+
+
+#### Level 4 : File compression operations
+
+As a helper around file operations,
+the library has been recently extended with `lz4file.c` and `lz4file.h`
+(still considered experimental at the time of this writing).
+These helpers allow opening, reading, writing, and closing files
+using transparent LZ4 compression / decompression.
+As a consequence, using `lz4file` adds a dependency on `<stdio.h>`.
+
+`lz4file` relies on `lz4frame` in order to produce compressed data
+conformant to the [LZ4 Frame format] specification.
+Consequently, to enable this capability,
+it's necessary to include all `*.c` and `*.h` files from `lib/` directory.
#### Advanced / Experimental API
Definitions which are not guaranteed to remain stable in future versions,
are protected behind macros, such as `LZ4_STATIC_LINKING_ONLY`.
-As the name strongly implies, these definitions should only be invoked
+As the name suggests, these definitions should only be invoked
in the context of static linking ***only***.
Otherwise, dependent application may fail on API or ABI break in the future.
The associated symbols are also not exposed by the dynamic library by default.
@@ -58,7 +77,7 @@
Set to 65535 by default, which is the maximum value supported by lz4 format.
Reducing maximum distance will reduce opportunities for LZ4 to find matches,
hence will produce a worse compression ratio.
- However, a smaller max distance can allow compatibility with specific decoders using limited memory budget.
+ Setting a smaller max distance could allow compatibility with specific decoders with limited memory budget.
This build macro only influences the compressed output of the compressor.
- `LZ4_DISABLE_DEPRECATE_WARNINGS` : invoking a deprecated function will make the compiler generate a warning.
@@ -69,15 +88,11 @@
This build macro offers another project-specific method
by defining `LZ4_DISABLE_DEPRECATE_WARNINGS` before including the LZ4 header files.
-- `LZ4_USER_MEMORY_FUNCTIONS` : replace calls to <stdlib>'s `malloc`, `calloc` and `free`
- by user-defined functions, which must be called `LZ4_malloc()`, `LZ4_calloc()` and `LZ4_free()`.
- User functions must be available at link time.
-
- `LZ4_FORCE_SW_BITCOUNT` : by default, the compression algorithm tries to determine lengths
by using bitcount instructions, generally implemented as fast single instructions in many cpus.
In case the target cpus doesn't support it, or compiler intrinsic doesn't work, or feature bad performance,
it's possible to use an optimized software path instead.
- This is achieved by setting this build macros .
+ This is achieved by setting this build macros.
In most cases, it's not expected to be necessary,
but it can be legitimately considered for less common platforms.
@@ -85,6 +100,22 @@
passed as argument to become a compression state is suitably aligned.
This test can be disabled if it proves flaky, by setting this value to 0.
+- `LZ4_USER_MEMORY_FUNCTIONS` : replace calls to `<stdlib,h>`'s `malloc()`, `calloc()` and `free()`
+ by user-defined functions, which must be named `LZ4_malloc()`, `LZ4_calloc()` and `LZ4_free()`.
+ User functions must be available at link time.
+
+- `LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION` :
+ Remove support of dynamic memory allocation.
+ For more details, see description of this macro in `lib/lz4.c`.
+
+- `LZ4_FREESTANDING` : by setting this build macro to 1,
+ LZ4/HC removes dependencies on the C standard library,
+ including allocation functions and `memmove()`, `memcpy()`, and `memset()`.
+ This build macro is designed to help use LZ4/HC in restricted environments
+ (embedded, bootloader, etc).
+ For more details, see description of this macro in `lib/lz4.h`.
+
+
#### Amalgamation
@@ -101,7 +132,7 @@
DLL can be created using MinGW+MSYS with the `make liblz4` command.
This command creates `dll\liblz4.dll` and the import library `dll\liblz4.lib`.
-To override the `dlltool` command when cross-compiling on Linux, just set the `DLLTOOL` variable. Example of cross compilation on Linux with mingw-w64 64 bits:
+To override the `dlltool` command when cross-compiling on Linux, just set the `DLLTOOL` variable. Example of cross compilation on Linux with mingw-w64 64 bits:
```
make BUILD_STATIC=no CC=x86_64-w64-mingw32-gcc DLLTOOL=x86_64-w64-mingw32-dlltool OS=Windows_NT
```
@@ -127,6 +158,7 @@
- `README.md` : this file
[official interoperable frame format]: ../doc/lz4_Frame_format.md
+[LZ4 Frame format]: ../doc/lz4_Frame_format.md
[LZ4 block format]: ../doc/lz4_Block_format.md
diff --git a/lib/dll/example/Makefile b/lib/dll/example/Makefile
index e987956..eb8cc1e 100644
--- a/lib/dll/example/Makefile
+++ b/lib/dll/example/Makefile
@@ -1,6 +1,6 @@
# ##########################################################################
# LZ4 programs - Makefile
-# Copyright (C) Yann Collet 2016
+# Copyright (C) Yann Collet 2016-2020
#
# GPL v2 License
#
diff --git a/lib/dll/example/README.md b/lib/dll/example/README.md
index 223e473..b93914b 100644
--- a/lib/dll/example/README.md
+++ b/lib/dll/example/README.md
@@ -4,8 +4,8 @@
#### The package contents
- `lz4.exe` : Command Line Utility, supporting gzip-like arguments
-- `dll\liblz4.dll` : The DLL of LZ4 library
-- `dll\liblz4.lib` : The import library of LZ4 library for Visual C++
+- `dll\msys-lz4-1.dll` : The DLL of LZ4 library, compiled by msys
+- `dll\liblz4.dll.a` : The import library of LZ4 library for Visual C++
- `example\` : The example of usage of LZ4 library
- `include\` : Header files required with LZ4 library
- `static\liblz4_static.lib` : The static LZ4 library
@@ -35,15 +35,15 @@
#### Using LZ4 DLL with gcc/MinGW
-The header files from `include\` and the dynamic library `dll\liblz4.dll`
+The header files from `include\` and the dynamic library `dll\msys-lz4-1.dll`
are required to compile a project using gcc/MinGW.
The dynamic library has to be added to linking options.
It means that if a project that uses LZ4 consists of a single `test-dll.c`
-file it should be linked with `dll\liblz4.dll`. For example:
+file it should be linked with `dll\msys-lz4-1.dll`. For example:
```
- gcc $(CFLAGS) -Iinclude\ test-dll.c -o test-dll dll\liblz4.dll
+ gcc $(CFLAGS) -Iinclude\ test-dll.c -o test-dll dll\msys-lz4-1.dll
```
-The compiled executable will require LZ4 DLL which is available at `dll\liblz4.dll`.
+The compiled executable will require LZ4 DLL which is available at `dll\msys-lz4-1.dll`.
#### The example of usage of static and dynamic LZ4 libraries with Visual C++
@@ -51,19 +51,19 @@
Open `example\fullbench-dll.sln` to compile `fullbench-dll` that uses a
dynamic LZ4 library from the `dll` directory. The solution works with Visual C++
2010 or newer. When one will open the solution with Visual C++ newer than 2010
-then the solution will upgraded to the current version.
+then the solution will be upgraded to the current version.
#### Using LZ4 DLL with Visual C++
-The header files from `include\` and the import library `dll\liblz4.lib`
+The header files from `include\` and the import library `dll\liblz4.dll.a`
are required to compile a project using Visual C++.
1. The header files should be added to `Additional Include Directories` that can
be found in project properties `C/C++` then `General`.
2. The import library has to be added to `Additional Dependencies` that can
be found in project properties `Linker` then `Input`.
- If one will provide only the name `liblz4.lib` without a full path to the library
+ If one will provide only the name `liblz4.dll.a` without a full path to the library
the directory has to be added to `Linker\General\Additional Library Directories`.
-The compiled executable will require LZ4 DLL which is available at `dll\liblz4.dll`.
+The compiled executable will require LZ4 DLL which is available at `dll\msys-lz4-1.dll`.
diff --git a/lib/liblz4-dll.rc.in b/lib/liblz4-dll.rc.in
index bf9adf5..e2d84b6 100644
--- a/lib/liblz4-dll.rc.in
+++ b/lib/liblz4-dll.rc.in
@@ -22,7 +22,7 @@
VALUE "FileDescription", "Extremely fast compression"
VALUE "FileVersion", "@LIBVER_MAJOR@.@LIBVER_MINOR@.@LIBVER_PATCH@.0"
VALUE "InternalName", "@LIBLZ4@"
- VALUE "LegalCopyright", "Copyright (C) 2013-2016, Yann Collet"
+ VALUE "LegalCopyright", "Copyright (C) 2013-2020, Yann Collet"
VALUE "OriginalFilename", "@LIBLZ4@.dll"
VALUE "ProductName", "LZ4"
VALUE "ProductVersion", "@LIBVER_MAJOR@.@LIBVER_MINOR@.@LIBVER_PATCH@.0"
diff --git a/lib/liblz4.pc.in b/lib/liblz4.pc.in
index cb31cd7..ed52214 100644
--- a/lib/liblz4.pc.in
+++ b/lib/liblz4.pc.in
@@ -1,5 +1,5 @@
# LZ4 - Fast LZ compression algorithm
-# Copyright (C) 2011-2014, Yann Collet.
+# Copyright (C) 2011-2020, Yann Collet.
# BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
prefix=@PREFIX@
@@ -10,5 +10,5 @@
Description: extremely fast lossless compression algorithm library
URL: http://www.lz4.org/
Version: @VERSION@
-Libs: -L@LIBDIR@ -llz4
-Cflags: -I@INCLUDEDIR@
+Libs: -L${libdir} -llz4
+Cflags: -I${includedir}
diff --git a/lib/lz4.c b/lib/lz4.c
index 9f5e9bf..654bfdf 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -1,6 +1,6 @@
/*
LZ4 - Fast LZ compression algorithm
- Copyright (C) 2011-present, Yann Collet.
+ Copyright (C) 2011-2020, Yann Collet.
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
@@ -124,6 +124,7 @@
#if defined(_MSC_VER) && (_MSC_VER >= 1400) /* Visual Studio 2005+ */
# include <intrin.h> /* only present in VS2005+ */
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
+# pragma warning(disable : 6237) /* disable: C6237: conditional expression is always 0 */
#endif /* _MSC_VER */
#ifndef LZ4_FORCE_INLINE
@@ -187,7 +188,27 @@
/*-************************************
* Memory routines
**************************************/
-#ifdef LZ4_USER_MEMORY_FUNCTIONS
+
+/*! LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION :
+ * Disable relatively high-level LZ4/HC functions that use dynamic memory
+ * allocation functions (malloc(), calloc(), free()).
+ *
+ * Note that this is a compile-time switch. And since it disables
+ * public/stable LZ4 v1 API functions, we don't recommend using this
+ * symbol to generate a library for distribution.
+ *
+ * The following public functions are removed when this symbol is defined.
+ * - lz4 : LZ4_createStream, LZ4_freeStream,
+ * LZ4_createStreamDecode, LZ4_freeStreamDecode, LZ4_create (deprecated)
+ * - lz4hc : LZ4_createStreamHC, LZ4_freeStreamHC,
+ * LZ4_createHC (deprecated), LZ4_freeHC (deprecated)
+ * - lz4frame, lz4file : All LZ4F_* functions
+ */
+#if defined(LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION)
+# define ALLOC(s) lz4_error_memory_allocation_is_disabled
+# define ALLOC_AND_ZERO(s) lz4_error_memory_allocation_is_disabled
+# define FREEMEM(p) lz4_error_memory_allocation_is_disabled
+#elif defined(LZ4_USER_MEMORY_FUNCTIONS)
/* memory management functions can be customized by user project.
* Below functions must exist somewhere in the Project
* and be available at link time */
@@ -204,8 +225,13 @@
# define FREEMEM(p) free(p)
#endif
-#include <string.h> /* memset, memcpy */
-#define MEM_INIT(p,v,s) memset((p),(v),(s))
+#if ! LZ4_FREESTANDING
+# include <string.h> /* memset, memcpy */
+#endif
+#if !defined(LZ4_memset)
+# define LZ4_memset(p,v,s) memset((p),(v),(s))
+#endif
+#define MEM_INIT(p,v,s) LZ4_memset((p),(v),(s))
/*-************************************
@@ -316,10 +342,20 @@
* memcpy() as if it were standard compliant, so it can inline it in freestanding
* environments. This is needed when decompressing the Linux Kernel, for example.
*/
-#if defined(__GNUC__) && (__GNUC__ >= 4)
-#define LZ4_memcpy(dst, src, size) __builtin_memcpy(dst, src, size)
-#else
-#define LZ4_memcpy(dst, src, size) memcpy(dst, src, size)
+#if !defined(LZ4_memcpy)
+# if defined(__GNUC__) && (__GNUC__ >= 4)
+# define LZ4_memcpy(dst, src, size) __builtin_memcpy(dst, src, size)
+# else
+# define LZ4_memcpy(dst, src, size) memcpy(dst, src, size)
+# endif
+#endif
+
+#if !defined(LZ4_memmove)
+# if defined(__GNUC__) && (__GNUC__ >= 4)
+# define LZ4_memmove __builtin_memmove
+# else
+# define LZ4_memmove memmove
+# endif
#endif
static unsigned LZ4_isLittleEndian(void)
@@ -343,14 +379,14 @@
/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */
/* currently only defined for gcc and icc */
-typedef union { U16 u16; U32 u32; reg_t uArch; } __attribute__((packed)) unalign;
+typedef union { U16 u16; U32 u32; reg_t uArch; } __attribute__((packed)) LZ4_unalign;
-static U16 LZ4_read16(const void* ptr) { return ((const unalign*)ptr)->u16; }
-static U32 LZ4_read32(const void* ptr) { return ((const unalign*)ptr)->u32; }
-static reg_t LZ4_read_ARCH(const void* ptr) { return ((const unalign*)ptr)->uArch; }
+static U16 LZ4_read16(const void* ptr) { return ((const LZ4_unalign*)ptr)->u16; }
+static U32 LZ4_read32(const void* ptr) { return ((const LZ4_unalign*)ptr)->u32; }
+static reg_t LZ4_read_ARCH(const void* ptr) { return ((const LZ4_unalign*)ptr)->uArch; }
-static void LZ4_write16(void* memPtr, U16 value) { ((unalign*)memPtr)->u16 = value; }
-static void LZ4_write32(void* memPtr, U32 value) { ((unalign*)memPtr)->u32 = value; }
+static void LZ4_write16(void* memPtr, U16 value) { ((LZ4_unalign*)memPtr)->u16 = value; }
+static void LZ4_write32(void* memPtr, U32 value) { ((LZ4_unalign*)memPtr)->u32 = value; }
#else /* safe and portable access using memcpy() */
@@ -421,10 +457,12 @@
#ifndef LZ4_FAST_DEC_LOOP
# if defined __i386__ || defined _M_IX86 || defined __x86_64__ || defined _M_X64
# define LZ4_FAST_DEC_LOOP 1
+# elif defined(__aarch64__) && defined(__APPLE__)
+# define LZ4_FAST_DEC_LOOP 1
# elif defined(__aarch64__) && !defined(__clang__)
- /* On aarch64, we disable this optimization for clang because on certain
- * mobile chipsets, performance is reduced with clang. For information
- * refer to https://github.com/lz4/lz4/pull/707 */
+ /* On non-Apple aarch64, we disable this optimization for clang because
+ * on certain mobile chipsets, performance is reduced with clang. For
+ * more information refer to https://github.com/lz4/lz4/pull/707 */
# define LZ4_FAST_DEC_LOOP 1
# else
# define LZ4_FAST_DEC_LOOP 0
@@ -486,7 +524,14 @@
case 2:
LZ4_memcpy(v, srcPtr, 2);
LZ4_memcpy(&v[2], srcPtr, 2);
+#if defined(_MSC_VER) && (_MSC_VER <= 1933) /* MSVC 2022 ver 17.3 or earlier */
+# pragma warning(push)
+# pragma warning(disable : 6385) /* warning C6385: Reading invalid data from 'v'. */
+#endif
LZ4_memcpy(&v[4], v, 4);
+#if defined(_MSC_VER) && (_MSC_VER <= 1933) /* MSVC 2022 ver 17.3 or earlier */
+# pragma warning(pop)
+#endif
break;
case 4:
LZ4_memcpy(v, srcPtr, 4);
@@ -515,9 +560,20 @@
assert(val != 0);
if (LZ4_isLittleEndian()) {
if (sizeof(val) == 8) {
-# if defined(_MSC_VER) && (_MSC_VER >= 1800) && defined(_M_AMD64) && !defined(LZ4_FORCE_SW_BITCOUNT)
+# if defined(_MSC_VER) && (_MSC_VER >= 1800) && (defined(_M_AMD64) && !defined(_M_ARM64EC)) && !defined(LZ4_FORCE_SW_BITCOUNT)
+/*-*************************************************************************************************
+* ARM64EC is a Microsoft-designed ARM64 ABI compatible with AMD64 applications on ARM64 Windows 11.
+* The ARM64EC ABI does not support AVX/AVX2/AVX512 instructions, nor their relevant intrinsics
+* including _tzcnt_u64. Therefore, we need to neuter the _tzcnt_u64 code path for ARM64EC.
+****************************************************************************************************/
+# if defined(__clang__) && (__clang_major__ < 10)
+ /* Avoid undefined clang-cl intrinsics issue.
+ * See https://github.com/lz4/lz4/pull/1017 for details. */
+ return (unsigned)__builtin_ia32_tzcnt_u64(val) >> 3;
+# else
/* x64 CPUS without BMI support interpret `TZCNT` as `REP BSF` */
return (unsigned)_tzcnt_u64(val) >> 3;
+# endif
# elif defined(_MSC_VER) && defined(_WIN64) && !defined(LZ4_FORCE_SW_BITCOUNT)
unsigned long r = 0;
_BitScanForward64(&r, (U64)val);
@@ -652,10 +708,10 @@
* - usingExtDict : Like withPrefix64k, but the preceding content is somewhere
* else in memory, starting at ctx->dictionary with length
* ctx->dictSize.
- * - usingDictCtx : Like usingExtDict, but everything concerning the preceding
- * content is in a separate context, pointed to by
- * ctx->dictCtx. ctx->dictionary, ctx->dictSize, and table
- * entries in the current context that refer to positions
+ * - usingDictCtx : Everything concerning the preceding content is
+ * in a separate context, pointed to by ctx->dictCtx.
+ * ctx->dictionary, ctx->dictSize, and table entries
+ * in the current context that refer to positions
* preceding the beginning of the current compression are
* ignored. Instead, ctx->dictCtx->dictionary and ctx->dictCtx
* ->dictSize describe the location and size of the preceding
@@ -672,12 +728,12 @@
int LZ4_versionNumber (void) { return LZ4_VERSION_NUMBER; }
const char* LZ4_versionString(void) { return LZ4_VERSION_STRING; }
int LZ4_compressBound(int isize) { return LZ4_COMPRESSBOUND(isize); }
-int LZ4_sizeofState(void) { return LZ4_STREAMSIZE; }
+int LZ4_sizeofState(void) { return sizeof(LZ4_stream_t); }
-/*-************************************
-* Internal Definitions used in Tests
-**************************************/
+/*-****************************************
+* Internal Definitions, used only in Tests
+*******************************************/
#if defined (__cplusplus)
extern "C" {
#endif
@@ -687,7 +743,9 @@
int LZ4_decompress_safe_forceExtDict(const char* source, char* dest,
int compressedSize, int maxOutputSize,
const void* dictStart, size_t dictSize);
-
+int LZ4_decompress_safe_partial_forceExtDict(const char* source, char* dest,
+ int compressedSize, int targetOutputSize, int dstCapacity,
+ const void* dictStart, size_t dictSize);
#if defined (__cplusplus)
}
#endif
@@ -827,9 +885,10 @@
}
}
- /* Adding a gap, so all previous entries are > LZ4_DISTANCE_MAX back, is faster
- * than compressing without a gap. However, compressing with
- * currentOffset == 0 is faster still, so we preserve that case.
+ /* Adding a gap, so all previous entries are > LZ4_DISTANCE_MAX back,
+ * is faster than compressing without a gap.
+ * However, compressing with currentOffset == 0 is faster still,
+ * so we preserve that case.
*/
if (cctx->currentOffset != 0 && tableType == byU32) {
DEBUGLOG(5, "LZ4_prepareTable: adding 64KB to currentOffset");
@@ -853,7 +912,7 @@
const char* const source,
char* const dest,
const int inputSize,
- int *inputConsumed, /* only written when outputDirective == fillOutput */
+ int* inputConsumed, /* only written when outputDirective == fillOutput */
const int maxOutputSize,
const limitedOutput_directive outputDirective,
const tableType_t tableType,
@@ -885,7 +944,8 @@
/* the dictCtx currentOffset is indexed on the start of the dictionary,
* while a dictionary in the current context precedes the currentOffset */
- const BYTE* dictBase = !dictionary ? NULL : (dictDirective == usingDictCtx) ?
+ const BYTE* dictBase = (dictionary == NULL) ? NULL :
+ (dictDirective == usingDictCtx) ?
dictionary + dictSize - dictCtx->currentOffset :
dictionary + dictSize - startIndex;
@@ -981,10 +1041,11 @@
match = base + matchIndex;
lowLimit = (const BYTE*)source;
}
- } else if (dictDirective==usingExtDict) {
+ } else if (dictDirective == usingExtDict) {
if (matchIndex < startIndex) {
DEBUGLOG(7, "extDict candidate: matchIndex=%5u < startIndex=%5u", matchIndex, startIndex);
assert(startIndex - matchIndex >= MINMATCH);
+ assert(dictBase);
match = dictBase + matchIndex;
lowLimit = dictionary;
} else {
@@ -1048,7 +1109,7 @@
_next_match:
/* at this stage, the following variables must be correctly set :
* - ip : at start of LZ operation
- * - match : at start of previous pattern occurence; can be within current prefix, or within extDict
+ * - match : at start of previous pattern occurrence; can be within current prefix, or within extDict
* - offset : if maybe_ext_memSegment==1 (constant)
* - lowLimit : must be == dictionary to mean "match is within extDict"; must be == source otherwise
* - token and *token : position to write 4-bits for match length; higher 4-bits for literal length supposed already written
@@ -1173,6 +1234,7 @@
}
} else if (dictDirective==usingExtDict) {
if (matchIndex < startIndex) {
+ assert(dictBase);
match = dictBase + matchIndex;
lowLimit = dictionary; /* required for match length counter */
} else {
@@ -1355,7 +1417,7 @@
{
int result;
#if (LZ4_HEAPMODE)
- LZ4_stream_t* ctxPtr = ALLOC(sizeof(LZ4_stream_t)); /* malloc-calloc always properly aligned */
+ LZ4_stream_t* ctxPtr = (LZ4_stream_t*)ALLOC(sizeof(LZ4_stream_t)); /* malloc-calloc always properly aligned */
if (ctxPtr == NULL) return 0;
#else
LZ4_stream_t ctx;
@@ -1420,15 +1482,17 @@
* Streaming functions
********************************/
+#if !defined(LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION)
LZ4_stream_t* LZ4_createStream(void)
{
LZ4_stream_t* const lz4s = (LZ4_stream_t*)ALLOC(sizeof(LZ4_stream_t));
- LZ4_STATIC_ASSERT(LZ4_STREAMSIZE >= sizeof(LZ4_stream_t_internal)); /* A compilation error here means LZ4_STREAMSIZE is not large enough */
+ LZ4_STATIC_ASSERT(sizeof(LZ4_stream_t) >= sizeof(LZ4_stream_t_internal));
DEBUGLOG(4, "LZ4_createStream %p", lz4s);
if (lz4s == NULL) return NULL;
LZ4_initStream(lz4s, sizeof(*lz4s));
return lz4s;
}
+#endif
static size_t LZ4_stream_t_alignment(void)
{
@@ -1462,6 +1526,7 @@
LZ4_prepareTable(&(ctx->internal_donotuse), 0, byU32);
}
+#if !defined(LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION)
int LZ4_freeStream (LZ4_stream_t* LZ4_stream)
{
if (!LZ4_stream) return 0; /* support free on NULL */
@@ -1469,6 +1534,7 @@
FREEMEM(LZ4_stream);
return (0);
}
+#endif
#define HASH_UNIT sizeof(reg_t)
@@ -1514,8 +1580,9 @@
return (int)dict->dictSize;
}
-void LZ4_attach_dictionary(LZ4_stream_t* workingStream, const LZ4_stream_t* dictionaryStream) {
- const LZ4_stream_t_internal* dictCtx = dictionaryStream == NULL ? NULL :
+void LZ4_attach_dictionary(LZ4_stream_t* workingStream, const LZ4_stream_t* dictionaryStream)
+{
+ const LZ4_stream_t_internal* dictCtx = (dictionaryStream == NULL) ? NULL :
&(dictionaryStream->internal_donotuse);
DEBUGLOG(4, "LZ4_attach_dictionary (%p, %p, size %u)",
@@ -1568,36 +1635,40 @@
int acceleration)
{
const tableType_t tableType = byU32;
- LZ4_stream_t_internal* streamPtr = &LZ4_stream->internal_donotuse;
- const BYTE* dictEnd = streamPtr->dictionary + streamPtr->dictSize;
+ LZ4_stream_t_internal* const streamPtr = &LZ4_stream->internal_donotuse;
+ const char* dictEnd = streamPtr->dictSize ? (const char*)streamPtr->dictionary + streamPtr->dictSize : NULL;
- DEBUGLOG(5, "LZ4_compress_fast_continue (inputSize=%i)", inputSize);
+ DEBUGLOG(5, "LZ4_compress_fast_continue (inputSize=%i, dictSize=%u)", inputSize, streamPtr->dictSize);
- LZ4_renormDictT(streamPtr, inputSize); /* avoid index overflow */
+ LZ4_renormDictT(streamPtr, inputSize); /* fix index overflow */
if (acceleration < 1) acceleration = LZ4_ACCELERATION_DEFAULT;
if (acceleration > LZ4_ACCELERATION_MAX) acceleration = LZ4_ACCELERATION_MAX;
/* invalidate tiny dictionaries */
- if ( (streamPtr->dictSize-1 < 4-1) /* intentional underflow */
- && (dictEnd != (const BYTE*)source) ) {
+ if ( (streamPtr->dictSize < 4) /* tiny dictionary : not enough for a hash */
+ && (dictEnd != source) /* prefix mode */
+ && (inputSize > 0) /* tolerance : don't lose history, in case next invocation would use prefix mode */
+ && (streamPtr->dictCtx == NULL) /* usingDictCtx */
+ ) {
DEBUGLOG(5, "LZ4_compress_fast_continue: dictSize(%u) at addr:%p is too small", streamPtr->dictSize, streamPtr->dictionary);
+ /* remove dictionary existence from history, to employ faster prefix mode */
streamPtr->dictSize = 0;
streamPtr->dictionary = (const BYTE*)source;
- dictEnd = (const BYTE*)source;
+ dictEnd = source;
}
/* Check overlapping input/dictionary space */
- { const BYTE* sourceEnd = (const BYTE*) source + inputSize;
- if ((sourceEnd > streamPtr->dictionary) && (sourceEnd < dictEnd)) {
+ { const char* const sourceEnd = source + inputSize;
+ if ((sourceEnd > (const char*)streamPtr->dictionary) && (sourceEnd < dictEnd)) {
streamPtr->dictSize = (U32)(dictEnd - sourceEnd);
if (streamPtr->dictSize > 64 KB) streamPtr->dictSize = 64 KB;
if (streamPtr->dictSize < 4) streamPtr->dictSize = 0;
- streamPtr->dictionary = dictEnd - streamPtr->dictSize;
+ streamPtr->dictionary = (const BYTE*)dictEnd - streamPtr->dictSize;
}
}
/* prefix mode : source data follows dictionary */
- if (dictEnd == (const BYTE*)source) {
+ if (dictEnd == source) {
if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset))
return LZ4_compress_generic(streamPtr, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, withPrefix64k, dictSmall, acceleration);
else
@@ -1623,7 +1694,7 @@
} else {
result = LZ4_compress_generic(streamPtr, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, usingDictCtx, noDictIssue, acceleration);
}
- } else {
+ } else { /* small data <= 4 KB */
if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) {
result = LZ4_compress_generic(streamPtr, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, usingExtDict, dictSmall, acceleration);
} else {
@@ -1661,21 +1732,25 @@
/*! LZ4_saveDict() :
* If previously compressed data block is not guaranteed to remain available at its memory location,
* save it into a safer place (char* safeBuffer).
- * Note : you don't need to call LZ4_loadDict() afterwards,
- * dictionary is immediately usable, you can therefore call LZ4_compress_fast_continue().
- * Return : saved dictionary size in bytes (necessarily <= dictSize), or 0 if error.
+ * Note : no need to call LZ4_loadDict() afterwards, dictionary is immediately usable,
+ * one can therefore call LZ4_compress_fast_continue() right after.
+ * @return : saved dictionary size in bytes (necessarily <= dictSize), or 0 if error.
*/
int LZ4_saveDict (LZ4_stream_t* LZ4_dict, char* safeBuffer, int dictSize)
{
LZ4_stream_t_internal* const dict = &LZ4_dict->internal_donotuse;
- const BYTE* const previousDictEnd = dict->dictionary + dict->dictSize;
+
+ DEBUGLOG(5, "LZ4_saveDict : dictSize=%i, safeBuffer=%p", dictSize, safeBuffer);
if ((U32)dictSize > 64 KB) { dictSize = 64 KB; } /* useless to define a dictionary > 64 KB */
if ((U32)dictSize > dict->dictSize) { dictSize = (int)dict->dictSize; }
if (safeBuffer == NULL) assert(dictSize == 0);
- if (dictSize > 0)
- memmove(safeBuffer, previousDictEnd - dictSize, dictSize);
+ if (dictSize > 0) {
+ const BYTE* const previousDictEnd = dict->dictionary + dict->dictSize;
+ assert(dict->dictionary);
+ LZ4_memmove(safeBuffer, previousDictEnd - dictSize, (size_t)dictSize);
+ }
dict->dictionary = (const BYTE*)safeBuffer;
dict->dictSize = (U32)dictSize;
@@ -1689,39 +1764,163 @@
* Decompression functions
********************************/
-typedef enum { endOnOutputSize = 0, endOnInputSize = 1 } endCondition_directive;
typedef enum { decode_full_block = 0, partial_decode = 1 } earlyEnd_directive;
#undef MIN
#define MIN(a,b) ( (a) < (b) ? (a) : (b) )
+
+/* variant for decompress_unsafe()
+ * does not know end of input
+ * presumes input is well formed
+ * note : will consume at least one byte */
+size_t read_long_length_no_check(const BYTE** pp)
+{
+ size_t b, l = 0;
+ do { b = **pp; (*pp)++; l += b; } while (b==255);
+ DEBUGLOG(6, "read_long_length_no_check: +length=%zu using %zu input bytes", l, l/255 + 1)
+ return l;
+}
+
+/* core decoder variant for LZ4_decompress_fast*()
+ * for legacy support only : these entry points are deprecated.
+ * - Presumes input is correctly formed (no defense vs malformed inputs)
+ * - Does not know input size (presume input buffer is "large enough")
+ * - Decompress a full block (only)
+ * @return : nb of bytes read from input.
+ * Note : this variant is not optimized for speed, just for maintenance.
+ * the goal is to remove support of decompress_fast*() variants by v2.0
+**/
+LZ4_FORCE_INLINE int
+LZ4_decompress_unsafe_generic(
+ const BYTE* const istart,
+ BYTE* const ostart,
+ int decompressedSize,
+
+ size_t prefixSize,
+ const BYTE* const dictStart, /* only if dict==usingExtDict */
+ const size_t dictSize /* note: =0 if dictStart==NULL */
+ )
+{
+ const BYTE* ip = istart;
+ BYTE* op = (BYTE*)ostart;
+ BYTE* const oend = ostart + decompressedSize;
+ const BYTE* const prefixStart = ostart - prefixSize;
+
+ DEBUGLOG(5, "LZ4_decompress_unsafe_generic");
+ if (dictStart == NULL) assert(dictSize == 0);
+
+ while (1) {
+ /* start new sequence */
+ unsigned token = *ip++;
+
+ /* literals */
+ { size_t ll = token >> ML_BITS;
+ if (ll==15) {
+ /* long literal length */
+ ll += read_long_length_no_check(&ip);
+ }
+ if ((size_t)(oend-op) < ll) return -1; /* output buffer overflow */
+ LZ4_memmove(op, ip, ll); /* support in-place decompression */
+ op += ll;
+ ip += ll;
+ if ((size_t)(oend-op) < MFLIMIT) {
+ if (op==oend) break; /* end of block */
+ DEBUGLOG(5, "invalid: literals end at distance %zi from end of block", oend-op);
+ /* incorrect end of block :
+ * last match must start at least MFLIMIT==12 bytes before end of output block */
+ return -1;
+ } }
+
+ /* match */
+ { size_t ml = token & 15;
+ size_t const offset = LZ4_readLE16(ip);
+ ip+=2;
+
+ if (ml==15) {
+ /* long literal length */
+ ml += read_long_length_no_check(&ip);
+ }
+ ml += MINMATCH;
+
+ if ((size_t)(oend-op) < ml) return -1; /* output buffer overflow */
+
+ { const BYTE* match = op - offset;
+
+ /* out of range */
+ if (offset > (size_t)(op - prefixStart) + dictSize) {
+ DEBUGLOG(6, "offset out of range");
+ return -1;
+ }
+
+ /* check special case : extDict */
+ if (offset > (size_t)(op - prefixStart)) {
+ /* extDict scenario */
+ const BYTE* const dictEnd = dictStart + dictSize;
+ const BYTE* extMatch = dictEnd - (offset - (size_t)(op-prefixStart));
+ size_t const extml = (size_t)(dictEnd - extMatch);
+ if (extml > ml) {
+ /* match entirely within extDict */
+ LZ4_memmove(op, extMatch, ml);
+ op += ml;
+ ml = 0;
+ } else {
+ /* match split between extDict & prefix */
+ LZ4_memmove(op, extMatch, extml);
+ op += extml;
+ ml -= extml;
+ }
+ match = prefixStart;
+ }
+
+ /* match copy - slow variant, supporting overlap copy */
+ { size_t u;
+ for (u=0; u<ml; u++) {
+ op[u] = match[u];
+ } } }
+ op += ml;
+ if ((size_t)(oend-op) < LASTLITERALS) {
+ DEBUGLOG(5, "invalid: match ends at distance %zi from end of block", oend-op);
+ /* incorrect end of block :
+ * last match must stop at least LASTLITERALS==5 bytes before end of output block */
+ return -1;
+ }
+ } /* match */
+ } /* main loop */
+ return (int)(ip - istart);
+}
+
+
/* Read the variable-length literal or match length.
*
- * ip - pointer to use as input.
- * lencheck - end ip. Return an error if ip advances >= lencheck.
- * loop_check - check ip >= lencheck in body of loop. Returns loop_error if so.
- * initial_check - check ip >= lencheck before start of loop. Returns initial_error if so.
- * error (output) - error code. Should be set to 0 before call.
- */
-typedef enum { loop_error = -2, initial_error = -1, ok = 0 } variable_length_error;
-LZ4_FORCE_INLINE unsigned
-read_variable_length(const BYTE**ip, const BYTE* lencheck,
- int loop_check, int initial_check,
- variable_length_error* error)
+ * @ip : input pointer
+ * @ilimit : position after which if length is not decoded, the input is necessarily corrupted.
+ * @initial_check - check ip >= ipmax before start of loop. Returns initial_error if so.
+ * @error (output) - error code. Must be set to 0 before call.
+**/
+typedef size_t Rvl_t;
+static const Rvl_t rvl_error = (Rvl_t)(-1);
+LZ4_FORCE_INLINE Rvl_t
+read_variable_length(const BYTE** ip, const BYTE* ilimit,
+ int initial_check)
{
- U32 length = 0;
- U32 s;
- if (initial_check && unlikely((*ip) >= lencheck)) { /* overflow detection */
- *error = initial_error;
- return length;
+ Rvl_t s, length = 0;
+ assert(ip != NULL);
+ assert(*ip != NULL);
+ assert(ilimit != NULL);
+ if (initial_check && unlikely((*ip) >= ilimit)) { /* read limit reached */
+ return rvl_error;
}
do {
s = **ip;
(*ip)++;
length += s;
- if (loop_check && unlikely((*ip) >= lencheck)) { /* overflow detection */
- *error = loop_error;
- return length;
+ if (unlikely((*ip) > ilimit)) { /* read limit reached */
+ return rvl_error;
+ }
+ /* accumulator overflow detection (32-bit mode only) */
+ if ((sizeof(length)<8) && unlikely(length > ((Rvl_t)(-1)/2)) ) {
+ return rvl_error;
}
} while (s==255);
@@ -1741,7 +1940,6 @@
int srcSize,
int outputSize, /* If endOnInput==endOnInputSize, this value is `dstCapacity` */
- endCondition_directive endOnInput, /* endOnOutputSize, endOnInputSize */
earlyEnd_directive partialDecoding, /* full, partial */
dict_directive dict, /* noDict, withPrefix64k, usingExtDict */
const BYTE* const lowPrefix, /* always <= dst, == dst when no prefix */
@@ -1749,7 +1947,7 @@
const size_t dictSize /* note : = 0 if noDict */
)
{
- if (src == NULL) { return -1; }
+ if ((src == NULL) || (outputSize < 0)) { return -1; }
{ const BYTE* ip = (const BYTE*) src;
const BYTE* const iend = ip + srcSize;
@@ -1760,13 +1958,12 @@
const BYTE* const dictEnd = (dictStart == NULL) ? NULL : dictStart + dictSize;
- const int safeDecode = (endOnInput==endOnInputSize);
- const int checkOffset = ((safeDecode) && (dictSize < (int)(64 KB)));
+ const int checkOffset = (dictSize < (int)(64 KB));
/* Set up the "end" pointers for the shortcut. */
- const BYTE* const shortiend = iend - (endOnInput ? 14 : 8) /*maxLL*/ - 2 /*offset*/;
- const BYTE* const shortoend = oend - (endOnInput ? 14 : 8) /*maxLL*/ - 18 /*maxML*/;
+ const BYTE* const shortiend = iend - 14 /*maxLL*/ - 2 /*offset*/;
+ const BYTE* const shortoend = oend - 14 /*maxLL*/ - 18 /*maxML*/;
const BYTE* match;
size_t offset;
@@ -1778,83 +1975,70 @@
/* Special cases */
assert(lowPrefix <= op);
- if ((endOnInput) && (unlikely(outputSize==0))) {
+ if (unlikely(outputSize==0)) {
/* Empty output buffer */
if (partialDecoding) return 0;
return ((srcSize==1) && (*ip==0)) ? 0 : -1;
}
- if ((!endOnInput) && (unlikely(outputSize==0))) { return (*ip==0 ? 1 : -1); }
- if ((endOnInput) && unlikely(srcSize==0)) { return -1; }
+ if (unlikely(srcSize==0)) { return -1; }
- /* Currently the fast loop shows a regression on qualcomm arm chips. */
+ /* LZ4_FAST_DEC_LOOP:
+ * designed for modern OoO performance cpus,
+ * where copying reliably 32-bytes is preferable to an unpredictable branch.
+ * note : fast loop may show a regression for some client arm chips. */
#if LZ4_FAST_DEC_LOOP
if ((oend - op) < FASTLOOP_SAFE_DISTANCE) {
DEBUGLOG(6, "skip fast decode loop");
goto safe_decode;
}
- /* Fast loop : decode sequences as long as output < iend-FASTLOOP_SAFE_DISTANCE */
+ /* Fast loop : decode sequences as long as output < oend-FASTLOOP_SAFE_DISTANCE */
while (1) {
/* Main fastloop assertion: We can always wildcopy FASTLOOP_SAFE_DISTANCE */
assert(oend - op >= FASTLOOP_SAFE_DISTANCE);
- if (endOnInput) { assert(ip < iend); }
+ assert(ip < iend);
token = *ip++;
length = token >> ML_BITS; /* literal length */
- assert(!endOnInput || ip <= iend); /* ip < iend before the increment */
-
/* decode literal length */
if (length == RUN_MASK) {
- variable_length_error error = ok;
- length += read_variable_length(&ip, iend-RUN_MASK, (int)endOnInput, (int)endOnInput, &error);
- if (error == initial_error) { goto _output_error; }
- if ((safeDecode) && unlikely((uptrval)(op)+length<(uptrval)(op))) { goto _output_error; } /* overflow detection */
- if ((safeDecode) && unlikely((uptrval)(ip)+length<(uptrval)(ip))) { goto _output_error; } /* overflow detection */
+ size_t const addl = read_variable_length(&ip, iend-RUN_MASK, 1);
+ if (addl == rvl_error) { goto _output_error; }
+ length += addl;
+ if (unlikely((uptrval)(op)+length<(uptrval)(op))) { goto _output_error; } /* overflow detection */
+ if (unlikely((uptrval)(ip)+length<(uptrval)(ip))) { goto _output_error; } /* overflow detection */
/* copy literals */
cpy = op+length;
LZ4_STATIC_ASSERT(MFLIMIT >= WILDCOPYLENGTH);
- if (endOnInput) { /* LZ4_decompress_safe() */
- if ((cpy>oend-32) || (ip+length>iend-32)) { goto safe_literal_copy; }
- LZ4_wildCopy32(op, ip, cpy);
- } else { /* LZ4_decompress_fast() */
- if (cpy>oend-8) { goto safe_literal_copy; }
- LZ4_wildCopy8(op, ip, cpy); /* LZ4_decompress_fast() cannot copy more than 8 bytes at a time :
- * it doesn't know input length, and only relies on end-of-block properties */
- }
+ if ((cpy>oend-32) || (ip+length>iend-32)) { goto safe_literal_copy; }
+ LZ4_wildCopy32(op, ip, cpy);
ip += length; op = cpy;
} else {
cpy = op+length;
- if (endOnInput) { /* LZ4_decompress_safe() */
- DEBUGLOG(7, "copy %u bytes in a 16-bytes stripe", (unsigned)length);
- /* We don't need to check oend, since we check it once for each loop below */
- if (ip > iend-(16 + 1/*max lit + offset + nextToken*/)) { goto safe_literal_copy; }
- /* Literals can only be 14, but hope compilers optimize if we copy by a register size */
- LZ4_memcpy(op, ip, 16);
- } else { /* LZ4_decompress_fast() */
- /* LZ4_decompress_fast() cannot copy more than 8 bytes at a time :
- * it doesn't know input length, and relies on end-of-block properties */
- LZ4_memcpy(op, ip, 8);
- if (length > 8) { LZ4_memcpy(op+8, ip+8, 8); }
- }
+ DEBUGLOG(7, "copy %u bytes in a 16-bytes stripe", (unsigned)length);
+ /* We don't need to check oend, since we check it once for each loop below */
+ if (ip > iend-(16 + 1/*max lit + offset + nextToken*/)) { goto safe_literal_copy; }
+ /* Literals can only be <= 14, but hope compilers optimize better when copy by a register size */
+ LZ4_memcpy(op, ip, 16);
ip += length; op = cpy;
}
/* get offset */
offset = LZ4_readLE16(ip); ip+=2;
match = op - offset;
- assert(match <= op);
+ assert(match <= op); /* overflow check */
/* get matchlength */
length = token & ML_MASK;
if (length == ML_MASK) {
- variable_length_error error = ok;
- if ((checkOffset) && (unlikely(match + dictSize < lowPrefix))) { goto _output_error; } /* Error : offset outside buffers */
- length += read_variable_length(&ip, iend - LASTLITERALS + 1, (int)endOnInput, 0, &error);
- if (error != ok) { goto _output_error; }
- if ((safeDecode) && unlikely((uptrval)(op)+length<(uptrval)op)) { goto _output_error; } /* overflow detection */
+ size_t const addl = read_variable_length(&ip, iend - LASTLITERALS + 1, 0);
+ if (addl == rvl_error) { goto _output_error; }
+ length += addl;
length += MINMATCH;
+ if (unlikely((uptrval)(op)+length<(uptrval)op)) { goto _output_error; } /* overflow detection */
+ if ((checkOffset) && (unlikely(match + dictSize < lowPrefix))) { goto _output_error; } /* Error : offset outside buffers */
if (op + length >= oend - FASTLOOP_SAFE_DISTANCE) {
goto safe_match_copy;
}
@@ -1864,7 +2048,7 @@
goto safe_match_copy;
}
- /* Fastpath check: Avoids a branch in LZ4_wildCopy32 if true */
+ /* Fastpath check: skip LZ4_wildCopy32 when true */
if ((dict == withPrefix64k) || (match >= lowPrefix)) {
if (offset >= 8) {
assert(match >= lowPrefix);
@@ -1881,6 +2065,7 @@
if (checkOffset && (unlikely(match + dictSize < lowPrefix))) { goto _output_error; } /* Error : offset outside buffers */
/* match starting within external dictionary */
if ((dict==usingExtDict) && (match < lowPrefix)) {
+ assert(dictEnd != NULL);
if (unlikely(op+length > oend-LASTLITERALS)) {
if (partialDecoding) {
DEBUGLOG(7, "partialDecoding: dictionary match, close to dstEnd");
@@ -1891,7 +2076,7 @@
if (length <= (size_t)(lowPrefix-match)) {
/* match fits entirely within external dictionary : just copy */
- memmove(op, dictEnd - (lowPrefix-match), length);
+ LZ4_memmove(op, dictEnd - (lowPrefix-match), length);
op += length;
} else {
/* match stretches into both external dictionary and current block */
@@ -1927,11 +2112,10 @@
/* Main Loop : decode remaining sequences where output < FASTLOOP_SAFE_DISTANCE */
while (1) {
+ assert(ip < iend);
token = *ip++;
length = token >> ML_BITS; /* literal length */
- assert(!endOnInput || ip <= iend); /* ip < iend before the increment */
-
/* A two-stage shortcut for the most common case:
* 1) If the literal length is 0..14, and there is enough space,
* enter the shortcut and copy 16 bytes on behalf of the literals
@@ -1941,11 +2125,11 @@
* those 18 bytes earlier, upon entering the shortcut (in other words,
* there is a combined check for both stages).
*/
- if ( (endOnInput ? length != RUN_MASK : length <= 8)
+ if ( (length != RUN_MASK)
/* strictly "less than" on input, to re-enter the loop with at least one byte */
- && likely((endOnInput ? ip < shortiend : 1) & (op <= shortoend)) ) {
+ && likely((ip < shortiend) & (op <= shortoend)) ) {
/* Copy the literals */
- LZ4_memcpy(op, ip, endOnInput ? 16 : 8);
+ LZ4_memcpy(op, ip, 16);
op += length; ip += length;
/* The second stage: prepare for match copying, decode full info.
@@ -1975,11 +2159,11 @@
/* decode literal length */
if (length == RUN_MASK) {
- variable_length_error error = ok;
- length += read_variable_length(&ip, iend-RUN_MASK, (int)endOnInput, (int)endOnInput, &error);
- if (error == initial_error) { goto _output_error; }
- if ((safeDecode) && unlikely((uptrval)(op)+length<(uptrval)(op))) { goto _output_error; } /* overflow detection */
- if ((safeDecode) && unlikely((uptrval)(ip)+length<(uptrval)(ip))) { goto _output_error; } /* overflow detection */
+ size_t const addl = read_variable_length(&ip, iend-RUN_MASK, 1);
+ if (addl == rvl_error) { goto _output_error; }
+ length += addl;
+ if (unlikely((uptrval)(op)+length<(uptrval)(op))) { goto _output_error; } /* overflow detection */
+ if (unlikely((uptrval)(ip)+length<(uptrval)(ip))) { goto _output_error; } /* overflow detection */
}
/* copy literals */
@@ -1988,9 +2172,7 @@
safe_literal_copy:
#endif
LZ4_STATIC_ASSERT(MFLIMIT >= WILDCOPYLENGTH);
- if ( ((endOnInput) && ((cpy>oend-MFLIMIT) || (ip+length>iend-(2+1+LASTLITERALS))) )
- || ((!endOnInput) && (cpy>oend-WILDCOPYLENGTH)) )
- {
+ if ((cpy>oend-MFLIMIT) || (ip+length>iend-(2+1+LASTLITERALS))) {
/* We've either hit the input parsing restriction or the output parsing restriction.
* In the normal scenario, decoding a full block, it must be the last sequence,
* otherwise it's an error (invalid input or dimensions).
@@ -2000,7 +2182,6 @@
/* Since we are partial decoding we may be in this block because of the output parsing
* restriction, which is not valid since the output buffer is allowed to be undersized.
*/
- assert(endOnInput);
DEBUGLOG(7, "partialDecoding: copying literals, close to input or output end")
DEBUGLOG(7, "partialDecoding: literal length = %u", (unsigned)length);
DEBUGLOG(7, "partialDecoding: remaining space in dstBuffer : %i", (int)(oend - op));
@@ -2021,21 +2202,17 @@
length = (size_t)(oend-op);
}
} else {
- /* We must be on the last sequence because of the parsing limitations so check
- * that we exactly regenerate the original size (must be exact when !endOnInput).
- */
- if ((!endOnInput) && (cpy != oend)) { goto _output_error; }
/* We must be on the last sequence (or invalid) because of the parsing limitations
* so check that we exactly consume the input and don't overrun the output buffer.
*/
- if ((endOnInput) && ((ip+length != iend) || (cpy > oend))) {
+ if ((ip+length != iend) || (cpy > oend)) {
DEBUGLOG(6, "should have been last run of literals")
DEBUGLOG(6, "ip(%p) + length(%i) = %p != iend (%p)", ip, (int)length, ip+length, iend);
DEBUGLOG(6, "or cpy(%p) > oend(%p)", cpy, oend);
goto _output_error;
}
}
- memmove(op, ip, length); /* supports overlapping memory regions; only matters for in-place decompression scenarios */
+ LZ4_memmove(op, ip, length); /* supports overlapping memory regions, for in-place decompression scenarios */
ip += length;
op += length;
/* Necessarily EOF when !partialDecoding.
@@ -2047,7 +2224,7 @@
break;
}
} else {
- LZ4_wildCopy8(op, ip, cpy); /* may overwrite up to WILDCOPYLENGTH beyond cpy */
+ LZ4_wildCopy8(op, ip, cpy); /* can overwrite up to 8 bytes beyond cpy */
ip += length; op = cpy;
}
@@ -2060,10 +2237,10 @@
_copy_match:
if (length == ML_MASK) {
- variable_length_error error = ok;
- length += read_variable_length(&ip, iend - LASTLITERALS + 1, (int)endOnInput, 0, &error);
- if (error != ok) goto _output_error;
- if ((safeDecode) && unlikely((uptrval)(op)+length<(uptrval)op)) goto _output_error; /* overflow detection */
+ size_t const addl = read_variable_length(&ip, iend - LASTLITERALS + 1, 0);
+ if (addl == rvl_error) { goto _output_error; }
+ length += addl;
+ if (unlikely((uptrval)(op)+length<(uptrval)op)) goto _output_error; /* overflow detection */
}
length += MINMATCH;
@@ -2073,6 +2250,7 @@
if ((checkOffset) && (unlikely(match + dictSize < lowPrefix))) goto _output_error; /* Error : offset outside buffers */
/* match starting within external dictionary */
if ((dict==usingExtDict) && (match < lowPrefix)) {
+ assert(dictEnd != NULL);
if (unlikely(op+length > oend-LASTLITERALS)) {
if (partialDecoding) length = MIN(length, (size_t)(oend-op));
else goto _output_error; /* doesn't respect parsing restriction */
@@ -2080,7 +2258,7 @@
if (length <= (size_t)(lowPrefix-match)) {
/* match fits entirely within external dictionary : just copy */
- memmove(op, dictEnd - (lowPrefix-match), length);
+ LZ4_memmove(op, dictEnd - (lowPrefix-match), length);
op += length;
} else {
/* match stretches into both external dictionary and current block */
@@ -2151,12 +2329,8 @@
}
/* end of decoding */
- if (endOnInput) {
- DEBUGLOG(5, "decoded %i bytes", (int) (((char*)op)-dst));
- return (int) (((char*)op)-dst); /* Nb of output bytes decoded */
- } else {
- return (int) (((const char*)ip)-src); /* Nb of input bytes read */
- }
+ DEBUGLOG(5, "decoded %i bytes", (int) (((char*)op)-dst));
+ return (int) (((char*)op)-dst); /* Nb of output bytes decoded */
/* Overflow error detected */
_output_error:
@@ -2171,7 +2345,7 @@
int LZ4_decompress_safe(const char* source, char* dest, int compressedSize, int maxDecompressedSize)
{
return LZ4_decompress_generic(source, dest, compressedSize, maxDecompressedSize,
- endOnInputSize, decode_full_block, noDict,
+ decode_full_block, noDict,
(BYTE*)dest, NULL, 0);
}
@@ -2180,16 +2354,17 @@
{
dstCapacity = MIN(targetOutputSize, dstCapacity);
return LZ4_decompress_generic(src, dst, compressedSize, dstCapacity,
- endOnInputSize, partial_decode,
+ partial_decode,
noDict, (BYTE*)dst, NULL, 0);
}
LZ4_FORCE_O2
int LZ4_decompress_fast(const char* source, char* dest, int originalSize)
{
- return LZ4_decompress_generic(source, dest, 0, originalSize,
- endOnOutputSize, decode_full_block, withPrefix64k,
- (BYTE*)dest - 64 KB, NULL, 0);
+ DEBUGLOG(5, "LZ4_decompress_fast");
+ return LZ4_decompress_unsafe_generic(
+ (const BYTE*)source, (BYTE*)dest, originalSize,
+ 0, NULL, 0);
}
/*===== Instantiate a few more decoding cases, used more than once. =====*/
@@ -2198,16 +2373,25 @@
int LZ4_decompress_safe_withPrefix64k(const char* source, char* dest, int compressedSize, int maxOutputSize)
{
return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize,
- endOnInputSize, decode_full_block, withPrefix64k,
+ decode_full_block, withPrefix64k,
+ (BYTE*)dest - 64 KB, NULL, 0);
+}
+
+LZ4_FORCE_O2
+static int LZ4_decompress_safe_partial_withPrefix64k(const char* source, char* dest, int compressedSize, int targetOutputSize, int dstCapacity)
+{
+ dstCapacity = MIN(targetOutputSize, dstCapacity);
+ return LZ4_decompress_generic(source, dest, compressedSize, dstCapacity,
+ partial_decode, withPrefix64k,
(BYTE*)dest - 64 KB, NULL, 0);
}
/* Another obsolete API function, paired with the previous one. */
int LZ4_decompress_fast_withPrefix64k(const char* source, char* dest, int originalSize)
{
- /* LZ4_decompress_fast doesn't validate match offsets,
- * and thus serves well with any prefixed dictionary. */
- return LZ4_decompress_fast(source, dest, originalSize);
+ return LZ4_decompress_unsafe_generic(
+ (const BYTE*)source, (BYTE*)dest, originalSize,
+ 64 KB, NULL, 0);
}
LZ4_FORCE_O2
@@ -2215,7 +2399,17 @@
size_t prefixSize)
{
return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize,
- endOnInputSize, decode_full_block, noDict,
+ decode_full_block, noDict,
+ (BYTE*)dest-prefixSize, NULL, 0);
+}
+
+LZ4_FORCE_O2
+static int LZ4_decompress_safe_partial_withSmallPrefix(const char* source, char* dest, int compressedSize, int targetOutputSize, int dstCapacity,
+ size_t prefixSize)
+{
+ dstCapacity = MIN(targetOutputSize, dstCapacity);
+ return LZ4_decompress_generic(source, dest, compressedSize, dstCapacity,
+ partial_decode, noDict,
(BYTE*)dest-prefixSize, NULL, 0);
}
@@ -2225,7 +2419,18 @@
const void* dictStart, size_t dictSize)
{
return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize,
- endOnInputSize, decode_full_block, usingExtDict,
+ decode_full_block, usingExtDict,
+ (BYTE*)dest, (const BYTE*)dictStart, dictSize);
+}
+
+LZ4_FORCE_O2
+int LZ4_decompress_safe_partial_forceExtDict(const char* source, char* dest,
+ int compressedSize, int targetOutputSize, int dstCapacity,
+ const void* dictStart, size_t dictSize)
+{
+ dstCapacity = MIN(targetOutputSize, dstCapacity);
+ return LZ4_decompress_generic(source, dest, compressedSize, dstCapacity,
+ partial_decode, usingExtDict,
(BYTE*)dest, (const BYTE*)dictStart, dictSize);
}
@@ -2233,9 +2438,9 @@
static int LZ4_decompress_fast_extDict(const char* source, char* dest, int originalSize,
const void* dictStart, size_t dictSize)
{
- return LZ4_decompress_generic(source, dest, 0, originalSize,
- endOnOutputSize, decode_full_block, usingExtDict,
- (BYTE*)dest, (const BYTE*)dictStart, dictSize);
+ return LZ4_decompress_unsafe_generic(
+ (const BYTE*)source, (BYTE*)dest, originalSize,
+ 0, (const BYTE*)dictStart, dictSize);
}
/* The "double dictionary" mode, for use with e.g. ring buffers: the first part
@@ -2247,26 +2452,17 @@
size_t prefixSize, const void* dictStart, size_t dictSize)
{
return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize,
- endOnInputSize, decode_full_block, usingExtDict,
- (BYTE*)dest-prefixSize, (const BYTE*)dictStart, dictSize);
-}
-
-LZ4_FORCE_INLINE
-int LZ4_decompress_fast_doubleDict(const char* source, char* dest, int originalSize,
- size_t prefixSize, const void* dictStart, size_t dictSize)
-{
- return LZ4_decompress_generic(source, dest, 0, originalSize,
- endOnOutputSize, decode_full_block, usingExtDict,
+ decode_full_block, usingExtDict,
(BYTE*)dest-prefixSize, (const BYTE*)dictStart, dictSize);
}
/*===== streaming decompression functions =====*/
+#if !defined(LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION)
LZ4_streamDecode_t* LZ4_createStreamDecode(void)
{
- LZ4_streamDecode_t* lz4s = (LZ4_streamDecode_t*) ALLOC_AND_ZERO(sizeof(LZ4_streamDecode_t));
- LZ4_STATIC_ASSERT(LZ4_STREAMDECODESIZE >= sizeof(LZ4_streamDecode_t_internal)); /* A compilation error here means LZ4_STREAMDECODESIZE is not large enough */
- return lz4s;
+ LZ4_STATIC_ASSERT(sizeof(LZ4_streamDecode_t) >= sizeof(LZ4_streamDecode_t_internal));
+ return (LZ4_streamDecode_t*) ALLOC_AND_ZERO(sizeof(LZ4_streamDecode_t));
}
int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream)
@@ -2275,6 +2471,7 @@
FREEMEM(LZ4_stream);
return 0;
}
+#endif
/*! LZ4_setStreamDecode() :
* Use this function to instruct where to find the dictionary.
@@ -2285,8 +2482,13 @@
int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dictionary, int dictSize)
{
LZ4_streamDecode_t_internal* lz4sd = &LZ4_streamDecode->internal_donotuse;
- lz4sd->prefixSize = (size_t) dictSize;
- lz4sd->prefixEnd = (const BYTE*) dictionary + dictSize;
+ lz4sd->prefixSize = (size_t)dictSize;
+ if (dictSize) {
+ assert(dictionary != NULL);
+ lz4sd->prefixEnd = (const BYTE*) dictionary + dictSize;
+ } else {
+ lz4sd->prefixEnd = (const BYTE*) dictionary;
+ }
lz4sd->externalDict = NULL;
lz4sd->extDictSize = 0;
return 1;
@@ -2358,29 +2560,35 @@
return result;
}
-LZ4_FORCE_O2
-int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int originalSize)
+LZ4_FORCE_O2 int
+LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode,
+ const char* source, char* dest, int originalSize)
{
- LZ4_streamDecode_t_internal* lz4sd = &LZ4_streamDecode->internal_donotuse;
+ LZ4_streamDecode_t_internal* const lz4sd =
+ (assert(LZ4_streamDecode!=NULL), &LZ4_streamDecode->internal_donotuse);
int result;
+
+ DEBUGLOG(5, "LZ4_decompress_fast_continue (toDecodeSize=%i)", originalSize);
assert(originalSize >= 0);
if (lz4sd->prefixSize == 0) {
+ DEBUGLOG(5, "first invocation : no prefix nor extDict");
assert(lz4sd->extDictSize == 0);
result = LZ4_decompress_fast(source, dest, originalSize);
if (result <= 0) return result;
lz4sd->prefixSize = (size_t)originalSize;
lz4sd->prefixEnd = (BYTE*)dest + originalSize;
} else if (lz4sd->prefixEnd == (BYTE*)dest) {
- if (lz4sd->prefixSize >= 64 KB - 1 || lz4sd->extDictSize == 0)
- result = LZ4_decompress_fast(source, dest, originalSize);
- else
- result = LZ4_decompress_fast_doubleDict(source, dest, originalSize,
- lz4sd->prefixSize, lz4sd->externalDict, lz4sd->extDictSize);
+ DEBUGLOG(5, "continue using existing prefix");
+ result = LZ4_decompress_unsafe_generic(
+ (const BYTE*)source, (BYTE*)dest, originalSize,
+ lz4sd->prefixSize,
+ lz4sd->externalDict, lz4sd->extDictSize);
if (result <= 0) return result;
lz4sd->prefixSize += (size_t)originalSize;
lz4sd->prefixEnd += originalSize;
} else {
+ DEBUGLOG(5, "prefix becomes extDict");
lz4sd->extDictSize = lz4sd->prefixSize;
lz4sd->externalDict = lz4sd->prefixEnd - lz4sd->extDictSize;
result = LZ4_decompress_fast_extDict(source, dest, originalSize,
@@ -2416,10 +2624,27 @@
return LZ4_decompress_safe_forceExtDict(source, dest, compressedSize, maxOutputSize, dictStart, (size_t)dictSize);
}
+int LZ4_decompress_safe_partial_usingDict(const char* source, char* dest, int compressedSize, int targetOutputSize, int dstCapacity, const char* dictStart, int dictSize)
+{
+ if (dictSize==0)
+ return LZ4_decompress_safe_partial(source, dest, compressedSize, targetOutputSize, dstCapacity);
+ if (dictStart+dictSize == dest) {
+ if (dictSize >= 64 KB - 1) {
+ return LZ4_decompress_safe_partial_withPrefix64k(source, dest, compressedSize, targetOutputSize, dstCapacity);
+ }
+ assert(dictSize >= 0);
+ return LZ4_decompress_safe_partial_withSmallPrefix(source, dest, compressedSize, targetOutputSize, dstCapacity, (size_t)dictSize);
+ }
+ assert(dictSize >= 0);
+ return LZ4_decompress_safe_partial_forceExtDict(source, dest, compressedSize, targetOutputSize, dstCapacity, dictStart, (size_t)dictSize);
+}
+
int LZ4_decompress_fast_usingDict(const char* source, char* dest, int originalSize, const char* dictStart, int dictSize)
{
if (dictSize==0 || dictStart+dictSize == dest)
- return LZ4_decompress_fast(source, dest, originalSize);
+ return LZ4_decompress_unsafe_generic(
+ (const BYTE*)source, (BYTE*)dest, originalSize,
+ (size_t)dictSize, NULL, 0);
assert(dictSize >= 0);
return LZ4_decompress_fast_extDict(source, dest, originalSize, dictStart, (size_t)dictSize);
}
@@ -2471,7 +2696,7 @@
/* Obsolete Streaming functions */
-int LZ4_sizeofStreamState(void) { return LZ4_STREAMSIZE; }
+int LZ4_sizeofStreamState(void) { return sizeof(LZ4_stream_t); }
int LZ4_resetStreamState(void* state, char* inputBuffer)
{
@@ -2480,11 +2705,13 @@
return 0;
}
+#if !defined(LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION)
void* LZ4_create (char* inputBuffer)
{
(void)inputBuffer;
return LZ4_createStream();
}
+#endif
char* LZ4_slideInputBuffer (void* state)
{
diff --git a/lib/lz4.h b/lib/lz4.h
index 7ab1e48..491c608 100644
--- a/lib/lz4.h
+++ b/lib/lz4.h
@@ -1,7 +1,7 @@
/*
* LZ4 - Fast LZ compression algorithm
* Header File
- * Copyright (C) 2011-present, Yann Collet.
+ * Copyright (C) 2011-2020, Yann Collet.
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
@@ -97,36 +97,77 @@
# define LZ4LIB_API LZ4LIB_VISIBILITY
#endif
+/*! LZ4_FREESTANDING :
+ * When this macro is set to 1, it enables "freestanding mode" that is
+ * suitable for typical freestanding environment which doesn't support
+ * standard C library.
+ *
+ * - LZ4_FREESTANDING is a compile-time switch.
+ * - It requires the following macros to be defined:
+ * LZ4_memcpy, LZ4_memmove, LZ4_memset.
+ * - It only enables LZ4/HC functions which don't use heap.
+ * All LZ4F_* functions are not supported.
+ * - See tests/freestanding.c to check its basic setup.
+ */
+#if defined(LZ4_FREESTANDING) && (LZ4_FREESTANDING == 1)
+# define LZ4_HEAPMODE 0
+# define LZ4HC_HEAPMODE 0
+# define LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION 1
+# if !defined(LZ4_memcpy)
+# error "LZ4_FREESTANDING requires macro 'LZ4_memcpy'."
+# endif
+# if !defined(LZ4_memset)
+# error "LZ4_FREESTANDING requires macro 'LZ4_memset'."
+# endif
+# if !defined(LZ4_memmove)
+# error "LZ4_FREESTANDING requires macro 'LZ4_memmove'."
+# endif
+#elif ! defined(LZ4_FREESTANDING)
+# define LZ4_FREESTANDING 0
+#endif
+
+
/*------ Version ------*/
#define LZ4_VERSION_MAJOR 1 /* for breaking interface changes */
#define LZ4_VERSION_MINOR 9 /* for new (non-breaking) interface capabilities */
-#define LZ4_VERSION_RELEASE 3 /* for tweaks, bug-fixes, or development */
+#define LZ4_VERSION_RELEASE 4 /* for tweaks, bug-fixes, or development */
#define LZ4_VERSION_NUMBER (LZ4_VERSION_MAJOR *100*100 + LZ4_VERSION_MINOR *100 + LZ4_VERSION_RELEASE)
#define LZ4_LIB_VERSION LZ4_VERSION_MAJOR.LZ4_VERSION_MINOR.LZ4_VERSION_RELEASE
#define LZ4_QUOTE(str) #str
#define LZ4_EXPAND_AND_QUOTE(str) LZ4_QUOTE(str)
-#define LZ4_VERSION_STRING LZ4_EXPAND_AND_QUOTE(LZ4_LIB_VERSION)
+#define LZ4_VERSION_STRING LZ4_EXPAND_AND_QUOTE(LZ4_LIB_VERSION) /* requires v1.7.3+ */
-LZ4LIB_API int LZ4_versionNumber (void); /**< library version number; useful to check dll version */
-LZ4LIB_API const char* LZ4_versionString (void); /**< library version string; useful to check dll version */
+LZ4LIB_API int LZ4_versionNumber (void); /**< library version number; useful to check dll version; requires v1.3.0+ */
+LZ4LIB_API const char* LZ4_versionString (void); /**< library version string; useful to check dll version; requires v1.7.5+ */
/*-************************************
* Tuning parameter
**************************************/
+#define LZ4_MEMORY_USAGE_MIN 10
+#define LZ4_MEMORY_USAGE_DEFAULT 14
+#define LZ4_MEMORY_USAGE_MAX 20
+
/*!
* LZ4_MEMORY_USAGE :
- * Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.)
- * Increasing memory usage improves compression ratio.
- * Reduced memory usage may improve speed, thanks to better cache locality.
+ * Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; )
+ * Increasing memory usage improves compression ratio, at the cost of speed.
+ * Reduced memory usage may improve speed at the cost of ratio, thanks to better cache locality.
* Default value is 14, for 16KB, which nicely fits into Intel x86 L1 cache
*/
#ifndef LZ4_MEMORY_USAGE
-# define LZ4_MEMORY_USAGE 14
+# define LZ4_MEMORY_USAGE LZ4_MEMORY_USAGE_DEFAULT
#endif
+#if (LZ4_MEMORY_USAGE < LZ4_MEMORY_USAGE_MIN)
+# error "LZ4_MEMORY_USAGE is too small !"
+#endif
+
+#if (LZ4_MEMORY_USAGE > LZ4_MEMORY_USAGE_MAX)
+# error "LZ4_MEMORY_USAGE is too large !"
+#endif
/*-************************************
* Simple Functions
@@ -270,8 +311,25 @@
***********************************************/
typedef union LZ4_stream_u LZ4_stream_t; /* incomplete type (defined later) */
+/**
+ Note about RC_INVOKED
+
+ - RC_INVOKED is predefined symbol of rc.exe (the resource compiler which is part of MSVC/Visual Studio).
+ https://docs.microsoft.com/en-us/windows/win32/menurc/predefined-macros
+
+ - Since rc.exe is a legacy compiler, it truncates long symbol (> 30 chars)
+ and reports warning "RC4011: identifier truncated".
+
+ - To eliminate the warning, we surround long preprocessor symbol with
+ "#if !defined(RC_INVOKED) ... #endif" block that means
+ "skip this block when rc.exe is trying to read it".
+*/
+#if !defined(RC_INVOKED) /* https://docs.microsoft.com/en-us/windows/win32/menurc/predefined-macros */
+#if !defined(LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION)
LZ4LIB_API LZ4_stream_t* LZ4_createStream(void);
LZ4LIB_API int LZ4_freeStream (LZ4_stream_t* streamPtr);
+#endif /* !defined(LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION) */
+#endif
/*! LZ4_resetStream_fast() : v1.9.0+
* Use this to prepare an LZ4_stream_t for a new chain of dependent blocks
@@ -355,8 +413,12 @@
* creation / destruction of streaming decompression tracking context.
* A tracking context can be re-used multiple times.
*/
+#if !defined(RC_INVOKED) /* https://docs.microsoft.com/en-us/windows/win32/menurc/predefined-macros */
+#if !defined(LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION)
LZ4LIB_API LZ4_streamDecode_t* LZ4_createStreamDecode(void);
LZ4LIB_API int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream);
+#endif /* !defined(LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION) */
+#endif
/*! LZ4_setStreamDecode() :
* An LZ4_streamDecode_t context can be allocated once and re-used multiple times.
@@ -406,7 +468,10 @@
* save the last 64KB of decoded data into a safe buffer where it can't be modified during decompression,
* then indicate where this data is saved using LZ4_setStreamDecode(), before decompressing next block.
*/
-LZ4LIB_API int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* src, char* dst, int srcSize, int dstCapacity);
+LZ4LIB_API int
+LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode,
+ const char* src, char* dst,
+ int srcSize, int dstCapacity);
/*! LZ4_decompress_*_usingDict() :
@@ -417,7 +482,16 @@
* Performance tip : Decompression speed can be substantially increased
* when dst == dictStart + dictSize.
*/
-LZ4LIB_API int LZ4_decompress_safe_usingDict (const char* src, char* dst, int srcSize, int dstCapcity, const char* dictStart, int dictSize);
+LZ4LIB_API int
+LZ4_decompress_safe_usingDict(const char* src, char* dst,
+ int srcSize, int dstCapacity,
+ const char* dictStart, int dictSize);
+
+LZ4LIB_API int
+LZ4_decompress_safe_partial_usingDict(const char* src, char* dst,
+ int compressedSize,
+ int targetOutputSize, int maxOutputSize,
+ const char* dictStart, int dictSize);
#endif /* LZ4_H_2983827168210 */
@@ -496,13 +570,15 @@
* stream (and source buffer) must remain in-place / accessible / unchanged
* through the completion of the first compression call on the stream.
*/
-LZ4LIB_STATIC_API void LZ4_attach_dictionary(LZ4_stream_t* workingStream, const LZ4_stream_t* dictionaryStream);
+LZ4LIB_STATIC_API void
+LZ4_attach_dictionary(LZ4_stream_t* workingStream,
+ const LZ4_stream_t* dictionaryStream);
/*! In-place compression and decompression
*
* It's possible to have input and output sharing the same buffer,
- * for highly contrained memory environments.
+ * for highly constrained memory environments.
* In both cases, it requires input to lay at the end of the buffer,
* and decompression to start at beginning of the buffer.
* Buffer size must feature some margin, hence be larger than final size.
@@ -592,38 +668,26 @@
typedef unsigned int LZ4_u32;
#endif
+/*! LZ4_stream_t :
+ * Never ever use below internal definitions directly !
+ * These definitions are not API/ABI safe, and may change in future versions.
+ * If you need static allocation, declare or allocate an LZ4_stream_t object.
+**/
+
typedef struct LZ4_stream_t_internal LZ4_stream_t_internal;
struct LZ4_stream_t_internal {
LZ4_u32 hashTable[LZ4_HASH_SIZE_U32];
- LZ4_u32 currentOffset;
- LZ4_u32 tableType;
const LZ4_byte* dictionary;
const LZ4_stream_t_internal* dictCtx;
+ LZ4_u32 currentOffset;
+ LZ4_u32 tableType;
LZ4_u32 dictSize;
+ /* Implicit padding to ensure structure is aligned */
};
-typedef struct {
- const LZ4_byte* externalDict;
- size_t extDictSize;
- const LZ4_byte* prefixEnd;
- size_t prefixSize;
-} LZ4_streamDecode_t_internal;
-
-
-/*! LZ4_stream_t :
- * Do not use below internal definitions directly !
- * Declare or allocate an LZ4_stream_t instead.
- * LZ4_stream_t can also be created using LZ4_createStream(), which is recommended.
- * The structure definition can be convenient for static allocation
- * (on stack, or as part of larger structure).
- * Init this structure with LZ4_initStream() before first use.
- * note : only use this definition in association with static linking !
- * this definition is not API/ABI safe, and may change in future versions.
- */
-#define LZ4_STREAMSIZE 16416 /* static size, for inter-version compatibility */
-#define LZ4_STREAMSIZE_VOIDP (LZ4_STREAMSIZE / sizeof(void*))
+#define LZ4_STREAM_MINSIZE ((1UL << LZ4_MEMORY_USAGE) + 32) /* static size, for inter-version compatibility */
union LZ4_stream_u {
- void* table[LZ4_STREAMSIZE_VOIDP];
+ char minStateSize[LZ4_STREAM_MINSIZE];
LZ4_stream_t_internal internal_donotuse;
}; /* previously typedef'd to LZ4_stream_t */
@@ -641,21 +705,25 @@
* In which case, the function will @return NULL.
* Note2: An LZ4_stream_t structure guarantees correct alignment and size.
* Note3: Before v1.9.0, use LZ4_resetStream() instead
- */
+**/
LZ4LIB_API LZ4_stream_t* LZ4_initStream (void* buffer, size_t size);
/*! LZ4_streamDecode_t :
- * information structure to track an LZ4 stream during decompression.
- * init this structure using LZ4_setStreamDecode() before first use.
- * note : only use in association with static linking !
- * this definition is not API/ABI safe,
- * and may change in a future version !
- */
-#define LZ4_STREAMDECODESIZE_U64 (4 + ((sizeof(void*)==16) ? 2 : 0) /*AS-400*/ )
-#define LZ4_STREAMDECODESIZE (LZ4_STREAMDECODESIZE_U64 * sizeof(unsigned long long))
+ * Never ever use below internal definitions directly !
+ * These definitions are not API/ABI safe, and may change in future versions.
+ * If you need static allocation, declare or allocate an LZ4_streamDecode_t object.
+**/
+typedef struct {
+ const LZ4_byte* externalDict;
+ const LZ4_byte* prefixEnd;
+ size_t extDictSize;
+ size_t prefixSize;
+} LZ4_streamDecode_t_internal;
+
+#define LZ4_STREAMDECODE_MINSIZE 32
union LZ4_streamDecode_u {
- unsigned long long table[LZ4_STREAMDECODESIZE_U64];
+ char minStateSize[LZ4_STREAMDECODE_MINSIZE];
LZ4_streamDecode_t_internal internal_donotuse;
} ; /* previously typedef'd to LZ4_streamDecode_t */
diff --git a/lib/lz4file.c b/lib/lz4file.c
new file mode 100644
index 0000000..eaf9b17
--- /dev/null
+++ b/lib/lz4file.c
@@ -0,0 +1,311 @@
+/*
+ * LZ4 file library
+ * Copyright (C) 2022, Xiaomi Inc.
+ *
+ * BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ * - Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * You can contact the author at :
+ * - LZ4 homepage : http://www.lz4.org
+ * - LZ4 source repository : https://github.com/lz4/lz4
+ */
+#include <stdlib.h>
+#include <string.h>
+#include "lz4.h"
+#include "lz4file.h"
+
+struct LZ4_readFile_s {
+ LZ4F_dctx* dctxPtr;
+ FILE* fp;
+ LZ4_byte* srcBuf;
+ size_t srcBufNext;
+ size_t srcBufSize;
+ size_t srcBufMaxSize;
+};
+
+struct LZ4_writeFile_s {
+ LZ4F_cctx* cctxPtr;
+ FILE* fp;
+ LZ4_byte* dstBuf;
+ size_t maxWriteSize;
+ size_t dstBufMaxSize;
+ LZ4F_errorCode_t errCode;
+};
+
+LZ4F_errorCode_t LZ4F_readOpen(LZ4_readFile_t** lz4fRead, FILE* fp)
+{
+ char buf[LZ4F_HEADER_SIZE_MAX];
+ size_t consumedSize;
+ LZ4F_errorCode_t ret;
+ LZ4F_frameInfo_t info;
+
+ if (fp == NULL || lz4fRead == NULL) {
+ return -LZ4F_ERROR_GENERIC;
+ }
+
+ *lz4fRead = (LZ4_readFile_t*)calloc(1, sizeof(LZ4_readFile_t));
+ if (*lz4fRead == NULL) {
+ return -LZ4F_ERROR_allocation_failed;
+ }
+
+ ret = LZ4F_createDecompressionContext(&(*lz4fRead)->dctxPtr, LZ4F_getVersion());
+ if (LZ4F_isError(ret)) {
+ free(*lz4fRead);
+ return ret;
+ }
+
+ (*lz4fRead)->fp = fp;
+ consumedSize = fread(buf, 1, sizeof(buf), (*lz4fRead)->fp);
+ if (consumedSize != sizeof(buf)) {
+ free(*lz4fRead);
+ return -LZ4F_ERROR_GENERIC;
+ }
+
+ ret = LZ4F_getFrameInfo((*lz4fRead)->dctxPtr, &info, buf, &consumedSize);
+ if (LZ4F_isError(ret)) {
+ LZ4F_freeDecompressionContext((*lz4fRead)->dctxPtr);
+ free(*lz4fRead);
+ return ret;
+ }
+
+ switch (info.blockSizeID) {
+ case LZ4F_default :
+ case LZ4F_max64KB :
+ (*lz4fRead)->srcBufMaxSize = 64 * 1024;
+ break;
+ case LZ4F_max256KB:
+ (*lz4fRead)->srcBufMaxSize = 256 * 1024;
+ break;
+ case LZ4F_max1MB:
+ (*lz4fRead)->srcBufMaxSize = 1 * 1024 * 1024;
+ break;
+ case LZ4F_max4MB:
+ (*lz4fRead)->srcBufMaxSize = 4 * 1024 * 1024;
+ break;
+ default:
+ LZ4F_freeDecompressionContext((*lz4fRead)->dctxPtr);
+ free(*lz4fRead);
+ return -LZ4F_ERROR_maxBlockSize_invalid;
+ }
+
+ (*lz4fRead)->srcBuf = (LZ4_byte*)malloc((*lz4fRead)->srcBufMaxSize);
+ if ((*lz4fRead)->srcBuf == NULL) {
+ LZ4F_freeDecompressionContext((*lz4fRead)->dctxPtr);
+ free(lz4fRead);
+ return -LZ4F_ERROR_allocation_failed;
+ }
+
+ (*lz4fRead)->srcBufSize = sizeof(buf) - consumedSize;
+ memcpy((*lz4fRead)->srcBuf, buf + consumedSize, (*lz4fRead)->srcBufSize);
+
+ return ret;
+}
+
+size_t LZ4F_read(LZ4_readFile_t* lz4fRead, void* buf, size_t size)
+{
+ LZ4_byte* p = (LZ4_byte*)buf;
+ size_t next = 0;
+
+ if (lz4fRead == NULL || buf == NULL)
+ return -LZ4F_ERROR_GENERIC;
+
+ while (next < size) {
+ size_t srcsize = lz4fRead->srcBufSize - lz4fRead->srcBufNext;
+ size_t dstsize = size - next;
+ size_t ret;
+
+ if (srcsize == 0) {
+ ret = fread(lz4fRead->srcBuf, 1, lz4fRead->srcBufMaxSize, lz4fRead->fp);
+ if (ret > 0) {
+ lz4fRead->srcBufSize = ret;
+ srcsize = lz4fRead->srcBufSize;
+ lz4fRead->srcBufNext = 0;
+ }
+ else if (ret == 0) {
+ break;
+ }
+ else {
+ return -LZ4F_ERROR_GENERIC;
+ }
+ }
+
+ ret = LZ4F_decompress(lz4fRead->dctxPtr,
+ p, &dstsize,
+ lz4fRead->srcBuf + lz4fRead->srcBufNext,
+ &srcsize,
+ NULL);
+ if (LZ4F_isError(ret)) {
+ return ret;
+ }
+
+ lz4fRead->srcBufNext += srcsize;
+ next += dstsize;
+ p += dstsize;
+ }
+
+ return next;
+}
+
+LZ4F_errorCode_t LZ4F_readClose(LZ4_readFile_t* lz4fRead)
+{
+ if (lz4fRead == NULL)
+ return -LZ4F_ERROR_GENERIC;
+ LZ4F_freeDecompressionContext(lz4fRead->dctxPtr);
+ free(lz4fRead->srcBuf);
+ free(lz4fRead);
+ return LZ4F_OK_NoError;
+}
+
+LZ4F_errorCode_t LZ4F_writeOpen(LZ4_writeFile_t** lz4fWrite, FILE* fp, const LZ4F_preferences_t* prefsPtr)
+{
+ LZ4_byte buf[LZ4F_HEADER_SIZE_MAX];
+ size_t ret;
+
+ if (fp == NULL || lz4fWrite == NULL)
+ return -LZ4F_ERROR_GENERIC;
+
+ *lz4fWrite = (LZ4_writeFile_t*)malloc(sizeof(LZ4_writeFile_t));
+ if (*lz4fWrite == NULL) {
+ return -LZ4F_ERROR_allocation_failed;
+ }
+ if (prefsPtr != NULL) {
+ switch (prefsPtr->frameInfo.blockSizeID) {
+ case LZ4F_default :
+ case LZ4F_max64KB :
+ (*lz4fWrite)->maxWriteSize = 64 * 1024;
+ break;
+ case LZ4F_max256KB:
+ (*lz4fWrite)->maxWriteSize = 256 * 1024;
+ break;
+ case LZ4F_max1MB:
+ (*lz4fWrite)->maxWriteSize = 1 * 1024 * 1024;
+ break;
+ case LZ4F_max4MB:
+ (*lz4fWrite)->maxWriteSize = 4 * 1024 * 1024;
+ break;
+ default:
+ free(lz4fWrite);
+ return -LZ4F_ERROR_maxBlockSize_invalid;
+ }
+ } else {
+ (*lz4fWrite)->maxWriteSize = 64 * 1024;
+ }
+
+ (*lz4fWrite)->dstBufMaxSize = LZ4F_compressBound((*lz4fWrite)->maxWriteSize, prefsPtr);
+ (*lz4fWrite)->dstBuf = (LZ4_byte*)malloc((*lz4fWrite)->dstBufMaxSize);
+ if ((*lz4fWrite)->dstBuf == NULL) {
+ free(*lz4fWrite);
+ return -LZ4F_ERROR_allocation_failed;
+ }
+
+ ret = LZ4F_createCompressionContext(&(*lz4fWrite)->cctxPtr, LZ4F_getVersion());
+ if (LZ4F_isError(ret)) {
+ free((*lz4fWrite)->dstBuf);
+ free(*lz4fWrite);
+ return ret;
+ }
+
+ ret = LZ4F_compressBegin((*lz4fWrite)->cctxPtr, buf, LZ4F_HEADER_SIZE_MAX, prefsPtr);
+ if (LZ4F_isError(ret)) {
+ LZ4F_freeCompressionContext((*lz4fWrite)->cctxPtr);
+ free((*lz4fWrite)->dstBuf);
+ free(*lz4fWrite);
+ return ret;
+ }
+
+ if (ret != fwrite(buf, 1, ret, fp)) {
+ LZ4F_freeCompressionContext((*lz4fWrite)->cctxPtr);
+ free((*lz4fWrite)->dstBuf);
+ free(*lz4fWrite);
+ return -LZ4F_ERROR_GENERIC;
+ }
+
+ (*lz4fWrite)->fp = fp;
+ (*lz4fWrite)->errCode = LZ4F_OK_NoError;
+ return LZ4F_OK_NoError;
+}
+
+size_t LZ4F_write(LZ4_writeFile_t* lz4fWrite, void* buf, size_t size)
+{
+ LZ4_byte* p = (LZ4_byte*)buf;
+ size_t remain = size;
+ size_t chunk;
+ size_t ret;
+
+ if (lz4fWrite == NULL || buf == NULL)
+ return -LZ4F_ERROR_GENERIC;
+ while (remain) {
+ if (remain > lz4fWrite->maxWriteSize)
+ chunk = lz4fWrite->maxWriteSize;
+ else
+ chunk = remain;
+
+ ret = LZ4F_compressUpdate(lz4fWrite->cctxPtr,
+ lz4fWrite->dstBuf, lz4fWrite->dstBufMaxSize,
+ p, chunk,
+ NULL);
+ if (LZ4F_isError(ret)) {
+ lz4fWrite->errCode = ret;
+ return ret;
+ }
+
+ if(ret != fwrite(lz4fWrite->dstBuf, 1, ret, lz4fWrite->fp)) {
+ lz4fWrite->errCode = -LZ4F_ERROR_GENERIC;
+ return -LZ4F_ERROR_GENERIC;
+ }
+
+ p += chunk;
+ remain -= chunk;
+ }
+
+ return size;
+}
+
+LZ4F_errorCode_t LZ4F_writeClose(LZ4_writeFile_t* lz4fWrite)
+{
+ LZ4F_errorCode_t ret = LZ4F_OK_NoError;
+
+ if (lz4fWrite == NULL)
+ return -LZ4F_ERROR_GENERIC;
+
+ if (lz4fWrite->errCode == LZ4F_OK_NoError) {
+ ret = LZ4F_compressEnd(lz4fWrite->cctxPtr,
+ lz4fWrite->dstBuf, lz4fWrite->dstBufMaxSize,
+ NULL);
+ if (LZ4F_isError(ret)) {
+ goto out;
+ }
+
+ if (ret != fwrite(lz4fWrite->dstBuf, 1, ret, lz4fWrite->fp)) {
+ ret = -LZ4F_ERROR_GENERIC;
+ }
+ }
+
+out:
+ LZ4F_freeCompressionContext(lz4fWrite->cctxPtr);
+ free(lz4fWrite->dstBuf);
+ free(lz4fWrite);
+ return ret;
+}
diff --git a/lib/lz4file.h b/lib/lz4file.h
new file mode 100644
index 0000000..5527130
--- /dev/null
+++ b/lib/lz4file.h
@@ -0,0 +1,93 @@
+/*
+ LZ4 file library
+ Header File
+ Copyright (C) 2022, Xiaomi Inc.
+ BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions are
+ met:
+
+ * Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ * Redistributions in binary form must reproduce the above
+ copyright notice, this list of conditions and the following disclaimer
+ in the documentation and/or other materials provided with the
+ distribution.
+
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+ You can contact the author at :
+ - LZ4 source repository : https://github.com/lz4/lz4
+ - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c
+*/
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+#ifndef LZ4FILE_H
+#define LZ4FILE_H
+
+#include <stdio.h>
+#include "lz4frame_static.h"
+
+typedef struct LZ4_readFile_s LZ4_readFile_t;
+typedef struct LZ4_writeFile_s LZ4_writeFile_t;
+
+/*! LZ4F_readOpen() :
+ * Set read lz4file handle.
+ * `lz4f` will set a lz4file handle.
+ * `fp` must be the return value of the lz4 file opened by fopen.
+ */
+LZ4FLIB_STATIC_API LZ4F_errorCode_t LZ4F_readOpen(LZ4_readFile_t** lz4fRead, FILE* fp);
+
+/*! LZ4F_read() :
+ * Read lz4file content to buffer.
+ * `lz4f` must use LZ4_readOpen to set first.
+ * `buf` read data buffer.
+ * `size` read data buffer size.
+ */
+LZ4FLIB_STATIC_API size_t LZ4F_read(LZ4_readFile_t* lz4fRead, void* buf, size_t size);
+
+/*! LZ4F_readClose() :
+ * Close lz4file handle.
+ * `lz4f` must use LZ4_readOpen to set first.
+ */
+LZ4FLIB_STATIC_API LZ4F_errorCode_t LZ4F_readClose(LZ4_readFile_t* lz4fRead);
+
+/*! LZ4F_writeOpen() :
+ * Set write lz4file handle.
+ * `lz4f` will set a lz4file handle.
+ * `fp` must be the return value of the lz4 file opened by fopen.
+ */
+LZ4FLIB_STATIC_API LZ4F_errorCode_t LZ4F_writeOpen(LZ4_writeFile_t** lz4fWrite, FILE* fp, const LZ4F_preferences_t* prefsPtr);
+
+/*! LZ4F_write() :
+ * Write buffer to lz4file.
+ * `lz4f` must use LZ4F_writeOpen to set first.
+ * `buf` write data buffer.
+ * `size` write data buffer size.
+ */
+LZ4FLIB_STATIC_API size_t LZ4F_write(LZ4_writeFile_t* lz4fWrite, void* buf, size_t size);
+
+/*! LZ4F_writeClose() :
+ * Close lz4file handle.
+ * `lz4f` must use LZ4F_writeOpen to set first.
+ */
+LZ4FLIB_STATIC_API LZ4F_errorCode_t LZ4F_writeClose(LZ4_writeFile_t* lz4fWrite);
+
+#endif /* LZ4FILE_H */
+
+#if defined (__cplusplus)
+}
+#endif
diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index ec02c92..174f9ae 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -45,7 +45,7 @@
* Compiler Options
**************************************/
#ifdef _MSC_VER /* Visual Studio */
-# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
+# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
#endif
@@ -63,28 +63,6 @@
/*-************************************
-* Memory routines
-**************************************/
-/*
- * User may redirect invocations of
- * malloc(), calloc() and free()
- * towards another library or solution of their choice
- * by modifying below section.
- */
-#ifndef LZ4_SRC_INCLUDED /* avoid redefinition when sources are coalesced */
-# include <stdlib.h> /* malloc, calloc, free */
-# define ALLOC(s) malloc(s)
-# define ALLOC_AND_ZERO(s) calloc(1,(s))
-# define FREEMEM(p) free(p)
-#endif
-
-#include <string.h> /* memset, memcpy, memmove */
-#ifndef LZ4_SRC_INCLUDED /* avoid redefinition when sources are coalesced */
-# define MEM_INIT(p,v,s) memset((p),(v),(s))
-#endif
-
-
-/*-************************************
* Library declarations
**************************************/
#define LZ4F_STATIC_LINKING_ONLY
@@ -98,6 +76,66 @@
/*-************************************
+* Memory routines
+**************************************/
+/*
+ * User may redirect invocations of
+ * malloc(), calloc() and free()
+ * towards another library or solution of their choice
+ * by modifying below section.
+**/
+
+#include <string.h> /* memset, memcpy, memmove */
+#ifndef LZ4_SRC_INCLUDED /* avoid redefinition when sources are coalesced */
+# define MEM_INIT(p,v,s) memset((p),(v),(s))
+#endif
+
+#ifndef LZ4_SRC_INCLUDED /* avoid redefinition when sources are coalesced */
+# include <stdlib.h> /* malloc, calloc, free */
+# define ALLOC(s) malloc(s)
+# define ALLOC_AND_ZERO(s) calloc(1,(s))
+# define FREEMEM(p) free(p)
+#endif
+
+static void* LZ4F_calloc(size_t s, LZ4F_CustomMem cmem)
+{
+ /* custom calloc defined : use it */
+ if (cmem.customCalloc != NULL) {
+ return cmem.customCalloc(cmem.opaqueState, s);
+ }
+ /* nothing defined : use default <stdlib.h>'s calloc() */
+ if (cmem.customAlloc == NULL) {
+ return ALLOC_AND_ZERO(s);
+ }
+ /* only custom alloc defined : use it, and combine it with memset() */
+ { void* const p = cmem.customAlloc(cmem.opaqueState, s);
+ if (p != NULL) MEM_INIT(p, 0, s);
+ return p;
+} }
+
+static void* LZ4F_malloc(size_t s, LZ4F_CustomMem cmem)
+{
+ /* custom malloc defined : use it */
+ if (cmem.customAlloc != NULL) {
+ return cmem.customAlloc(cmem.opaqueState, s);
+ }
+ /* nothing defined : use default <stdlib.h>'s malloc() */
+ return ALLOC(s);
+}
+
+static void LZ4F_free(void* p, LZ4F_CustomMem cmem)
+{
+ /* custom malloc defined : use it */
+ if (cmem.customFree != NULL) {
+ cmem.customFree(cmem.opaqueState, p);
+ return;
+ }
+ /* nothing defined : use default <stdlib.h>'s free() */
+ FREEMEM(p);
+}
+
+
+/*-************************************
* Debug
**************************************/
#if defined(LZ4_DEBUG) && (LZ4_DEBUG>=1)
@@ -143,7 +181,7 @@
#endif
-/* unoptimized version; solves endianess & alignment issues */
+/* unoptimized version; solves endianness & alignment issues */
static U32 LZ4F_readLE32 (const void* src)
{
const BYTE* const srcPtr = (const BYTE*)src;
@@ -206,8 +244,6 @@
#define _4BITS 0x0F
#define _8BITS 0xFF
-#define LZ4F_MAGIC_SKIPPABLE_START 0x184D2A50U
-#define LZ4F_MAGICNUMBER 0x184D2204U
#define LZ4F_BLOCKUNCOMPRESSED_FLAG 0x80000000U
#define LZ4F_BLOCKSIZEID_DEFAULT LZ4F_max64KB
@@ -220,22 +256,27 @@
/*-************************************
* Structures and local types
**************************************/
+
+typedef enum { LZ4B_COMPRESSED, LZ4B_UNCOMPRESSED} LZ4F_blockCompression_t;
+
typedef struct LZ4F_cctx_s
{
+ LZ4F_CustomMem cmem;
LZ4F_preferences_t prefs;
U32 version;
U32 cStage;
const LZ4F_CDict* cdict;
size_t maxBlockSize;
size_t maxBufferSize;
- BYTE* tmpBuff;
- BYTE* tmpIn;
- size_t tmpInSize;
+ BYTE* tmpBuff; /* internal buffer, for streaming */
+ BYTE* tmpIn; /* starting position of data compress within internal buffer (>= tmpBuff) */
+ size_t tmpInSize; /* amount of data to compress after tmpIn */
U64 totalInSize;
XXH32_state_t xxh;
void* lz4CtxPtr;
U16 lz4CtxAlloc; /* sized for: 0 = none, 1 = lz4 ctx, 2 = lz4hc ctx */
U16 lz4CtxState; /* in use as: 0 = none, 1 = lz4 ctx, 2 = lz4hc ctx */
+ LZ4F_blockCompression_t blockCompression;
} LZ4F_cctx_t;
@@ -264,27 +305,33 @@
return (LZ4F_errorCodes)(-(ptrdiff_t)functionResult);
}
-static LZ4F_errorCode_t err0r(LZ4F_errorCodes code)
+static LZ4F_errorCode_t LZ4F_returnErrorCode(LZ4F_errorCodes code)
{
/* A compilation error here means sizeof(ptrdiff_t) is not large enough */
LZ4F_STATIC_ASSERT(sizeof(ptrdiff_t) >= sizeof(size_t));
return (LZ4F_errorCode_t)-(ptrdiff_t)code;
}
+#define RETURN_ERROR(e) return LZ4F_returnErrorCode(LZ4F_ERROR_ ## e)
+
+#define RETURN_ERROR_IF(c,e) if (c) RETURN_ERROR(e)
+
+#define FORWARD_IF_ERROR(r) if (LZ4F_isError(r)) return (r)
+
unsigned LZ4F_getVersion(void) { return LZ4F_VERSION; }
int LZ4F_compressionLevel_max(void) { return LZ4HC_CLEVEL_MAX; }
-size_t LZ4F_getBlockSize(unsigned blockSizeID)
+size_t LZ4F_getBlockSize(LZ4F_blockSizeID_t blockSizeID)
{
static const size_t blockSizes[4] = { 64 KB, 256 KB, 1 MB, 4 MB };
if (blockSizeID == 0) blockSizeID = LZ4F_BLOCKSIZEID_DEFAULT;
if (blockSizeID < LZ4F_max64KB || blockSizeID > LZ4F_max4MB)
- return err0r(LZ4F_ERROR_maxBlockSize_invalid);
- blockSizeID -= LZ4F_max64KB;
- return blockSizes[blockSizeID];
-}
+ RETURN_ERROR(maxBlockSize_invalid);
+ { int const blockSizeIdx = (int)blockSizeID - (int)LZ4F_max64KB;
+ return blockSizes[blockSizeIdx];
+} }
/*-************************************
* Private functions
@@ -397,21 +444,20 @@
MEM_INIT(&options, 0, sizeof(options));
options.stableSrc = 1;
- if (dstCapacity < LZ4F_compressFrameBound(srcSize, &prefs)) /* condition to guarantee success */
- return err0r(LZ4F_ERROR_dstMaxSize_tooSmall);
+ RETURN_ERROR_IF(dstCapacity < LZ4F_compressFrameBound(srcSize, &prefs), dstMaxSize_tooSmall);
{ size_t const headerSize = LZ4F_compressBegin_usingCDict(cctx, dstBuffer, dstCapacity, cdict, &prefs); /* write header */
- if (LZ4F_isError(headerSize)) return headerSize;
+ FORWARD_IF_ERROR(headerSize);
dstPtr += headerSize; /* header size */ }
assert(dstEnd >= dstPtr);
{ size_t const cSize = LZ4F_compressUpdate(cctx, dstPtr, (size_t)(dstEnd-dstPtr), srcBuffer, srcSize, &options);
- if (LZ4F_isError(cSize)) return cSize;
+ FORWARD_IF_ERROR(cSize);
dstPtr += cSize; }
assert(dstEnd >= dstPtr);
{ size_t const tailSize = LZ4F_compressEnd(cctx, dstPtr, (size_t)(dstEnd-dstPtr), &options); /* flush last block, and generate suffix */
- if (LZ4F_isError(tailSize)) return tailSize;
+ FORWARD_IF_ERROR(tailSize);
dstPtr += tailSize; }
assert(dstEnd >= dstStart);
@@ -432,27 +478,26 @@
{
size_t result;
#if (LZ4F_HEAPMODE)
- LZ4F_cctx_t *cctxPtr;
+ LZ4F_cctx_t* cctxPtr;
result = LZ4F_createCompressionContext(&cctxPtr, LZ4F_VERSION);
- if (LZ4F_isError(result)) return result;
+ FORWARD_IF_ERROR(result);
#else
LZ4F_cctx_t cctx;
LZ4_stream_t lz4ctx;
- LZ4F_cctx_t *cctxPtr = &cctx;
+ LZ4F_cctx_t* const cctxPtr = &cctx;
- DEBUGLOG(4, "LZ4F_compressFrame");
MEM_INIT(&cctx, 0, sizeof(cctx));
cctx.version = LZ4F_VERSION;
cctx.maxBufferSize = 5 MB; /* mess with real buffer size to prevent dynamic allocation; works only because autoflush==1 & stableSrc==1 */
- if (preferencesPtr == NULL ||
- preferencesPtr->compressionLevel < LZ4HC_CLEVEL_MIN)
- {
+ if ( preferencesPtr == NULL
+ || preferencesPtr->compressionLevel < LZ4HC_CLEVEL_MIN ) {
LZ4_initStream(&lz4ctx, sizeof(lz4ctx));
cctxPtr->lz4CtxPtr = &lz4ctx;
cctxPtr->lz4CtxAlloc = 1;
cctxPtr->lz4CtxState = 1;
}
#endif
+ DEBUGLOG(4, "LZ4F_compressFrame");
result = LZ4F_compressFrame_usingCDict(cctxPtr, dstBuffer, dstCapacity,
srcBuffer, srcSize,
@@ -461,10 +506,9 @@
#if (LZ4F_HEAPMODE)
LZ4F_freeCompressionContext(cctxPtr);
#else
- if (preferencesPtr != NULL &&
- preferencesPtr->compressionLevel >= LZ4HC_CLEVEL_MIN)
- {
- FREEMEM(cctxPtr->lz4CtxPtr);
+ if ( preferencesPtr != NULL
+ && preferencesPtr->compressionLevel >= LZ4HC_CLEVEL_MIN ) {
+ LZ4F_free(cctxPtr->lz4CtxPtr, cctxPtr->cmem);
}
#endif
return result;
@@ -476,30 +520,31 @@
*****************************************************/
struct LZ4F_CDict_s {
+ LZ4F_CustomMem cmem;
void* dictContent;
LZ4_stream_t* fastCtx;
LZ4_streamHC_t* HCCtx;
}; /* typedef'd to LZ4F_CDict within lz4frame_static.h */
-/*! LZ4F_createCDict() :
- * When compressing multiple messages / blocks with the same dictionary, it's recommended to load it just once.
- * LZ4F_createCDict() will create a digested dictionary, ready to start future compression operations without startup delay.
- * LZ4F_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only.
- * `dictBuffer` can be released after LZ4F_CDict creation, since its content is copied within CDict
- * @return : digested dictionary for compression, or NULL if failed */
-LZ4F_CDict* LZ4F_createCDict(const void* dictBuffer, size_t dictSize)
+LZ4F_CDict*
+LZ4F_createCDict_advanced(LZ4F_CustomMem cmem, const void* dictBuffer, size_t dictSize)
{
const char* dictStart = (const char*)dictBuffer;
- LZ4F_CDict* cdict = (LZ4F_CDict*) ALLOC(sizeof(*cdict));
- DEBUGLOG(4, "LZ4F_createCDict");
+ LZ4F_CDict* const cdict = (LZ4F_CDict*)LZ4F_malloc(sizeof(*cdict), cmem);
+ DEBUGLOG(4, "LZ4F_createCDict_advanced");
if (!cdict) return NULL;
+ cdict->cmem = cmem;
if (dictSize > 64 KB) {
dictStart += dictSize - 64 KB;
dictSize = 64 KB;
}
- cdict->dictContent = ALLOC(dictSize);
- cdict->fastCtx = LZ4_createStream();
- cdict->HCCtx = LZ4_createStreamHC();
+ cdict->dictContent = LZ4F_malloc(dictSize, cmem);
+ cdict->fastCtx = (LZ4_stream_t*)LZ4F_malloc(sizeof(LZ4_stream_t), cmem);
+ if (cdict->fastCtx)
+ LZ4_initStream(cdict->fastCtx, sizeof(LZ4_stream_t));
+ cdict->HCCtx = (LZ4_streamHC_t*)LZ4F_malloc(sizeof(LZ4_streamHC_t), cmem);
+ if (cdict->HCCtx)
+ LZ4_initStream(cdict->HCCtx, sizeof(LZ4_streamHC_t));
if (!cdict->dictContent || !cdict->fastCtx || !cdict->HCCtx) {
LZ4F_freeCDict(cdict);
return NULL;
@@ -511,13 +556,25 @@
return cdict;
}
+/*! LZ4F_createCDict() :
+ * When compressing multiple messages / blocks with the same dictionary, it's recommended to load it just once.
+ * LZ4F_createCDict() will create a digested dictionary, ready to start future compression operations without startup delay.
+ * LZ4F_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only.
+ * @dictBuffer can be released after LZ4F_CDict creation, since its content is copied within CDict
+ * @return : digested dictionary for compression, or NULL if failed */
+LZ4F_CDict* LZ4F_createCDict(const void* dictBuffer, size_t dictSize)
+{
+ DEBUGLOG(4, "LZ4F_createCDict");
+ return LZ4F_createCDict_advanced(LZ4F_defaultCMem, dictBuffer, dictSize);
+}
+
void LZ4F_freeCDict(LZ4F_CDict* cdict)
{
if (cdict==NULL) return; /* support free on NULL */
- FREEMEM(cdict->dictContent);
- LZ4_freeStream(cdict->fastCtx);
- LZ4_freeStreamHC(cdict->HCCtx);
- FREEMEM(cdict);
+ LZ4F_free(cdict->dictContent, cdict->cmem);
+ LZ4F_free(cdict->fastCtx, cdict->cmem);
+ LZ4F_free(cdict->HCCtx, cdict->cmem);
+ LZ4F_free(cdict, cdict->cmem);
}
@@ -525,6 +582,20 @@
* Advanced compression functions
***********************************/
+LZ4F_cctx*
+LZ4F_createCompressionContext_advanced(LZ4F_CustomMem customMem, unsigned version)
+{
+ LZ4F_cctx* const cctxPtr =
+ (LZ4F_cctx*)LZ4F_calloc(sizeof(LZ4F_cctx), customMem);
+ if (cctxPtr==NULL) return NULL;
+
+ cctxPtr->cmem = customMem;
+ cctxPtr->version = version;
+ cctxPtr->cStage = 0; /* Uninitialized. Next stage : init cctx */
+
+ return cctxPtr;
+}
+
/*! LZ4F_createCompressionContext() :
* The first thing to do is to create a compressionContext object, which will be used in all compression operations.
* This is achieved using LZ4F_createCompressionContext(), which takes as argument a version and an LZ4F_preferences_t structure.
@@ -532,17 +603,16 @@
* The function will provide a pointer to an allocated LZ4F_compressionContext_t object.
* If the result LZ4F_errorCode_t is not OK_NoError, there was an error during context creation.
* Object can release its memory using LZ4F_freeCompressionContext();
- */
-LZ4F_errorCode_t LZ4F_createCompressionContext(LZ4F_cctx** LZ4F_compressionContextPtr, unsigned version)
+**/
+LZ4F_errorCode_t
+LZ4F_createCompressionContext(LZ4F_cctx** LZ4F_compressionContextPtr, unsigned version)
{
- LZ4F_cctx_t* const cctxPtr = (LZ4F_cctx_t*)ALLOC_AND_ZERO(sizeof(LZ4F_cctx_t));
- if (cctxPtr==NULL) return err0r(LZ4F_ERROR_allocation_failed);
+ assert(LZ4F_compressionContextPtr != NULL); /* considered a violation of narrow contract */
+ /* in case it nonetheless happen in production */
+ RETURN_ERROR_IF(LZ4F_compressionContextPtr == NULL, parameter_null);
- cctxPtr->version = version;
- cctxPtr->cStage = 0; /* Next stage : init stream */
-
- *LZ4F_compressionContextPtr = cctxPtr;
-
+ *LZ4F_compressionContextPtr = LZ4F_createCompressionContext_advanced(LZ4F_defaultCMem, version);
+ RETURN_ERROR_IF(*LZ4F_compressionContextPtr==NULL, allocation_failed);
return LZ4F_OK_NoError;
}
@@ -550,11 +620,10 @@
LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctxPtr)
{
if (cctxPtr != NULL) { /* support free on NULL */
- FREEMEM(cctxPtr->lz4CtxPtr); /* note: LZ4_streamHC_t and LZ4_stream_t are simple POD types */
- FREEMEM(cctxPtr->tmpBuff);
- FREEMEM(cctxPtr);
+ LZ4F_free(cctxPtr->lz4CtxPtr, cctxPtr->cmem); /* note: LZ4_streamHC_t and LZ4_stream_t are simple POD types */
+ LZ4F_free(cctxPtr->tmpBuff, cctxPtr->cmem);
+ LZ4F_free(cctxPtr, cctxPtr->cmem);
}
-
return LZ4F_OK_NoError;
}
@@ -588,11 +657,21 @@
}
}
+static int ctxTypeID_to_size(int ctxTypeID) {
+ switch(ctxTypeID) {
+ case 1:
+ return LZ4_sizeofState();
+ case 2:
+ return LZ4_sizeofStateHC();
+ default:
+ return 0;
+ }
+}
/*! LZ4F_compressBegin_usingCDict() :
- * init streaming compression and writes frame header into dstBuffer.
- * dstBuffer must be >= LZ4F_HEADER_SIZE_MAX bytes.
- * @return : number of bytes written into dstBuffer for the header
+ * init streaming compression AND writes frame header into @dstBuffer.
+ * @dstCapacity must be >= LZ4F_HEADER_SIZE_MAX bytes.
+ * @return : number of bytes written into @dstBuffer for the header
* or an error code (can be tested using LZ4F_isError())
*/
size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr,
@@ -600,41 +679,46 @@
const LZ4F_CDict* cdict,
const LZ4F_preferences_t* preferencesPtr)
{
- LZ4F_preferences_t prefNull;
+ LZ4F_preferences_t const prefNull = LZ4F_INIT_PREFERENCES;
BYTE* const dstStart = (BYTE*)dstBuffer;
BYTE* dstPtr = dstStart;
- BYTE* headerStart;
- if (dstCapacity < maxFHSize) return err0r(LZ4F_ERROR_dstMaxSize_tooSmall);
- MEM_INIT(&prefNull, 0, sizeof(prefNull));
+ RETURN_ERROR_IF(dstCapacity < maxFHSize, dstMaxSize_tooSmall);
if (preferencesPtr == NULL) preferencesPtr = &prefNull;
cctxPtr->prefs = *preferencesPtr;
- /* Ctx Management */
+ /* cctx Management */
{ U16 const ctxTypeID = (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) ? 1 : 2;
- if (cctxPtr->lz4CtxAlloc < ctxTypeID) {
- FREEMEM(cctxPtr->lz4CtxPtr);
+ int requiredSize = ctxTypeID_to_size(ctxTypeID);
+ int allocatedSize = ctxTypeID_to_size(cctxPtr->lz4CtxAlloc);
+ if (allocatedSize < requiredSize) {
+ /* not enough space allocated */
+ LZ4F_free(cctxPtr->lz4CtxPtr, cctxPtr->cmem);
if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) {
- cctxPtr->lz4CtxPtr = LZ4_createStream();
+ /* must take ownership of memory allocation,
+ * in order to respect custom allocator contract */
+ cctxPtr->lz4CtxPtr = LZ4F_malloc(sizeof(LZ4_stream_t), cctxPtr->cmem);
+ if (cctxPtr->lz4CtxPtr)
+ LZ4_initStream(cctxPtr->lz4CtxPtr, sizeof(LZ4_stream_t));
} else {
- cctxPtr->lz4CtxPtr = LZ4_createStreamHC();
+ cctxPtr->lz4CtxPtr = LZ4F_malloc(sizeof(LZ4_streamHC_t), cctxPtr->cmem);
+ if (cctxPtr->lz4CtxPtr)
+ LZ4_initStreamHC(cctxPtr->lz4CtxPtr, sizeof(LZ4_streamHC_t));
}
- if (cctxPtr->lz4CtxPtr == NULL)
- return err0r(LZ4F_ERROR_allocation_failed);
+ RETURN_ERROR_IF(cctxPtr->lz4CtxPtr == NULL, allocation_failed);
cctxPtr->lz4CtxAlloc = ctxTypeID;
cctxPtr->lz4CtxState = ctxTypeID;
} else if (cctxPtr->lz4CtxState != ctxTypeID) {
- /* otherwise, a sufficient buffer is allocated, but we need to
- * reset it to the correct context type */
+ /* otherwise, a sufficient buffer is already allocated,
+ * but we need to reset it to the correct context type */
if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) {
- LZ4_initStream((LZ4_stream_t *) cctxPtr->lz4CtxPtr, sizeof (LZ4_stream_t));
+ LZ4_initStream((LZ4_stream_t*)cctxPtr->lz4CtxPtr, sizeof(LZ4_stream_t));
} else {
- LZ4_initStreamHC((LZ4_streamHC_t *) cctxPtr->lz4CtxPtr, sizeof(LZ4_streamHC_t));
- LZ4_setCompressionLevel((LZ4_streamHC_t *) cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel);
+ LZ4_initStreamHC((LZ4_streamHC_t*)cctxPtr->lz4CtxPtr, sizeof(LZ4_streamHC_t));
+ LZ4_setCompressionLevel((LZ4_streamHC_t*)cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel);
}
cctxPtr->lz4CtxState = ctxTypeID;
- }
- }
+ } }
/* Buffer Management */
if (cctxPtr->prefs.frameInfo.blockSizeID == 0)
@@ -647,9 +731,9 @@
if (cctxPtr->maxBufferSize < requiredBuffSize) {
cctxPtr->maxBufferSize = 0;
- FREEMEM(cctxPtr->tmpBuff);
- cctxPtr->tmpBuff = (BYTE*)ALLOC_AND_ZERO(requiredBuffSize);
- if (cctxPtr->tmpBuff == NULL) return err0r(LZ4F_ERROR_allocation_failed);
+ LZ4F_free(cctxPtr->tmpBuff, cctxPtr->cmem);
+ cctxPtr->tmpBuff = (BYTE*)LZ4F_calloc(requiredBuffSize, cctxPtr->cmem);
+ RETURN_ERROR_IF(cctxPtr->tmpBuff == NULL, allocation_failed);
cctxPtr->maxBufferSize = requiredBuffSize;
} }
cctxPtr->tmpIn = cctxPtr->tmpBuff;
@@ -669,31 +753,32 @@
/* Magic Number */
LZ4F_writeLE32(dstPtr, LZ4F_MAGICNUMBER);
dstPtr += 4;
- headerStart = dstPtr;
+ { BYTE* const headerStart = dstPtr;
- /* FLG Byte */
- *dstPtr++ = (BYTE)(((1 & _2BITS) << 6) /* Version('01') */
- + ((cctxPtr->prefs.frameInfo.blockMode & _1BIT ) << 5)
- + ((cctxPtr->prefs.frameInfo.blockChecksumFlag & _1BIT ) << 4)
- + ((unsigned)(cctxPtr->prefs.frameInfo.contentSize > 0) << 3)
- + ((cctxPtr->prefs.frameInfo.contentChecksumFlag & _1BIT ) << 2)
- + (cctxPtr->prefs.frameInfo.dictID > 0) );
- /* BD Byte */
- *dstPtr++ = (BYTE)((cctxPtr->prefs.frameInfo.blockSizeID & _3BITS) << 4);
- /* Optional Frame content size field */
- if (cctxPtr->prefs.frameInfo.contentSize) {
- LZ4F_writeLE64(dstPtr, cctxPtr->prefs.frameInfo.contentSize);
- dstPtr += 8;
- cctxPtr->totalInSize = 0;
+ /* FLG Byte */
+ *dstPtr++ = (BYTE)(((1 & _2BITS) << 6) /* Version('01') */
+ + ((cctxPtr->prefs.frameInfo.blockMode & _1BIT ) << 5)
+ + ((cctxPtr->prefs.frameInfo.blockChecksumFlag & _1BIT ) << 4)
+ + ((unsigned)(cctxPtr->prefs.frameInfo.contentSize > 0) << 3)
+ + ((cctxPtr->prefs.frameInfo.contentChecksumFlag & _1BIT ) << 2)
+ + (cctxPtr->prefs.frameInfo.dictID > 0) );
+ /* BD Byte */
+ *dstPtr++ = (BYTE)((cctxPtr->prefs.frameInfo.blockSizeID & _3BITS) << 4);
+ /* Optional Frame content size field */
+ if (cctxPtr->prefs.frameInfo.contentSize) {
+ LZ4F_writeLE64(dstPtr, cctxPtr->prefs.frameInfo.contentSize);
+ dstPtr += 8;
+ cctxPtr->totalInSize = 0;
+ }
+ /* Optional dictionary ID field */
+ if (cctxPtr->prefs.frameInfo.dictID) {
+ LZ4F_writeLE32(dstPtr, cctxPtr->prefs.frameInfo.dictID);
+ dstPtr += 4;
+ }
+ /* Header CRC Byte */
+ *dstPtr = LZ4F_headerChecksum(headerStart, (size_t)(dstPtr - headerStart));
+ dstPtr++;
}
- /* Optional dictionary ID field */
- if (cctxPtr->prefs.frameInfo.dictID) {
- LZ4F_writeLE32(dstPtr, cctxPtr->prefs.frameInfo.dictID);
- dstPtr += 4;
- }
- /* Header CRC Byte */
- *dstPtr = LZ4F_headerChecksum(headerStart, (size_t)(dstPtr - headerStart));
- dstPtr++;
cctxPtr->cStage = 1; /* header written, now request input data block */
return (size_t)(dstPtr - dstStart);
@@ -701,9 +786,9 @@
/*! LZ4F_compressBegin() :
- * init streaming compression and writes frame header into dstBuffer.
- * dstBuffer must be >= LZ4F_HEADER_SIZE_MAX bytes.
- * preferencesPtr can be NULL, in which case default parameters are selected.
+ * init streaming compression AND writes frame header into @dstBuffer.
+ * @dstCapacity must be >= LZ4F_HEADER_SIZE_MAX bytes.
+ * @preferencesPtr can be NULL, in which case default parameters are selected.
* @return : number of bytes written into dstBuffer for the header
* or an error code (can be tested using LZ4F_isError())
*/
@@ -744,11 +829,13 @@
LZ4F_blockChecksum_t crcFlag)
{
BYTE* const cSizePtr = (BYTE*)dst;
- U32 cSize = (U32)compress(lz4ctx, (const char*)src, (char*)(cSizePtr+BHSize),
- (int)(srcSize), (int)(srcSize-1),
- level, cdict);
- if (cSize == 0) { /* compression failed */
- DEBUGLOG(5, "LZ4F_makeBlock: compression failed, creating a raw block (size %u)", (U32)srcSize);
+ U32 cSize;
+ assert(compress != NULL);
+ cSize = (U32)compress(lz4ctx, (const char*)src, (char*)(cSizePtr+BHSize),
+ (int)(srcSize), (int)(srcSize-1),
+ level, cdict);
+
+ if (cSize == 0 || cSize >= srcSize) {
cSize = (U32)srcSize;
LZ4F_writeLE32(cSizePtr, cSize | LZ4F_BLOCKUNCOMPRESSED_FLAG);
memcpy(cSizePtr+BHSize, src, srcSize);
@@ -766,6 +853,7 @@
static int LZ4F_compressBlock(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level, const LZ4F_CDict* cdict)
{
int const acceleration = (level < 0) ? -level + 1 : 1;
+ DEBUGLOG(5, "LZ4F_compressBlock (srcSize=%i)", srcSize);
LZ4F_initStream(ctx, cdict, level, LZ4F_blockIndependent);
if (cdict) {
return LZ4_compress_fast_continue((LZ4_stream_t*)ctx, src, dst, srcSize, dstCapacity, acceleration);
@@ -778,6 +866,7 @@
{
int const acceleration = (level < 0) ? -level + 1 : 1;
(void)cdict; /* init once at beginning of frame */
+ DEBUGLOG(5, "LZ4F_compressBlock_continue (srcSize=%i)", srcSize);
return LZ4_compress_fast_continue((LZ4_stream_t*)ctx, src, dst, srcSize, dstCapacity, acceleration);
}
@@ -796,8 +885,15 @@
return LZ4_compress_HC_continue((LZ4_streamHC_t*)ctx, src, dst, srcSize, dstCapacity);
}
-static compressFunc_t LZ4F_selectCompression(LZ4F_blockMode_t blockMode, int level)
+static int LZ4F_doNotCompressBlock(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level, const LZ4F_CDict* cdict)
{
+ (void)ctx; (void)src; (void)dst; (void)srcSize; (void)dstCapacity; (void)level; (void)cdict;
+ return 0;
+}
+
+static compressFunc_t LZ4F_selectCompression(LZ4F_blockMode_t blockMode, int level, LZ4F_blockCompression_t compressMode)
+{
+ if (compressMode == LZ4B_UNCOMPRESSED) return LZ4F_doNotCompressBlock;
if (level < LZ4HC_CLEVEL_MIN) {
if (blockMode == LZ4F_blockIndependent) return LZ4F_compressBlock;
return LZ4F_compressBlock_continue;
@@ -806,6 +902,7 @@
return LZ4F_compressBlockHC_continue;
}
+/* Save history (up to 64KB) into @tmpBuff */
static int LZ4F_localSaveDict(LZ4F_cctx_t* cctxPtr)
{
if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN)
@@ -815,38 +912,57 @@
typedef enum { notDone, fromTmpBuffer, fromSrcBuffer } LZ4F_lastBlockStatus;
-/*! LZ4F_compressUpdate() :
+static const LZ4F_compressOptions_t k_cOptionsNull = { 0, { 0, 0, 0 } };
+
+
+ /*! LZ4F_compressUpdateImpl() :
* LZ4F_compressUpdate() can be called repetitively to compress as much data as necessary.
- * dstBuffer MUST be >= LZ4F_compressBound(srcSize, preferencesPtr).
- * LZ4F_compressOptions_t structure is optional : you can provide NULL as argument.
+ * When successful, the function always entirely consumes @srcBuffer.
+ * src data is either buffered or compressed into @dstBuffer.
+ * If the block compression does not match the compression of the previous block, the old data is flushed
+ * and operations continue with the new compression mode.
+ * @dstCapacity MUST be >= LZ4F_compressBound(srcSize, preferencesPtr) when block compression is turned on.
+ * @compressOptionsPtr is optional : provide NULL to mean "default".
* @return : the number of bytes written into dstBuffer. It can be zero, meaning input data was just buffered.
* or an error code if it fails (which can be tested using LZ4F_isError())
+ * After an error, the state is left in a UB state, and must be re-initialized.
*/
-size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr,
- void* dstBuffer, size_t dstCapacity,
+static size_t LZ4F_compressUpdateImpl(LZ4F_cctx* cctxPtr,
+ void* dstBuffer, size_t dstCapacity,
const void* srcBuffer, size_t srcSize,
- const LZ4F_compressOptions_t* compressOptionsPtr)
-{
- LZ4F_compressOptions_t cOptionsNull;
+ const LZ4F_compressOptions_t* compressOptionsPtr,
+ LZ4F_blockCompression_t blockCompression)
+ {
size_t const blockSize = cctxPtr->maxBlockSize;
const BYTE* srcPtr = (const BYTE*)srcBuffer;
const BYTE* const srcEnd = srcPtr + srcSize;
BYTE* const dstStart = (BYTE*)dstBuffer;
BYTE* dstPtr = dstStart;
LZ4F_lastBlockStatus lastBlockCompressed = notDone;
- compressFunc_t const compress = LZ4F_selectCompression(cctxPtr->prefs.frameInfo.blockMode, cctxPtr->prefs.compressionLevel);
-
+ compressFunc_t const compress = LZ4F_selectCompression(cctxPtr->prefs.frameInfo.blockMode, cctxPtr->prefs.compressionLevel, blockCompression);
+ size_t bytesWritten;
DEBUGLOG(4, "LZ4F_compressUpdate (srcSize=%zu)", srcSize);
- if (cctxPtr->cStage != 1) return err0r(LZ4F_ERROR_GENERIC);
+ RETURN_ERROR_IF(cctxPtr->cStage != 1, compressionState_uninitialized); /* state must be initialized and waiting for next block */
if (dstCapacity < LZ4F_compressBound_internal(srcSize, &(cctxPtr->prefs), cctxPtr->tmpInSize))
- return err0r(LZ4F_ERROR_dstMaxSize_tooSmall);
- MEM_INIT(&cOptionsNull, 0, sizeof(cOptionsNull));
- if (compressOptionsPtr == NULL) compressOptionsPtr = &cOptionsNull;
+ RETURN_ERROR(dstMaxSize_tooSmall);
+
+ if (blockCompression == LZ4B_UNCOMPRESSED && dstCapacity < srcSize)
+ RETURN_ERROR(dstMaxSize_tooSmall);
+
+ /* flush currently written block, to continue with new block compression */
+ if (cctxPtr->blockCompression != blockCompression) {
+ bytesWritten = LZ4F_flush(cctxPtr, dstBuffer, dstCapacity, compressOptionsPtr);
+ dstPtr += bytesWritten;
+ cctxPtr->blockCompression = blockCompression;
+ }
+
+ if (compressOptionsPtr == NULL) compressOptionsPtr = &k_cOptionsNull;
/* complete tmp buffer */
if (cctxPtr->tmpInSize > 0) { /* some data already within tmp buffer */
size_t const sizeToCopy = blockSize - cctxPtr->tmpInSize;
+ assert(blockSize > cctxPtr->tmpInSize);
if (sizeToCopy > srcSize) {
/* add src to tmpIn buffer */
memcpy(cctxPtr->tmpIn + cctxPtr->tmpInSize, srcBuffer, srcSize);
@@ -864,11 +980,9 @@
compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel,
cctxPtr->cdict,
cctxPtr->prefs.frameInfo.blockChecksumFlag);
-
if (cctxPtr->prefs.frameInfo.blockMode==LZ4F_blockLinked) cctxPtr->tmpIn += blockSize;
cctxPtr->tmpInSize = 0;
- }
- }
+ } }
while ((size_t)(srcEnd - srcPtr) >= blockSize) {
/* compress full blocks */
@@ -882,33 +996,38 @@
}
if ((cctxPtr->prefs.autoFlush) && (srcPtr < srcEnd)) {
- /* compress remaining input < blockSize */
+ /* autoFlush : remaining input (< blockSize) is compressed */
lastBlockCompressed = fromSrcBuffer;
dstPtr += LZ4F_makeBlock(dstPtr,
srcPtr, (size_t)(srcEnd - srcPtr),
compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel,
cctxPtr->cdict,
cctxPtr->prefs.frameInfo.blockChecksumFlag);
- srcPtr = srcEnd;
+ srcPtr = srcEnd;
}
- /* preserve dictionary if necessary */
+ /* preserve dictionary within @tmpBuff whenever necessary */
if ((cctxPtr->prefs.frameInfo.blockMode==LZ4F_blockLinked) && (lastBlockCompressed==fromSrcBuffer)) {
+ /* linked blocks are only supported in compressed mode, see LZ4F_uncompressedUpdate */
+ assert(blockCompression == LZ4B_COMPRESSED);
if (compressOptionsPtr->stableSrc) {
- cctxPtr->tmpIn = cctxPtr->tmpBuff;
+ cctxPtr->tmpIn = cctxPtr->tmpBuff; /* src is stable : dictionary remains in src across invocations */
} else {
int const realDictSize = LZ4F_localSaveDict(cctxPtr);
- if (realDictSize==0) return err0r(LZ4F_ERROR_GENERIC);
+ assert(0 <= realDictSize && realDictSize <= 64 KB);
cctxPtr->tmpIn = cctxPtr->tmpBuff + realDictSize;
}
}
/* keep tmpIn within limits */
- if ((cctxPtr->tmpIn + blockSize) > (cctxPtr->tmpBuff + cctxPtr->maxBufferSize) /* necessarily LZ4F_blockLinked && lastBlockCompressed==fromTmpBuffer */
- && !(cctxPtr->prefs.autoFlush))
+ if (!(cctxPtr->prefs.autoFlush) /* no autoflush : there may be some data left within internal buffer */
+ && (cctxPtr->tmpIn + blockSize) > (cctxPtr->tmpBuff + cctxPtr->maxBufferSize) ) /* not enough room to store next block */
{
+ /* only preserve 64KB within internal buffer. Ensures there is enough room for next block.
+ * note: this situation necessarily implies lastBlockCompressed==fromTmpBuffer */
int const realDictSize = LZ4F_localSaveDict(cctxPtr);
cctxPtr->tmpIn = cctxPtr->tmpBuff + realDictSize;
+ assert((cctxPtr->tmpIn + blockSize) <= (cctxPtr->tmpBuff + cctxPtr->maxBufferSize));
}
/* some input data left, necessarily < blockSize */
@@ -926,6 +1045,53 @@
return (size_t)(dstPtr - dstStart);
}
+/*! LZ4F_compressUpdate() :
+ * LZ4F_compressUpdate() can be called repetitively to compress as much data as necessary.
+ * When successful, the function always entirely consumes @srcBuffer.
+ * src data is either buffered or compressed into @dstBuffer.
+ * If previously an uncompressed block was written, buffered data is flushed
+ * before appending compressed data is continued.
+ * @dstCapacity MUST be >= LZ4F_compressBound(srcSize, preferencesPtr).
+ * @compressOptionsPtr is optional : provide NULL to mean "default".
+ * @return : the number of bytes written into dstBuffer. It can be zero, meaning input data was just buffered.
+ * or an error code if it fails (which can be tested using LZ4F_isError())
+ * After an error, the state is left in a UB state, and must be re-initialized.
+ */
+size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr,
+ void* dstBuffer, size_t dstCapacity,
+ const void* srcBuffer, size_t srcSize,
+ const LZ4F_compressOptions_t* compressOptionsPtr)
+{
+ return LZ4F_compressUpdateImpl(cctxPtr,
+ dstBuffer, dstCapacity,
+ srcBuffer, srcSize,
+ compressOptionsPtr, LZ4B_COMPRESSED);
+}
+
+/*! LZ4F_compressUpdate() :
+ * LZ4F_compressUpdate() can be called repetitively to compress as much data as necessary.
+ * When successful, the function always entirely consumes @srcBuffer.
+ * src data is either buffered or compressed into @dstBuffer.
+ * If previously an uncompressed block was written, buffered data is flushed
+ * before appending compressed data is continued.
+ * This is only supported when LZ4F_blockIndependent is used
+ * @dstCapacity MUST be >= LZ4F_compressBound(srcSize, preferencesPtr).
+ * @compressOptionsPtr is optional : provide NULL to mean "default".
+ * @return : the number of bytes written into dstBuffer. It can be zero, meaning input data was just buffered.
+ * or an error code if it fails (which can be tested using LZ4F_isError())
+ * After an error, the state is left in a UB state, and must be re-initialized.
+ */
+size_t LZ4F_uncompressedUpdate(LZ4F_cctx* cctxPtr,
+ void* dstBuffer, size_t dstCapacity,
+ const void* srcBuffer, size_t srcSize,
+ const LZ4F_compressOptions_t* compressOptionsPtr) {
+ RETURN_ERROR_IF(cctxPtr->prefs.frameInfo.blockMode != LZ4F_blockIndependent, blockMode_invalid);
+ return LZ4F_compressUpdateImpl(cctxPtr,
+ dstBuffer, dstCapacity,
+ srcBuffer, srcSize,
+ compressOptionsPtr, LZ4B_UNCOMPRESSED);
+}
+
/*! LZ4F_flush() :
* When compressed data must be sent immediately, without waiting for a block to be filled,
@@ -944,13 +1110,12 @@
compressFunc_t compress;
if (cctxPtr->tmpInSize == 0) return 0; /* nothing to flush */
- if (cctxPtr->cStage != 1) return err0r(LZ4F_ERROR_GENERIC);
- if (dstCapacity < (cctxPtr->tmpInSize + BHSize + BFSize))
- return err0r(LZ4F_ERROR_dstMaxSize_tooSmall);
- (void)compressOptionsPtr; /* not yet useful */
+ RETURN_ERROR_IF(cctxPtr->cStage != 1, compressionState_uninitialized);
+ RETURN_ERROR_IF(dstCapacity < (cctxPtr->tmpInSize + BHSize + BFSize), dstMaxSize_tooSmall);
+ (void)compressOptionsPtr; /* not useful (yet) */
/* select compression function */
- compress = LZ4F_selectCompression(cctxPtr->prefs.frameInfo.blockMode, cctxPtr->prefs.compressionLevel);
+ compress = LZ4F_selectCompression(cctxPtr->prefs.frameInfo.blockMode, cctxPtr->prefs.compressionLevel, cctxPtr->blockCompression);
/* compress tmp buffer */
dstPtr += LZ4F_makeBlock(dstPtr,
@@ -992,19 +1157,19 @@
size_t const flushSize = LZ4F_flush(cctxPtr, dstBuffer, dstCapacity, compressOptionsPtr);
DEBUGLOG(5,"LZ4F_compressEnd: dstCapacity=%u", (unsigned)dstCapacity);
- if (LZ4F_isError(flushSize)) return flushSize;
+ FORWARD_IF_ERROR(flushSize);
dstPtr += flushSize;
assert(flushSize <= dstCapacity);
dstCapacity -= flushSize;
- if (dstCapacity < 4) return err0r(LZ4F_ERROR_dstMaxSize_tooSmall);
+ RETURN_ERROR_IF(dstCapacity < 4, dstMaxSize_tooSmall);
LZ4F_writeLE32(dstPtr, 0);
dstPtr += 4; /* endMark */
if (cctxPtr->prefs.frameInfo.contentChecksumFlag == LZ4F_contentChecksumEnabled) {
U32 const xxh = XXH32_digest(&(cctxPtr->xxh));
- if (dstCapacity < 8) return err0r(LZ4F_ERROR_dstMaxSize_tooSmall);
+ RETURN_ERROR_IF(dstCapacity < 8, dstMaxSize_tooSmall);
DEBUGLOG(5,"Writing 32-bit content checksum");
LZ4F_writeLE32(dstPtr, xxh);
dstPtr+=4; /* content Checksum */
@@ -1015,7 +1180,7 @@
if (cctxPtr->prefs.frameInfo.contentSize) {
if (cctxPtr->prefs.frameInfo.contentSize != cctxPtr->totalInSize)
- return err0r(LZ4F_ERROR_frameSize_wrong);
+ RETURN_ERROR(frameSize_wrong);
}
return (size_t)(dstPtr - dstStart);
@@ -1039,6 +1204,7 @@
} dStage_t;
struct LZ4F_dctx_s {
+ LZ4F_CustomMem cmem;
LZ4F_frameInfo_t frameInfo;
U32 version;
dStage_t dStage;
@@ -1056,26 +1222,37 @@
size_t tmpOutStart;
XXH32_state_t xxh;
XXH32_state_t blockChecksum;
+ int skipChecksum;
BYTE header[LZ4F_HEADER_SIZE_MAX];
}; /* typedef'd to LZ4F_dctx in lz4frame.h */
+LZ4F_dctx* LZ4F_createDecompressionContext_advanced(LZ4F_CustomMem customMem, unsigned version)
+{
+ LZ4F_dctx* const dctx = (LZ4F_dctx*)LZ4F_calloc(sizeof(LZ4F_dctx), customMem);
+ if (dctx == NULL) return NULL;
+
+ dctx->cmem = customMem;
+ dctx->version = version;
+ return dctx;
+}
+
/*! LZ4F_createDecompressionContext() :
* Create a decompressionContext object, which will track all decompression operations.
* Provides a pointer to a fully allocated and initialized LZ4F_decompressionContext object.
* Object can later be released using LZ4F_freeDecompressionContext().
* @return : if != 0, there was an error during context creation.
*/
-LZ4F_errorCode_t LZ4F_createDecompressionContext(LZ4F_dctx** LZ4F_decompressionContextPtr, unsigned versionNumber)
+LZ4F_errorCode_t
+LZ4F_createDecompressionContext(LZ4F_dctx** LZ4F_decompressionContextPtr, unsigned versionNumber)
{
- LZ4F_dctx* const dctx = (LZ4F_dctx*)ALLOC_AND_ZERO(sizeof(LZ4F_dctx));
- if (dctx == NULL) { /* failed allocation */
- *LZ4F_decompressionContextPtr = NULL;
- return err0r(LZ4F_ERROR_allocation_failed);
- }
+ assert(LZ4F_decompressionContextPtr != NULL); /* violation of narrow contract */
+ RETURN_ERROR_IF(LZ4F_decompressionContextPtr == NULL, parameter_null); /* in case it nonetheless happen in production */
- dctx->version = versionNumber;
- *LZ4F_decompressionContextPtr = dctx;
+ *LZ4F_decompressionContextPtr = LZ4F_createDecompressionContext_advanced(LZ4F_defaultCMem, versionNumber);
+ if (*LZ4F_decompressionContextPtr == NULL) { /* failed allocation */
+ RETURN_ERROR(allocation_failed);
+ }
return LZ4F_OK_NoError;
}
@@ -1084,9 +1261,9 @@
LZ4F_errorCode_t result = LZ4F_OK_NoError;
if (dctx != NULL) { /* can accept NULL input, like free() */
result = (LZ4F_errorCode_t)dctx->dStage;
- FREEMEM(dctx->tmpIn);
- FREEMEM(dctx->tmpOutBuffer);
- FREEMEM(dctx);
+ LZ4F_free(dctx->tmpIn, dctx->cmem);
+ LZ4F_free(dctx->tmpOutBuffer, dctx->cmem);
+ LZ4F_free(dctx, dctx->cmem);
}
return result;
}
@@ -1099,6 +1276,7 @@
dctx->dStage = dstage_getFrameHeader;
dctx->dict = NULL;
dctx->dictSize = 0;
+ dctx->skipChecksum = 0;
}
@@ -1118,7 +1296,7 @@
DEBUGLOG(5, "LZ4F_decodeHeader");
/* need to decode header to get frameInfo */
- if (srcSize < minFHSize) return err0r(LZ4F_ERROR_frameHeader_incomplete); /* minimal frame header size */
+ RETURN_ERROR_IF(srcSize < minFHSize, frameHeader_incomplete); /* minimal frame header size */
MEM_INIT(&(dctx->frameInfo), 0, sizeof(dctx->frameInfo));
/* special case : skippable frames */
@@ -1132,14 +1310,13 @@
} else {
dctx->dStage = dstage_getSFrameSize;
return 4;
- }
- }
+ } }
/* control magic number */
#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
if (LZ4F_readLE32(srcPtr) != LZ4F_MAGICNUMBER) {
DEBUGLOG(4, "frame header error : unknown magic number");
- return err0r(LZ4F_ERROR_frameType_unknown);
+ RETURN_ERROR(frameType_unknown);
}
#endif
dctx->frameInfo.frameType = LZ4F_frame;
@@ -1153,8 +1330,8 @@
contentChecksumFlag = (FLG>>2) & _1BIT;
dictIDFlag = FLG & _1BIT;
/* validate */
- if (((FLG>>1)&_1BIT) != 0) return err0r(LZ4F_ERROR_reservedFlag_set); /* Reserved bit */
- if (version != 1) return err0r(LZ4F_ERROR_headerVersion_wrong); /* Version Number, only supported value */
+ if (((FLG>>1)&_1BIT) != 0) RETURN_ERROR(reservedFlag_set); /* Reserved bit */
+ if (version != 1) RETURN_ERROR(headerVersion_wrong); /* Version Number, only supported value */
}
/* Frame Header Size */
@@ -1173,17 +1350,16 @@
{ U32 const BD = srcPtr[5];
blockSizeID = (BD>>4) & _3BITS;
/* validate */
- if (((BD>>7)&_1BIT) != 0) return err0r(LZ4F_ERROR_reservedFlag_set); /* Reserved bit */
- if (blockSizeID < 4) return err0r(LZ4F_ERROR_maxBlockSize_invalid); /* 4-7 only supported values for the time being */
- if (((BD>>0)&_4BITS) != 0) return err0r(LZ4F_ERROR_reservedFlag_set); /* Reserved bits */
+ if (((BD>>7)&_1BIT) != 0) RETURN_ERROR(reservedFlag_set); /* Reserved bit */
+ if (blockSizeID < 4) RETURN_ERROR(maxBlockSize_invalid); /* 4-7 only supported values for the time being */
+ if (((BD>>0)&_4BITS) != 0) RETURN_ERROR(reservedFlag_set); /* Reserved bits */
}
/* check header */
assert(frameHeaderSize > 5);
#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
{ BYTE const HC = LZ4F_headerChecksum(srcPtr+4, frameHeaderSize-5);
- if (HC != srcPtr[frameHeaderSize-1])
- return err0r(LZ4F_ERROR_headerChecksum_invalid);
+ RETURN_ERROR_IF(HC != srcPtr[frameHeaderSize-1], headerChecksum_invalid);
}
#endif
@@ -1192,10 +1368,9 @@
dctx->frameInfo.blockChecksumFlag = (LZ4F_blockChecksum_t)blockChecksumFlag;
dctx->frameInfo.contentChecksumFlag = (LZ4F_contentChecksum_t)contentChecksumFlag;
dctx->frameInfo.blockSizeID = (LZ4F_blockSizeID_t)blockSizeID;
- dctx->maxBlockSize = LZ4F_getBlockSize(blockSizeID);
+ dctx->maxBlockSize = LZ4F_getBlockSize((LZ4F_blockSizeID_t)blockSizeID);
if (contentSizeFlag)
- dctx->frameRemainingSize =
- dctx->frameInfo.contentSize = LZ4F_readLE64(srcPtr+6);
+ dctx->frameRemainingSize = dctx->frameInfo.contentSize = LZ4F_readLE64(srcPtr+6);
if (dictIDFlag)
dctx->frameInfo.dictID = LZ4F_readLE32(srcPtr + frameHeaderSize - 5);
@@ -1211,11 +1386,11 @@
*/
size_t LZ4F_headerSize(const void* src, size_t srcSize)
{
- if (src == NULL) return err0r(LZ4F_ERROR_srcPtr_wrong);
+ RETURN_ERROR_IF(src == NULL, srcPtr_wrong);
/* minimal srcSize to determine header size */
if (srcSize < LZ4F_MIN_SIZE_TO_KNOW_HEADER_LENGTH)
- return err0r(LZ4F_ERROR_frameHeader_incomplete);
+ RETURN_ERROR(frameHeader_incomplete);
/* special case : skippable frames */
if ((LZ4F_readLE32(src) & 0xFFFFFFF0U) == LZ4F_MAGIC_SKIPPABLE_START)
@@ -1224,7 +1399,7 @@
/* control magic number */
#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
if (LZ4F_readLE32(src) != LZ4F_MAGICNUMBER)
- return err0r(LZ4F_ERROR_frameType_unknown);
+ RETURN_ERROR(frameType_unknown);
#endif
/* Frame Header Size */
@@ -1266,13 +1441,13 @@
if (dctx->dStage == dstage_storeFrameHeader) {
/* frame decoding already started, in the middle of header => automatic fail */
*srcSizePtr = 0;
- return err0r(LZ4F_ERROR_frameDecoding_alreadyStarted);
+ RETURN_ERROR(frameDecoding_alreadyStarted);
} else {
size_t const hSize = LZ4F_headerSize(srcBuffer, *srcSizePtr);
if (LZ4F_isError(hSize)) { *srcSizePtr=0; return hSize; }
if (*srcSizePtr < hSize) {
*srcSizePtr=0;
- return err0r(LZ4F_ERROR_frameHeader_incomplete);
+ RETURN_ERROR(frameHeader_incomplete);
}
{ size_t decodeResult = LZ4F_decodeHeader(dctx, srcBuffer, hSize);
@@ -1290,16 +1465,14 @@
/* LZ4F_updateDict() :
* only used for LZ4F_blockLinked mode
- * Condition : dstPtr != NULL
+ * Condition : @dstPtr != NULL
*/
static void LZ4F_updateDict(LZ4F_dctx* dctx,
const BYTE* dstPtr, size_t dstSize, const BYTE* dstBufferStart,
unsigned withinTmp)
{
assert(dstPtr != NULL);
- if (dctx->dictSize==0) {
- dctx->dict = (const BYTE*)dstPtr; /* priority to prefix mode */
- }
+ if (dctx->dictSize==0) dctx->dict = (const BYTE*)dstPtr; /* will lead to prefix mode */
assert(dctx->dict != NULL);
if (dctx->dict + dctx->dictSize == dstPtr) { /* prefix mode, everything within dstBuffer */
@@ -1362,7 +1535,6 @@
}
-
/*! LZ4F_decompress() :
* Call this function repetitively to regenerate compressed data in srcBuffer.
* The function will attempt to decode up to *srcSizePtr bytes from srcBuffer
@@ -1406,6 +1578,7 @@
*srcSizePtr = 0;
*dstSizePtr = 0;
assert(dctx != NULL);
+ dctx->skipChecksum |= (decompressOptionsPtr->skipChecksums != 0); /* once set, disable for the remainder of the frame */
/* behaves as a state machine */
@@ -1418,7 +1591,7 @@
DEBUGLOG(6, "dstage_getFrameHeader");
if ((size_t)(srcEnd-srcPtr) >= maxFHSize) { /* enough to decode - shortcut */
size_t const hSize = LZ4F_decodeHeader(dctx, srcPtr, (size_t)(srcEnd-srcPtr)); /* will update dStage appropriately */
- if (LZ4F_isError(hSize)) return hSize;
+ FORWARD_IF_ERROR(hSize);
srcPtr += hSize;
break;
}
@@ -1440,9 +1613,7 @@
doAnotherStage = 0; /* not enough src data, ask for some more */
break;
}
- { size_t const hSize = LZ4F_decodeHeader(dctx, dctx->header, dctx->tmpInTarget); /* will update dStage appropriately */
- if (LZ4F_isError(hSize)) return hSize;
- }
+ FORWARD_IF_ERROR( LZ4F_decodeHeader(dctx, dctx->header, dctx->tmpInTarget) ); /* will update dStage appropriately */
break;
case dstage_init:
@@ -1453,14 +1624,12 @@
+ ((dctx->frameInfo.blockMode==LZ4F_blockLinked) ? 128 KB : 0);
if (bufferNeeded > dctx->maxBufferSize) { /* tmp buffers too small */
dctx->maxBufferSize = 0; /* ensure allocation will be re-attempted on next entry*/
- FREEMEM(dctx->tmpIn);
- dctx->tmpIn = (BYTE*)ALLOC(dctx->maxBlockSize + BFSize /* block checksum */);
- if (dctx->tmpIn == NULL)
- return err0r(LZ4F_ERROR_allocation_failed);
- FREEMEM(dctx->tmpOutBuffer);
- dctx->tmpOutBuffer= (BYTE*)ALLOC(bufferNeeded);
- if (dctx->tmpOutBuffer== NULL)
- return err0r(LZ4F_ERROR_allocation_failed);
+ LZ4F_free(dctx->tmpIn, dctx->cmem);
+ dctx->tmpIn = (BYTE*)LZ4F_malloc(dctx->maxBlockSize + BFSize /* block checksum */, dctx->cmem);
+ RETURN_ERROR_IF(dctx->tmpIn == NULL, allocation_failed);
+ LZ4F_free(dctx->tmpOutBuffer, dctx->cmem);
+ dctx->tmpOutBuffer= (BYTE*)LZ4F_malloc(bufferNeeded, dctx->cmem);
+ RETURN_ERROR_IF(dctx->tmpOutBuffer== NULL, allocation_failed);
dctx->maxBufferSize = bufferNeeded;
} }
dctx->tmpInSize = 0;
@@ -1509,7 +1678,7 @@
break;
}
if (nextCBlockSize > dctx->maxBlockSize) {
- return err0r(LZ4F_ERROR_maxBlockSize_invalid);
+ RETURN_ERROR(maxBlockSize_invalid);
}
if (blockHeader & LZ4F_BLOCKUNCOMPRESSED_FLAG) {
/* next block is uncompressed */
@@ -1540,11 +1709,13 @@
size_t const minBuffSize = MIN((size_t)(srcEnd-srcPtr), (size_t)(dstEnd-dstPtr));
sizeToCopy = MIN(dctx->tmpInTarget, minBuffSize);
memcpy(dstPtr, srcPtr, sizeToCopy);
- if (dctx->frameInfo.blockChecksumFlag) {
- (void)XXH32_update(&dctx->blockChecksum, srcPtr, sizeToCopy);
+ if (!dctx->skipChecksum) {
+ if (dctx->frameInfo.blockChecksumFlag) {
+ (void)XXH32_update(&dctx->blockChecksum, srcPtr, sizeToCopy);
+ }
+ if (dctx->frameInfo.contentChecksumFlag)
+ (void)XXH32_update(&dctx->xxh, srcPtr, sizeToCopy);
}
- if (dctx->frameInfo.contentChecksumFlag)
- (void)XXH32_update(&dctx->xxh, srcPtr, sizeToCopy);
if (dctx->frameInfo.contentSize)
dctx->frameRemainingSize -= sizeToCopy;
@@ -1590,14 +1761,15 @@
}
crcSrc = dctx->header;
}
- { U32 const readCRC = LZ4F_readLE32(crcSrc);
+ if (!dctx->skipChecksum) {
+ U32 const readCRC = LZ4F_readLE32(crcSrc);
U32 const calcCRC = XXH32_digest(&dctx->blockChecksum);
#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
DEBUGLOG(6, "compare block checksum");
if (readCRC != calcCRC) {
DEBUGLOG(4, "incorrect block checksum: %08X != %08X",
readCRC, calcCRC);
- return err0r(LZ4F_ERROR_blockChecksum_invalid);
+ RETURN_ERROR(blockChecksum_invalid);
}
#else
(void)readCRC;
@@ -1637,37 +1809,44 @@
}
/* At this stage, input is large enough to decode a block */
+
+ /* First, decode and control block checksum if it exists */
if (dctx->frameInfo.blockChecksumFlag) {
+ assert(dctx->tmpInTarget >= 4);
dctx->tmpInTarget -= 4;
assert(selectedIn != NULL); /* selectedIn is defined at this stage (either srcPtr, or dctx->tmpIn) */
{ U32 const readBlockCrc = LZ4F_readLE32(selectedIn + dctx->tmpInTarget);
U32 const calcBlockCrc = XXH32(selectedIn, dctx->tmpInTarget, 0);
#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
- if (readBlockCrc != calcBlockCrc)
- return err0r(LZ4F_ERROR_blockChecksum_invalid);
+ RETURN_ERROR_IF(readBlockCrc != calcBlockCrc, blockChecksum_invalid);
#else
(void)readBlockCrc;
(void)calcBlockCrc;
#endif
} }
- if ((size_t)(dstEnd-dstPtr) >= dctx->maxBlockSize) {
+ /* decode directly into destination buffer if there is enough room */
+ if ( ((size_t)(dstEnd-dstPtr) >= dctx->maxBlockSize)
+ /* unless the dictionary is stored in tmpOut:
+ * in which case it's faster to decode within tmpOut
+ * to benefit from prefix speedup */
+ && !(dctx->dict!= NULL && (const BYTE*)dctx->dict + dctx->dictSize == dctx->tmpOut) )
+ {
const char* dict = (const char*)dctx->dict;
size_t dictSize = dctx->dictSize;
int decodedSize;
assert(dstPtr != NULL);
if (dict && dictSize > 1 GB) {
- /* the dictSize param is an int, avoid truncation / sign issues */
+ /* overflow control : dctx->dictSize is an int, avoid truncation / sign issues */
dict += dictSize - 64 KB;
dictSize = 64 KB;
}
- /* enough capacity in `dst` to decompress directly there */
decodedSize = LZ4_decompress_safe_usingDict(
(const char*)selectedIn, (char*)dstPtr,
(int)dctx->tmpInTarget, (int)dctx->maxBlockSize,
dict, (int)dictSize);
- if (decodedSize < 0) return err0r(LZ4F_ERROR_GENERIC); /* decompression failed */
- if (dctx->frameInfo.contentChecksumFlag)
+ RETURN_ERROR_IF(decodedSize < 0, decompressionFailed);
+ if ((dctx->frameInfo.contentChecksumFlag) && (!dctx->skipChecksum))
XXH32_update(&(dctx->xxh), dstPtr, (size_t)decodedSize);
if (dctx->frameInfo.contentSize)
dctx->frameRemainingSize -= (size_t)decodedSize;
@@ -1678,25 +1857,27 @@
}
dstPtr += decodedSize;
- dctx->dStage = dstage_getBlockHeader;
+ dctx->dStage = dstage_getBlockHeader; /* end of block, let's get another one */
break;
}
/* not enough place into dst : decode into tmpOut */
- /* ensure enough place for tmpOut */
+
+ /* manage dictionary */
if (dctx->frameInfo.blockMode == LZ4F_blockLinked) {
if (dctx->dict == dctx->tmpOutBuffer) {
+ /* truncate dictionary to 64 KB if too big */
if (dctx->dictSize > 128 KB) {
memcpy(dctx->tmpOutBuffer, dctx->dict + dctx->dictSize - 64 KB, 64 KB);
dctx->dictSize = 64 KB;
}
dctx->tmpOut = dctx->tmpOutBuffer + dctx->dictSize;
- } else { /* dict not within tmp */
+ } else { /* dict not within tmpOut */
size_t const reservedDictSpace = MIN(dctx->dictSize, 64 KB);
dctx->tmpOut = dctx->tmpOutBuffer + reservedDictSpace;
} }
- /* Decode block */
+ /* Decode block into tmpOut */
{ const char* dict = (const char*)dctx->dict;
size_t dictSize = dctx->dictSize;
int decodedSize;
@@ -1709,9 +1890,8 @@
(const char*)selectedIn, (char*)dctx->tmpOut,
(int)dctx->tmpInTarget, (int)dctx->maxBlockSize,
dict, (int)dictSize);
- if (decodedSize < 0) /* decompression failed */
- return err0r(LZ4F_ERROR_decompressionFailed);
- if (dctx->frameInfo.contentChecksumFlag)
+ RETURN_ERROR_IF(decodedSize < 0, decompressionFailed);
+ if (dctx->frameInfo.contentChecksumFlag && !dctx->skipChecksum)
XXH32_update(&(dctx->xxh), dctx->tmpOut, (size_t)decodedSize);
if (dctx->frameInfo.contentSize)
dctx->frameRemainingSize -= (size_t)decodedSize;
@@ -1744,8 +1924,7 @@
break;
case dstage_getSuffix:
- if (dctx->frameRemainingSize)
- return err0r(LZ4F_ERROR_frameSize_wrong); /* incorrect frame size decoded */
+ RETURN_ERROR_IF(dctx->frameRemainingSize, frameSize_wrong); /* incorrect frame size decoded */
if (!dctx->frameInfo.contentChecksumFlag) { /* no checksum, frame is completed */
nextSrcSizeHint = 0;
LZ4F_resetDecompressionContext(dctx);
@@ -1777,20 +1956,20 @@
} /* if (dctx->dStage == dstage_storeSuffix) */
/* case dstage_checkSuffix: */ /* no direct entry, avoid initialization risks */
- { U32 const readCRC = LZ4F_readLE32(selectedIn);
+ if (!dctx->skipChecksum) {
+ U32 const readCRC = LZ4F_readLE32(selectedIn);
U32 const resultCRC = XXH32_digest(&(dctx->xxh));
#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
- if (readCRC != resultCRC)
- return err0r(LZ4F_ERROR_contentChecksum_invalid);
+ RETURN_ERROR_IF(readCRC != resultCRC, contentChecksum_invalid);
#else
(void)readCRC;
(void)resultCRC;
#endif
- nextSrcSizeHint = 0;
- LZ4F_resetDecompressionContext(dctx);
- doAnotherStage = 0;
- break;
}
+ nextSrcSizeHint = 0;
+ LZ4F_resetDecompressionContext(dctx);
+ doAnotherStage = 0;
+ break;
case dstage_getSFrameSize:
if ((srcEnd - srcPtr) >= 4) {
@@ -1841,7 +2020,7 @@
} /* switch (dctx->dStage) */
} /* while (doAnotherStage) */
- /* preserve history within tmp whenever necessary */
+ /* preserve history within tmpOut whenever necessary */
LZ4F_STATIC_ASSERT((unsigned)dstage_init == 2);
if ( (dctx->frameInfo.blockMode==LZ4F_blockLinked) /* next block will use up to 64KB from previous ones */
&& (dctx->dict != dctx->tmpOutBuffer) /* dictionary is not already within tmp */
diff --git a/lib/lz4frame.h b/lib/lz4frame.h
index 4573317..1bdf6c4 100644
--- a/lib/lz4frame.h
+++ b/lib/lz4frame.h
@@ -1,7 +1,7 @@
/*
- LZ4 auto-framing library
+ LZ4F - LZ4-Frame library
Header File
- Copyright (C) 2011-2017, Yann Collet.
+ Copyright (C) 2011-2020, Yann Collet.
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
Redistribution and use in source and binary forms, with or without
@@ -39,7 +39,7 @@
* LZ4F also offers streaming capabilities.
*
* lz4.h is not required when using lz4frame.h,
- * except to extract common constant such as LZ4_VERSION_NUMBER.
+ * except to extract common constants such as LZ4_VERSION_NUMBER.
* */
#ifndef LZ4F_H_09782039843
@@ -54,12 +54,12 @@
/**
- Introduction
-
- lz4frame.h implements LZ4 frame specification (doc/lz4_Frame_format.md).
- lz4frame.h provides frame compression functions that take care
- of encoding standard metadata alongside LZ4-compressed blocks.
-*/
+ * Introduction
+ *
+ * lz4frame.h implements LZ4 frame specification: see doc/lz4_Frame_format.md .
+ * LZ4 Frames are compatible with `lz4` CLI,
+ * and designed to be interoperable with any system.
+**/
/*-***************************************************************
* Compiler specifics
@@ -210,7 +210,7 @@
* Returns the maximum possible compressed size with LZ4F_compressFrame() given srcSize and preferences.
* `preferencesPtr` is optional. It can be replaced by NULL, in which case, the function will assume default preferences.
* Note : this result is only usable with LZ4F_compressFrame().
- * It may also be used with LZ4F_compressUpdate() _if no flush() operation_ is performed.
+ * It may also be relevant to LZ4F_compressUpdate() _only if_ no flush() operation is ever performed.
*/
LZ4FLIB_API size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_preferences_t* preferencesPtr);
@@ -230,7 +230,7 @@
* Advanced compression functions
*************************************/
typedef struct LZ4F_cctx_s LZ4F_cctx; /* incomplete type */
-typedef LZ4F_cctx* LZ4F_compressionContext_t; /* for compatibility with previous API version */
+typedef LZ4F_cctx* LZ4F_compressionContext_t; /* for compatibility with older APIs, prefer using LZ4F_cctx */
typedef struct {
unsigned stableSrc; /* 1 == src content will remain present on future calls to LZ4F_compress(); skip copying src content within tmp buffer */
@@ -243,20 +243,27 @@
LZ4FLIB_API unsigned LZ4F_getVersion(void);
/*! LZ4F_createCompressionContext() :
- * The first thing to do is to create a compressionContext object, which will be used in all compression operations.
- * This is achieved using LZ4F_createCompressionContext(), which takes as argument a version.
- * The version provided MUST be LZ4F_VERSION. It is intended to track potential version mismatch, notably when using DLL.
- * The function will provide a pointer to a fully allocated LZ4F_cctx object.
- * If @return != zero, there was an error during context creation.
- * Object can release its memory using LZ4F_freeCompressionContext();
- */
+ * The first thing to do is to create a compressionContext object,
+ * which will keep track of operation state during streaming compression.
+ * This is achieved using LZ4F_createCompressionContext(), which takes as argument a version,
+ * and a pointer to LZ4F_cctx*, to write the resulting pointer into.
+ * @version provided MUST be LZ4F_VERSION. It is intended to track potential version mismatch, notably when using DLL.
+ * The function provides a pointer to a fully allocated LZ4F_cctx object.
+ * @cctxPtr MUST be != NULL.
+ * If @return != zero, context creation failed.
+ * A created compression context can be employed multiple times for consecutive streaming operations.
+ * Once all streaming compression jobs are completed,
+ * the state object can be released using LZ4F_freeCompressionContext().
+ * Note1 : LZ4F_freeCompressionContext() is always successful. Its return value can be ignored.
+ * Note2 : LZ4F_freeCompressionContext() works fine with NULL input pointers (do nothing).
+**/
LZ4FLIB_API LZ4F_errorCode_t LZ4F_createCompressionContext(LZ4F_cctx** cctxPtr, unsigned version);
LZ4FLIB_API LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctx);
/*---- Compression ----*/
-#define LZ4F_HEADER_SIZE_MIN 7 /* LZ4 Frame header size can vary, depending on selected paramaters */
+#define LZ4F_HEADER_SIZE_MIN 7 /* LZ4 Frame header size can vary, depending on selected parameters */
#define LZ4F_HEADER_SIZE_MAX 19
/* Size in bytes of a block header in little-endian format. Highest bit indicates if block data is uncompressed */
@@ -301,8 +308,9 @@
* Important rule: dstCapacity MUST be large enough to ensure operation success even in worst case situations.
* This value is provided by LZ4F_compressBound().
* If this condition is not respected, LZ4F_compress() will fail (result is an errorCode).
- * LZ4F_compressUpdate() doesn't guarantee error recovery.
- * When an error occurs, compression context must be freed or resized.
+ * After an error, the state is left in a UB state, and must be re-initialized or freed.
+ * If previously an uncompressed block was written, buffered data is flushed
+ * before appending compressed data is continued.
* `cOptPtr` is optional : NULL can be provided, in which case all options are set to default.
* @return : number of bytes written into `dstBuffer` (it can be zero, meaning input data was just buffered).
* or an error code if it fails (which can be tested using LZ4F_isError())
@@ -347,8 +355,12 @@
typedef LZ4F_dctx* LZ4F_decompressionContext_t; /* compatibility with previous API versions */
typedef struct {
- unsigned stableDst; /* pledges that last 64KB decompressed data will remain available unmodified. This optimization skips storage operations in tmp buffers. */
- unsigned reserved[3]; /* must be set to zero for forward compatibility */
+ unsigned stableDst; /* pledges that last 64KB decompressed data will remain available unmodified between invocations.
+ * This optimization skips storage operations in tmp buffers. */
+ unsigned skipChecksums; /* disable checksum calculation and verification, even when one is present in frame, to save CPU time.
+ * Setting this option to 1 once disables all checksums for the rest of the frame. */
+ unsigned reserved1; /* must be set to zero for forward compatibility */
+ unsigned reserved0; /* idem */
} LZ4F_decompressOptions_t;
@@ -356,9 +368,10 @@
/*! LZ4F_createDecompressionContext() :
* Create an LZ4F_dctx object, to track all decompression operations.
- * The version provided MUST be LZ4F_VERSION.
- * The function provides a pointer to an allocated and initialized LZ4F_dctx object.
- * The result is an errorCode, which can be tested using LZ4F_isError().
+ * @version provided MUST be LZ4F_VERSION.
+ * @dctxPtr MUST be valid.
+ * The function fills @dctxPtr with the value of a pointer to an allocated and initialized LZ4F_dctx object.
+ * The @return is an errorCode, which can be tested using LZ4F_isError().
* dctx memory can be released using LZ4F_freeDecompressionContext();
* Result of LZ4F_freeDecompressionContext() indicates current state of decompressionContext when being released.
* That is, it should be == 0 if decompression has been completed fully and correctly.
@@ -371,6 +384,8 @@
* Streaming decompression functions
*************************************/
+#define LZ4F_MAGICNUMBER 0x184D2204U
+#define LZ4F_MAGIC_SKIPPABLE_START 0x184D2A50U
#define LZ4F_MIN_SIZE_TO_KNOW_HEADER_LENGTH 5
/*! LZ4F_headerSize() : v1.9.0+
@@ -386,7 +401,7 @@
/*! LZ4F_getFrameInfo() :
* This function extracts frame parameters (max blockSize, dictID, etc.).
- * Its usage is optional: user can call LZ4F_decompress() directly.
+ * Its usage is optional: user can also invoke LZ4F_decompress() directly.
*
* Extracted information will fill an existing LZ4F_frameInfo_t structure.
* This can be useful for allocation and dictionary identification purposes.
@@ -427,9 +442,10 @@
* note 1 : in case of error, dctx is not modified. Decoding operation can resume from beginning safely.
* note 2 : frame parameters are *copied into* an already allocated LZ4F_frameInfo_t structure.
*/
-LZ4FLIB_API size_t LZ4F_getFrameInfo(LZ4F_dctx* dctx,
- LZ4F_frameInfo_t* frameInfoPtr,
- const void* srcBuffer, size_t* srcSizePtr);
+LZ4FLIB_API size_t
+LZ4F_getFrameInfo(LZ4F_dctx* dctx,
+ LZ4F_frameInfo_t* frameInfoPtr,
+ const void* srcBuffer, size_t* srcSizePtr);
/*! LZ4F_decompress() :
* Call this function repetitively to regenerate data compressed in `srcBuffer`.
@@ -462,10 +478,11 @@
*
* After a frame is fully decoded, dctx can be used again to decompress another frame.
*/
-LZ4FLIB_API size_t LZ4F_decompress(LZ4F_dctx* dctx,
- void* dstBuffer, size_t* dstSizePtr,
- const void* srcBuffer, size_t* srcSizePtr,
- const LZ4F_decompressOptions_t* dOptPtr);
+LZ4FLIB_API size_t
+LZ4F_decompress(LZ4F_dctx* dctx,
+ void* dstBuffer, size_t* dstSizePtr,
+ const void* srcBuffer, size_t* srcSizePtr,
+ const LZ4F_decompressOptions_t* dOptPtr);
/*! LZ4F_resetDecompressionContext() : added in v1.8.0
@@ -529,6 +546,8 @@
ITEM(ERROR_headerChecksum_invalid) \
ITEM(ERROR_contentChecksum_invalid) \
ITEM(ERROR_frameDecoding_alreadyStarted) \
+ ITEM(ERROR_compressionState_uninitialized) \
+ ITEM(ERROR_parameter_null) \
ITEM(ERROR_maxCode)
#define LZ4F_GENERATE_ENUM(ENUM) LZ4F_##ENUM,
@@ -539,7 +558,31 @@
LZ4FLIB_STATIC_API LZ4F_errorCodes LZ4F_getErrorCode(size_t functionResult);
-LZ4FLIB_STATIC_API size_t LZ4F_getBlockSize(unsigned);
+
+/*! LZ4F_getBlockSize() :
+ * Return, in scalar format (size_t),
+ * the maximum block size associated with blockSizeID.
+**/
+LZ4FLIB_STATIC_API size_t LZ4F_getBlockSize(LZ4F_blockSizeID_t blockSizeID);
+
+/*! LZ4F_uncompressedUpdate() :
+ * LZ4F_uncompressedUpdate() can be called repetitively to add as much data uncompressed data as necessary.
+ * Important rule: dstCapacity MUST be large enough to store the entire source buffer as
+ * no compression is done for this operation
+ * If this condition is not respected, LZ4F_uncompressedUpdate() will fail (result is an errorCode).
+ * After an error, the state is left in a UB state, and must be re-initialized or freed.
+ * If previously a compressed block was written, buffered data is flushed
+ * before appending uncompressed data is continued.
+ * This is only supported when LZ4F_blockIndependent is used
+ * `cOptPtr` is optional : NULL can be provided, in which case all options are set to default.
+ * @return : number of bytes written into `dstBuffer` (it can be zero, meaning input data was just buffered).
+ * or an error code if it fails (which can be tested using LZ4F_isError())
+ */
+LZ4FLIB_STATIC_API size_t
+LZ4F_uncompressedUpdate(LZ4F_cctx* cctx,
+ void* dstBuffer, size_t dstCapacity,
+ const void* srcBuffer, size_t srcSize,
+ const LZ4F_compressOptions_t* cOptPtr);
/**********************************
* Bulk processing dictionary API
@@ -583,12 +626,12 @@
* but it's not recommended, as it's the only way to provide dictID in the frame header.
* @return : number of bytes written into dstBuffer.
* or an error code if it fails (can be tested using LZ4F_isError()) */
-LZ4FLIB_STATIC_API size_t LZ4F_compressFrame_usingCDict(
- LZ4F_cctx* cctx,
- void* dst, size_t dstCapacity,
- const void* src, size_t srcSize,
- const LZ4F_CDict* cdict,
- const LZ4F_preferences_t* preferencesPtr);
+LZ4FLIB_STATIC_API size_t
+LZ4F_compressFrame_usingCDict(LZ4F_cctx* cctx,
+ void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize,
+ const LZ4F_CDict* cdict,
+ const LZ4F_preferences_t* preferencesPtr);
/*! LZ4F_compressBegin_usingCDict() :
@@ -598,23 +641,49 @@
* however, it's the only way to provide dictID in the frame header.
* @return : number of bytes written into dstBuffer for the header,
* or an error code (which can be tested using LZ4F_isError()) */
-LZ4FLIB_STATIC_API size_t LZ4F_compressBegin_usingCDict(
- LZ4F_cctx* cctx,
- void* dstBuffer, size_t dstCapacity,
- const LZ4F_CDict* cdict,
- const LZ4F_preferences_t* prefsPtr);
+LZ4FLIB_STATIC_API size_t
+LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctx,
+ void* dstBuffer, size_t dstCapacity,
+ const LZ4F_CDict* cdict,
+ const LZ4F_preferences_t* prefsPtr);
/*! LZ4F_decompress_usingDict() :
* Same as LZ4F_decompress(), using a predefined dictionary.
* Dictionary is used "in place", without any preprocessing.
- * It must remain accessible throughout the entire frame decoding. */
-LZ4FLIB_STATIC_API size_t LZ4F_decompress_usingDict(
- LZ4F_dctx* dctxPtr,
- void* dstBuffer, size_t* dstSizePtr,
- const void* srcBuffer, size_t* srcSizePtr,
- const void* dict, size_t dictSize,
- const LZ4F_decompressOptions_t* decompressOptionsPtr);
+** It must remain accessible throughout the entire frame decoding. */
+LZ4FLIB_STATIC_API size_t
+LZ4F_decompress_usingDict(LZ4F_dctx* dctxPtr,
+ void* dstBuffer, size_t* dstSizePtr,
+ const void* srcBuffer, size_t* srcSizePtr,
+ const void* dict, size_t dictSize,
+ const LZ4F_decompressOptions_t* decompressOptionsPtr);
+
+
+/*! Custom memory allocation :
+ * These prototypes make it possible to pass custom allocation/free functions.
+ * LZ4F_customMem is provided at state creation time, using LZ4F_create*_advanced() listed below.
+ * All allocation/free operations will be completed using these custom variants instead of regular <stdlib.h> ones.
+ */
+typedef void* (*LZ4F_AllocFunction) (void* opaqueState, size_t size);
+typedef void* (*LZ4F_CallocFunction) (void* opaqueState, size_t size);
+typedef void (*LZ4F_FreeFunction) (void* opaqueState, void* address);
+typedef struct {
+ LZ4F_AllocFunction customAlloc;
+ LZ4F_CallocFunction customCalloc; /* optional; when not defined, uses customAlloc + memset */
+ LZ4F_FreeFunction customFree;
+ void* opaqueState;
+} LZ4F_CustomMem;
+static
+#ifdef __GNUC__
+__attribute__((__unused__))
+#endif
+LZ4F_CustomMem const LZ4F_defaultCMem = { NULL, NULL, NULL, NULL }; /**< this constant defers to stdlib's functions */
+
+LZ4FLIB_STATIC_API LZ4F_cctx* LZ4F_createCompressionContext_advanced(LZ4F_CustomMem customMem, unsigned version);
+LZ4FLIB_STATIC_API LZ4F_dctx* LZ4F_createDecompressionContext_advanced(LZ4F_CustomMem customMem, unsigned version);
+LZ4FLIB_STATIC_API LZ4F_CDict* LZ4F_createCDict_advanced(LZ4F_CustomMem customMem, const void* dictBuffer, size_t dictSize);
+
#if defined (__cplusplus)
}
diff --git a/lib/lz4frame_static.h b/lib/lz4frame_static.h
index 925a2c5..2b44a63 100644
--- a/lib/lz4frame_static.h
+++ b/lib/lz4frame_static.h
@@ -1,7 +1,7 @@
/*
LZ4 auto-framing library
Header File for static linking only
- Copyright (C) 2011-2016, Yann Collet.
+ Copyright (C) 2011-2020, Yann Collet.
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 77c9f43..b21ad6b 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -1,6 +1,6 @@
/*
LZ4 HC - High Compression Mode of LZ4
- Copyright (C) 2011-2017, Yann Collet.
+ Copyright (C) 2011-2020, Yann Collet.
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
@@ -42,7 +42,7 @@
* Select how default compression function will allocate workplace memory,
* in stack (0:fastest), or in heap (1:requires malloc()).
* Since workplace is rather large, heap mode is recommended.
- */
+**/
#ifndef LZ4HC_HEAPMODE
# define LZ4HC_HEAPMODE 1
#endif
@@ -99,18 +99,20 @@
static void LZ4HC_init_internal (LZ4HC_CCtx_internal* hc4, const BYTE* start)
{
- uptrval startingOffset = (uptrval)(hc4->end - hc4->base);
- if (startingOffset > 1 GB) {
+ size_t const bufferSize = (size_t)(hc4->end - hc4->prefixStart);
+ size_t newStartingOffset = bufferSize + hc4->dictLimit;
+ assert(newStartingOffset >= bufferSize); /* check overflow */
+ if (newStartingOffset > 1 GB) {
LZ4HC_clearTables(hc4);
- startingOffset = 0;
+ newStartingOffset = 0;
}
- startingOffset += 64 KB;
- hc4->nextToUpdate = (U32) startingOffset;
- hc4->base = start - startingOffset;
+ newStartingOffset += 64 KB;
+ hc4->nextToUpdate = (U32)newStartingOffset;
+ hc4->prefixStart = start;
hc4->end = start;
- hc4->dictBase = start - startingOffset;
- hc4->dictLimit = (U32) startingOffset;
- hc4->lowLimit = (U32) startingOffset;
+ hc4->dictStart = start;
+ hc4->dictLimit = (U32)newStartingOffset;
+ hc4->lowLimit = (U32)newStartingOffset;
}
@@ -119,12 +121,15 @@
{
U16* const chainTable = hc4->chainTable;
U32* const hashTable = hc4->hashTable;
- const BYTE* const base = hc4->base;
- U32 const target = (U32)(ip - base);
+ const BYTE* const prefixPtr = hc4->prefixStart;
+ U32 const prefixIdx = hc4->dictLimit;
+ U32 const target = (U32)(ip - prefixPtr) + prefixIdx;
U32 idx = hc4->nextToUpdate;
+ assert(ip >= prefixPtr);
+ assert(target >= prefixIdx);
while (idx < target) {
- U32 const h = LZ4HC_hashPtr(base+idx);
+ U32 const h = LZ4HC_hashPtr(prefixPtr+idx-prefixIdx);
size_t delta = idx - hashTable[h];
if (delta>LZ4_DISTANCE_MAX) delta = LZ4_DISTANCE_MAX;
DELTANEXTU16(chainTable, idx) = (U16)delta;
@@ -193,15 +198,14 @@
BYTE const byte = (BYTE)(pattern >> bitOffset);
if (*ip != byte) break;
ip ++; bitOffset -= 8;
- }
- }
+ } }
return (unsigned)(ip - iStart);
}
/* LZ4HC_reverseCountPattern() :
* pattern must be a sample of repetitive pattern of length 1, 2 or 4 (but not 3!)
- * read using natural platform endianess */
+ * read using natural platform endianness */
static unsigned
LZ4HC_reverseCountPattern(const BYTE* ip, const BYTE* const iLow, U32 pattern)
{
@@ -211,7 +215,7 @@
if (LZ4_read32(ip-4) != pattern) break;
ip -= 4;
}
- { const BYTE* bytePtr = (const BYTE*)(&pattern) + 3; /* works for any endianess */
+ { const BYTE* bytePtr = (const BYTE*)(&pattern) + 3; /* works for any endianness */
while (likely(ip>iLow)) {
if (ip[-1] != *bytePtr) break;
ip--; bytePtr--;
@@ -234,28 +238,28 @@
LZ4_FORCE_INLINE int
LZ4HC_InsertAndGetWiderMatch (
- LZ4HC_CCtx_internal* hc4,
- const BYTE* const ip,
- const BYTE* const iLowLimit,
- const BYTE* const iHighLimit,
- int longest,
- const BYTE** matchpos,
- const BYTE** startpos,
- const int maxNbAttempts,
- const int patternAnalysis,
- const int chainSwap,
- const dictCtx_directive dict,
- const HCfavor_e favorDecSpeed)
+ LZ4HC_CCtx_internal* const hc4,
+ const BYTE* const ip,
+ const BYTE* const iLowLimit, const BYTE* const iHighLimit,
+ int longest,
+ const BYTE** matchpos,
+ const BYTE** startpos,
+ const int maxNbAttempts,
+ const int patternAnalysis, const int chainSwap,
+ const dictCtx_directive dict,
+ const HCfavor_e favorDecSpeed)
{
U16* const chainTable = hc4->chainTable;
U32* const HashTable = hc4->hashTable;
const LZ4HC_CCtx_internal * const dictCtx = hc4->dictCtx;
- const BYTE* const base = hc4->base;
- const U32 dictLimit = hc4->dictLimit;
- const BYTE* const lowPrefixPtr = base + dictLimit;
- const U32 ipIndex = (U32)(ip - base);
- const U32 lowestMatchIndex = (hc4->lowLimit + (LZ4_DISTANCE_MAX + 1) > ipIndex) ? hc4->lowLimit : ipIndex - LZ4_DISTANCE_MAX;
- const BYTE* const dictBase = hc4->dictBase;
+ const BYTE* const prefixPtr = hc4->prefixStart;
+ const U32 prefixIdx = hc4->dictLimit;
+ const U32 ipIndex = (U32)(ip - prefixPtr) + prefixIdx;
+ const int withinStartDistance = (hc4->lowLimit + (LZ4_DISTANCE_MAX + 1) > ipIndex);
+ const U32 lowestMatchIndex = (withinStartDistance) ? hc4->lowLimit : ipIndex - LZ4_DISTANCE_MAX;
+ const BYTE* const dictStart = hc4->dictStart;
+ const U32 dictIdx = hc4->lowLimit;
+ const BYTE* const dictEnd = dictStart + prefixIdx - dictIdx;
int const lookBackLength = (int)(ip-iLowLimit);
int nbAttempts = maxNbAttempts;
U32 matchChainPos = 0;
@@ -277,14 +281,13 @@
assert(matchIndex < ipIndex);
if (favorDecSpeed && (ipIndex - matchIndex < 8)) {
/* do nothing */
- } else if (matchIndex >= dictLimit) { /* within current Prefix */
- const BYTE* const matchPtr = base + matchIndex;
- assert(matchPtr >= lowPrefixPtr);
+ } else if (matchIndex >= prefixIdx) { /* within current Prefix */
+ const BYTE* const matchPtr = prefixPtr + matchIndex - prefixIdx;
assert(matchPtr < ip);
assert(longest >= 1);
if (LZ4_read16(iLowLimit + longest - 1) == LZ4_read16(matchPtr - lookBackLength + longest - 1)) {
if (LZ4_read32(matchPtr) == pattern) {
- int const back = lookBackLength ? LZ4HC_countBack(ip, matchPtr, iLowLimit, lowPrefixPtr) : 0;
+ int const back = lookBackLength ? LZ4HC_countBack(ip, matchPtr, iLowLimit, prefixPtr) : 0;
matchLength = MINMATCH + (int)LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, iHighLimit);
matchLength -= back;
if (matchLength > longest) {
@@ -293,24 +296,25 @@
*startpos = ip + back;
} } }
} else { /* lowestMatchIndex <= matchIndex < dictLimit */
- const BYTE* const matchPtr = dictBase + matchIndex;
- if (LZ4_read32(matchPtr) == pattern) {
- const BYTE* const dictStart = dictBase + hc4->lowLimit;
+ const BYTE* const matchPtr = dictStart + (matchIndex - dictIdx);
+ assert(matchIndex >= dictIdx);
+ if ( likely(matchIndex <= prefixIdx - 4)
+ && (LZ4_read32(matchPtr) == pattern) ) {
int back = 0;
- const BYTE* vLimit = ip + (dictLimit - matchIndex);
+ const BYTE* vLimit = ip + (prefixIdx - matchIndex);
if (vLimit > iHighLimit) vLimit = iHighLimit;
matchLength = (int)LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, vLimit) + MINMATCH;
if ((ip+matchLength == vLimit) && (vLimit < iHighLimit))
- matchLength += LZ4_count(ip+matchLength, lowPrefixPtr, iHighLimit);
+ matchLength += LZ4_count(ip+matchLength, prefixPtr, iHighLimit);
back = lookBackLength ? LZ4HC_countBack(ip, matchPtr, iLowLimit, dictStart) : 0;
matchLength -= back;
if (matchLength > longest) {
longest = matchLength;
- *matchpos = base + matchIndex + back; /* virtual pos, relative to ip, to retrieve offset */
+ *matchpos = prefixPtr - prefixIdx + matchIndex + back; /* virtual pos, relative to ip, to retrieve offset */
*startpos = ip + back;
} } }
- if (chainSwap && matchLength==longest) { /* better match => select a better chain */
+ if (chainSwap && matchLength==longest) { /* better match => select a better chain */
assert(lookBackLength==0); /* search forward only */
if (matchIndex + (U32)longest <= ipIndex) {
int const kTrigger = 4;
@@ -326,8 +330,7 @@
distanceToNextMatch = candidateDist;
matchChainPos = (U32)pos;
accel = 1 << kTrigger;
- }
- }
+ } }
if (distanceToNextMatch > 1) {
if (distanceToNextMatch > matchIndex) break; /* avoid overflow */
matchIndex -= distanceToNextMatch;
@@ -347,23 +350,24 @@
repeat = rep_not;
} }
if ( (repeat == rep_confirmed) && (matchCandidateIdx >= lowestMatchIndex)
- && LZ4HC_protectDictEnd(dictLimit, matchCandidateIdx) ) {
- const int extDict = matchCandidateIdx < dictLimit;
- const BYTE* const matchPtr = (extDict ? dictBase : base) + matchCandidateIdx;
+ && LZ4HC_protectDictEnd(prefixIdx, matchCandidateIdx) ) {
+ const int extDict = matchCandidateIdx < prefixIdx;
+ const BYTE* const matchPtr = (extDict ? dictStart - dictIdx : prefixPtr - prefixIdx) + matchCandidateIdx;
if (LZ4_read32(matchPtr) == pattern) { /* good candidate */
- const BYTE* const dictStart = dictBase + hc4->lowLimit;
- const BYTE* const iLimit = extDict ? dictBase + dictLimit : iHighLimit;
+ const BYTE* const iLimit = extDict ? dictEnd : iHighLimit;
size_t forwardPatternLength = LZ4HC_countPattern(matchPtr+sizeof(pattern), iLimit, pattern) + sizeof(pattern);
if (extDict && matchPtr + forwardPatternLength == iLimit) {
U32 const rotatedPattern = LZ4HC_rotatePattern(forwardPatternLength, pattern);
- forwardPatternLength += LZ4HC_countPattern(lowPrefixPtr, iHighLimit, rotatedPattern);
+ forwardPatternLength += LZ4HC_countPattern(prefixPtr, iHighLimit, rotatedPattern);
}
- { const BYTE* const lowestMatchPtr = extDict ? dictStart : lowPrefixPtr;
+ { const BYTE* const lowestMatchPtr = extDict ? dictStart : prefixPtr;
size_t backLength = LZ4HC_reverseCountPattern(matchPtr, lowestMatchPtr, pattern);
size_t currentSegmentLength;
- if (!extDict && matchPtr - backLength == lowPrefixPtr && hc4->lowLimit < dictLimit) {
+ if (!extDict
+ && matchPtr - backLength == prefixPtr
+ && dictIdx < prefixIdx) {
U32 const rotatedPattern = LZ4HC_rotatePattern((U32)(-(int)backLength), pattern);
- backLength += LZ4HC_reverseCountPattern(dictBase + dictLimit, dictStart, rotatedPattern);
+ backLength += LZ4HC_reverseCountPattern(dictEnd, dictStart, rotatedPattern);
}
/* Limit backLength not go further than lowestMatchIndex */
backLength = matchCandidateIdx - MAX(matchCandidateIdx - (U32)backLength, lowestMatchIndex);
@@ -373,28 +377,28 @@
if ( (currentSegmentLength >= srcPatternLength) /* current pattern segment large enough to contain full srcPatternLength */
&& (forwardPatternLength <= srcPatternLength) ) { /* haven't reached this position yet */
U32 const newMatchIndex = matchCandidateIdx + (U32)forwardPatternLength - (U32)srcPatternLength; /* best position, full pattern, might be followed by more match */
- if (LZ4HC_protectDictEnd(dictLimit, newMatchIndex))
+ if (LZ4HC_protectDictEnd(prefixIdx, newMatchIndex))
matchIndex = newMatchIndex;
else {
/* Can only happen if started in the prefix */
- assert(newMatchIndex >= dictLimit - 3 && newMatchIndex < dictLimit && !extDict);
- matchIndex = dictLimit;
+ assert(newMatchIndex >= prefixIdx - 3 && newMatchIndex < prefixIdx && !extDict);
+ matchIndex = prefixIdx;
}
} else {
U32 const newMatchIndex = matchCandidateIdx - (U32)backLength; /* farthest position in current segment, will find a match of length currentSegmentLength + maybe some back */
- if (!LZ4HC_protectDictEnd(dictLimit, newMatchIndex)) {
- assert(newMatchIndex >= dictLimit - 3 && newMatchIndex < dictLimit && !extDict);
- matchIndex = dictLimit;
+ if (!LZ4HC_protectDictEnd(prefixIdx, newMatchIndex)) {
+ assert(newMatchIndex >= prefixIdx - 3 && newMatchIndex < prefixIdx && !extDict);
+ matchIndex = prefixIdx;
} else {
matchIndex = newMatchIndex;
if (lookBackLength==0) { /* no back possible */
size_t const maxML = MIN(currentSegmentLength, srcPatternLength);
if ((size_t)longest < maxML) {
- assert(base + matchIndex != ip);
- if ((size_t)(ip - base) - matchIndex > LZ4_DISTANCE_MAX) break;
+ assert(prefixPtr - prefixIdx + matchIndex != ip);
+ if ((size_t)(ip - prefixPtr) + prefixIdx - matchIndex > LZ4_DISTANCE_MAX) break;
assert(maxML < 2 GB);
longest = (int)maxML;
- *matchpos = base + matchIndex; /* virtual pos, relative to ip, to retrieve offset */
+ *matchpos = prefixPtr - prefixIdx + matchIndex; /* virtual pos, relative to ip, to retrieve offset */
*startpos = ip;
}
{ U32 const distToNextPattern = DELTANEXTU16(chainTable, matchIndex);
@@ -413,12 +417,12 @@
if ( dict == usingDictCtxHc
&& nbAttempts > 0
&& ipIndex - lowestMatchIndex < LZ4_DISTANCE_MAX) {
- size_t const dictEndOffset = (size_t)(dictCtx->end - dictCtx->base);
+ size_t const dictEndOffset = (size_t)(dictCtx->end - dictCtx->prefixStart) + dictCtx->dictLimit;
U32 dictMatchIndex = dictCtx->hashTable[LZ4HC_hashPtr(ip)];
assert(dictEndOffset <= 1 GB);
matchIndex = dictMatchIndex + lowestMatchIndex - (U32)dictEndOffset;
while (ipIndex - matchIndex <= LZ4_DISTANCE_MAX && nbAttempts--) {
- const BYTE* const matchPtr = dictCtx->base + dictMatchIndex;
+ const BYTE* const matchPtr = dictCtx->prefixStart - dictCtx->dictLimit + dictMatchIndex;
if (LZ4_read32(matchPtr) == pattern) {
int mlt;
@@ -426,11 +430,11 @@
const BYTE* vLimit = ip + (dictEndOffset - dictMatchIndex);
if (vLimit > iHighLimit) vLimit = iHighLimit;
mlt = (int)LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, vLimit) + MINMATCH;
- back = lookBackLength ? LZ4HC_countBack(ip, matchPtr, iLowLimit, dictCtx->base + dictCtx->dictLimit) : 0;
+ back = lookBackLength ? LZ4HC_countBack(ip, matchPtr, iLowLimit, dictCtx->prefixStart) : 0;
mlt -= back;
if (mlt > longest) {
longest = mlt;
- *matchpos = base + matchIndex + back;
+ *matchpos = prefixPtr - prefixIdx + matchIndex + back;
*startpos = ip + back;
} }
@@ -442,13 +446,13 @@
return longest;
}
-LZ4_FORCE_INLINE
-int LZ4HC_InsertAndFindBestMatch(LZ4HC_CCtx_internal* const hc4, /* Index table will be updated */
- const BYTE* const ip, const BYTE* const iLimit,
- const BYTE** matchpos,
- const int maxNbAttempts,
- const int patternAnalysis,
- const dictCtx_directive dict)
+LZ4_FORCE_INLINE int
+LZ4HC_InsertAndFindBestMatch(LZ4HC_CCtx_internal* const hc4, /* Index table will be updated */
+ const BYTE* const ip, const BYTE* const iLimit,
+ const BYTE** matchpos,
+ const int maxNbAttempts,
+ const int patternAnalysis,
+ const dictCtx_directive dict)
{
const BYTE* uselessPtr = ip;
/* note : LZ4HC_InsertAndGetWiderMatch() is able to modify the starting position of a match (*startpos),
@@ -751,7 +755,7 @@
} else {
*op++ = (BYTE)(lastRunSize << ML_BITS);
}
- memcpy(op, anchor, lastRunSize);
+ LZ4_memcpy(op, anchor, lastRunSize);
op += lastRunSize;
}
@@ -884,13 +888,13 @@
limitedOutput_directive limit
)
{
- const size_t position = (size_t)(ctx->end - ctx->base) - ctx->lowLimit;
+ const size_t position = (size_t)(ctx->end - ctx->prefixStart) + (ctx->dictLimit - ctx->lowLimit);
assert(ctx->dictCtx != NULL);
if (position >= 64 KB) {
ctx->dictCtx = NULL;
return LZ4HC_compress_generic_noDictCtx(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit);
} else if (position == 0 && *srcSizePtr > 4 KB) {
- memcpy(ctx, ctx->dictCtx, sizeof(LZ4HC_CCtx_internal));
+ LZ4_memcpy(ctx, ctx->dictCtx, sizeof(LZ4HC_CCtx_internal));
LZ4HC_setExternalDict(ctx, (const BYTE *)src);
ctx->compressionLevel = (short)cLevel;
return LZ4HC_compress_generic_noDictCtx(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit);
@@ -953,13 +957,15 @@
int LZ4_compress_HC(const char* src, char* dst, int srcSize, int dstCapacity, int compressionLevel)
{
+ int cSize;
#if defined(LZ4HC_HEAPMODE) && LZ4HC_HEAPMODE==1
LZ4_streamHC_t* const statePtr = (LZ4_streamHC_t*)ALLOC(sizeof(LZ4_streamHC_t));
+ if (statePtr==NULL) return 0;
#else
LZ4_streamHC_t state;
LZ4_streamHC_t* const statePtr = &state;
#endif
- int const cSize = LZ4_compress_HC_extStateHC(statePtr, src, dst, srcSize, dstCapacity, compressionLevel);
+ cSize = LZ4_compress_HC_extStateHC(statePtr, src, dst, srcSize, dstCapacity, compressionLevel);
#if defined(LZ4HC_HEAPMODE) && LZ4HC_HEAPMODE==1
FREEMEM(statePtr);
#endif
@@ -982,6 +988,7 @@
* Streaming Functions
**************************************/
/* allocation */
+#if !defined(LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION)
LZ4_streamHC_t* LZ4_createStreamHC(void)
{
LZ4_streamHC_t* const state =
@@ -998,13 +1005,12 @@
FREEMEM(LZ4_streamHCPtr);
return 0;
}
+#endif
LZ4_streamHC_t* LZ4_initStreamHC (void* buffer, size_t size)
{
LZ4_streamHC_t* const LZ4_streamHCPtr = (LZ4_streamHC_t*)buffer;
- /* if compilation fails here, LZ4_STREAMHCSIZE must be increased */
- LZ4_STATIC_ASSERT(sizeof(LZ4HC_CCtx_internal) <= LZ4_STREAMHCSIZE);
DEBUGLOG(4, "LZ4_initStreamHC(%p, %u)", buffer, (unsigned)size);
/* check conditions */
if (buffer == NULL) return NULL;
@@ -1030,9 +1036,13 @@
if (LZ4_streamHCPtr->internal_donotuse.dirty) {
LZ4_initStreamHC(LZ4_streamHCPtr, sizeof(*LZ4_streamHCPtr));
} else {
- /* preserve end - base : can trigger clearTable's threshold */
- LZ4_streamHCPtr->internal_donotuse.end -= (uptrval)LZ4_streamHCPtr->internal_donotuse.base;
- LZ4_streamHCPtr->internal_donotuse.base = NULL;
+ /* preserve end - prefixStart : can trigger clearTable's threshold */
+ if (LZ4_streamHCPtr->internal_donotuse.end != NULL) {
+ LZ4_streamHCPtr->internal_donotuse.end -= (uptrval)LZ4_streamHCPtr->internal_donotuse.prefixStart;
+ } else {
+ assert(LZ4_streamHCPtr->internal_donotuse.prefixStart == NULL);
+ }
+ LZ4_streamHCPtr->internal_donotuse.prefixStart = NULL;
LZ4_streamHCPtr->internal_donotuse.dictCtx = NULL;
}
LZ4_setCompressionLevel(LZ4_streamHCPtr, compressionLevel);
@@ -1083,14 +1093,14 @@
static void LZ4HC_setExternalDict(LZ4HC_CCtx_internal* ctxPtr, const BYTE* newBlock)
{
DEBUGLOG(4, "LZ4HC_setExternalDict(%p, %p)", ctxPtr, newBlock);
- if (ctxPtr->end >= ctxPtr->base + ctxPtr->dictLimit + 4)
+ if (ctxPtr->end >= ctxPtr->prefixStart + 4)
LZ4HC_Insert (ctxPtr, ctxPtr->end-3); /* Referencing remaining dictionary content */
/* Only one memory segment for extDict, so any previous extDict is lost at this stage */
ctxPtr->lowLimit = ctxPtr->dictLimit;
- ctxPtr->dictLimit = (U32)(ctxPtr->end - ctxPtr->base);
- ctxPtr->dictBase = ctxPtr->base;
- ctxPtr->base = newBlock - ctxPtr->dictLimit;
+ ctxPtr->dictStart = ctxPtr->prefixStart;
+ ctxPtr->dictLimit += (U32)(ctxPtr->end - ctxPtr->prefixStart);
+ ctxPtr->prefixStart = newBlock;
ctxPtr->end = newBlock;
ctxPtr->nextToUpdate = ctxPtr->dictLimit; /* match referencing will resume from there */
@@ -1109,11 +1119,11 @@
LZ4_streamHCPtr, src, *srcSizePtr, limit);
assert(ctxPtr != NULL);
/* auto-init if forgotten */
- if (ctxPtr->base == NULL) LZ4HC_init_internal (ctxPtr, (const BYTE*) src);
+ if (ctxPtr->prefixStart == NULL) LZ4HC_init_internal (ctxPtr, (const BYTE*) src);
/* Check overflow */
- if ((size_t)(ctxPtr->end - ctxPtr->base) > 2 GB) {
- size_t dictSize = (size_t)(ctxPtr->end - ctxPtr->base) - ctxPtr->dictLimit;
+ if ((size_t)(ctxPtr->end - ctxPtr->prefixStart) + ctxPtr->dictLimit > 2 GB) {
+ size_t dictSize = (size_t)(ctxPtr->end - ctxPtr->prefixStart);
if (dictSize > 64 KB) dictSize = 64 KB;
LZ4_loadDictHC(LZ4_streamHCPtr, (const char*)(ctxPtr->end) - dictSize, (int)dictSize);
}
@@ -1124,13 +1134,16 @@
/* Check overlapping input/dictionary space */
{ const BYTE* sourceEnd = (const BYTE*) src + *srcSizePtr;
- const BYTE* const dictBegin = ctxPtr->dictBase + ctxPtr->lowLimit;
- const BYTE* const dictEnd = ctxPtr->dictBase + ctxPtr->dictLimit;
+ const BYTE* const dictBegin = ctxPtr->dictStart;
+ const BYTE* const dictEnd = ctxPtr->dictStart + (ctxPtr->dictLimit - ctxPtr->lowLimit);
if ((sourceEnd > dictBegin) && ((const BYTE*)src < dictEnd)) {
if (sourceEnd > dictEnd) sourceEnd = dictEnd;
- ctxPtr->lowLimit = (U32)(sourceEnd - ctxPtr->dictBase);
- if (ctxPtr->dictLimit - ctxPtr->lowLimit < 4) ctxPtr->lowLimit = ctxPtr->dictLimit;
- } }
+ ctxPtr->lowLimit += (U32)(sourceEnd - ctxPtr->dictStart);
+ ctxPtr->dictStart += (U32)(sourceEnd - ctxPtr->dictStart);
+ if (ctxPtr->dictLimit - ctxPtr->lowLimit < 4) {
+ ctxPtr->lowLimit = ctxPtr->dictLimit;
+ ctxPtr->dictStart = ctxPtr->prefixStart;
+ } } }
return LZ4HC_compress_generic (ctxPtr, src, dst, srcSizePtr, dstCapacity, ctxPtr->compressionLevel, limit);
}
@@ -1158,7 +1171,7 @@
int LZ4_saveDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, char* safeBuffer, int dictSize)
{
LZ4HC_CCtx_internal* const streamPtr = &LZ4_streamHCPtr->internal_donotuse;
- int const prefixSize = (int)(streamPtr->end - (streamPtr->base + streamPtr->dictLimit));
+ int const prefixSize = (int)(streamPtr->end - streamPtr->prefixStart);
DEBUGLOG(5, "LZ4_saveDictHC(%p, %p, %d)", LZ4_streamHCPtr, safeBuffer, dictSize);
assert(prefixSize >= 0);
if (dictSize > 64 KB) dictSize = 64 KB;
@@ -1166,12 +1179,13 @@
if (dictSize > prefixSize) dictSize = prefixSize;
if (safeBuffer == NULL) assert(dictSize == 0);
if (dictSize > 0)
- memmove(safeBuffer, streamPtr->end - dictSize, dictSize);
- { U32 const endIndex = (U32)(streamPtr->end - streamPtr->base);
+ LZ4_memmove(safeBuffer, streamPtr->end - dictSize, dictSize);
+ { U32 const endIndex = (U32)(streamPtr->end - streamPtr->prefixStart) + streamPtr->dictLimit;
streamPtr->end = (const BYTE*)safeBuffer + dictSize;
- streamPtr->base = streamPtr->end - endIndex;
+ streamPtr->prefixStart = streamPtr->end - dictSize;
streamPtr->dictLimit = endIndex - (U32)dictSize;
streamPtr->lowLimit = endIndex - (U32)dictSize;
+ streamPtr->dictStart = streamPtr->prefixStart;
if (streamPtr->nextToUpdate < streamPtr->dictLimit)
streamPtr->nextToUpdate = streamPtr->dictLimit;
}
@@ -1199,7 +1213,7 @@
/* Deprecated streaming functions */
-int LZ4_sizeofStreamStateHC(void) { return LZ4_STREAMHCSIZE; }
+int LZ4_sizeofStreamStateHC(void) { return sizeof(LZ4_streamHC_t); }
/* state is presumed correctly sized, aka >= sizeof(LZ4_streamHC_t)
* @return : 0 on success, !=0 if error */
@@ -1211,6 +1225,7 @@
return 0;
}
+#if !defined(LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION)
void* LZ4_createHC (const char* inputBuffer)
{
LZ4_streamHC_t* const hc4 = LZ4_createStreamHC();
@@ -1225,6 +1240,7 @@
FREEMEM(LZ4HC_Data);
return 0;
}
+#endif
int LZ4_compressHC2_continue (void* LZ4HC_Data, const char* src, char* dst, int srcSize, int cLevel)
{
@@ -1238,11 +1254,11 @@
char* LZ4_slideInputBufferHC(void* LZ4HC_Data)
{
- LZ4_streamHC_t *ctx = (LZ4_streamHC_t*)LZ4HC_Data;
- const BYTE *bufferStart = ctx->internal_donotuse.base + ctx->internal_donotuse.lowLimit;
+ LZ4_streamHC_t* const ctx = (LZ4_streamHC_t*)LZ4HC_Data;
+ const BYTE* bufferStart = ctx->internal_donotuse.prefixStart - ctx->internal_donotuse.dictLimit + ctx->internal_donotuse.lowLimit;
LZ4_resetStreamHC_fast(ctx, ctx->internal_donotuse.compressionLevel);
/* avoid const char * -> char * conversion warning :( */
- return (char *)(uptrval)bufferStart;
+ return (char*)(uptrval)bufferStart;
}
@@ -1325,7 +1341,7 @@
{
int retval = 0;
#define TRAILING_LITERALS 3
-#ifdef LZ4HC_HEAPMODE
+#if defined(LZ4HC_HEAPMODE) && LZ4HC_HEAPMODE==1
LZ4HC_optimal_t* const opt = (LZ4HC_optimal_t*)ALLOC(sizeof(LZ4HC_optimal_t) * (LZ4_OPT_NUM + TRAILING_LITERALS));
#else
LZ4HC_optimal_t opt[LZ4_OPT_NUM + TRAILING_LITERALS]; /* ~64 KB, which is a bit large for stack... */
@@ -1343,7 +1359,7 @@
const BYTE* ovref = NULL;
/* init */
-#ifdef LZ4HC_HEAPMODE
+#if defined(LZ4HC_HEAPMODE) && LZ4HC_HEAPMODE==1
if (opt == NULL) goto _return_label;
#endif
DEBUGLOG(5, "LZ4HC_compress_optimal(dst=%p, dstCapa=%u)", dst, (unsigned)dstCapacity);
@@ -1575,7 +1591,7 @@
} else {
*op++ = (BYTE)(lastRunSize << ML_BITS);
}
- memcpy(op, anchor, lastRunSize);
+ LZ4_memcpy(op, anchor, lastRunSize);
op += lastRunSize;
}
@@ -1608,7 +1624,7 @@
goto _last_literals;
}
_return_label:
-#ifdef LZ4HC_HEAPMODE
+#if defined(LZ4HC_HEAPMODE) && LZ4HC_HEAPMODE==1
FREEMEM(opt);
#endif
return retval;
diff --git a/lib/lz4hc.h b/lib/lz4hc.h
index 3d441fb..e937acf 100644
--- a/lib/lz4hc.h
+++ b/lib/lz4hc.h
@@ -1,7 +1,7 @@
/*
LZ4 HC - High Compression Mode of LZ4
Header File
- Copyright (C) 2011-2017, Yann Collet.
+ Copyright (C) 2011-2020, Yann Collet.
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
Redistribution and use in source and binary forms, with or without
@@ -198,14 +198,17 @@
#define LZ4HC_HASH_MASK (LZ4HC_HASHTABLESIZE - 1)
+/* Never ever use these definitions directly !
+ * Declare or allocate an LZ4_streamHC_t instead.
+**/
typedef struct LZ4HC_CCtx_internal LZ4HC_CCtx_internal;
struct LZ4HC_CCtx_internal
{
LZ4_u32 hashTable[LZ4HC_HASHTABLESIZE];
LZ4_u16 chainTable[LZ4HC_MAXD];
const LZ4_byte* end; /* next block here to continue on current prefix */
- const LZ4_byte* base; /* All index relative to this position */
- const LZ4_byte* dictBase; /* alternate base for extDict */
+ const LZ4_byte* prefixStart; /* Indexes relative to this position */
+ const LZ4_byte* dictStart; /* alternate reference for extDict */
LZ4_u32 dictLimit; /* below that point, need extDict */
LZ4_u32 lowLimit; /* below that point, no more dict */
LZ4_u32 nextToUpdate; /* index from which to continue dictionary update */
@@ -216,20 +219,15 @@
const LZ4HC_CCtx_internal* dictCtx;
};
-
-/* Do not use these definitions directly !
- * Declare or allocate an LZ4_streamHC_t instead.
- */
-#define LZ4_STREAMHCSIZE 262200 /* static size, for inter-version compatibility */
-#define LZ4_STREAMHCSIZE_VOIDP (LZ4_STREAMHCSIZE / sizeof(void*))
+#define LZ4_STREAMHC_MINSIZE 262200 /* static size, for inter-version compatibility */
union LZ4_streamHC_u {
- void* table[LZ4_STREAMHCSIZE_VOIDP];
+ char minStateSize[LZ4_STREAMHC_MINSIZE];
LZ4HC_CCtx_internal internal_donotuse;
}; /* previously typedef'd to LZ4_streamHC_t */
/* LZ4_streamHC_t :
* This structure allows static allocation of LZ4 HC streaming state.
- * This can be used to allocate statically, on state, or as part of a larger structure.
+ * This can be used to allocate statically on stack, or as part of a larger structure.
*
* Such state **must** be initialized using LZ4_initStreamHC() before first use.
*
@@ -244,7 +242,7 @@
* Required before first use of a statically allocated LZ4_streamHC_t.
* Before v1.9.0 : use LZ4_resetStreamHC() instead
*/
-LZ4LIB_API LZ4_streamHC_t* LZ4_initStreamHC (void* buffer, size_t size);
+LZ4LIB_API LZ4_streamHC_t* LZ4_initStreamHC(void* buffer, size_t size);
/*-************************************
@@ -272,9 +270,11 @@
* LZ4_slideInputBufferHC() will truncate the history of the stream, rather
* than preserve a window-sized chunk of history.
*/
+#if !defined(LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION)
LZ4_DEPRECATED("use LZ4_createStreamHC() instead") LZ4LIB_API void* LZ4_createHC (const char* inputBuffer);
-LZ4_DEPRECATED("use LZ4_saveDictHC() instead") LZ4LIB_API char* LZ4_slideInputBufferHC (void* LZ4HC_Data);
LZ4_DEPRECATED("use LZ4_freeStreamHC() instead") LZ4LIB_API int LZ4_freeHC (void* LZ4HC_Data);
+#endif
+LZ4_DEPRECATED("use LZ4_saveDictHC() instead") LZ4LIB_API char* LZ4_slideInputBufferHC (void* LZ4HC_Data);
LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") LZ4LIB_API int LZ4_compressHC2_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int compressionLevel);
LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") LZ4LIB_API int LZ4_compressHC2_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel);
LZ4_DEPRECATED("use LZ4_createStreamHC() instead") LZ4LIB_API int LZ4_sizeofStreamStateHC(void);
@@ -305,7 +305,7 @@
* They should not be linked from DLL,
* as there is no guarantee of API stability yet.
* Prototypes will be promoted to "stable" status
- * after successfull usage in real-life scenarios.
+ * after successful usage in real-life scenarios.
***************************************************/
#ifdef LZ4_HC_STATIC_LINKING_ONLY /* protection macro */
#ifndef LZ4_HC_SLO_098092834
diff --git a/ossfuzz/.gitignore b/ossfuzz/.gitignore
new file mode 100644
index 0000000..0ef0d2b
--- /dev/null
+++ b/ossfuzz/.gitignore
@@ -0,0 +1,8 @@
+
+# build artefacts
+round_trip_frame_uncompressed_fuzzer
+
+# test artefacts
+
+# local tests
+
diff --git a/ossfuzz/Makefile b/ossfuzz/Makefile
index 2ec1675..deb2938 100644
--- a/ossfuzz/Makefile
+++ b/ossfuzz/Makefile
@@ -45,6 +45,7 @@
round_trip_hc_fuzzer \
compress_frame_fuzzer \
round_trip_frame_fuzzer \
+ round_trip_frame_uncompressed_fuzzer \
decompress_frame_fuzzer
.PHONY: all
diff --git a/ossfuzz/decompress_fuzzer.c b/ossfuzz/decompress_fuzzer.c
index 6f48e30..490b3fd 100644
--- a/ossfuzz/decompress_fuzzer.c
+++ b/ossfuzz/decompress_fuzzer.c
@@ -39,7 +39,7 @@
/* No dictionary. */
LZ4_decompress_safe_usingDict((char const*)data, dst, size,
dstCapacity, NULL, 0);
- /* Small external dictonary. */
+ /* Small external dictionary. */
LZ4_decompress_safe_usingDict((char const*)data, dst, size,
dstCapacity, smallDict, smallDictSize);
/* Large external dictionary. */
@@ -49,11 +49,27 @@
LZ4_decompress_safe_usingDict((char const*)dataAfterDict, dst, size,
dstCapacity, smallDict, smallDictSize);
/* Large prefix. */
- LZ4_decompress_safe_usingDict((char const*)data, dst, size,
+ LZ4_decompress_safe_usingDict((char const*)dataAfterDict, dst, size,
dstCapacity, largeDict, largeDictSize);
/* Partial decompression. */
LZ4_decompress_safe_partial((char const*)data, dst, size,
dstCapacity, dstCapacity);
+ /* Partial decompression using each possible dictionary configuration. */
+ /* Partial decompression with no dictionary. */
+ LZ4_decompress_safe_partial_usingDict((char const*)data, dst, size,
+ dstCapacity, dstCapacity, NULL, 0);
+ /* Partial decompression with small external dictionary. */
+ LZ4_decompress_safe_partial_usingDict((char const*)data, dst, size,
+ dstCapacity, dstCapacity, smallDict, smallDictSize);
+ /* Partial decompression with large external dictionary. */
+ LZ4_decompress_safe_partial_usingDict((char const*)data, dst, size,
+ dstCapacity, dstCapacity, largeDict, largeDictSize);
+ /* Partial decompression with small prefix. */
+ LZ4_decompress_safe_partial_usingDict((char const*)dataAfterDict, dst, size,
+ dstCapacity, dstCapacity, smallDict, smallDictSize);
+ /* Partial decompression wtih large prefix. */
+ LZ4_decompress_safe_partial_usingDict((char const*)dataAfterDict, dst, size,
+ dstCapacity, dstCapacity, largeDict, largeDictSize);
free(dst);
free(dict);
FUZZ_dataProducer_free(producer);
diff --git a/ossfuzz/fuzz_helpers.h b/ossfuzz/fuzz_helpers.h
index c4a8645..efd9acf 100644
--- a/ossfuzz/fuzz_helpers.h
+++ b/ossfuzz/fuzz_helpers.h
@@ -4,7 +4,8 @@
*
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
- * in the COPYING file in the root directory of this source tree).
+ * in the COPYING file in the root directory of this source tree),
+ * meaning you may select, at your option, one of the above-listed licenses.
*/
/**
@@ -81,7 +82,7 @@
return rand32 >> 5;
}
-/* Returns a random numer in the range [min, max]. */
+/* Returns a random number in the range [min, max]. */
FUZZ_STATIC uint32_t FUZZ_rand32(uint32_t *state, uint32_t min, uint32_t max) {
uint32_t random = FUZZ_rand(state);
return min + (random % (max - min + 1));
diff --git a/ossfuzz/round_trip_frame_uncompressed_fuzzer.c b/ossfuzz/round_trip_frame_uncompressed_fuzzer.c
new file mode 100644
index 0000000..76a99d2
--- /dev/null
+++ b/ossfuzz/round_trip_frame_uncompressed_fuzzer.c
@@ -0,0 +1,134 @@
+/**
+ * This fuzz target performs a lz4 round-trip test (compress & decompress),
+ * compares the result with the original, and calls abort() on corruption.
+ */
+
+#include <stddef.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "fuzz_data_producer.h"
+#include "fuzz_helpers.h"
+#include "lz4.h"
+#include "lz4_helpers.h"
+#include "lz4frame.h"
+#include "lz4frame_static.h"
+
+static void decompress(LZ4F_dctx *dctx, void *src, void *dst,
+ size_t dstCapacity, size_t readSize) {
+ size_t ret = 1;
+ const void *srcPtr = (const char *) src;
+ void *dstPtr = (char *) dst;
+ const void *const srcEnd = (const char *) srcPtr + readSize;
+
+ while (ret != 0) {
+ while (srcPtr < srcEnd && ret != 0) {
+ /* Any data within dst has been flushed at this stage */
+ size_t dstSize = dstCapacity;
+ size_t srcSize = (const char *) srcEnd - (const char *) srcPtr;
+ ret = LZ4F_decompress(dctx, dstPtr, &dstSize, srcPtr, &srcSize,
+ /* LZ4F_decompressOptions_t */ NULL);
+ FUZZ_ASSERT(!LZ4F_isError(ret));
+
+ /* Update input */
+ srcPtr = (const char *) srcPtr + srcSize;
+ dstPtr = (char *) dstPtr + dstSize;
+ }
+
+ FUZZ_ASSERT(srcPtr <= srcEnd);
+ }
+}
+
+static void compress_round_trip(const uint8_t *data, size_t size,
+ FUZZ_dataProducer_t *producer, LZ4F_preferences_t const prefs) {
+
+ // Choose random uncompressed offset start and end by producing seeds from random data, calculate the remaining
+ // data size that will be used for compression later and use the seeds to actually calculate the offsets
+ size_t const uncompressedOffsetSeed = FUZZ_dataProducer_retrieve32(producer);
+ size_t const uncompressedEndOffsetSeed = FUZZ_dataProducer_retrieve32(producer);
+ size = FUZZ_dataProducer_remainingBytes(producer);
+
+ size_t const uncompressedOffset = FUZZ_getRange_from_uint32(uncompressedOffsetSeed, 0, size);
+ size_t const uncompressedEndOffset = FUZZ_getRange_from_uint32(uncompressedEndOffsetSeed, uncompressedOffset, size);
+ size_t const uncompressedSize = uncompressedEndOffset - uncompressedOffset;
+ FUZZ_ASSERT(uncompressedOffset <= uncompressedEndOffset);
+ FUZZ_ASSERT(uncompressedEndOffset <= size);
+
+ const uint8_t *const uncompressedData = data + uncompressedOffset;
+
+ size_t const dstCapacity =
+ LZ4F_compressFrameBound(LZ4_compressBound(size), &prefs) +
+ uncompressedSize;
+ char *const dst = (char *) malloc(dstCapacity);
+ size_t rtCapacity = dstCapacity;
+ char *const rt = (char *) malloc(rtCapacity);
+
+ FUZZ_ASSERT(dst);
+ FUZZ_ASSERT(rt);
+
+ /* Compression must succeed and round trip correctly. */
+ LZ4F_compressionContext_t ctx;
+ size_t const ctxCreation = LZ4F_createCompressionContext(&ctx, LZ4F_VERSION);
+ FUZZ_ASSERT(!LZ4F_isError(ctxCreation));
+
+ size_t const headerSize = LZ4F_compressBegin(ctx, dst, dstCapacity, &prefs);
+ FUZZ_ASSERT(!LZ4F_isError(headerSize));
+ size_t compressedSize = headerSize;
+
+ /* Compress data before uncompressed offset */
+ size_t lz4Return = LZ4F_compressUpdate(ctx, dst + compressedSize, dstCapacity,
+ data, uncompressedOffset, NULL);
+ FUZZ_ASSERT(!LZ4F_isError(lz4Return));
+ compressedSize += lz4Return;
+
+ /* Add uncompressed data */
+ lz4Return = LZ4F_uncompressedUpdate(ctx, dst + compressedSize, dstCapacity,
+ uncompressedData, uncompressedSize, NULL);
+ FUZZ_ASSERT(!LZ4F_isError(lz4Return));
+ compressedSize += lz4Return;
+
+ /* Compress data after uncompressed offset */
+ lz4Return = LZ4F_compressUpdate(ctx, dst + compressedSize, dstCapacity,
+ data + uncompressedEndOffset,
+ size - uncompressedEndOffset, NULL);
+ FUZZ_ASSERT(!LZ4F_isError(lz4Return));
+ compressedSize += lz4Return;
+
+ /* Finish compression */
+ lz4Return = LZ4F_compressEnd(ctx, dst + compressedSize, dstCapacity, NULL);
+ FUZZ_ASSERT(!LZ4F_isError(lz4Return));
+ compressedSize += lz4Return;
+
+ LZ4F_decompressOptions_t opts;
+ memset(&opts, 0, sizeof(opts));
+ opts.stableDst = 1;
+ LZ4F_dctx *dctx;
+ LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION);
+ FUZZ_ASSERT(dctx);
+
+ decompress(dctx, dst, rt, rtCapacity, compressedSize);
+
+ LZ4F_freeDecompressionContext(dctx);
+
+ FUZZ_ASSERT_MSG(!memcmp(data, rt, size), "Corruption!");
+
+ free(dst);
+ free(rt);
+
+ FUZZ_dataProducer_free(producer);
+ LZ4F_freeCompressionContext(ctx);
+}
+
+static void compress_independent_block_mode(const uint8_t *data, size_t size) {
+ FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(data, size);
+ LZ4F_preferences_t prefs = FUZZ_dataProducer_preferences(producer);
+ prefs.frameInfo.blockMode = LZ4F_blockIndependent;
+ compress_round_trip(data, size, producer, prefs);
+}
+
+
+int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
+ compress_independent_block_mode(data, size);
+ return 0;
+}
diff --git a/ossfuzz/round_trip_fuzzer.c b/ossfuzz/round_trip_fuzzer.c
index 6307058..6236201 100644
--- a/ossfuzz/round_trip_fuzzer.c
+++ b/ossfuzz/round_trip_fuzzer.c
@@ -20,11 +20,16 @@
size_t const partialCapacity = FUZZ_getRange_from_uint32(partialCapacitySeed, 0, size);
size_t const dstCapacity = LZ4_compressBound(size);
-
- char* const dst = (char*)malloc(dstCapacity);
+ size_t const largeSize = 64 * 1024 - 1;
+ size_t const smallSize = 1024;
+ char* const dstPlusLargePrefix = (char*)malloc(dstCapacity + largeSize);
+ FUZZ_ASSERT(dstPlusLargePrefix);
+ char* const dstPlusSmallPrefix = dstPlusLargePrefix + largeSize - smallSize;
+ char* const largeDict = (char*)malloc(largeSize);
+ FUZZ_ASSERT(largeDict);
+ char* const smallDict = largeDict + largeSize - smallSize;
+ char* const dst = dstPlusLargePrefix + largeSize;
char* const rt = (char*)malloc(size);
-
- FUZZ_ASSERT(dst);
FUZZ_ASSERT(rt);
/* Compression must succeed and round trip correctly. */
@@ -47,9 +52,64 @@
FUZZ_ASSERT_MSG(!memcmp(data, partial, partialSize), "Corruption!");
free(partial);
}
+ /* Partial decompression using dict with no dict. */
+ {
+ char* const partial = (char*)malloc(partialCapacity);
+ FUZZ_ASSERT(partial);
+ int const partialSize = LZ4_decompress_safe_partial_usingDict(
+ dst, partial, dstSize, partialCapacity, partialCapacity, NULL, 0);
+ FUZZ_ASSERT(partialSize >= 0);
+ FUZZ_ASSERT_MSG(partialSize == partialCapacity, "Incorrect size");
+ FUZZ_ASSERT_MSG(!memcmp(data, partial, partialSize), "Corruption!");
+ free(partial);
+ }
+ /* Partial decompression using dict with small prefix as dict */
+ {
+ char* const partial = (char*)malloc(partialCapacity);
+ FUZZ_ASSERT(partial);
+ int const partialSize = LZ4_decompress_safe_partial_usingDict(
+ dst, partial, dstSize, partialCapacity, partialCapacity, dstPlusSmallPrefix, smallSize);
+ FUZZ_ASSERT(partialSize >= 0);
+ FUZZ_ASSERT_MSG(partialSize == partialCapacity, "Incorrect size");
+ FUZZ_ASSERT_MSG(!memcmp(data, partial, partialSize), "Corruption!");
+ free(partial);
+ }
+ /* Partial decompression using dict with large prefix as dict */
+ {
+ char* const partial = (char*)malloc(partialCapacity);
+ FUZZ_ASSERT(partial);
+ int const partialSize = LZ4_decompress_safe_partial_usingDict(
+ dst, partial, dstSize, partialCapacity, partialCapacity, dstPlusLargePrefix, largeSize);
+ FUZZ_ASSERT(partialSize >= 0);
+ FUZZ_ASSERT_MSG(partialSize == partialCapacity, "Incorrect size");
+ FUZZ_ASSERT_MSG(!memcmp(data, partial, partialSize), "Corruption!");
+ free(partial);
+ }
+ /* Partial decompression using dict with small external dict */
+ {
+ char* const partial = (char*)malloc(partialCapacity);
+ FUZZ_ASSERT(partial);
+ int const partialSize = LZ4_decompress_safe_partial_usingDict(
+ dst, partial, dstSize, partialCapacity, partialCapacity, smallDict, smallSize);
+ FUZZ_ASSERT(partialSize >= 0);
+ FUZZ_ASSERT_MSG(partialSize == partialCapacity, "Incorrect size");
+ FUZZ_ASSERT_MSG(!memcmp(data, partial, partialSize), "Corruption!");
+ free(partial);
+ }
+ /* Partial decompression using dict with large external dict */
+ {
+ char* const partial = (char*)malloc(partialCapacity);
+ FUZZ_ASSERT(partial);
+ int const partialSize = LZ4_decompress_safe_partial_usingDict(
+ dst, partial, dstSize, partialCapacity, partialCapacity, largeDict, largeSize);
+ FUZZ_ASSERT(partialSize >= 0);
+ FUZZ_ASSERT_MSG(partialSize == partialCapacity, "Incorrect size");
+ FUZZ_ASSERT_MSG(!memcmp(data, partial, partialSize), "Corruption!");
+ free(partial);
+ }
-
- free(dst);
+ free(dstPlusLargePrefix);
+ free(largeDict);
free(rt);
FUZZ_dataProducer_free(producer);
diff --git a/programs/Makefile b/programs/Makefile
index c1053f6..ace0d03 100644
--- a/programs/Makefile
+++ b/programs/Makefile
@@ -1,6 +1,6 @@
# ##########################################################################
# LZ4 programs - Makefile
-# Copyright (C) Yann Collet 2011-2017
+# Copyright (C) Yann Collet 2011-2020
#
# This Makefile is validated for Linux, macOS, *BSD, Hurd, Solaris, MSYS2 targets
#
@@ -28,13 +28,14 @@
# lz4c : CLU, supporting also legacy lz4demo arguments
# lz4c32: Same as lz4c, but forced to compile in 32-bits mode
# ##########################################################################
+SED = sed
# Version numbers
LZ4DIR := ../lib
LIBVER_SRC := $(LZ4DIR)/lz4.h
-LIBVER_MAJOR_SCRIPT:=`sed -n '/define LZ4_VERSION_MAJOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < $(LIBVER_SRC)`
-LIBVER_MINOR_SCRIPT:=`sed -n '/define LZ4_VERSION_MINOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < $(LIBVER_SRC)`
-LIBVER_PATCH_SCRIPT:=`sed -n '/define LZ4_VERSION_RELEASE/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < $(LIBVER_SRC)`
+LIBVER_MAJOR_SCRIPT:=`$(SED) -n '/define LZ4_VERSION_MAJOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < $(LIBVER_SRC)`
+LIBVER_MINOR_SCRIPT:=`$(SED) -n '/define LZ4_VERSION_MINOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < $(LIBVER_SRC)`
+LIBVER_PATCH_SCRIPT:=`$(SED) -n '/define LZ4_VERSION_RELEASE/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < $(LIBVER_SRC)`
LIBVER_SCRIPT:= $(LIBVER_MAJOR_SCRIPT).$(LIBVER_MINOR_SCRIPT).$(LIBVER_PATCH_SCRIPT)
LIBVER_MAJOR := $(shell echo $(LIBVER_MAJOR_SCRIPT))
LIBVER_MINOR := $(shell echo $(LIBVER_MINOR_SCRIPT))
@@ -51,16 +52,26 @@
-Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes \
-Wpointer-arith -Wstrict-aliasing=1
CFLAGS += $(DEBUGFLAGS) $(MOREFLAGS)
+
+include ../Makefile.inc
+
+OS_VERSION ?= $(UNAME) -r
+ifeq ($(TARGET_OS)$(shell $(OS_VERSION)),SunOS5.10)
+LDFLAGS += -lrt
+endif
+
FLAGS = $(CFLAGS) $(CPPFLAGS) $(LDFLAGS)
LZ4_VERSION=$(LIBVER)
MD2ROFF = ronn
MD2ROFF_FLAGS = --roff --warnings --manual="User Commands" --organization="lz4 $(LZ4_VERSION)"
-include ../Makefile.inc
default: lz4-release
+# silent mode by default; verbose can be triggered by V=1 or VERBOSE=1
+$(V)$(VERBOSE).SILENT:
+
all: lz4 lz4c
all32: CFLAGS+=-m32
@@ -69,7 +80,7 @@
ifeq ($(WINBASED),yes)
lz4-exe.rc: lz4-exe.rc.in
@echo creating executable resource
- $(Q)sed -e 's|@PROGNAME@|lz4|' \
+ $(SED) -e 's|@PROGNAME@|lz4|' \
-e 's|@LIBVER_MAJOR@|$(LIBVER_MAJOR)|g' \
-e 's|@LIBVER_MINOR@|$(LIBVER_MINOR)|g' \
-e 's|@LIBVER_PATCH@|$(LIBVER_PATCH)|g' \
@@ -110,7 +121,7 @@
$(CC) $(FLAGS) $^ -o $@$(EXT)
lz4.1: lz4.1.md $(LIBVER_SRC)
- cat $< | $(MD2ROFF) $(MD2ROFF_FLAGS) | sed -n '/^\.\\\".*/!p' > $@
+ cat $< | $(MD2ROFF) $(MD2ROFF_FLAGS) | $(SED) -n '/^\.\\\".*/!p' > $@
man: lz4.1
@@ -122,10 +133,10 @@
clean:
ifeq ($(WINBASED),yes)
- $(Q)$(RM) *.rc
+ $(RM) *.rc
endif
- @$(MAKE) -C $(LZ4DIR) $@ > $(VOID)
- @$(RM) core *.o *.test tmp* \
+ $(MAKE) -C $(LZ4DIR) $@ > $(VOID)
+ $(RM) core *.o *.test tmp* \
lz4$(EXT) lz4c$(EXT) lz4c32$(EXT) lz4-wlib$(EXT) \
unlz4$(EXT) lz4cat$(EXT)
@echo Cleaning completed
@@ -160,28 +171,28 @@
man1dir ?= $(MAN1DIR)
install: lz4
- @echo Installing binaries
- @$(INSTALL_DIR) $(DESTDIR)$(bindir)/ $(DESTDIR)$(man1dir)/
- @$(INSTALL_PROGRAM) lz4$(EXT) $(DESTDIR)$(bindir)/lz4$(EXT)
- @$(LN_S) lz4$(EXT) $(DESTDIR)$(bindir)/lz4c$(EXT)
- @$(LN_S) lz4$(EXT) $(DESTDIR)$(bindir)/lz4cat$(EXT)
- @$(LN_S) lz4$(EXT) $(DESTDIR)$(bindir)/unlz4$(EXT)
- @echo Installing man pages
- @$(INSTALL_DATA) lz4.1 $(DESTDIR)$(man1dir)/lz4.1
- @$(LN_SF) lz4.1 $(DESTDIR)$(man1dir)/lz4c.1
- @$(LN_SF) lz4.1 $(DESTDIR)$(man1dir)/lz4cat.1
- @$(LN_SF) lz4.1 $(DESTDIR)$(man1dir)/unlz4.1
+ @echo Installing binaries in $(DESTDIR)$(bindir)
+ $(INSTALL_DIR) $(DESTDIR)$(bindir)/ $(DESTDIR)$(man1dir)/
+ $(INSTALL_PROGRAM) lz4$(EXT) $(DESTDIR)$(bindir)/lz4$(EXT)
+ $(LN_SF) lz4$(EXT) $(DESTDIR)$(bindir)/lz4c$(EXT)
+ $(LN_SF) lz4$(EXT) $(DESTDIR)$(bindir)/lz4cat$(EXT)
+ $(LN_SF) lz4$(EXT) $(DESTDIR)$(bindir)/unlz4$(EXT)
+ @echo Installing man pages in $(DESTDIR)$(man1dir)
+ $(INSTALL_DATA) lz4.1 $(DESTDIR)$(man1dir)/lz4.1
+ $(LN_SF) lz4.1 $(DESTDIR)$(man1dir)/lz4c.1
+ $(LN_SF) lz4.1 $(DESTDIR)$(man1dir)/lz4cat.1
+ $(LN_SF) lz4.1 $(DESTDIR)$(man1dir)/unlz4.1
@echo lz4 installation completed
uninstall:
- @$(RM) $(DESTDIR)$(bindir)/lz4cat$(EXT)
- @$(RM) $(DESTDIR)$(bindir)/unlz4$(EXT)
- @$(RM) $(DESTDIR)$(bindir)/lz4$(EXT)
- @$(RM) $(DESTDIR)$(bindir)/lz4c$(EXT)
- @$(RM) $(DESTDIR)$(man1dir)/lz4.1
- @$(RM) $(DESTDIR)$(man1dir)/lz4c.1
- @$(RM) $(DESTDIR)$(man1dir)/lz4cat.1
- @$(RM) $(DESTDIR)$(man1dir)/unlz4.1
+ $(RM) $(DESTDIR)$(bindir)/lz4cat$(EXT)
+ $(RM) $(DESTDIR)$(bindir)/unlz4$(EXT)
+ $(RM) $(DESTDIR)$(bindir)/lz4$(EXT)
+ $(RM) $(DESTDIR)$(bindir)/lz4c$(EXT)
+ $(RM) $(DESTDIR)$(man1dir)/lz4.1
+ $(RM) $(DESTDIR)$(man1dir)/lz4c.1
+ $(RM) $(DESTDIR)$(man1dir)/lz4cat.1
+ $(RM) $(DESTDIR)$(man1dir)/unlz4.1
@echo lz4 programs successfully uninstalled
endif
diff --git a/programs/bench.c b/programs/bench.c
index 3357d14..4d35ef9 100644
--- a/programs/bench.c
+++ b/programs/bench.c
@@ -1,6 +1,6 @@
/*
bench.c - Demo program to benchmark open-source compression algorithms
- Copyright (C) Yann Collet 2012-2016
+ Copyright (C) Yann Collet 2012-2020
GPL v2 License
@@ -51,171 +51,7 @@
#include "lz4.h"
#define LZ4_HC_STATIC_LINKING_ONLY
#include "lz4hc.h"
-
-
-/* *************************************
-* Compression parameters and functions
-***************************************/
-
-struct compressionParameters
-{
- int cLevel;
- const char* dictBuf;
- int dictSize;
-
- LZ4_stream_t* LZ4_stream;
- LZ4_stream_t* LZ4_dictStream;
- LZ4_streamHC_t* LZ4_streamHC;
- LZ4_streamHC_t* LZ4_dictStreamHC;
-
- void (*initFunction)(
- struct compressionParameters* pThis);
- void (*resetFunction)(
- const struct compressionParameters* pThis);
- int (*blockFunction)(
- const struct compressionParameters* pThis,
- const char* src, char* dst, int srcSize, int dstSize);
- void (*cleanupFunction)(
- const struct compressionParameters* pThis);
-};
-
-static void LZ4_compressInitNoStream(
- struct compressionParameters* pThis)
-{
- pThis->LZ4_stream = NULL;
- pThis->LZ4_dictStream = NULL;
- pThis->LZ4_streamHC = NULL;
- pThis->LZ4_dictStreamHC = NULL;
-}
-
-static void LZ4_compressInitStream(
- struct compressionParameters* pThis)
-{
- pThis->LZ4_stream = LZ4_createStream();
- pThis->LZ4_dictStream = LZ4_createStream();
- pThis->LZ4_streamHC = NULL;
- pThis->LZ4_dictStreamHC = NULL;
- LZ4_loadDict(pThis->LZ4_dictStream, pThis->dictBuf, pThis->dictSize);
-}
-
-static void LZ4_compressInitStreamHC(
- struct compressionParameters* pThis)
-{
- pThis->LZ4_stream = NULL;
- pThis->LZ4_dictStream = NULL;
- pThis->LZ4_streamHC = LZ4_createStreamHC();
- pThis->LZ4_dictStreamHC = LZ4_createStreamHC();
- LZ4_loadDictHC(pThis->LZ4_dictStreamHC, pThis->dictBuf, pThis->dictSize);
-}
-
-static void LZ4_compressResetNoStream(
- const struct compressionParameters* pThis)
-{
- (void)pThis;
-}
-
-static void LZ4_compressResetStream(
- const struct compressionParameters* pThis)
-{
- LZ4_resetStream_fast(pThis->LZ4_stream);
- LZ4_attach_dictionary(pThis->LZ4_stream, pThis->LZ4_dictStream);
-}
-
-static void LZ4_compressResetStreamHC(
- const struct compressionParameters* pThis)
-{
- LZ4_resetStreamHC_fast(pThis->LZ4_streamHC, pThis->cLevel);
- LZ4_attach_HC_dictionary(pThis->LZ4_streamHC, pThis->LZ4_dictStreamHC);
-}
-
-static int LZ4_compressBlockNoStream(
- const struct compressionParameters* pThis,
- const char* src, char* dst,
- int srcSize, int dstSize)
-{
- int const acceleration = (pThis->cLevel < 0) ? -pThis->cLevel + 1 : 1;
- return LZ4_compress_fast(src, dst, srcSize, dstSize, acceleration);
-}
-
-static int LZ4_compressBlockNoStreamHC(
- const struct compressionParameters* pThis,
- const char* src, char* dst,
- int srcSize, int dstSize)
-{
- return LZ4_compress_HC(src, dst, srcSize, dstSize, pThis->cLevel);
-}
-
-static int LZ4_compressBlockStream(
- const struct compressionParameters* pThis,
- const char* src, char* dst,
- int srcSize, int dstSize)
-{
- int const acceleration = (pThis->cLevel < 0) ? -pThis->cLevel + 1 : 1;
- return LZ4_compress_fast_continue(pThis->LZ4_stream, src, dst, srcSize, dstSize, acceleration);
-}
-
-static int LZ4_compressBlockStreamHC(
- const struct compressionParameters* pThis,
- const char* src, char* dst,
- int srcSize, int dstSize)
-{
- return LZ4_compress_HC_continue(pThis->LZ4_streamHC, src, dst, srcSize, dstSize);
-}
-
-static void LZ4_compressCleanupNoStream(
- const struct compressionParameters* pThis)
-{
- (void)pThis;
-}
-
-static void LZ4_compressCleanupStream(
- const struct compressionParameters* pThis)
-{
- LZ4_freeStream(pThis->LZ4_stream);
- LZ4_freeStream(pThis->LZ4_dictStream);
-}
-
-static void LZ4_compressCleanupStreamHC(
- const struct compressionParameters* pThis)
-{
- LZ4_freeStreamHC(pThis->LZ4_streamHC);
- LZ4_freeStreamHC(pThis->LZ4_dictStreamHC);
-}
-
-static void LZ4_buildCompressionParameters(
- struct compressionParameters* pParams,
- int cLevel, const char* dictBuf, int dictSize)
-{
- pParams->cLevel = cLevel;
- pParams->dictBuf = dictBuf;
- pParams->dictSize = dictSize;
-
- if (dictSize) {
- if (cLevel < LZ4HC_CLEVEL_MIN) {
- pParams->initFunction = LZ4_compressInitStream;
- pParams->resetFunction = LZ4_compressResetStream;
- pParams->blockFunction = LZ4_compressBlockStream;
- pParams->cleanupFunction = LZ4_compressCleanupStream;
- } else {
- pParams->initFunction = LZ4_compressInitStreamHC;
- pParams->resetFunction = LZ4_compressResetStreamHC;
- pParams->blockFunction = LZ4_compressBlockStreamHC;
- pParams->cleanupFunction = LZ4_compressCleanupStreamHC;
- }
- } else {
- pParams->initFunction = LZ4_compressInitNoStream;
- pParams->resetFunction = LZ4_compressResetNoStream;
- pParams->cleanupFunction = LZ4_compressCleanupNoStream;
-
- if (cLevel < LZ4HC_CLEVEL_MIN) {
- pParams->blockFunction = LZ4_compressBlockNoStream;
- } else {
- pParams->blockFunction = LZ4_compressBlockNoStreamHC;
- }
- }
-}
-
-#define LZ4_isError(errcode) (errcode==0)
+#include "lz4frame.h" /* LZ4F_decompress */
/* *************************************
@@ -261,13 +97,13 @@
/* *************************************
-* Exceptions
+* DEBUG and error conditions
***************************************/
#ifndef DEBUG
# define DEBUG 0
#endif
#define DEBUGOUTPUT(...) if (DEBUG) DISPLAY(__VA_ARGS__);
-#define EXM_THROW(error, ...) \
+#define END_PROCESS(error, ...) \
{ \
DEBUGOUTPUT("Error defined at %s, line %i : \n", __FILE__, __LINE__); \
DISPLAYLEVEL(1, "Error %i : ", error); \
@@ -276,6 +112,8 @@
exit(error); \
}
+#define LZ4_isError(errcode) (errcode==0)
+
/* *************************************
* Benchmark Parameters
@@ -284,6 +122,8 @@
static size_t g_blockSize = 0;
int g_additionalParam = 0;
int g_benchSeparately = 0;
+int g_decodeOnly = 0;
+unsigned g_skipChecksums = 0;
void BMK_setNotificationLevel(unsigned level) { g_displayLevel=level; }
@@ -299,6 +139,203 @@
void BMK_setBenchSeparately(int separate) { g_benchSeparately = (separate!=0); }
+void BMK_setDecodeOnlyMode(int set) { g_decodeOnly = (set!=0); }
+
+void BMK_skipChecksums(int skip) { g_skipChecksums = (skip!=0); }
+
+
+/* *************************************
+ * Compression state management
+***************************************/
+
+struct compressionParameters
+{
+ int cLevel;
+ const char* dictBuf;
+ int dictSize;
+
+ LZ4_stream_t* LZ4_stream;
+ LZ4_stream_t* LZ4_dictStream;
+ LZ4_streamHC_t* LZ4_streamHC;
+ LZ4_streamHC_t* LZ4_dictStreamHC;
+
+ void (*initFunction)(
+ struct compressionParameters* pThis);
+ void (*resetFunction)(
+ const struct compressionParameters* pThis);
+ int (*blockFunction)(
+ const struct compressionParameters* pThis,
+ const char* src, char* dst, int srcSize, int dstSize);
+ void (*cleanupFunction)(
+ const struct compressionParameters* pThis);
+};
+
+static void
+LZ4_compressInitNoStream(struct compressionParameters* pThis)
+{
+ pThis->LZ4_stream = NULL;
+ pThis->LZ4_dictStream = NULL;
+ pThis->LZ4_streamHC = NULL;
+ pThis->LZ4_dictStreamHC = NULL;
+}
+
+static void
+LZ4_compressInitStream(struct compressionParameters* pThis)
+{
+ pThis->LZ4_stream = LZ4_createStream();
+ pThis->LZ4_dictStream = LZ4_createStream();
+ pThis->LZ4_streamHC = NULL;
+ pThis->LZ4_dictStreamHC = NULL;
+ LZ4_loadDict(pThis->LZ4_dictStream, pThis->dictBuf, pThis->dictSize);
+}
+
+static void
+LZ4_compressInitStreamHC(struct compressionParameters* pThis)
+{
+ pThis->LZ4_stream = NULL;
+ pThis->LZ4_dictStream = NULL;
+ pThis->LZ4_streamHC = LZ4_createStreamHC();
+ pThis->LZ4_dictStreamHC = LZ4_createStreamHC();
+ LZ4_loadDictHC(pThis->LZ4_dictStreamHC, pThis->dictBuf, pThis->dictSize);
+}
+
+static void
+LZ4_compressResetNoStream(const struct compressionParameters* pThis)
+{
+ (void)pThis;
+}
+
+static void
+LZ4_compressResetStream(const struct compressionParameters* pThis)
+{
+ LZ4_resetStream_fast(pThis->LZ4_stream);
+ LZ4_attach_dictionary(pThis->LZ4_stream, pThis->LZ4_dictStream);
+}
+
+static void
+LZ4_compressResetStreamHC(const struct compressionParameters* pThis)
+{
+ LZ4_resetStreamHC_fast(pThis->LZ4_streamHC, pThis->cLevel);
+ LZ4_attach_HC_dictionary(pThis->LZ4_streamHC, pThis->LZ4_dictStreamHC);
+}
+
+static int
+LZ4_compressBlockNoStream(const struct compressionParameters* pThis,
+ const char* src, char* dst,
+ int srcSize, int dstSize)
+{
+ int const acceleration = (pThis->cLevel < 0) ? -pThis->cLevel + 1 : 1;
+ return LZ4_compress_fast(src, dst, srcSize, dstSize, acceleration);
+}
+
+static int
+LZ4_compressBlockNoStreamHC(const struct compressionParameters* pThis,
+ const char* src, char* dst,
+ int srcSize, int dstSize)
+{
+ return LZ4_compress_HC(src, dst, srcSize, dstSize, pThis->cLevel);
+}
+
+static int
+LZ4_compressBlockStream(const struct compressionParameters* pThis,
+ const char* src, char* dst,
+ int srcSize, int dstSize)
+{
+ int const acceleration = (pThis->cLevel < 0) ? -pThis->cLevel + 1 : 1;
+ return LZ4_compress_fast_continue(pThis->LZ4_stream, src, dst, srcSize, dstSize, acceleration);
+}
+
+static int
+LZ4_compressBlockStreamHC(const struct compressionParameters* pThis,
+ const char* src, char* dst,
+ int srcSize, int dstSize)
+{
+ return LZ4_compress_HC_continue(pThis->LZ4_streamHC, src, dst, srcSize, dstSize);
+}
+
+static void
+LZ4_compressCleanupNoStream(const struct compressionParameters* pThis)
+{
+ (void)pThis;
+}
+
+static void
+LZ4_compressCleanupStream(const struct compressionParameters* pThis)
+{
+ LZ4_freeStream(pThis->LZ4_stream);
+ LZ4_freeStream(pThis->LZ4_dictStream);
+}
+
+static void
+LZ4_compressCleanupStreamHC(const struct compressionParameters* pThis)
+{
+ LZ4_freeStreamHC(pThis->LZ4_streamHC);
+ LZ4_freeStreamHC(pThis->LZ4_dictStreamHC);
+}
+
+static void
+LZ4_buildCompressionParameters(struct compressionParameters* pParams,
+ int cLevel,
+ const char* dictBuf, int dictSize)
+{
+ pParams->cLevel = cLevel;
+ pParams->dictBuf = dictBuf;
+ pParams->dictSize = dictSize;
+
+ if (dictSize) {
+ if (cLevel < LZ4HC_CLEVEL_MIN) {
+ pParams->initFunction = LZ4_compressInitStream;
+ pParams->resetFunction = LZ4_compressResetStream;
+ pParams->blockFunction = LZ4_compressBlockStream;
+ pParams->cleanupFunction = LZ4_compressCleanupStream;
+ } else {
+ pParams->initFunction = LZ4_compressInitStreamHC;
+ pParams->resetFunction = LZ4_compressResetStreamHC;
+ pParams->blockFunction = LZ4_compressBlockStreamHC;
+ pParams->cleanupFunction = LZ4_compressCleanupStreamHC;
+ }
+ } else {
+ pParams->initFunction = LZ4_compressInitNoStream;
+ pParams->resetFunction = LZ4_compressResetNoStream;
+ pParams->cleanupFunction = LZ4_compressCleanupNoStream;
+
+ if (cLevel < LZ4HC_CLEVEL_MIN) {
+ pParams->blockFunction = LZ4_compressBlockNoStream;
+ } else {
+ pParams->blockFunction = LZ4_compressBlockNoStreamHC;
+ }
+ }
+}
+
+
+typedef int (*DecFunction_f)(const char* src, char* dst,
+ int srcSize, int dstCapacity,
+ const char* dictStart, int dictSize);
+
+static LZ4F_dctx* g_dctx = NULL;
+
+static int
+LZ4F_decompress_binding(const char* src, char* dst,
+ int srcSize, int dstCapacity,
+ const char* dictStart, int dictSize)
+{
+ size_t dstSize = (size_t)dstCapacity;
+ size_t readSize = (size_t)srcSize;
+ LZ4F_decompressOptions_t dOpt = { 1, 0, 0, 0 };
+ size_t decStatus;
+ dOpt.skipChecksums = g_skipChecksums;
+ decStatus = LZ4F_decompress(g_dctx,
+ dst, &dstSize,
+ src, &readSize,
+ &dOpt);
+ if ( (decStatus == 0) /* decompression successful */
+ && ((int)readSize==srcSize) /* consume all input */ )
+ return (int)dstSize;
+ /* else, error */
+ return -1;
+ (void)dictStart; (void)dictSize; /* not compatible with dictionary yet */
+}
+
/* ********************************************************
* Bench functions
@@ -321,24 +358,32 @@
const size_t* fileSizes, U32 nbFiles,
const char* dictBuf, int dictSize)
{
- size_t const blockSize = (g_blockSize>=32 ? g_blockSize : srcSize) + (!srcSize) /* avoid div by 0 */ ;
- U32 const maxNbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize) + nbFiles;
+ size_t const blockSize = (g_blockSize>=32 && !g_decodeOnly ? g_blockSize : srcSize) + (!srcSize) /* avoid div by 0 */ ;
+ U32 const maxNbBlocks = (U32)((srcSize + (blockSize-1)) / blockSize) + nbFiles;
blockParam_t* const blockTable = (blockParam_t*) malloc(maxNbBlocks * sizeof(blockParam_t));
- size_t const maxCompressedSize = LZ4_compressBound((int)srcSize) + (maxNbBlocks * 1024); /* add some room for safety */
+ size_t const maxCompressedSize = (size_t)LZ4_compressBound((int)srcSize) + (maxNbBlocks * 1024); /* add some room for safety */
void* const compressedBuffer = malloc(maxCompressedSize);
- void* const resultBuffer = malloc(srcSize);
+ size_t const decMultiplier = g_decodeOnly ? 255 : 1;
+ size_t const maxInSize = (size_t)LZ4_MAX_INPUT_SIZE / decMultiplier;
+ size_t const maxDecSize = srcSize < maxInSize ? srcSize * decMultiplier : LZ4_MAX_INPUT_SIZE;
+ void* const resultBuffer = malloc(maxDecSize);
U32 nbBlocks;
struct compressionParameters compP;
/* checks */
if (!compressedBuffer || !resultBuffer || !blockTable)
- EXM_THROW(31, "allocation error : not enough memory");
+ END_PROCESS(31, "allocation error : not enough memory");
if (strlen(displayName)>17) displayName += strlen(displayName)-17; /* can only display 17 characters */
/* init */
LZ4_buildCompressionParameters(&compP, cLevel, dictBuf, dictSize);
compP.initFunction(&compP);
+ if (g_dctx==NULL) {
+ LZ4F_createDecompressionContext(&g_dctx, LZ4F_VERSION);
+ if (g_dctx==NULL)
+ END_PROCESS(1, "allocation error - decompression state");
+ }
/* Init blockTable data */
{ const char* srcPtr = (const char*)srcBuffer;
@@ -351,6 +396,8 @@
U32 const blockEnd = nbBlocks + nbBlocksforThisFile;
for ( ; nbBlocks<blockEnd; nbBlocks++) {
size_t const thisBlockSize = MIN(remaining, blockSize);
+ size_t const resMaxSize = thisBlockSize * decMultiplier;
+ size_t const resCapa = (thisBlockSize < maxInSize) ? resMaxSize : LZ4_MAX_INPUT_SIZE;
blockTable[nbBlocks].srcPtr = srcPtr;
blockTable[nbBlocks].cPtr = cPtr;
blockTable[nbBlocks].resPtr = resPtr;
@@ -358,29 +405,37 @@
blockTable[nbBlocks].cRoom = (size_t)LZ4_compressBound((int)thisBlockSize);
srcPtr += thisBlockSize;
cPtr += blockTable[nbBlocks].cRoom;
- resPtr += thisBlockSize;
+ resPtr += resCapa;
remaining -= thisBlockSize;
} } }
- /* warmimg up memory */
+ /* warming up memory */
RDG_genBuffer(compressedBuffer, maxCompressedSize, 0.10, 0.50, 1);
+ /* decode-only mode : copy input to @compressedBuffer */
+ if (g_decodeOnly) {
+ U32 blockNb;
+ for (blockNb=0; blockNb < nbBlocks; blockNb++) {
+ memcpy(blockTable[blockNb].cPtr, blockTable[blockNb].srcPtr, blockTable[blockNb].srcSize);
+ blockTable[blockNb].cSize = blockTable[blockNb].srcSize;
+ } }
+
/* Bench */
{ U64 fastestC = (U64)(-1LL), fastestD = (U64)(-1LL);
U64 const crcOrig = XXH64(srcBuffer, srcSize, 0);
- UTIL_time_t coolTime;
+ UTIL_time_t coolTime = UTIL_getTime();
U64 const maxTime = (g_nbSeconds * TIMELOOP_NANOSEC) + 100;
U32 nbCompressionLoops = (U32)((5 MB) / (srcSize+1)) + 1; /* conservative initial compression speed estimate */
U32 nbDecodeLoops = (U32)((200 MB) / (srcSize+1)) + 1; /* conservative initial decode speed estimate */
U64 totalCTime=0, totalDTime=0;
- U32 cCompleted=0, dCompleted=0;
+ U32 cCompleted=(g_decodeOnly==1), dCompleted=0;
# define NB_MARKS 4
const char* const marks[NB_MARKS] = { " |", " /", " =", "\\" };
U32 markNb = 0;
- size_t cSize = 0;
+ size_t cSize = srcSize;
+ size_t totalRSize = srcSize;
double ratio = 0.;
- coolTime = UTIL_getTime();
DISPLAYLEVEL(2, "\r%79s\r", "");
while (!cCompleted || !dCompleted) {
/* overheat protection */
@@ -391,8 +446,8 @@
}
/* Compression */
- DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->\r", marks[markNb], displayName, (U32)srcSize);
- if (!cCompleted) memset(compressedBuffer, 0xE5, maxCompressedSize); /* warm up and erase result buffer */
+ DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->\r", marks[markNb], displayName, (U32)totalRSize);
+ if (!cCompleted) memset(compressedBuffer, 0xE5, maxCompressedSize); /* warm up and erase compressed buffer */
UTIL_sleepMilli(1); /* give processor time to other processes */
UTIL_waitForNextTick();
@@ -408,7 +463,7 @@
&compP,
blockTable[blockNb].srcPtr, blockTable[blockNb].cPtr,
(int)blockTable[blockNb].srcSize, (int)blockTable[blockNb].cRoom);
- if (LZ4_isError(rSize)) EXM_THROW(1, "LZ4 compression failed");
+ if (LZ4_isError(rSize)) END_PROCESS(1, "LZ4 compression failed");
blockTable[blockNb].cSize = rSize;
} }
{ U64 const clockSpan = UTIL_clockSpanNano(clockStart);
@@ -423,17 +478,18 @@
}
totalCTime += clockSpan;
cCompleted = totalCTime>maxTime;
- } }
+ }
- cSize = 0;
- { U32 blockNb; for (blockNb=0; blockNb<nbBlocks; blockNb++) cSize += blockTable[blockNb].cSize; }
- cSize += !cSize; /* avoid div by 0 */
- ratio = (double)srcSize / (double)cSize;
- markNb = (markNb+1) % NB_MARKS;
- DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.3f),%6.1f MB/s\r",
- marks[markNb], displayName, (U32)srcSize, (U32)cSize, ratio,
- ((double)srcSize / fastestC) * 1000 );
-
+ cSize = 0;
+ { U32 blockNb; for (blockNb=0; blockNb<nbBlocks; blockNb++) cSize += blockTable[blockNb].cSize; }
+ cSize += !cSize; /* avoid div by 0 */
+ ratio = (double)totalRSize / (double)cSize;
+ markNb = (markNb+1) % NB_MARKS;
+ DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.3f),%6.1f MB/s\r",
+ marks[markNb], displayName,
+ (U32)totalRSize, (U32)cSize, ratio,
+ ((double)totalRSize / fastestC) * 1000 );
+ }
(void)fastestD; (void)crcOrig; /* unused when decompression disabled */
#if 1
/* Decompression */
@@ -443,17 +499,30 @@
UTIL_waitForNextTick();
if (!dCompleted) {
+ const DecFunction_f decFunction = g_decodeOnly ?
+ LZ4F_decompress_binding : LZ4_decompress_safe_usingDict;
+ const char* const decString = g_decodeOnly ?
+ "LZ4F_decompress" : "LZ4_decompress_safe_usingDict";
UTIL_time_t const clockStart = UTIL_getTime();
U32 nbLoops;
+
for (nbLoops=0; nbLoops < nbDecodeLoops; nbLoops++) {
U32 blockNb;
for (blockNb=0; blockNb<nbBlocks; blockNb++) {
- int const regenSize = LZ4_decompress_safe_usingDict(
+ size_t const inMaxSize = (size_t)INT_MAX / decMultiplier;
+ size_t const resCapa = (blockTable[blockNb].srcSize < inMaxSize) ?
+ blockTable[blockNb].srcSize * decMultiplier :
+ INT_MAX;
+ int const regenSize = decFunction(
blockTable[blockNb].cPtr, blockTable[blockNb].resPtr,
- (int)blockTable[blockNb].cSize, (int)blockTable[blockNb].srcSize,
+ (int)blockTable[blockNb].cSize, (int)resCapa,
dictBuf, dictSize);
if (regenSize < 0) {
- DISPLAY("LZ4_decompress_safe_usingDict() failed on block %u \n", blockNb);
+ DISPLAY("%s() failed on block %u of size %u \n",
+ decString, blockNb, (unsigned)blockTable[blockNb].srcSize);
+ if (g_decodeOnly)
+ DISPLAY("Is input using LZ4 Frame format ? \n");
+ END_PROCESS(2, "error during decoding");
break;
}
blockTable[blockNb].resSize = (size_t)regenSize;
@@ -472,14 +541,22 @@
dCompleted = totalDTime > (DECOMP_MULT*maxTime);
} }
+ if (g_decodeOnly) {
+ unsigned u;
+ totalRSize = 0;
+ for (u=0; u<nbBlocks; u++) totalRSize += blockTable[u].resSize;
+ }
markNb = (markNb+1) % NB_MARKS;
+ ratio = (double)totalRSize / (double)cSize;
DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.3f),%6.1f MB/s ,%6.1f MB/s\r",
- marks[markNb], displayName, (U32)srcSize, (U32)cSize, ratio,
- ((double)srcSize / fastestC) * 1000,
- ((double)srcSize / fastestD) * 1000);
+ marks[markNb], displayName,
+ (U32)totalRSize, (U32)cSize, ratio,
+ ((double)totalRSize / fastestC) * 1000,
+ ((double)totalRSize / fastestD) * 1000);
- /* CRC Checking */
- { U64 const crcCheck = XXH64(resultBuffer, srcSize, 0);
+ /* CRC Checking (not possible in decode-only mode)*/
+ if (!g_decodeOnly) {
+ U64 const crcCheck = XXH64(resultBuffer, srcSize, 0);
if (crcOrig!=crcCheck) {
size_t u;
DISPLAY("\n!!! WARNING !!! %17s : Invalid Checksum : %x != %x \n", displayName, (unsigned)crcOrig, (unsigned)crcCheck);
@@ -594,21 +671,21 @@
continue;
}
f = fopen(fileNamesTable[n], "rb");
- if (f==NULL) EXM_THROW(10, "impossible to open file %s", fileNamesTable[n]);
+ if (f==NULL) END_PROCESS(10, "impossible to open file %s", fileNamesTable[n]);
DISPLAYUPDATE(2, "Loading %s... \r", fileNamesTable[n]);
if (fileSize > bufferSize-pos) { /* buffer too small - stop after this file */
fileSize = bufferSize-pos;
nbFiles=n;
}
{ size_t const readSize = fread(((char*)buffer)+pos, 1, (size_t)fileSize, f);
- if (readSize != (size_t)fileSize) EXM_THROW(11, "could not read %s", fileNamesTable[n]);
+ if (readSize != (size_t)fileSize) END_PROCESS(11, "could not read %s", fileNamesTable[n]);
pos += readSize; }
fileSizes[n] = (size_t)fileSize;
totalSize += (size_t)fileSize;
fclose(f);
}
- if (totalSize == 0) EXM_THROW(12, "no data to bench");
+ if (totalSize == 0) END_PROCESS(12, "no data to bench");
}
static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles,
@@ -621,11 +698,11 @@
U64 const totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, nbFiles);
char mfName[20] = {0};
- if (!fileSizes) EXM_THROW(12, "not enough memory for fileSizes");
+ if (!fileSizes) END_PROCESS(12, "not enough memory for fileSizes");
/* Memory allocation & restrictions */
benchedSize = BMK_findMaxMem(totalSizeToLoad * 3) / 3;
- if (benchedSize==0) EXM_THROW(12, "not enough memory");
+ if (benchedSize==0) END_PROCESS(12, "not enough memory");
if ((U64)benchedSize > totalSizeToLoad) benchedSize = (size_t)totalSizeToLoad;
if (benchedSize > LZ4_MAX_INPUT_SIZE) {
benchedSize = LZ4_MAX_INPUT_SIZE;
@@ -635,7 +712,7 @@
DISPLAY("Not enough memory; testing %u MB only...\n", (U32)(benchedSize >> 20));
}
srcBuffer = malloc(benchedSize + !benchedSize); /* avoid alloc of zero */
- if (!srcBuffer) EXM_THROW(12, "not enough memory");
+ if (!srcBuffer) END_PROCESS(12, "not enough memory");
/* Load input buffer */
BMK_loadFiles(srcBuffer, benchedSize, fileSizes, fileNamesTable, nbFiles);
@@ -663,7 +740,7 @@
void* const srcBuffer = malloc(benchedSize);
/* Memory allocation */
- if (!srcBuffer) EXM_THROW(21, "not enough memory");
+ if (!srcBuffer) END_PROCESS(21, "not enough memory");
/* Fill input buffer */
RDG_genBuffer(srcBuffer, benchedSize, compressibility, 0.0, 0);
@@ -677,7 +754,8 @@
}
-int BMK_benchFilesSeparately(const char** fileNamesTable, unsigned nbFiles,
+static int
+BMK_benchFilesSeparately(const char** fileNamesTable, unsigned nbFiles,
int cLevel, int cLevelLast,
const char* dictBuf, int dictSize)
{
@@ -685,7 +763,6 @@
if (cLevel > LZ4HC_CLEVEL_MAX) cLevel = LZ4HC_CLEVEL_MAX;
if (cLevelLast > LZ4HC_CLEVEL_MAX) cLevelLast = LZ4HC_CLEVEL_MAX;
if (cLevelLast < cLevel) cLevelLast = cLevel;
- if (cLevelLast > cLevel) DISPLAYLEVEL(2, "Benchmarking levels from %d to %d\n", cLevel, cLevelLast);
for (fileNb=0; fileNb<nbFiles; fileNb++)
BMK_benchFileTable(fileNamesTable+fileNb, 1, cLevel, cLevelLast, dictBuf, dictSize);
@@ -700,45 +777,59 @@
{
double const compressibility = (double)g_compressibilityDefault / 100;
char* dictBuf = NULL;
- int dictSize = 0;
+ size_t dictSize = 0;
if (cLevel > LZ4HC_CLEVEL_MAX) cLevel = LZ4HC_CLEVEL_MAX;
+ if (g_decodeOnly) {
+ DISPLAYLEVEL(2, "Benchmark Decompression of LZ4 Frame ");
+ if (g_skipChecksums) {
+ DISPLAYLEVEL(2, "_without_ checksum even when present \n");
+ } else {
+ DISPLAYLEVEL(2, "+ Checksum when present \n");
+ }
+ cLevelLast = cLevel;
+ }
if (cLevelLast > LZ4HC_CLEVEL_MAX) cLevelLast = LZ4HC_CLEVEL_MAX;
if (cLevelLast < cLevel) cLevelLast = cLevel;
- if (cLevelLast > cLevel) DISPLAYLEVEL(2, "Benchmarking levels from %d to %d\n", cLevel, cLevelLast);
+ if (cLevelLast > cLevel)
+ DISPLAYLEVEL(2, "Benchmarking levels from %d to %d\n", cLevel, cLevelLast);
if (dictFileName) {
FILE* dictFile = NULL;
- U64 dictFileSize = UTIL_getFileSize(dictFileName);
- if (!dictFileSize) EXM_THROW(25, "Dictionary error : could not stat dictionary file");
+ U64 const dictFileSize = UTIL_getFileSize(dictFileName);
+ if (!dictFileSize)
+ END_PROCESS(25, "Dictionary error : could not stat dictionary file");
+ if (g_decodeOnly)
+ END_PROCESS(26, "Error : LZ4 Frame decoder mode not compatible with dictionary yet");
dictFile = fopen(dictFileName, "rb");
- if (!dictFile) EXM_THROW(25, "Dictionary error : could not open dictionary file");
+ if (!dictFile)
+ END_PROCESS(25, "Dictionary error : could not open dictionary file");
if (dictFileSize > LZ4_MAX_DICT_SIZE) {
dictSize = LZ4_MAX_DICT_SIZE;
- if (UTIL_fseek(dictFile, dictFileSize - dictSize, SEEK_SET))
- EXM_THROW(25, "Dictionary error : could not seek dictionary file");
+ if (UTIL_fseek(dictFile, (long)(dictFileSize - dictSize), SEEK_SET))
+ END_PROCESS(25, "Dictionary error : could not seek dictionary file");
} else {
- dictSize = (int)dictFileSize;
+ dictSize = (size_t)dictFileSize;
}
- dictBuf = (char *)malloc(dictSize);
- if (!dictBuf) EXM_THROW(25, "Allocation error : not enough memory");
+ dictBuf = (char*)malloc(dictSize);
+ if (!dictBuf) END_PROCESS(25, "Allocation error : not enough memory");
- if (fread(dictBuf, 1, dictSize, dictFile) != (size_t)dictSize)
- EXM_THROW(25, "Dictionary error : could not read dictionary file");
+ if (fread(dictBuf, 1, dictSize, dictFile) != dictSize)
+ END_PROCESS(25, "Dictionary error : could not read dictionary file");
fclose(dictFile);
}
if (nbFiles == 0)
- BMK_syntheticTest(cLevel, cLevelLast, compressibility, dictBuf, dictSize);
+ BMK_syntheticTest(cLevel, cLevelLast, compressibility, dictBuf, (int)dictSize);
else {
if (g_benchSeparately)
- BMK_benchFilesSeparately(fileNamesTable, nbFiles, cLevel, cLevelLast, dictBuf, dictSize);
+ BMK_benchFilesSeparately(fileNamesTable, nbFiles, cLevel, cLevelLast, dictBuf, (int)dictSize);
else
- BMK_benchFileTable(fileNamesTable, nbFiles, cLevel, cLevelLast, dictBuf, dictSize);
+ BMK_benchFileTable(fileNamesTable, nbFiles, cLevel, cLevelLast, dictBuf, (int)dictSize);
}
free(dictBuf);
diff --git a/programs/bench.h b/programs/bench.h
index 22ebf60..1d81a99 100644
--- a/programs/bench.h
+++ b/programs/bench.h
@@ -1,6 +1,6 @@
/*
bench.h - Demo program to benchmark open-source compression algorithm
- Copyright (C) Yann Collet 2012-2016
+ Copyright (C) Yann Collet 2012-2020
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
@@ -25,15 +25,30 @@
#include <stddef.h>
+/* BMK_benchFiles() :
+ * Benchmark all files provided through array @fileNamesTable.
+ * All files must be valid, otherwise benchmark fails.
+ * Roundtrip measurements are done for each file individually, but
+ * unless BMK_setBenchSeparately() is set, all results are agglomerated.
+ * The method benchmarks all compression levels from @cLevelStart to @cLevelLast,
+ * both inclusive, providing one result per compression level.
+ * If @cLevelLast <= @cLevelStart, BMK_benchFiles() benchmarks @cLevelStart only.
+ * @dictFileName is optional, it's possible to provide NULL.
+ * When provided, compression and decompression use the specified file as dictionary.
+ * Only one dictionary can be provided, in which case it's applied to all benchmarked files.
+**/
int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles,
- int cLevel, int cLevelLast,
+ int cLevelStart, int cLevelLast,
const char* dictFileName);
/* Set Parameters */
-void BMK_setNbSeconds(unsigned nbLoops);
-void BMK_setBlockSize(size_t blockSize);
-void BMK_setAdditionalParam(int additionalParam);
-void BMK_setNotificationLevel(unsigned level);
-void BMK_setBenchSeparately(int separate);
+void BMK_setNbSeconds(unsigned nbSeconds); /* minimum benchmark duration, in seconds, for both compression and decompression */
+void BMK_setBlockSize(size_t blockSize); /* Internally cut input file(s) into independent blocks of specified size */
+void BMK_setNotificationLevel(unsigned level); /* Influence verbosity level */
+void BMK_setBenchSeparately(int separate); /* When providing multiple files, output one result per file */
+void BMK_setDecodeOnlyMode(int set); /* v1.9.4+: set benchmark mode to decode only */
+void BMK_skipChecksums(int skip); /* v1.9.4+: only useful for DecodeOnlyMode; do not calculate checksum when present, to save CPU time */
+
+void BMK_setAdditionalParam(int additionalParam); /* hidden param, influence output format, for python parsing */
#endif /* BENCH_H_125623623633 */
diff --git a/programs/datagen.c b/programs/datagen.c
index 24a2da2..f448640 100644
--- a/programs/datagen.c
+++ b/programs/datagen.c
@@ -1,6 +1,6 @@
/*
datagen.c - compressible data generator test tool
- Copyright (C) Yann Collet 2012-2016
+ Copyright (C) Yann Collet 2012-2020
GPL v2 License
diff --git a/programs/datagen.h b/programs/datagen.h
index 91c5b02..c20c9c7 100644
--- a/programs/datagen.h
+++ b/programs/datagen.h
@@ -1,6 +1,6 @@
/*
datagen.h - compressible data generator header
- Copyright (C) Yann Collet 2012-2016
+ Copyright (C) Yann Collet 2012-2020
GPL v2 License
diff --git a/programs/lz4-exe.rc.in b/programs/lz4-exe.rc.in
index 7b81030..bcf4d7d 100644
--- a/programs/lz4-exe.rc.in
+++ b/programs/lz4-exe.rc.in
@@ -13,7 +13,7 @@
VALUE "FileDescription", "Extremely fast compression"
VALUE "FileVersion", "@LIBVER_MAJOR@.@LIBVER_MINOR@.@LIBVER_PATCH@.0"
VALUE "InternalName", "@PROGNAME@"
- VALUE "LegalCopyright", "Copyright (C) 2013-2016, Yann Collet"
+ VALUE "LegalCopyright", "Copyright (C) 2013-2020, Yann Collet"
VALUE "OriginalFilename", "@PROGNAME@.@EXT@"
VALUE "ProductName", "LZ4"
VALUE "ProductVersion", "@LIBVER_MAJOR@.@LIBVER_MINOR@.@LIBVER_PATCH@.0"
@@ -24,4 +24,3 @@
VALUE "Translation", 0x0409, 1200
}
}
-
diff --git a/programs/lz4.1 b/programs/lz4.1
index d758ed5..7cb98d6 100644
--- a/programs/lz4.1
+++ b/programs/lz4.1
@@ -1,5 +1,5 @@
.
-.TH "LZ4" "1" "July 2019" "lz4 1.9.2" "User Commands"
+.TH "LZ4" "1" "August 2022" "lz4 v1.9.4" "User Commands"
.
.SH "NAME"
\fBlz4\fR \- lz4, unlz4, lz4cat \- Compress or decompress \.lz4 files
@@ -17,7 +17,7 @@
When writing scripts that need to decompress files, it is recommended to always use the name \fBlz4\fR with appropriate arguments (\fBlz4 \-d\fR or \fBlz4 \-dc\fR) instead of the names \fBunlz4\fR and \fBlz4cat\fR\.
.
.SH "DESCRIPTION"
-\fBlz4\fR is an extremely fast lossless compression algorithm, based on \fBbyte\-aligned LZ77\fR family of compression scheme\. \fBlz4\fR offers compression speeds of 400 MB/s per core, linearly scalable with multi\-core CPUs\. It features an extremely fast decoder, with speed in multiple GB/s per core, typically reaching RAM speed limit on multi\-core systems\. The native file format is the \fB\.lz4\fR format\.
+\fBlz4\fR is an extremely fast lossless compression algorithm, based on \fBbyte\-aligned LZ77\fR family of compression scheme\. \fBlz4\fR offers compression speeds > 500 MB/s per core, linearly scalable with multi\-core CPUs\. It features an extremely fast decoder, offering speed in multiple GB/s per core, typically reaching RAM speed limit on multi\-core systems\. The native file format is the \fB\.lz4\fR format\.
.
.SS "Difference between lz4 and gzip"
\fBlz4\fR supports a command line syntax similar \fIbut not identical\fR to \fBgzip(1)\fR\. Differences are :
@@ -32,7 +32,7 @@
\fBlz4 file\.lz4\fR will default to decompression (use \fB\-z\fR to force compression)
.
.IP "\(bu" 4
-\fBlz4\fR preserves original files
+\fBlz4\fR preserves original files (see \fB\-\-rm\fR to erase source file on completion)
.
.IP "\(bu" 4
\fBlz4\fR shows real\-time notification statistics during compression or decompression of a single file (use \fB\-q\fR to silence them)
@@ -121,7 +121,7 @@
.
.TP
\fB\-\-best\fR
-Set highest compression level\. Same as -12\.
+Set highest compression level\. Same as \-12\.
.
.TP
\fB\-\-favor\-decSpeed\fR
@@ -169,10 +169,18 @@
Blocks depend on predecessors (improves compression ratio, more noticeable on small blocks)
.
.TP
+\fB\-BX\fR
+Generate block checksums (default:disabled)
+.
+.TP
\fB\-\-[no\-]frame\-crc\fR
Select frame checksum (default:enabled)
.
.TP
+\fB\-\-no\-crc\fR
+Disable both frame and block checksums
+.
+.TP
\fB\-\-[no\-]content\-size\fR
Header includes original size (default:not present)
.
diff --git a/programs/lz4.1.md b/programs/lz4.1.md
index 56c0053..06c06cf 100644
--- a/programs/lz4.1.md
+++ b/programs/lz4.1.md
@@ -20,9 +20,9 @@
`lz4` is an extremely fast lossless compression algorithm,
based on **byte-aligned LZ77** family of compression scheme.
-`lz4` offers compression speeds of 400 MB/s per core, linearly scalable with
-multi-core CPUs.
-It features an extremely fast decoder, with speed in multiple GB/s per core,
+`lz4` offers compression speeds > 500 MB/s per core,
+linearly scalable with multi-core CPUs.
+It features an extremely fast decoder, offering speed in multiple GB/s per core,
typically reaching RAM speed limit on multi-core systems.
The native file format is the `.lz4` format.
@@ -34,7 +34,7 @@
* `lz4` compresses a single file by default (see `-m` for multiple files)
* `lz4 file1 file2` means : compress file1 _into_ file2
* `lz4 file.lz4` will default to decompression (use `-z` to force compression)
- * `lz4` preserves original files
+ * `lz4` preserves original files (see `--rm` to erase source file on completion)
* `lz4` shows real-time notification statistics
during compression or decompression of a single file
(use `-q` to silence them)
@@ -185,9 +185,15 @@
* `-BD`:
Blocks depend on predecessors (improves compression ratio, more noticeable on small blocks)
+* `-BX`:
+ Generate block checksums (default:disabled)
+
* `--[no-]frame-crc`:
Select frame checksum (default:enabled)
+* `--no-crc`:
+ Disable both frame and block checksums
+
* `--[no-]content-size`:
Header includes original size (default:not present)<br/>
Note : this option can only be activated when the original size can be
diff --git a/programs/lz4cli.c b/programs/lz4cli.c
index 523b8a8..8c3f9fd 100644
--- a/programs/lz4cli.c
+++ b/programs/lz4cli.c
@@ -1,6 +1,6 @@
/*
LZ4cli - LZ4 Command Line Interface
- Copyright (C) Yann Collet 2011-2016
+ Copyright (C) Yann Collet 2011-2020
GPL v2 License
@@ -186,7 +186,7 @@
DISPLAY( "\n");
DISPLAY( "Compression levels : \n");
DISPLAY( "---------------------\n");
- DISPLAY( "-0 ... -2 => Fast compression, all identicals\n");
+ DISPLAY( "-0 ... -2 => Fast compression, all identical\n");
DISPLAY( "-3 ... -%d => High compression; higher number == more compression but slower\n", LZ4HC_CLEVEL_MAX);
DISPLAY( "\n");
DISPLAY( "stdin, stdout and the console : \n");
@@ -314,6 +314,7 @@
cLevelLast=-10000,
legacy_format=0,
forceStdout=0,
+ forceOverwrite=0,
main_pause=0,
multiple_inputs=0,
all_arguments_are_files=0,
@@ -330,9 +331,8 @@
const char extension[] = LZ4_EXTENSION;
size_t blockSize = LZ4IO_setBlockSizeID(prefs, LZ4_BLOCKSIZEID_DEFAULT);
const char* const exeName = lastNameFromPath(argv[0]);
-#ifdef UTIL_HAS_CREATEFILELIST
- const char** extendedFileList = NULL;
char* fileNamesBuf = NULL;
+#ifdef UTIL_HAS_CREATEFILELIST
unsigned fileNamesNb, recursive=0;
#endif
@@ -377,16 +377,21 @@
if (argument[1]=='-') {
if (!strcmp(argument, "--")) { all_arguments_are_files = 1; continue; }
if (!strcmp(argument, "--compress")) { mode = om_compress; continue; }
- if ((!strcmp(argument, "--decompress"))
- || (!strcmp(argument, "--uncompress"))) { mode = om_decompress; continue; }
+ if ( (!strcmp(argument, "--decompress"))
+ || (!strcmp(argument, "--uncompress"))) {
+ if (mode != om_bench) mode = om_decompress;
+ BMK_setDecodeOnlyMode(1);
+ continue;
+ }
if (!strcmp(argument, "--multiple")) { multiple_inputs = 1; continue; }
if (!strcmp(argument, "--test")) { mode = om_test; continue; }
if (!strcmp(argument, "--force")) { LZ4IO_setOverwrite(prefs, 1); continue; }
if (!strcmp(argument, "--no-force")) { LZ4IO_setOverwrite(prefs, 0); continue; }
if ((!strcmp(argument, "--stdout"))
|| (!strcmp(argument, "--to-stdout"))) { forceStdout=1; output_filename=stdoutmark; continue; }
- if (!strcmp(argument, "--frame-crc")) { LZ4IO_setStreamChecksumMode(prefs, 1); continue; }
- if (!strcmp(argument, "--no-frame-crc")) { LZ4IO_setStreamChecksumMode(prefs, 0); continue; }
+ if (!strcmp(argument, "--frame-crc")) { LZ4IO_setStreamChecksumMode(prefs, 1); BMK_skipChecksums(0); continue; }
+ if (!strcmp(argument, "--no-frame-crc")) { LZ4IO_setStreamChecksumMode(prefs, 0); BMK_skipChecksums(1); continue; }
+ if (!strcmp(argument, "--no-crc")) { LZ4IO_setStreamChecksumMode(prefs, 0); LZ4IO_setBlockChecksumMode(prefs, 0); BMK_skipChecksums(1); continue; }
if (!strcmp(argument, "--content-size")) { LZ4IO_setContentSize(prefs, 1); continue; }
if (!strcmp(argument, "--no-content-size")) { LZ4IO_setContentSize(prefs, 0); continue; }
if (!strcmp(argument, "--list")) { mode = om_list; continue; }
@@ -478,7 +483,10 @@
case 'l': legacy_format = 1; blockSize = 8 MB; break;
/* Decoding */
- case 'd': mode = om_decompress; break;
+ case 'd':
+ if (mode != om_bench) mode = om_decompress;
+ BMK_setDecodeOnlyMode(1);
+ break;
/* Force stdout, even if stdout==console */
case 'c':
@@ -491,7 +499,7 @@
case 't': mode = om_test; break;
/* Overwrite */
- case 'f': LZ4IO_setOverwrite(prefs, 1); break;
+ case 'f': forceOverwrite=1; LZ4IO_setOverwrite(prefs, 1); break;
/* Verbose mode */
case 'v': displayLevel++; break;
@@ -581,20 +589,24 @@
}
/* Store in *inFileNames[] if -m is used. */
- if (multiple_inputs) { inFileNames[ifnIdx++]=argument; continue; }
+ if (multiple_inputs) { inFileNames[ifnIdx++] = argument; continue; }
- /* Store first non-option arg in input_filename to preserve original cli logic. */
- if (!input_filename) { input_filename=argument; continue; }
+ /* original cli logic : lz4 input output */
+ /* First non-option arg is input_filename. */
+ if (!input_filename) { input_filename = argument; continue; }
- /* Second non-option arg in output_filename to preserve original cli logic. */
+ /* Second non-option arg is output_filename */
if (!output_filename) {
- output_filename=argument;
+ output_filename = argument;
if (!strcmp (output_filename, nullOutput)) output_filename = nulmark;
continue;
}
- /* 3rd non-option arg should not exist */
- DISPLAYLEVEL(1, "Warning : %s won't be used ! Do you want multiple input files (-m) ? \n", argument);
+ /* 3rd+ non-option arg should not exist */
+ DISPLAYLEVEL(1, "%s : %s won't be used ! Do you want multiple input files (-m) ? \n",
+ forceOverwrite ? "Warning" : "Error",
+ argument);
+ if (!forceOverwrite) exit(1);
}
DISPLAYLEVEL(3, WELCOME_MESSAGE);
@@ -617,7 +629,7 @@
input_filename = inFileNames[0];
#ifdef UTIL_HAS_CREATEFILELIST
if (recursive) { /* at this stage, filenameTable is a list of paths, which can contain both files and directories */
- extendedFileList = UTIL_createFileList(inFileNames, ifnIdx, &fileNamesBuf, &fileNamesNb);
+ const char** extendedFileList = UTIL_createFileList(inFileNames, ifnIdx, &fileNamesBuf, &fileNamesNb);
if (extendedFileList) {
unsigned u;
for (u=0; u<fileNamesNb; u++) DISPLAYLEVEL(4, "%u %s\n", u, extendedFileList[u]);
@@ -649,27 +661,19 @@
mode = om_decompress; /* defer to decompress */
}
- /* compress or decompress */
+ /* No input provided => use stdin */
if (!input_filename) input_filename = stdinmark;
- /* Check if input is defined as console; trigger an error in this case */
+
+ /* Refuse to use the console as input */
if (!strcmp(input_filename, stdinmark) && IS_CONSOLE(stdin) ) {
DISPLAYLEVEL(1, "refusing to read from a console\n");
exit(1);
}
+
if (!strcmp(input_filename, stdinmark)) {
/* if input==stdin and no output defined, stdout becomes default output */
if (!output_filename) output_filename = stdoutmark;
}
- else{
-#ifdef UTIL_HAS_CREATEFILELIST
- if (!recursive && !UTIL_isRegFile(input_filename)) {
-#else
- if (!UTIL_isRegFile(input_filename)) {
-#endif
- DISPLAYLEVEL(1, "%s: is not a regular file \n", input_filename);
- exit(1);
- }
- }
/* No output filename ==> try to select one automatically (when possible) */
while ((!output_filename) && (multiple_inputs==0)) {
@@ -679,7 +683,7 @@
* To ensure `stdout` is explicitly selected, use `-c` command flag.
* Conversely, to ensure output will not become `stdout`, use `-m` command flag */
DISPLAYLEVEL(1, "Warning : using stdout as default output. Do not rely on this behavior: use explicit `-c` instead ! \n");
- output_filename=stdoutmark;
+ output_filename = stdoutmark;
break;
}
if (mode == om_auto) { /* auto-determine compression or decompression, based on file extension */
@@ -695,7 +699,7 @@
DISPLAYLEVEL(2, "Compressed filename will be : %s \n", output_filename);
break;
}
- if (mode == om_decompress) {/* decompression to file (automatic name will work only if input filename has correct format extension) */
+ if (mode == om_decompress) {/* decompress to file (automatic output name only works if input filename has correct format extension) */
size_t outl;
size_t const inl = strlen(input_filename);
dynNameSpace = (char*)calloc(1,inl+1);
@@ -704,32 +708,27 @@
outl = inl;
if (inl>4)
while ((outl >= inl-4) && (input_filename[outl] == extension[outl-inl+4])) dynNameSpace[outl--]=0;
- if (outl != inl-5) { DISPLAYLEVEL(1, "Cannot determine an output filename\n"); badusage(exeName); }
+ if (outl != inl-5) { DISPLAYLEVEL(1, "Cannot determine an output filename \n"); badusage(exeName); }
output_filename = dynNameSpace;
DISPLAYLEVEL(2, "Decoding file %s \n", output_filename);
}
break;
}
- if (mode == om_list){
- /* Exit if trying to read from stdin as this isn't supported in this mode */
- if(!strcmp(input_filename, stdinmark)){
- DISPLAYLEVEL(1, "refusing to read from standard input in --list mode\n");
- exit(1);
- }
- if(!multiple_inputs){
- inFileNames[ifnIdx++] = input_filename;
- }
- }
- else{
- if (multiple_inputs==0) assert(output_filename);
+ if (mode == om_list) {
+ if (!multiple_inputs) inFileNames[ifnIdx++] = input_filename;
+ } else {
+ if (!multiple_inputs) assert(output_filename != NULL);
}
/* when multiple_inputs==1, output_filename may simply be useless,
* however, output_filename must be !NULL for next strcmp() tests */
if (!output_filename) output_filename = "*\\dummy^!//";
/* Check if output is defined as console; trigger an error in this case */
- if (!strcmp(output_filename,stdoutmark) && IS_CONSOLE(stdout) && !forceStdout) {
+ if ( !strcmp(output_filename,stdoutmark)
+ && mode != om_list
+ && IS_CONSOLE(stdout)
+ && !forceStdout) {
DISPLAYLEVEL(1, "refusing to write to console without -c \n");
exit(1);
}
@@ -747,8 +746,10 @@
if (ifnIdx == 0) multiple_inputs = 0;
if (mode == om_decompress) {
if (multiple_inputs) {
- const char* const dec_extension = !strcmp(output_filename,stdoutmark) ? stdoutmark : LZ4_EXTENSION;
- assert(ifnIdx <= INT_MAX);
+ const char* dec_extension = LZ4_EXTENSION;
+ if (!strcmp(output_filename, stdoutmark)) dec_extension = stdoutmark;
+ if (!strcmp(output_filename, nulmark)) dec_extension = nulmark;
+ assert(ifnIdx < INT_MAX);
operationResult = LZ4IO_decompressMultipleFilenames(inFileNames, (int)ifnIdx, dec_extension, prefs);
} else {
operationResult = DEFAULT_DECOMPRESSOR(input_filename, output_filename, prefs);
@@ -776,12 +777,7 @@
_cleanup:
if (main_pause) waitEnter();
free(dynNameSpace);
-#ifdef UTIL_HAS_CREATEFILELIST
- if (extendedFileList) {
- UTIL_freeFileList(extendedFileList, fileNamesBuf);
- inFileNames = NULL;
- }
-#endif
+ free(fileNamesBuf);
LZ4IO_freePreferences(prefs);
free((void*)inFileNames);
return operationResult;
diff --git a/programs/lz4io.c b/programs/lz4io.c
index a274798..8b70b91 100644
--- a/programs/lz4io.c
+++ b/programs/lz4io.c
@@ -1,6 +1,6 @@
/*
LZ4io.c - LZ4 File/Stream Interface
- Copyright (C) Yann Collet 2011-2017
+ Copyright (C) Yann Collet 2011-2020
GPL v2 License
@@ -103,12 +103,30 @@
} }
static const clock_t refreshRate = CLOCKS_PER_SEC / 6;
static clock_t g_time = 0;
+
#define LZ4IO_STATIC_ASSERT(c) { enum { LZ4IO_static_assert = 1/(int)(!!(c)) }; } /* use after variable declarations */
/**************************************
-* Local Parameters
-**************************************/
+* Exceptions
+***************************************/
+#ifndef DEBUG
+# define DEBUG 0
+#endif
+#define DEBUGOUTPUT(...) if (DEBUG) DISPLAY(__VA_ARGS__);
+#define END_PROCESS(error, ...) \
+{ \
+ DEBUGOUTPUT("Error defined at %s, line %i : \n", __FILE__, __LINE__); \
+ DISPLAYLEVEL(1, "Error %i : ", error); \
+ DISPLAYLEVEL(1, __VA_ARGS__); \
+ DISPLAYLEVEL(1, " \n"); \
+ exit(error); \
+}
+
+
+/* ************************************************** */
+/* ****************** Parameters ******************** */
+/* ************************************************** */
struct LZ4IO_prefs_s {
int passThrough;
@@ -127,40 +145,10 @@
int removeSrcFile;
};
-/**************************************
-* Exceptions
-***************************************/
-#ifndef DEBUG
-# define DEBUG 0
-#endif
-#define DEBUGOUTPUT(...) if (DEBUG) DISPLAY(__VA_ARGS__);
-#define EXM_THROW(error, ...) \
-{ \
- DEBUGOUTPUT("Error defined at %s, line %i : \n", __FILE__, __LINE__); \
- DISPLAYLEVEL(1, "Error %i : ", error); \
- DISPLAYLEVEL(1, __VA_ARGS__); \
- DISPLAYLEVEL(1, " \n"); \
- exit(error); \
-}
-
-
-/**************************************
-* Version modifiers
-**************************************/
-#define EXTENDED_ARGUMENTS
-#define EXTENDED_HELP
-#define EXTENDED_FORMAT
-#define DEFAULT_DECOMPRESSOR LZ4IO_decompressLZ4F
-
-
-/* ************************************************** */
-/* ****************** Parameters ******************** */
-/* ************************************************** */
-
LZ4IO_prefs_t* LZ4IO_defaultPreferences(void)
{
LZ4IO_prefs_t* const ret = (LZ4IO_prefs_t*)malloc(sizeof(*ret));
- if (!ret) EXM_THROW(21, "Allocation error : not enough memory");
+ if (!ret) END_PROCESS(21, "Allocation error : not enough memory");
ret->passThrough = 0;
ret->overwrite = 1;
ret->testMode = 0;
@@ -298,6 +286,26 @@
/* ************************************************************************ **
+** ********************** String functions ********************* **
+** ************************************************************************ */
+
+static int LZ4IO_isDevNull(const char* s)
+{
+ return UTIL_sameString(s, nulmark);
+}
+
+static int LZ4IO_isStdin(const char* s)
+{
+ return UTIL_sameString(s, stdinmark);
+}
+
+static int LZ4IO_isStdout(const char* s)
+{
+ return UTIL_sameString(s, stdoutmark);
+}
+
+
+/* ************************************************************************ **
** ********************** LZ4 File / Pipe compression ********************* **
** ************************************************************************ */
@@ -313,13 +321,13 @@
{
FILE* f;
- if (!strcmp (srcFileName, stdinmark)) {
- DISPLAYLEVEL(4,"Using stdin for input\n");
+ if (LZ4IO_isStdin(srcFileName)) {
+ DISPLAYLEVEL(4,"Using stdin for input \n");
f = stdin;
SET_BINARY_MODE(stdin);
} else {
f = fopen(srcFileName, "rb");
- if ( f==NULL ) DISPLAYLEVEL(1, "%s: %s \n", srcFileName, strerror(errno));
+ if (f==NULL) DISPLAYLEVEL(1, "%s: %s \n", srcFileName, strerror(errno));
}
return f;
@@ -334,7 +342,7 @@
FILE* f;
assert(dstFileName != NULL);
- if (!strcmp (dstFileName, stdoutmark)) {
+ if (LZ4IO_isStdout(dstFileName)) {
DISPLAYLEVEL(4, "Using stdout for output \n");
f = stdout;
SET_BINARY_MODE(stdout);
@@ -343,7 +351,8 @@
" to force-enable it, add --sparse command \n");
}
} else {
- if (!prefs->overwrite && strcmp (dstFileName, nulmark)) { /* Check if destination file already exists */
+ if (!prefs->overwrite && !LZ4IO_isDevNull(dstFileName)) {
+ /* Check if destination file already exists */
FILE* const testf = fopen( dstFileName, "rb" );
if (testf != NULL) { /* dest exists, prompt for overwrite authorization */
fclose(testf);
@@ -351,7 +360,7 @@
DISPLAY("%s already exists; not overwritten \n", dstFileName);
return NULL;
}
- DISPLAY("%s already exists; do you wish to overwrite (y/N) ? ", dstFileName);
+ DISPLAY("%s already exists; do you want to overwrite (y/N) ? ", dstFileName);
{ int ch = getchar();
if ((ch!='Y') && (ch!='y')) {
DISPLAY(" not overwritten \n");
@@ -377,7 +386,11 @@
* Legacy Compression
***************************************/
-/* unoptimized version; solves endianess & alignment issues */
+/* Size in bytes of a legacy block header in little-endian format */
+#define LZ4IO_LEGACY_BLOCK_HEADER_SIZE 4
+#define LZ4IO_LEGACY_BLOCK_SIZE_MAX (8 MB)
+
+/* unoptimized version; solves endianness & alignment issues */
static void LZ4IO_writeLE32 (void* p, unsigned value32)
{
unsigned char* const dstPtr = (unsigned char*)p;
@@ -413,24 +426,24 @@
/* Init */
clock_t const clockStart = clock();
if (finput == NULL)
- EXM_THROW(20, "%s : open file error ", input_filename);
+ END_PROCESS(20, "%s : open file error ", input_filename);
foutput = LZ4IO_openDstFile(output_filename, prefs);
if (foutput == NULL) {
fclose(finput);
- EXM_THROW(20, "%s : open file error ", input_filename);
+ END_PROCESS(20, "%s : open file error ", input_filename);
}
/* Allocate Memory */
in_buff = (char*)malloc(LEGACY_BLOCKSIZE);
out_buff = (char*)malloc((size_t)outBuffSize + 4);
if (!in_buff || !out_buff)
- EXM_THROW(21, "Allocation error : not enough memory");
+ END_PROCESS(21, "Allocation error : not enough memory");
/* Write Archive Header */
LZ4IO_writeLE32(out_buff, LEGACY_MAGICNUMBER);
if (fwrite(out_buff, 1, MAGICNUMBER_SIZE, foutput) != MAGICNUMBER_SIZE)
- EXM_THROW(22, "Write error : cannot write header");
+ END_PROCESS(22, "Write error : cannot write header");
/* Main Loop */
while (1) {
@@ -445,7 +458,7 @@
outSize = compressionFunction(in_buff, out_buff+4, (int)inSize, outBuffSize, compressionlevel);
assert(outSize >= 0);
compressedfilesize += (unsigned long long)outSize+4;
- DISPLAYUPDATE(2, "\rRead : %i MB ==> %.2f%% ",
+ DISPLAYUPDATE(2, "\rRead : %i MiB ==> %.2f%% ",
(int)(filesize>>20), (double)compressedfilesize/filesize*100);
/* Write Block */
@@ -453,19 +466,19 @@
assert(outSize < outBuffSize);
LZ4IO_writeLE32(out_buff, (unsigned)outSize);
if (fwrite(out_buff, 1, (size_t)outSize+4, foutput) != (size_t)(outSize+4)) {
- EXM_THROW(24, "Write error : cannot write compressed block");
+ END_PROCESS(24, "Write error : cannot write compressed block");
} }
- if (ferror(finput)) EXM_THROW(25, "Error while reading %s ", input_filename);
+ if (ferror(finput)) END_PROCESS(24, "Error while reading %s ", input_filename);
/* Status */
clockEnd = clock();
- if (clockEnd==clockStart) clockEnd+=1; /* avoid division by zero (speed) */
+ clockEnd += (clockEnd==clockStart); /* avoid division by zero (speed) */
filesize += !filesize; /* avoid division by zero (ratio) */
DISPLAYLEVEL(2, "\r%79s\r", ""); /* blank line */
DISPLAYLEVEL(2,"Compressed %llu bytes into %llu bytes ==> %.2f%%\n",
filesize, compressedfilesize, (double)compressedfilesize / filesize * 100);
{ double const seconds = (double)(clockEnd - clockStart) / CLOCKS_PER_SEC;
- DISPLAYLEVEL(4,"Done in %.2f s ==> %.2f MB/s\n", seconds,
+ DISPLAYLEVEL(4,"Done in %.2f s ==> %.2f MiB/s\n", seconds,
(double)filesize / seconds / 1024 / 1024);
}
@@ -473,7 +486,7 @@
free(in_buff);
free(out_buff);
fclose(finput);
- if (strcmp(output_filename,stdoutmark)) fclose(foutput); /* do not close stdout */
+ if (!LZ4IO_isStdout(output_filename)) fclose(foutput); /* do not close stdout */
return 0;
}
@@ -498,7 +511,7 @@
/* loop on each file */
for (i=0; i<ifntSize; i++) {
size_t const ifnSize = strlen(inFileNamesTable[i]);
- if (!strcmp(suffix, stdoutmark)) {
+ if (LZ4IO_isStdout(suffix)) {
missed_files += LZ4IO_compressFilename_Legacy(
inFileNamesTable[i], stdoutmark,
compressionLevel, prefs);
@@ -530,7 +543,6 @@
/*********************************************
* Compression using Frame format
*********************************************/
-
typedef struct {
void* srcBuffer;
size_t srcBufferSize;
@@ -551,15 +563,15 @@
char* dictBuf;
FILE* dictFile;
- if (!circularBuf) EXM_THROW(25, "Allocation error : not enough memory for circular buffer");
- if (!dictFilename) EXM_THROW(25, "Dictionary error : no filename provided");
+ if (!circularBuf) END_PROCESS(25, "Allocation error : not enough memory for circular buffer");
+ if (!dictFilename) END_PROCESS(26, "Dictionary error : no filename provided");
dictFile = LZ4IO_openSrcFile(dictFilename);
- if (!dictFile) EXM_THROW(25, "Dictionary error : could not open dictionary file");
+ if (!dictFile) END_PROCESS(27, "Dictionary error : could not open dictionary file");
- /* opportunistically seek to the part of the file we care about. If this */
- /* fails it's not a problem since we'll just read everything anyways. */
- if (strcmp(dictFilename, stdinmark)) {
+ /* opportunistically seek to the part of the file we care about.
+ * If this fails it's not a problem since we'll just read everything anyways. */
+ if (!LZ4IO_isStdin(dictFilename)) {
(void)UTIL_fseek(dictFile, -LZ4_MAX_DICT_SIZE, SEEK_END);
}
@@ -584,7 +596,7 @@
} else {
/* Otherwise, we will alloc a new buffer and copy our dict into that. */
dictBuf = (char *)malloc(dictLen ? dictLen : 1);
- if (!dictBuf) EXM_THROW(25, "Allocation error : not enough memory");
+ if (!dictBuf) END_PROCESS(28, "Allocation error : not enough memory");
memcpy(dictBuf, circularBuf + dictStart, circularBufSize - dictStart);
memcpy(dictBuf + circularBufSize - dictStart, circularBuf, dictLen - (circularBufSize - dictStart));
@@ -603,7 +615,7 @@
LZ4F_CDict* cdict;
if (!prefs->useDictionary) return NULL;
dictionaryBuffer = LZ4IO_createDict(&dictionarySize, prefs->dictionaryFilename);
- if (!dictionaryBuffer) EXM_THROW(25, "Dictionary error : could not create dictionary");
+ if (!dictionaryBuffer) END_PROCESS(29, "Dictionary error : could not create dictionary");
cdict = LZ4F_createCDict(dictionaryBuffer, dictionarySize);
free(dictionaryBuffer);
return cdict;
@@ -615,14 +627,14 @@
cRess_t ress;
LZ4F_errorCode_t const errorCode = LZ4F_createCompressionContext(&(ress.ctx), LZ4F_VERSION);
- if (LZ4F_isError(errorCode)) EXM_THROW(30, "Allocation error : can't create LZ4F context : %s", LZ4F_getErrorName(errorCode));
+ if (LZ4F_isError(errorCode)) END_PROCESS(30, "Allocation error : can't create LZ4F context : %s", LZ4F_getErrorName(errorCode));
/* Allocate Memory */
ress.srcBuffer = malloc(blockSize);
ress.srcBufferSize = blockSize;
ress.dstBufferSize = LZ4F_compressFrameBound(blockSize, NULL); /* cover worst case */
ress.dstBuffer = malloc(ress.dstBufferSize);
- if (!ress.srcBuffer || !ress.dstBuffer) EXM_THROW(31, "Allocation error : not enough memory");
+ if (!ress.srcBuffer || !ress.dstBuffer) END_PROCESS(31, "Allocation error : not enough memory");
ress.cdict = LZ4IO_createCDict(prefs);
@@ -638,7 +650,7 @@
ress.cdict = NULL;
{ LZ4F_errorCode_t const errorCode = LZ4F_freeCompressionContext(ress.ctx);
- if (LZ4F_isError(errorCode)) EXM_THROW(38, "Error : can't free LZ4F context resource : %s", LZ4F_getErrorName(errorCode)); }
+ if (LZ4F_isError(errorCode)) END_PROCESS(35, "Error : can't free LZ4F context resource : %s", LZ4F_getErrorName(errorCode)); }
}
/*
@@ -686,7 +698,7 @@
/* read first block */
readSize = fread(srcBuffer, (size_t)1, blockSize, srcFile);
- if (ferror(srcFile)) EXM_THROW(30, "Error reading %s ", srcFileName);
+ if (ferror(srcFile)) END_PROCESS(40, "Error reading %s ", srcFileName);
filesize += readSize;
/* single-block file */
@@ -694,14 +706,14 @@
/* Compress in single pass */
size_t const cSize = LZ4F_compressFrame_usingCDict(ctx, dstBuffer, dstBufferSize, srcBuffer, readSize, ress.cdict, &prefs);
if (LZ4F_isError(cSize))
- EXM_THROW(31, "Compression failed : %s", LZ4F_getErrorName(cSize));
+ END_PROCESS(41, "Compression failed : %s", LZ4F_getErrorName(cSize));
compressedfilesize = cSize;
- DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%% ",
+ DISPLAYUPDATE(2, "\rRead : %u MiB ==> %.2f%% ",
(unsigned)(filesize>>20), (double)compressedfilesize/(filesize+!filesize)*100); /* avoid division by zero */
/* Write Block */
if (fwrite(dstBuffer, 1, cSize, dstFile) != cSize) {
- EXM_THROW(32, "Write error : failed writing single-block compressed frame");
+ END_PROCESS(42, "Write error : failed writing single-block compressed frame");
} }
else
@@ -710,55 +722,55 @@
{
/* Write Frame Header */
size_t const headerSize = LZ4F_compressBegin_usingCDict(ctx, dstBuffer, dstBufferSize, ress.cdict, &prefs);
- if (LZ4F_isError(headerSize)) EXM_THROW(33, "File header generation failed : %s", LZ4F_getErrorName(headerSize));
+ if (LZ4F_isError(headerSize)) END_PROCESS(43, "File header generation failed : %s", LZ4F_getErrorName(headerSize));
if (fwrite(dstBuffer, 1, headerSize, dstFile) != headerSize)
- EXM_THROW(34, "Write error : cannot write header");
+ END_PROCESS(44, "Write error : cannot write header");
compressedfilesize += headerSize;
/* Main Loop - one block at a time */
while (readSize>0) {
size_t const outSize = LZ4F_compressUpdate(ctx, dstBuffer, dstBufferSize, srcBuffer, readSize, NULL);
if (LZ4F_isError(outSize))
- EXM_THROW(35, "Compression failed : %s", LZ4F_getErrorName(outSize));
+ END_PROCESS(45, "Compression failed : %s", LZ4F_getErrorName(outSize));
compressedfilesize += outSize;
- DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%% ",
+ DISPLAYUPDATE(2, "\rRead : %u MiB ==> %.2f%% ",
(unsigned)(filesize>>20), (double)compressedfilesize/filesize*100);
/* Write Block */
if (fwrite(dstBuffer, 1, outSize, dstFile) != outSize)
- EXM_THROW(36, "Write error : cannot write compressed block");
+ END_PROCESS(46, "Write error : cannot write compressed block");
/* Read next block */
readSize = fread(srcBuffer, (size_t)1, (size_t)blockSize, srcFile);
filesize += readSize;
}
- if (ferror(srcFile)) EXM_THROW(37, "Error reading %s ", srcFileName);
+ if (ferror(srcFile)) END_PROCESS(47, "Error reading %s ", srcFileName);
/* End of Frame mark */
{ size_t const endSize = LZ4F_compressEnd(ctx, dstBuffer, dstBufferSize, NULL);
if (LZ4F_isError(endSize))
- EXM_THROW(38, "End of frame error : %s", LZ4F_getErrorName(endSize));
+ END_PROCESS(48, "End of frame error : %s", LZ4F_getErrorName(endSize));
if (fwrite(dstBuffer, 1, endSize, dstFile) != endSize)
- EXM_THROW(39, "Write error : cannot write end of frame");
+ END_PROCESS(49, "Write error : cannot write end of frame");
compressedfilesize += endSize;
} }
/* Release file handlers */
fclose (srcFile);
- if (strcmp(dstFileName,stdoutmark)) fclose (dstFile); /* do not close stdout */
+ if (!LZ4IO_isStdout(dstFileName)) fclose(dstFile); /* do not close stdout */
/* Copy owner, file permissions and modification time */
{ stat_t statbuf;
- if (strcmp (srcFileName, stdinmark)
- && strcmp (dstFileName, stdoutmark)
- && strcmp (dstFileName, nulmark)
+ if (!LZ4IO_isStdin(srcFileName)
+ && !LZ4IO_isStdout(dstFileName)
+ && !LZ4IO_isDevNull(dstFileName)
&& UTIL_getFileStat(srcFileName, &statbuf)) {
UTIL_setFileStat(dstFileName, &statbuf);
} }
if (io_prefs->removeSrcFile) { /* remove source file : --rm */
if (remove(srcFileName))
- EXM_THROW(40, "Remove error : %s: %s", srcFileName, strerror(errno));
+ END_PROCESS(50, "Remove error : %s: %s", srcFileName, strerror(errno));
}
/* Final Status */
@@ -814,12 +826,13 @@
/* loop on each file */
for (i=0; i<ifntSize; i++) {
size_t const ifnSize = strlen(inFileNamesTable[i]);
- if (!strcmp(suffix, stdoutmark)) {
+ if (LZ4IO_isStdout(suffix)) {
missed_files += LZ4IO_compressFilename_extRess(ress,
inFileNamesTable[i], stdoutmark,
compressionLevel, prefs);
continue;
}
+ /* suffix != stdout => compress into a file => generate its name */
if (ofnSize <= ifnSize+suffixSize+1) {
free(dstFileName);
ofnSize = ifnSize + 20;
@@ -877,14 +890,14 @@
if (!sparseMode) { /* normal write */
size_t const sizeCheck = fwrite(buffer, 1, bufferSize, file);
- if (sizeCheck != bufferSize) EXM_THROW(70, "Write error : cannot write decoded block");
+ if (sizeCheck != bufferSize) END_PROCESS(70, "Write error : cannot write decoded block");
return 0;
}
/* avoid int overflow */
if (storedSkips > 1 GB) {
int const seekResult = UTIL_fseek(file, 1 GB, SEEK_CUR);
- if (seekResult != 0) EXM_THROW(71, "1 GB skip error (sparse file support)");
+ if (seekResult != 0) END_PROCESS(71, "1 GB skip error (sparse file support)");
storedSkips -= 1 GB;
}
@@ -901,13 +914,13 @@
if (nb0T != seg0SizeT) { /* not all 0s */
errno = 0;
{ int const seekResult = UTIL_fseek(file, storedSkips, SEEK_CUR);
- if (seekResult) EXM_THROW(72, "Sparse skip error(%d): %s ; try --no-sparse", (int)errno, strerror(errno));
+ if (seekResult) END_PROCESS(72, "Sparse skip error(%d): %s ; try --no-sparse", (int)errno, strerror(errno));
}
storedSkips = 0;
seg0SizeT -= nb0T;
ptrT += nb0T;
{ size_t const sizeCheck = fwrite(ptrT, sizeT, seg0SizeT, file);
- if (sizeCheck != seg0SizeT) EXM_THROW(73, "Write error : cannot write decoded block");
+ if (sizeCheck != seg0SizeT) END_PROCESS(73, "Write error : cannot write decoded block");
} }
ptrT += seg0SizeT;
}
@@ -921,10 +934,10 @@
storedSkips += (unsigned) (restPtr - restStart);
if (restPtr != restEnd) {
int const seekResult = UTIL_fseek(file, storedSkips, SEEK_CUR);
- if (seekResult) EXM_THROW(74, "Sparse skip error ; try --no-sparse");
+ if (seekResult) END_PROCESS(74, "Sparse skip error ; try --no-sparse");
storedSkips = 0;
{ size_t const sizeCheck = fwrite(restPtr, 1, (size_t)(restEnd - restPtr), file);
- if (sizeCheck != (size_t)(restEnd - restPtr)) EXM_THROW(75, "Write error : cannot write decoded end of block");
+ if (sizeCheck != (size_t)(restEnd - restPtr)) END_PROCESS(75, "Write error : cannot write decoded end of block");
} }
}
@@ -936,15 +949,17 @@
if (storedSkips>0) { /* implies sparseFileSupport>0 */
const char lastZeroByte[1] = { 0 };
if (UTIL_fseek(file, storedSkips-1, SEEK_CUR) != 0)
- EXM_THROW(69, "Final skip error (sparse file)\n");
+ END_PROCESS(68, "Final skip error (sparse file)\n");
if (fwrite(lastZeroByte, 1, 1, file) != 1)
- EXM_THROW(69, "Write error : cannot write last zero\n");
+ END_PROCESS(69, "Write error : cannot write last zero\n");
}
}
static unsigned g_magicRead = 0; /* out-parameter of LZ4IO_decodeLegacyStream() */
-static unsigned long long LZ4IO_decodeLegacyStream(FILE* finput, FILE* foutput, const LZ4IO_prefs_t* prefs)
+
+static unsigned long long
+LZ4IO_decodeLegacyStream(FILE* finput, FILE* foutput, const LZ4IO_prefs_t* prefs)
{
unsigned long long streamSize = 0;
unsigned storedSkips = 0;
@@ -952,18 +967,19 @@
/* Allocate Memory */
char* const in_buff = (char*)malloc((size_t)LZ4_compressBound(LEGACY_BLOCKSIZE));
char* const out_buff = (char*)malloc(LEGACY_BLOCKSIZE);
- if (!in_buff || !out_buff) EXM_THROW(51, "Allocation error : not enough memory");
+ if (!in_buff || !out_buff) END_PROCESS(51, "Allocation error : not enough memory");
/* Main Loop */
while (1) {
unsigned int blockSize;
/* Block Size */
- { size_t const sizeCheck = fread(in_buff, 1, 4, finput);
+ { size_t const sizeCheck = fread(in_buff, 1, LZ4IO_LEGACY_BLOCK_HEADER_SIZE, finput);
if (sizeCheck == 0) break; /* Nothing to read : file read is completed */
- if (sizeCheck != 4) EXM_THROW(52, "Read error : cannot access block size "); }
- blockSize = LZ4IO_readLE32(in_buff); /* Convert to Little Endian */
- if (blockSize > LZ4_COMPRESSBOUND(LEGACY_BLOCKSIZE)) {
+ if (sizeCheck != LZ4IO_LEGACY_BLOCK_HEADER_SIZE) END_PROCESS(52, "Read error : cannot access block size ");
+ }
+ blockSize = LZ4IO_readLE32(in_buff); /* Convert to Little Endian */
+ if (blockSize > LZ4_COMPRESSBOUND(LEGACY_BLOCKSIZE)) {
/* Cannot read next block : maybe new stream ? */
g_magicRead = blockSize;
break;
@@ -971,16 +987,16 @@
/* Read Block */
{ size_t const sizeCheck = fread(in_buff, 1, blockSize, finput);
- if (sizeCheck!=blockSize) EXM_THROW(52, "Read error : cannot access compressed block !"); }
+ if (sizeCheck != blockSize) END_PROCESS(53, "Read error : cannot access compressed block !"); }
/* Decode Block */
{ int const decodeSize = LZ4_decompress_safe(in_buff, out_buff, (int)blockSize, LEGACY_BLOCKSIZE);
- if (decodeSize < 0) EXM_THROW(53, "Decoding Failed ! Corrupted input detected !");
+ if (decodeSize < 0) END_PROCESS(54, "Decoding Failed ! Corrupted input detected !");
streamSize += (unsigned long long)decodeSize;
/* Write Block */
storedSkips = LZ4IO_fwriteSparse(foutput, out_buff, (size_t)decodeSize, prefs->sparseFileSupport, storedSkips); /* success or die */
} }
- if (ferror(finput)) EXM_THROW(54, "Read error : ferror");
+ if (ferror(finput)) END_PROCESS(55, "Read error : ferror");
LZ4IO_fwriteSparseEnd(foutput, storedSkips);
@@ -1013,7 +1029,7 @@
}
ress->dictBuffer = LZ4IO_createDict(&ress->dictBufferSize, prefs->dictionaryFilename);
- if (!ress->dictBuffer) EXM_THROW(25, "Dictionary error : could not create dictionary");
+ if (!ress->dictBuffer) END_PROCESS(25, "Dictionary error : could not create dictionary");
}
static const size_t LZ4IO_dBufferSize = 64 KB;
@@ -1023,14 +1039,14 @@
/* init */
LZ4F_errorCode_t const errorCode = LZ4F_createDecompressionContext(&ress.dCtx, LZ4F_VERSION);
- if (LZ4F_isError(errorCode)) EXM_THROW(60, "Can't create LZ4F context : %s", LZ4F_getErrorName(errorCode));
+ if (LZ4F_isError(errorCode)) END_PROCESS(60, "Can't create LZ4F context : %s", LZ4F_getErrorName(errorCode));
/* Allocate Memory */
ress.srcBufferSize = LZ4IO_dBufferSize;
ress.srcBuffer = malloc(ress.srcBufferSize);
ress.dstBufferSize = LZ4IO_dBufferSize;
ress.dstBuffer = malloc(ress.dstBufferSize);
- if (!ress.srcBuffer || !ress.dstBuffer) EXM_THROW(61, "Allocation error : not enough memory");
+ if (!ress.srcBuffer || !ress.dstBuffer) END_PROCESS(61, "Allocation error : not enough memory");
LZ4IO_loadDDict(&ress, prefs);
@@ -1041,7 +1057,7 @@
static void LZ4IO_freeDResources(dRess_t ress)
{
LZ4F_errorCode_t errorCode = LZ4F_freeDecompressionContext(ress.dCtx);
- if (LZ4F_isError(errorCode)) EXM_THROW(69, "Error : can't free LZ4F context resource : %s", LZ4F_getErrorName(errorCode));
+ if (LZ4F_isError(errorCode)) END_PROCESS(69, "Error : can't free LZ4F context resource : %s", LZ4F_getErrorName(errorCode));
free(ress.srcBuffer);
free(ress.dstBuffer);
free(ress.dictBuffer);
@@ -1056,13 +1072,22 @@
unsigned long long filesize = 0;
LZ4F_errorCode_t nextToLoad;
unsigned storedSkips = 0;
+ LZ4F_decompressOptions_t const dOpt_skipCrc = { 0, 1, 0, 0 };
+ const LZ4F_decompressOptions_t* const dOptPtr =
+ ((prefs->blockChecksum==0) && (prefs->streamChecksum==0)) ?
+ &dOpt_skipCrc : NULL;
/* Init feed with magic number (already consumed from FILE* sFile) */
{ size_t inSize = MAGICNUMBER_SIZE;
size_t outSize= 0;
LZ4IO_writeLE32(ress.srcBuffer, LZ4IO_MAGICNUMBER);
- nextToLoad = LZ4F_decompress_usingDict(ress.dCtx, ress.dstBuffer, &outSize, ress.srcBuffer, &inSize, ress.dictBuffer, ress.dictBufferSize, NULL);
- if (LZ4F_isError(nextToLoad)) EXM_THROW(62, "Header error : %s", LZ4F_getErrorName(nextToLoad));
+ nextToLoad = LZ4F_decompress_usingDict(ress.dCtx,
+ ress.dstBuffer, &outSize,
+ ress.srcBuffer, &inSize,
+ ress.dictBuffer, ress.dictBufferSize,
+ dOptPtr); /* set it once, it's enough */
+ if (LZ4F_isError(nextToLoad))
+ END_PROCESS(62, "Header error : %s", LZ4F_getErrorName(nextToLoad));
}
/* Main Loop */
@@ -1080,8 +1105,13 @@
/* Decode Input (at least partially) */
size_t remaining = readSize - pos;
decodedBytes = ress.dstBufferSize;
- nextToLoad = LZ4F_decompress_usingDict(ress.dCtx, ress.dstBuffer, &decodedBytes, (char*)(ress.srcBuffer)+pos, &remaining, ress.dictBuffer, ress.dictBufferSize, NULL);
- if (LZ4F_isError(nextToLoad)) EXM_THROW(66, "Decompression error : %s", LZ4F_getErrorName(nextToLoad));
+ nextToLoad = LZ4F_decompress_usingDict(ress.dCtx,
+ ress.dstBuffer, &decodedBytes,
+ (char*)(ress.srcBuffer)+pos, &remaining,
+ ress.dictBuffer, ress.dictBufferSize,
+ NULL);
+ if (LZ4F_isError(nextToLoad))
+ END_PROCESS(66, "Decompression error : %s", LZ4F_getErrorName(nextToLoad));
pos += remaining;
/* Write Block */
@@ -1089,17 +1119,17 @@
if (!prefs->testMode)
storedSkips = LZ4IO_fwriteSparse(dstFile, ress.dstBuffer, decodedBytes, prefs->sparseFileSupport, storedSkips);
filesize += decodedBytes;
- DISPLAYUPDATE(2, "\rDecompressed : %u MB ", (unsigned)(filesize>>20));
+ DISPLAYUPDATE(2, "\rDecompressed : %u MiB ", (unsigned)(filesize>>20));
}
if (!nextToLoad) break;
}
}
/* can be out because readSize == 0, which could be an fread() error */
- if (ferror(srcFile)) EXM_THROW(67, "Read error");
+ if (ferror(srcFile)) END_PROCESS(67, "Read error");
if (!prefs->testMode) LZ4IO_fwriteSparseEnd(dstFile, storedSkips);
- if (nextToLoad!=0) EXM_THROW(68, "Unfinished stream");
+ if (nextToLoad!=0) END_PROCESS(68, "Unfinished stream");
return filesize;
}
@@ -1123,19 +1153,36 @@
unsigned storedSkips = 0;
if (fwrite(MNstore, 1, MAGICNUMBER_SIZE, foutput) != MAGICNUMBER_SIZE) {
- EXM_THROW(50, "Pass-through write error");
+ END_PROCESS(50, "Pass-through write error");
}
while (readBytes) {
readBytes = fread(buffer, 1, sizeof(buffer), finput);
total += readBytes;
storedSkips = LZ4IO_fwriteSparse(foutput, buffer, readBytes, sparseFileSupport, storedSkips);
}
- if (ferror(finput)) EXM_THROW(51, "Read Error");
+ if (ferror(finput)) END_PROCESS(51, "Read Error");
LZ4IO_fwriteSparseEnd(foutput, storedSkips);
return total;
}
+/* when fseek() doesn't work (pipe scenario),
+ * read and forget from input.
+**/
+#define SKIP_BUFF_SIZE (16 KB)
+#define MIN(a,b) ( ((a)<(b)) ? (a) : (b) )
+static int skipStream(FILE* f, unsigned offset)
+{
+ char buf[SKIP_BUFF_SIZE];
+ while (offset > 0) {
+ size_t const tr = MIN(offset, sizeof(buf));
+ size_t const r = fread(buf, 1, tr, f);
+ if (r != tr) return 1; /* error reading f */
+ offset -= (unsigned)tr;
+ }
+ assert(offset == 0);
+ return 0;
+}
/** Safely handle cases when (unsigned)offset > LONG_MAX */
static int fseek_u32(FILE *fp, unsigned offset, int where)
@@ -1147,14 +1194,17 @@
while (offset > 0) {
unsigned s = offset;
if (s > stepMax) s = stepMax;
- errorNb = UTIL_fseek(fp, (long) s, SEEK_CUR);
- if (errorNb != 0) break;
- offset -= s;
+ errorNb = UTIL_fseek(fp, (long)s, SEEK_CUR);
+ if (errorNb==0) { offset -= s; continue; }
+ errorNb = skipStream(fp, offset);
+ offset = 0;
}
return errorNb;
}
+
#define ENDOFSTREAM ((unsigned long long)-1)
+#define DECODING_ERROR ((unsigned long long)-2)
static unsigned long long
selectDecoder(dRess_t ress,
FILE* finput, FILE* foutput,
@@ -1175,7 +1225,7 @@
size_t const nbReadBytes = fread(MNstore, 1, MAGICNUMBER_SIZE, finput);
if (nbReadBytes==0) { nbFrames = 0; return ENDOFSTREAM; } /* EOF */
if (nbReadBytes != MAGICNUMBER_SIZE)
- EXM_THROW(40, "Unrecognized header : Magic Number unreadable");
+ END_PROCESS(40, "Unrecognized header : Magic Number unreadable");
magicNumber = LZ4IO_readLE32(MNstore); /* Little Endian format */
}
if (LZ4IO_isSkippableMagicNumber(magicNumber))
@@ -1192,15 +1242,14 @@
DISPLAYLEVEL(4, "Skipping detected skippable area \n");
{ size_t const nbReadBytes = fread(MNstore, 1, 4, finput);
if (nbReadBytes != 4)
- EXM_THROW(42, "Stream error : skippable size unreadable");
+ END_PROCESS(42, "Stream error : skippable size unreadable");
}
{ unsigned const size = LZ4IO_readLE32(MNstore);
int const errorNb = fseek_u32(finput, size, SEEK_CUR);
if (errorNb != 0)
- EXM_THROW(43, "Stream error : cannot skip skippable area");
+ END_PROCESS(43, "Stream error : cannot skip skippable area");
}
return 0;
- EXTENDED_FORMAT; /* macro extension for custom formats */
default:
if (nbFrames == 1) { /* just started */
/* Wrong magic number at the beginning of 1st stream */
@@ -1208,7 +1257,7 @@
nbFrames = 0;
return LZ4IO_passThrough(finput, foutput, MNstore, prefs->sparseFileSupport);
}
- EXM_THROW(44,"Unrecognized header : file cannot be decoded");
+ END_PROCESS(44,"Unrecognized header : file cannot be decoded");
}
{ long int const position = ftell(finput); /* only works for files < 2 GB */
DISPLAYLEVEL(2, "Stream followed by undecodable data ");
@@ -1216,7 +1265,7 @@
DISPLAYLEVEL(2, "at position %i ", (int)position);
DISPLAYLEVEL(2, "\n");
}
- return ENDOFSTREAM;
+ return DECODING_ERROR;
}
}
@@ -1228,6 +1277,7 @@
{
FILE* const foutput = ress.dstFile;
unsigned long long filesize = 0;
+ int result = 0;
/* Init */
FILE* const finput = LZ4IO_openSrcFile(input_filename);
@@ -1239,6 +1289,7 @@
unsigned long long const decodedSize =
selectDecoder(ress, finput, foutput, prefs);
if (decodedSize == ENDOFSTREAM) break;
+ if (decodedSize == DECODING_ERROR) { result=1; break; }
filesize += decodedSize;
}
@@ -1246,7 +1297,7 @@
fclose(finput);
if (prefs->removeSrcFile) { /* --rm */
if (remove(input_filename))
- EXM_THROW(45, "Remove error : %s: %s", input_filename, strerror(errno));
+ END_PROCESS(45, "Remove error : %s: %s", input_filename, strerror(errno));
}
/* Final Status */
@@ -1254,7 +1305,7 @@
DISPLAYLEVEL(2, "%-20.20s : decoded %llu bytes \n", input_filename, filesize);
(void)output_filename;
- return 0;
+ return result;
}
@@ -1263,45 +1314,50 @@
const char* input_filename, const char* output_filename,
const LZ4IO_prefs_t* const prefs)
{
+ int result;
stat_t statbuf;
int stat_result = 0;
FILE* const foutput = LZ4IO_openDstFile(output_filename, prefs);
if (foutput==NULL) return 1; /* failure */
- if ( strcmp(input_filename, stdinmark)
+ if ( !LZ4IO_isStdin(input_filename)
&& UTIL_getFileStat(input_filename, &statbuf))
stat_result = 1;
ress.dstFile = foutput;
- LZ4IO_decompressSrcFile(ress, input_filename, output_filename, prefs);
+ result = LZ4IO_decompressSrcFile(ress, input_filename, output_filename, prefs);
fclose(foutput);
/* Copy owner, file permissions and modification time */
if ( stat_result != 0
- && strcmp (output_filename, stdoutmark)
- && strcmp (output_filename, nulmark)) {
+ && !LZ4IO_isStdout(output_filename)
+ && !LZ4IO_isDevNull(output_filename)) {
UTIL_setFileStat(output_filename, &statbuf);
/* should return value be read ? or is silent fail good enough ? */
}
- return 0;
+ return result;
}
+/* Note : LZ4IO_decompressFilename()
+ * can provide total decompression time for the specified fileName.
+ * This information is not available with LZ4IO_decompressMultipleFilenames().
+ */
int LZ4IO_decompressFilename(const char* input_filename, const char* output_filename, const LZ4IO_prefs_t* prefs)
{
dRess_t const ress = LZ4IO_createDResources(prefs);
clock_t const start = clock();
- int const missingFiles = LZ4IO_decompressDstFile(ress, input_filename, output_filename, prefs);
+ int const status = LZ4IO_decompressDstFile(ress, input_filename, output_filename, prefs);
clock_t const end = clock();
double const seconds = (double)(end - start) / CLOCKS_PER_SEC;
DISPLAYLEVEL(4, "Done in %.2f sec \n", seconds);
LZ4IO_freeDResources(ress);
- return missingFiles;
+ return status;
}
@@ -1318,23 +1374,26 @@
size_t const suffixSize = strlen(suffix);
dRess_t ress = LZ4IO_createDResources(prefs);
- if (outFileName==NULL) EXM_THROW(70, "Memory allocation error");
+ if (outFileName==NULL) END_PROCESS(70, "Memory allocation error");
+ if (prefs->blockChecksum==0 && prefs->streamChecksum==0) {
+ DISPLAYLEVEL(4, "disabling checksum validation during decoding \n");
+ }
ress.dstFile = LZ4IO_openDstFile(stdoutmark, prefs);
for (i=0; i<ifntSize; i++) {
size_t const ifnSize = strlen(inFileNamesTable[i]);
const char* const suffixPtr = inFileNamesTable[i] + ifnSize - suffixSize;
- if (!strcmp(suffix, stdoutmark)) {
- missingFiles += LZ4IO_decompressSrcFile(ress, inFileNamesTable[i], stdoutmark, prefs);
+ if (LZ4IO_isStdout(suffix) || LZ4IO_isDevNull(suffix)) {
+ missingFiles += LZ4IO_decompressSrcFile(ress, inFileNamesTable[i], suffix, prefs);
continue;
}
if (ofnSize <= ifnSize-suffixSize+1) {
free(outFileName);
ofnSize = ifnSize + 20;
outFileName = (char*)malloc(ofnSize);
- if (outFileName==NULL) EXM_THROW(71, "Memory allocation error");
+ if (outFileName==NULL) END_PROCESS(71, "Memory allocation error");
}
- if (ifnSize <= suffixSize || strcmp(suffixPtr, suffix) != 0) {
+ if (ifnSize <= suffixSize || !UTIL_sameString(suffixPtr, suffix) ) {
DISPLAYLEVEL(1, "File extension doesn't match expected LZ4_EXTENSION (%4s); will not process file: %s\n", suffix, inFileNamesTable[i]);
skippedFiles++;
continue;
@@ -1387,7 +1446,7 @@
/* Read block headers and skip block data
Return total blocks size for this frame including block headers,
block checksums and content checksums.
- returns 0 in case it can't succesfully skip block data.
+ returns 0 in case it can't successfully skip block data.
Assumes SEEK_CUR after frame header.
*/
static unsigned long long
@@ -1424,37 +1483,47 @@
return totalBlocksSize;
}
+static const unsigned long long legacyFrameUndecodable = (0ULL-1);
/* For legacy frames only.
Read block headers and skip block data.
Return total blocks size for this frame including block headers.
- or 0 in case it can't succesfully skip block data.
+ or legacyFrameUndecodable in case it can't successfully skip block data.
This works as long as legacy block header size = magic number size.
Assumes SEEK_CUR after frame header.
*/
static unsigned long long LZ4IO_skipLegacyBlocksData(FILE* finput)
{
- unsigned char blockInfo[LZIO_LEGACY_BLOCK_HEADER_SIZE];
+ unsigned char blockInfo[LZ4IO_LEGACY_BLOCK_HEADER_SIZE];
unsigned long long totalBlocksSize = 0;
- LZ4IO_STATIC_ASSERT(LZIO_LEGACY_BLOCK_HEADER_SIZE == MAGICNUMBER_SIZE);
+ LZ4IO_STATIC_ASSERT(LZ4IO_LEGACY_BLOCK_HEADER_SIZE == MAGICNUMBER_SIZE);
for (;;) {
- if (!fread(blockInfo, 1, LZIO_LEGACY_BLOCK_HEADER_SIZE, finput)) {
+ size_t const bhs = fread(blockInfo, 1, LZ4IO_LEGACY_BLOCK_HEADER_SIZE, finput);
+ if (bhs == 0) {
if (feof(finput)) return totalBlocksSize;
- return 0;
+ return legacyFrameUndecodable;
+ }
+ if (bhs != 4) {
+ return legacyFrameUndecodable;
}
{ const unsigned int nextCBlockSize = LZ4IO_readLE32(&blockInfo);
- if ( nextCBlockSize == LEGACY_MAGICNUMBER ||
- nextCBlockSize == LZ4IO_MAGICNUMBER ||
- LZ4IO_isSkippableMagicNumber(nextCBlockSize)) {
- /* Rewind back. we want cursor at the begining of next frame.*/
- if (fseek(finput, -LZIO_LEGACY_BLOCK_HEADER_SIZE, SEEK_CUR) != 0) {
- return 0;
+ if ( nextCBlockSize == LEGACY_MAGICNUMBER
+ || nextCBlockSize == LZ4IO_MAGICNUMBER
+ || LZ4IO_isSkippableMagicNumber(nextCBlockSize) ) {
+ /* Rewind back. we want cursor at the beginning of next frame */
+ if (UTIL_fseek(finput, -LZ4IO_LEGACY_BLOCK_HEADER_SIZE, SEEK_CUR) != 0) {
+ END_PROCESS(37, "impossible to skip backward");
}
break;
}
- totalBlocksSize += LZIO_LEGACY_BLOCK_HEADER_SIZE + nextCBlockSize;
- /* skip to the next block */
+ if (nextCBlockSize > LZ4IO_LEGACY_BLOCK_SIZE_MAX) {
+ DISPLAYLEVEL(4, "Error : block in legacy frame is too large \n");
+ return legacyFrameUndecodable;
+ }
+ totalBlocksSize += LZ4IO_LEGACY_BLOCK_HEADER_SIZE + nextCBlockSize;
+ /* skip to the next block
+ * note : this won't fail if nextCBlockSize is too large, skipping past the end of finput */
if (UTIL_fseek(finput, nextCBlockSize, SEEK_CUR) != 0) {
- return 0;
+ return legacyFrameUndecodable;
} } }
return totalBlocksSize;
}
@@ -1514,7 +1583,7 @@
if (nbReadBytes == 0) { break; } /* EOF */
result = LZ4IO_format_not_known; /* default result (error) */
if (nbReadBytes != MAGICNUMBER_SIZE) {
- EXM_THROW(40, "Unrecognized header : Magic Number unreadable");
+ END_PROCESS(40, "Unrecognized header : Magic Number unreadable");
} }
magicNumber = LZ4IO_readLE32(buffer); /* Little Endian format */
if (LZ4IO_isSkippableMagicNumber(magicNumber))
@@ -1525,14 +1594,14 @@
if (cfinfo->frameSummary.frameType != lz4Frame) cfinfo->eqFrameTypes = 0;
/* Get frame info */
{ const size_t readBytes = fread(buffer + MAGICNUMBER_SIZE, 1, LZ4F_HEADER_SIZE_MIN - MAGICNUMBER_SIZE, finput);
- if (!readBytes || ferror(finput)) EXM_THROW(71, "Error reading %s", input_filename);
+ if (!readBytes || ferror(finput)) END_PROCESS(71, "Error reading %s", input_filename);
}
{ size_t hSize = LZ4F_headerSize(&buffer, LZ4F_HEADER_SIZE_MIN);
if (LZ4F_isError(hSize)) break;
if (hSize > (LZ4F_HEADER_SIZE_MIN + MAGICNUMBER_SIZE)) {
/* We've already read LZ4F_HEADER_SIZE_MIN so read any extra until hSize*/
const size_t readBytes = fread(buffer + LZ4F_HEADER_SIZE_MIN, 1, hSize - LZ4F_HEADER_SIZE_MIN, finput);
- if (!readBytes || ferror(finput)) EXM_THROW(72, "Error reading %s", input_filename);
+ if (!readBytes || ferror(finput)) END_PROCESS(72, "Error reading %s", input_filename);
}
/* Create decompression context */
{ LZ4F_dctx* dctx;
@@ -1578,6 +1647,11 @@
cfinfo->eqBlockTypes = 0;
cfinfo->allContentSize = 0;
{ const unsigned long long totalBlocksSize = LZ4IO_skipLegacyBlocksData(finput);
+ if (totalBlocksSize == legacyFrameUndecodable) {
+ DISPLAYLEVEL(1, "Corrupted legacy frame \n");
+ result = LZ4IO_format_not_known;
+ break;
+ }
if (totalBlocksSize) {
DISPLAYLEVEL(3, " %6llu %14s %5s %8s %20llu %20s %9s\n",
cfinfo->frameCount + 1,
@@ -1595,12 +1669,12 @@
cfinfo->allContentSize = 0;
{ size_t const nbReadBytes = fread(buffer, 1, 4, finput);
if (nbReadBytes != 4)
- EXM_THROW(42, "Stream error : skippable size unreadable");
+ END_PROCESS(42, "Stream error : skippable size unreadable");
}
{ unsigned const size = LZ4IO_readLE32(buffer);
int const errorNb = fseek_u32(finput, size, SEEK_CUR);
if (errorNb != 0)
- EXM_THROW(43, "Stream error : cannot skip skippable area");
+ END_PROCESS(43, "Stream error : cannot skip skippable area");
DISPLAYLEVEL(3, " %6llu %14s %5s %8s %20u %20s %9s\n",
cfinfo->frameCount + 1,
"SkippableFrame",
@@ -1614,6 +1688,7 @@
DISPLAYLEVEL(3, "Stream followed by undecodable data ");
if (position != -1L)
DISPLAYLEVEL(3, "at position %i ", (int)position);
+ result = LZ4IO_format_not_known;
DISPLAYLEVEL(3, "\n");
}
break;
@@ -1639,9 +1714,9 @@
/* Get file info */
LZ4IO_cFileInfo_t cfinfo = LZ4IO_INIT_CFILEINFO;
cfinfo.fileName = LZ4IO_baseName(inFileNames[idx]);
- if (!UTIL_isRegFile(inFileNames[idx])) {
+ if (LZ4IO_isStdin(inFileNames[idx]) ? !UTIL_isRegFD(0) : !UTIL_isRegFile(inFileNames[idx])) {
DISPLAYLEVEL(1, "lz4: %s is not a regular file \n", inFileNames[idx]);
- return 0;
+ return 1;
}
DISPLAYLEVEL(3, "%s(%llu/%llu)\n", cfinfo.fileName, (unsigned long long)idx + 1, (unsigned long long)ifnIdx);
DISPLAYLEVEL(3, " %6s %14s %5s %8s %20s %20s %9s\n",
@@ -1650,7 +1725,7 @@
if (op_result != LZ4IO_LZ4F_OK) {
assert(op_result == LZ4IO_format_not_known);
DISPLAYLEVEL(1, "lz4: %s: File format not recognized \n", inFileNames[idx]);
- return 0;
+ return 1;
} }
DISPLAYLEVEL(3, "\n");
if (g_displayLevel < 3) {
diff --git a/programs/lz4io.h b/programs/lz4io.h
index d6d7eee..0cfb1d2 100644
--- a/programs/lz4io.h
+++ b/programs/lz4io.h
@@ -1,6 +1,6 @@
/*
LZ4io.h - LZ4 File/Stream Interface
- Copyright (C) Yann Collet 2011-2016
+ Copyright (C) Yann Collet 2011-2020
GPL v2 License
This program is free software; you can redistribute it and/or modify
@@ -57,8 +57,6 @@
LZ4IO_prefs_t* LZ4IO_defaultPreferences(void);
void LZ4IO_freePreferences(LZ4IO_prefs_t* prefs);
-/* Size in bytes of a legacy block header in little-endian format */
-#define LZIO_LEGACY_BLOCK_HEADER_SIZE 4
/* ************************************************** */
/* ****************** Functions ********************* */
diff --git a/programs/platform.h b/programs/platform.h
index ab8300d..43a171b 100644
--- a/programs/platform.h
+++ b/programs/platform.h
@@ -1,6 +1,6 @@
/*
platform.h - compiler and OS detection
- Copyright (C) 2016-present, Przemyslaw Skibinski, Yann Collet
+ Copyright (C) 2016-2020, Przemyslaw Skibinski, Yann Collet
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
@@ -80,7 +80,7 @@
************************************************************** */
#if !defined(_WIN32) && (defined(__unix__) || defined(__unix) || (defined(__APPLE__) && defined(__MACH__)) /* UNIX-like OS */ \
|| defined(__midipix__) || defined(__VMS))
-# if (defined(__APPLE__) && defined(__MACH__)) || defined(__SVR4) || defined(_AIX) || defined(__hpux) /* POSIX.1–2001 (SUSv3) conformant */ \
+# if (defined(__APPLE__) && defined(__MACH__)) || defined(__SVR4) || defined(_AIX) || defined(__hpux) /* POSIX.1-2001 (SUSv3) conformant */ \
|| defined(__DragonFly__) || defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) || defined(__MidnightBSD__) /* BSD distros */ \
|| defined(__HAIKU__)
# define PLATFORM_POSIX_VERSION 200112L
diff --git a/programs/util.h b/programs/util.h
index 733c1ca..3192ddc 100644
--- a/programs/util.h
+++ b/programs/util.h
@@ -1,6 +1,6 @@
/*
util.h - utility functions
- Copyright (C) 2016-present, Przemyslaw Skibinski, Yann Collet
+ Copyright (C) 2016-2020, Przemyslaw Skibinski, Yann Collet
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
@@ -175,6 +175,39 @@
#endif
+
+/*-****************************************
+* Allocation functions
+******************************************/
+/*
+ * A modified version of realloc().
+ * If UTIL_realloc() fails the original block is freed.
+*/
+UTIL_STATIC void* UTIL_realloc(void* ptr, size_t size)
+{
+ void* const newptr = realloc(ptr, size);
+ if (newptr) return newptr;
+ free(ptr);
+ return NULL;
+}
+
+
+/*-****************************************
+* String functions
+******************************************/
+/*
+ * A modified version of realloc().
+ * If UTIL_realloc() fails the original block is freed.
+*/
+UTIL_STATIC int UTIL_sameString(const char* a, const char* b)
+{
+ assert(a!=NULL && b!=NULL); /* unsupported scenario */
+ if (a==NULL) return 0;
+ if (b==NULL) return 0;
+ return !strcmp(a,b);
+}
+
+
/*-****************************************
* Time functions
******************************************/
@@ -317,6 +350,7 @@
UTIL_STATIC int UTIL_isRegFile(const char* infilename);
+UTIL_STATIC int UTIL_isRegFD(int fd);
UTIL_STATIC int UTIL_setFileStat(const char *filename, stat_t *statbuf)
@@ -333,7 +367,8 @@
timebuf.modtime = statbuf->st_mtime;
res += utime(filename, &timebuf); /* set access and modification times */
#else
- struct timespec timebuf[2] = {};
+ struct timespec timebuf[2];
+ memset(timebuf, 0, sizeof(timebuf));
timebuf[0].tv_nsec = UTIME_NOW;
timebuf[1].tv_sec = statbuf->st_mtime;
res += utimensat(AT_FDCWD, filename, timebuf, 0); /* set access and modification times */
@@ -351,6 +386,19 @@
}
+UTIL_STATIC int UTIL_getFDStat(int fd, stat_t *statbuf)
+{
+ int r;
+#if defined(_MSC_VER)
+ r = _fstat64(fd, statbuf);
+ if (r || !(statbuf->st_mode & S_IFREG)) return 0; /* No good... */
+#else
+ r = fstat(fd, statbuf);
+ if (r || !S_ISREG(statbuf->st_mode)) return 0; /* No good... */
+#endif
+ return 1;
+}
+
UTIL_STATIC int UTIL_getFileStat(const char* infilename, stat_t *statbuf)
{
int r;
@@ -365,6 +413,17 @@
}
+UTIL_STATIC int UTIL_isRegFD(int fd)
+{
+ stat_t statbuf;
+#ifdef _WIN32
+ /* Windows runtime library always open file descriptors 0, 1 and 2 in text mode, therefore we can't use them for binary I/O */
+ if(fd < 3) return 0;
+#endif
+ return UTIL_getFDStat(fd, &statbuf); /* Only need to know whether it is a regular file */
+}
+
+
UTIL_STATIC int UTIL_isRegFile(const char* infilename)
{
stat_t statbuf;
@@ -425,19 +484,6 @@
}
-/*
- * A modified version of realloc().
- * If UTIL_realloc() fails the original block is freed.
-*/
-UTIL_STATIC void* UTIL_realloc(void* ptr, size_t size)
-{
- void* const newptr = realloc(ptr, size);
- if (newptr) return newptr;
- free(ptr);
- return NULL;
-}
-
-
#ifdef _WIN32
# define UTIL_HAS_CREATEFILELIST
@@ -511,22 +557,23 @@
{
DIR* dir;
struct dirent * entry;
- int dirLength, nbFiles = 0;
+ size_t dirLength;
+ int nbFiles = 0;
if (!(dir = opendir(dirName))) {
fprintf(stderr, "Cannot open directory '%s': %s\n", dirName, strerror(errno));
return 0;
}
- dirLength = (int)strlen(dirName);
+ dirLength = strlen(dirName);
errno = 0;
while ((entry = readdir(dir)) != NULL) {
char* path;
- int fnameLength, pathLength;
+ size_t fnameLength, pathLength;
if (strcmp (entry->d_name, "..") == 0 ||
strcmp (entry->d_name, ".") == 0) continue;
- fnameLength = (int)strlen(entry->d_name);
- path = (char*) malloc(dirLength + fnameLength + 2);
+ fnameLength = strlen(entry->d_name);
+ path = (char*)malloc(dirLength + fnameLength + 2);
if (!path) { closedir(dir); return 0; }
memcpy(path, dirName, dirLength);
path[dirLength] = '/';
@@ -539,7 +586,7 @@
if (*bufStart == NULL) { free(path); closedir(dir); return 0; }
} else {
if (*bufStart + *pos + pathLength >= *bufEnd) {
- ptrdiff_t newListSize = (*bufEnd - *bufStart) + LIST_SIZE_INCREASE;
+ size_t const newListSize = (size_t)(*bufEnd - *bufStart) + LIST_SIZE_INCREASE;
*bufStart = (char*)UTIL_realloc(*bufStart, newListSize);
*bufEnd = *bufStart + newListSize;
if (*bufStart == NULL) { free(path); closedir(dir); return 0; }
@@ -638,8 +685,8 @@
UTIL_STATIC void
UTIL_freeFileList(const char** filenameTable, char* allocatedBuffer)
{
- if (allocatedBuffer) free(allocatedBuffer);
- if (filenameTable) free((void*)filenameTable);
+ free(allocatedBuffer);
+ free((void*)filenameTable);
}
diff --git a/tests/.gitignore b/tests/.gitignore
index 99351af..c7d8f19 100644
--- a/tests/.gitignore
+++ b/tests/.gitignore
@@ -12,10 +12,14 @@
checkTag
checkFrame
decompress-partial
+decompress-partial-usingDict
+abiTest
+freestanding
# test artefacts
tmp*
versionsTest
+abiTests
lz4_all.c
# local tests
diff --git a/tests/Makefile b/tests/Makefile
index 6eee132..93a5581 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -1,6 +1,6 @@
# ##########################################################################
# LZ4 programs - Makefile
-# Copyright (C) Yann Collet 2011-present
+# Copyright (C) Yann Collet 2011-2020
#
# GPL v2 License
#
@@ -112,6 +112,12 @@
decompress-partial: lz4.o decompress-partial.c
$(CC) $(FLAGS) $^ -o $@$(EXT)
+decompress-partial-usingDict: lz4.o decompress-partial-usingDict.c
+ $(CC) $(FLAGS) $^ -o $@$(EXT)
+
+freestanding: freestanding.c
+ $(CC) -ffreestanding -nostdlib $^ -o $@$(EXT)
+
.PHONY: clean
clean:
@$(MAKE) -C $(LZ4DIR) $@ > $(VOID)
@@ -124,7 +130,8 @@
fasttest$(EXT) roundTripTest$(EXT) \
datagen$(EXT) checkTag$(EXT) \
frameTest$(EXT) decompress-partial$(EXT) \
- lz4_all.c
+ abiTest$(EXT) freestanding$(EXT) \
+ lz4_all.c
@$(RM) -rf $(TESTDIR)
@echo Cleaning completed
@@ -136,6 +143,12 @@
listTest: lz4
QEMU_SYS=$(QEMU_SYS) $(PYTHON) test-lz4-list.py
+abiTest: LDLIBS += -llz4
+
+.PHONY: abiTests
+abiTests:
+ $(PYTHON) test-lz4-abi.py
+
checkTag: checkTag.c $(LZ4DIR)/lz4.h
$(CC) $(FLAGS) $< -o $@$(EXT)
@@ -145,14 +158,15 @@
ifeq ($(POSIX_ENV),Yes)
MD5:=md5sum
-ifneq (,$(filter $(shell uname), Darwin ))
+ifneq (,$(filter $(shell $(UNAME)), Darwin ))
MD5:=md5 -r
endif
# note : we should probably settle on a single compare utility
CMP:=cmp
+GREP:=grep
DIFF:=diff
-ifneq (,$(filter $(shell uname),SunOS))
+ifneq (,$(filter $(shell $(UNAME)),SunOS))
DIFF:=gdiff
endif
@@ -168,278 +182,327 @@
check: test-lz4-essentials
.PHONY: test
-test: test-lz4 test-lz4c test-frametest test-fullbench test-fuzzer test-install test-amalgamation listTest test-decompress-partial
+test: test-lz4 test-lz4c test-frametest test-fullbench test-fuzzer test-amalgamation listTest test-decompress-partial
.PHONY: test32
test32: CFLAGS+=-m32
test32: test
+.PHONY: test-amalgamation
test-amalgamation: lz4_all.o
lz4_all.c: $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c $(LZ4DIR)/lz4frame.c
$(CAT) $^ > $@
+.PHONY: test-install
test-install: lz4 lib liblz4.pc
lz4_root=.. ./test_install.sh
+.PHONY: test-compile-with-lz4-memory-usage
+test-compile-with-lz4-memory-usage:
+ $(MAKE) clean; CFLAGS=-O0 CPPFLAGS=-D'LZ4_MEMORY_USAGE=LZ4_MEMORY_USAGE_MIN' $(MAKE) all
+ $(MAKE) clean; CFLAGS=-O0 CPPFLAGS=-D'LZ4_MEMORY_USAGE=LZ4_MEMORY_USAGE_MAX' $(MAKE) all
+
+.PHONY: test-lz4-sparse
+# Rules regarding Temporary test files :
+# Each test must use its own unique set of names during execution.
+# Each temporary test file must begin by an FPREFIX.
+# Each FPREFIX must be unique for each test.
+# All FPREFIX must start with `tmp`, for `make clean`
+# All tests must clean their temporary test files on successful completion,
+# and only their test files : do not employ sweeping statements such `rm tmp*` or `rm *.lz4`
+test-lz4-sparse: FPREFIX = tmp-tls
test-lz4-sparse: lz4 datagen
@echo "\n ---- test sparse file support ----"
- $(DATAGEN) -g5M -P100 > tmplsdg5M
- $(LZ4) -B4D tmplsdg5M -c | $(LZ4) -dv --sparse > tmplscB4
- $(DIFF) -s tmplsdg5M tmplscB4
- $(LZ4) -B5D tmplsdg5M -c | $(LZ4) -dv --sparse > tmplscB5
- $(DIFF) -s tmplsdg5M tmplscB5
- $(LZ4) -B6D tmplsdg5M -c | $(LZ4) -dv --sparse > tmplscB6
- $(DIFF) -s tmplsdg5M tmplscB6
- $(LZ4) -B7D tmplsdg5M -c | $(LZ4) -dv --sparse > tmplscB7
- $(DIFF) -s tmplsdg5M tmplscB7
- $(LZ4) tmplsdg5M -c | $(LZ4) -dv --no-sparse > tmplsnosparse
- $(DIFF) -s tmplsdg5M tmplsnosparse
- ls -ls tmpls*
- $(DATAGEN) -s1 -g1200007 -P100 | $(LZ4) | $(LZ4) -dv --sparse > tmplsodd # Odd size file (to generate non-full last block)
- $(DATAGEN) -s1 -g1200007 -P100 | $(DIFF) -s - tmplsodd
- ls -ls tmplsodd
- @$(RM) tmpls*
+ $(DATAGEN) -g5M -P100 > $(FPREFIX)dg5M
+ $(LZ4) -B4D $(FPREFIX)dg5M -c | $(LZ4) -dv --sparse > $(FPREFIX)cB4
+ $(DIFF) -s $(FPREFIX)dg5M $(FPREFIX)cB4
+ $(LZ4) -B5D $(FPREFIX)dg5M -c | $(LZ4) -dv --sparse > $(FPREFIX)cB5
+ $(DIFF) -s $(FPREFIX)dg5M $(FPREFIX)cB5
+ $(LZ4) -B6D $(FPREFIX)dg5M -c | $(LZ4) -dv --sparse > $(FPREFIX)cB6
+ $(DIFF) -s $(FPREFIX)dg5M $(FPREFIX)cB6
+ $(LZ4) -B7D $(FPREFIX)dg5M -c | $(LZ4) -dv --sparse > $(FPREFIX)cB7
+ $(DIFF) -s $(FPREFIX)dg5M $(FPREFIX)cB7
+ $(LZ4) $(FPREFIX)dg5M -c | $(LZ4) -dv --no-sparse > $(FPREFIX)nosparse
+ $(DIFF) -s $(FPREFIX)dg5M $(FPREFIX)nosparse
+ ls -ls $(FPREFIX)*
+ $(DATAGEN) -s1 -g1200007 -P100 | $(LZ4) | $(LZ4) -dv --sparse > $(FPREFIX)odd # Odd size file (to generate non-full last block)
+ $(DATAGEN) -s1 -g1200007 -P100 | $(DIFF) -s - $(FPREFIX)odd
+ ls -ls $(FPREFIX)odd
+ @$(RM) $(FPREFIX)*
@echo "\n Compatibility with Console :"
echo "Hello World 1 !" | $(LZ4) | $(LZ4) -d -c
echo "Hello World 2 !" | $(LZ4) | $(LZ4) -d | $(CAT)
echo "Hello World 3 !" | $(LZ4) --no-frame-crc | $(LZ4) -d -c
@echo "\n Compatibility with Append :"
- $(DATAGEN) -P100 -g1M > tmplsdg1M
- $(CAT) tmplsdg1M tmplsdg1M > tmpls2M
- $(LZ4) -B5 -v tmplsdg1M tmplsc
- $(LZ4) -d -v tmplsc tmplsr
- $(LZ4) -d -v tmplsc -c >> tmplsr
- ls -ls tmp*
- $(DIFF) tmpls2M tmplsr
- @$(RM) tmpls*
+ $(DATAGEN) -P100 -g1M > $(FPREFIX)dg1M
+ $(CAT) $(FPREFIX)dg1M $(FPREFIX)dg1M > $(FPREFIX)2M
+ $(LZ4) -B5 -v $(FPREFIX)dg1M $(FPREFIX)c
+ $(LZ4) -d -v $(FPREFIX)c $(FPREFIX)r
+ $(LZ4) -d -v $(FPREFIX)c -c >> $(FPREFIX)r
+ ls -ls $(FPREFIX)*
+ $(DIFF) $(FPREFIX)2M $(FPREFIX)r
+ @$(RM) $(FPREFIX)*
+test-lz4-contentSize: FPREFIX = tmp-lzc
test-lz4-contentSize: lz4 datagen
@echo "\n ---- test original size support ----"
- $(DATAGEN) -g15M > tmplc1
- $(LZ4) -v tmplc1 -c | $(LZ4) -t
- $(LZ4) -v --content-size tmplc1 -c | $(LZ4) -d > tmplc2
- $(DIFF) tmplc1 tmplc2
- $(LZ4) -f tmplc1 -c > tmplc1.lz4
- $(LZ4) --content-size tmplc1 -c > tmplc2.lz4
- ! $(DIFF) tmplc1.lz4 tmplc2.lz4 # must differ, due to content size
- $(LZ4) --content-size < tmplc1 > tmplc3.lz4
- $(DIFF) tmplc2.lz4 tmplc3.lz4 # both must contain content size
- $(CAT) tmplc1 | $(LZ4) > tmplc4.lz4
- $(DIFF) tmplc1.lz4 tmplc4.lz4 # both don't have content size
- $(CAT) tmplc1 | $(LZ4) --content-size > tmplc5.lz4 # can't determine content size
- $(DIFF) tmplc1.lz4 tmplc5.lz4 # both don't have content size
- @$(RM) tmplc*
+ $(DATAGEN) -g15M > $(FPREFIX)
+ $(LZ4) -v $(FPREFIX) -c | $(LZ4) -t
+ $(LZ4) -v --content-size $(FPREFIX) -c | $(LZ4) -d > $(FPREFIX)-dup
+ $(DIFF) $(FPREFIX) $(FPREFIX)-dup
+ $(LZ4) -f $(FPREFIX) -c > $(FPREFIX).lz4 # compressed with content size
+ $(LZ4) --content-size $(FPREFIX) -c > $(FPREFIX)-wcz.lz4
+ ! $(DIFF) $(FPREFIX).lz4 $(FPREFIX)-wcz.lz4 # must differ, due to content size
+ $(LZ4) --content-size < $(FPREFIX) > $(FPREFIX)-wcz2.lz4 # can determine content size because stdin is just a file
+ $(DIFF) $(FPREFIX)-wcz.lz4 $(FPREFIX)-wcz2.lz4 # both must contain content size
+ $(CAT) $(FPREFIX) | $(LZ4) > $(FPREFIX)-ncz.lz4
+ $(DIFF) $(FPREFIX).lz4 $(FPREFIX)-ncz.lz4 # both don't have content size
+ $(CAT) $(FPREFIX) | $(LZ4) --content-size > $(FPREFIX)-ncz2.lz4 # can't determine content size
+ $(DIFF) $(FPREFIX).lz4 $(FPREFIX)-ncz2.lz4 # both don't have content size
+ @$(RM) $(FPREFIX)*
+test-lz4-frame-concatenation: FPREFIX = tmp-lfc
test-lz4-frame-concatenation: lz4 datagen
@echo "\n ---- test frame concatenation ----"
- @echo -n > tmp-lfc-empty
- @echo hi > tmp-lfc-nonempty
- $(CAT) tmp-lfc-nonempty tmp-lfc-empty tmp-lfc-nonempty > tmp-lfc-src
- $(LZ4) -zq tmp-lfc-empty -c > tmp-lfc-empty.lz4
- $(LZ4) -zq tmp-lfc-nonempty -c > tmp-lfc-nonempty.lz4
- $(CAT) tmp-lfc-nonempty.lz4 tmp-lfc-empty.lz4 tmp-lfc-nonempty.lz4 > tmp-lfc-concat.lz4
- $(LZ4) -d tmp-lfc-concat.lz4 -c > tmp-lfc-result
- $(CMP) tmp-lfc-src tmp-lfc-result
- @$(RM) tmp-lfc-*
+ @echo -n > $(FPREFIX)-empty
+ @echo hi > $(FPREFIX)-nonempty
+ $(CAT) $(FPREFIX)-nonempty $(FPREFIX)-empty $(FPREFIX)-nonempty > $(FPREFIX)-src
+ $(LZ4) -zq $(FPREFIX)-empty -c > $(FPREFIX)-empty.lz4
+ $(LZ4) -zq $(FPREFIX)-nonempty -c > $(FPREFIX)-nonempty.lz4
+ $(CAT) $(FPREFIX)-nonempty.lz4 $(FPREFIX)-empty.lz4 $(FPREFIX)-nonempty.lz4 > $(FPREFIX)-concat.lz4
+ $(LZ4) -d $(FPREFIX)-concat.lz4 -c > $(FPREFIX)-result
+ $(CMP) $(FPREFIX)-src $(FPREFIX)-result
+ @$(RM) $(FPREFIX)*
@echo frame concatenation test completed
+test-lz4-multiple: FPREFIX = tmp-tml
test-lz4-multiple: lz4 datagen
@echo "\n ---- test multiple files ----"
- @$(DATAGEN) -s1 > tmp-tlm1 2> $(VOID)
- @$(DATAGEN) -s2 -g100K > tmp-tlm2 2> $(VOID)
- @$(DATAGEN) -s3 -g200K > tmp-tlm3 2> $(VOID)
+ @$(DATAGEN) -s1 > $(FPREFIX)1 2> $(VOID)
+ @$(DATAGEN) -s2 -g100K > $(FPREFIX)2 2> $(VOID)
+ @$(DATAGEN) -s3 -g200K > $(FPREFIX)3 2> $(VOID)
# compress multiple files : one .lz4 per source file
- $(LZ4) -f -m tmp-tlm*
- test -f tmp-tlm1.lz4
- test -f tmp-tlm2.lz4
- test -f tmp-tlm3.lz4
+ $(LZ4) -f -m $(FPREFIX)*
+ test -f $(FPREFIX)1.lz4
+ test -f $(FPREFIX)2.lz4
+ test -f $(FPREFIX)3.lz4
# decompress multiple files : one output file per .lz4
- mv tmp-tlm1 tmp-tlm1-orig
- mv tmp-tlm2 tmp-tlm2-orig
- mv tmp-tlm3 tmp-tlm3-orig
- $(LZ4) -d -f -m tmp-tlm*.lz4
- $(CMP) tmp-tlm1 tmp-tlm1-orig # must be identical
- $(CMP) tmp-tlm2 tmp-tlm2-orig
- $(CMP) tmp-tlm3 tmp-tlm3-orig
+ mv $(FPREFIX)1 $(FPREFIX)1-orig
+ mv $(FPREFIX)2 $(FPREFIX)2-orig
+ mv $(FPREFIX)3 $(FPREFIX)3-orig
+ $(LZ4) -d -f -m $(FPREFIX)*.lz4
+ $(CMP) $(FPREFIX)1 $(FPREFIX)1-orig # must be identical
+ $(CMP) $(FPREFIX)2 $(FPREFIX)2-orig
+ $(CMP) $(FPREFIX)3 $(FPREFIX)3-orig
# compress multiple files into stdout
- $(CAT) tmp-tlm1.lz4 tmp-tlm2.lz4 tmp-tlm3.lz4 > tmp-tlm-concat1
- $(RM) *.lz4
- $(LZ4) -m tmp-tlm1 tmp-tlm2 tmp-tlm3 -c > tmp-tlm-concat2
- test ! -f tmp-tlm1.lz4 # must not create .lz4 artefact
- $(CMP) tmp-tlm-concat1 tmp-tlm-concat2 # must be equivalent
+ $(CAT) $(FPREFIX)1.lz4 $(FPREFIX)2.lz4 $(FPREFIX)3.lz4 > $(FPREFIX)-concat1
+ $(RM) $(FPREFIX)*.lz4
+ $(LZ4) -m $(FPREFIX)1 $(FPREFIX)2 $(FPREFIX)3 -c > $(FPREFIX)-concat2
+ test ! -f $(FPREFIX)1.lz4 # must not create .lz4 artefact
+ $(CMP) $(FPREFIX)-concat1 $(FPREFIX)-concat2 # must be equivalent
# decompress multiple files into stdout
- $(RM) tmp-tlm-concat1 tmp-tlm-concat2
- $(LZ4) -f -m tmp-tlm1 tmp-tlm2 tmp-tlm3 # generate .lz4 to decompress
- $(CAT) tmp-tlm1 tmp-tlm2 tmp-tlm3 > tmp-tlm-concat1 # create concatenated reference
- $(RM) tmp-tlm1 tmp-tlm2 tmp-tlm3
- $(LZ4) -d -m tmp-tlm1.lz4 tmp-tlm2.lz4 tmp-tlm3.lz4 -c > tmp-tlm-concat2
- test ! -f tmp-tlm1 # must not create file artefact
- $(CMP) tmp-tlm-concat1 tmp-tlm-concat2 # must be equivalent
+ $(RM) $(FPREFIX)-concat1 $(FPREFIX)-concat2
+ $(LZ4) -f -m $(FPREFIX)1 $(FPREFIX)2 $(FPREFIX)3 # generate .lz4 to decompress
+ $(CAT) $(FPREFIX)1 $(FPREFIX)2 $(FPREFIX)3 > $(FPREFIX)-concat1 # create concatenated reference
+ $(RM) $(FPREFIX)1 $(FPREFIX)2 $(FPREFIX)3
+ $(LZ4) -d -m $(FPREFIX)1.lz4 $(FPREFIX)2.lz4 $(FPREFIX)3.lz4 -c > $(FPREFIX)-concat2
+ test ! -f $(FPREFIX)1 # must not create file artefact
+ $(CMP) $(FPREFIX)-concat1 $(FPREFIX)-concat2 # must be equivalent
# compress multiple files, one of which is absent (must fail)
- ! $(LZ4) -f -m tmp-tlm-concat1 notHere tmp-tlm-concat2 # must fail : notHere not present
- @$(RM) tmp-tlm*
+ ! $(LZ4) -f -m $(FPREFIX)-concat1 notHere $(FPREFIX)-concat2 # must fail : notHere not present
+ # test lz4-compressed file
+ $(LZ4) -tm $(FPREFIX)-concat1.lz4
+ $(LZ4) -tm $(FPREFIX)-concat1.lz4 $(FPREFIX)-concat2.lz4
+ # test multiple lz4 files, one of which is absent (must fail)
+ ! $(LZ4) -tm $(FPREFIX)-concat1.lz4 notHere.lz4 $(FPREFIX)-concat2.lz4
+ @$(RM) $(FPREFIX)*
+test-lz4-multiple-legacy: FPREFIX = tmp-lml
test-lz4-multiple-legacy: lz4 datagen
@echo "\n ---- test multiple files (Legacy format) ----"
- @$(DATAGEN) -s1 > tmp-tlm1 2> $(VOID)
- @$(DATAGEN) -s2 -g100K > tmp-tlm2 2> $(VOID)
- @$(DATAGEN) -s3 -g200K > tmp-tlm3 2> $(VOID)
+ @$(DATAGEN) -s1 > $(FPREFIX)1 2> $(VOID)
+ @$(DATAGEN) -s2 -g100K > $(FPREFIX)2 2> $(VOID)
+ @$(DATAGEN) -s3 -g200K > $(FPREFIX)3 2> $(VOID)
# compress multiple files using legacy format: one .lz4 per source file
- $(LZ4) -f -l -m tmp-tlm*
- test -f tmp-tlm1.lz4
- test -f tmp-tlm2.lz4
- test -f tmp-tlm3.lz4
+ $(LZ4) -f -l -m $(FPREFIX)*
+ test -f $(FPREFIX)1.lz4
+ test -f $(FPREFIX)2.lz4
+ test -f $(FPREFIX)3.lz4
# decompress multiple files compressed using legacy format: one output file per .lz4
- mv tmp-tlm1 tmp-tlm1-orig
- mv tmp-tlm2 tmp-tlm2-orig
- mv tmp-tlm3 tmp-tlm3-orig
- $(LZ4) -d -f -m tmp-tlm*.lz4
- $(LZ4) -l -d -f -m tmp-tlm*.lz4 # -l mustn't impact -d option
- $(CMP) tmp-tlm1 tmp-tlm1-orig # must be identical
- $(CMP) tmp-tlm2 tmp-tlm2-orig
- $(CMP) tmp-tlm3 tmp-tlm3-orig
+ mv $(FPREFIX)1 $(FPREFIX)1-orig
+ mv $(FPREFIX)2 $(FPREFIX)2-orig
+ mv $(FPREFIX)3 $(FPREFIX)3-orig
+ $(LZ4) -d -f -m $(FPREFIX)*.lz4
+ $(LZ4) -l -d -f -m $(FPREFIX)*.lz4 # -l mustn't impact -d option
+ $(CMP) $(FPREFIX)1 $(FPREFIX)1-orig # must be identical
+ $(CMP) $(FPREFIX)2 $(FPREFIX)2-orig
+ $(CMP) $(FPREFIX)3 $(FPREFIX)3-orig
# compress multiple files into stdout using legacy format
- $(CAT) tmp-tlm1.lz4 tmp-tlm2.lz4 tmp-tlm3.lz4 > tmp-tlm-concat1
- $(RM) *.lz4
- $(LZ4) -l -m tmp-tlm1 tmp-tlm2 tmp-tlm3 -c > tmp-tlm-concat2
- test ! -f tmp-tlm1.lz4 # must not create .lz4 artefact
- $(CMP) tmp-tlm-concat1 tmp-tlm-concat2 # must be equivalent
+ $(CAT) $(FPREFIX)1.lz4 $(FPREFIX)2.lz4 $(FPREFIX)3.lz4 > $(FPREFIX)-concat1
+ $(RM) $(FPREFIX)*.lz4
+ $(LZ4) -l -m $(FPREFIX)1 $(FPREFIX)2 $(FPREFIX)3 -c > $(FPREFIX)-concat2
+ test ! -f $(FPREFIX)1.lz4 # must not create .lz4 artefact
+ $(CMP) $(FPREFIX)-concat1 $(FPREFIX)-concat2 # must be equivalent
# # # decompress multiple files into stdout using legacy format
- $(RM) tmp-tlm-concat1 tmp-tlm-concat2
- $(LZ4) -l -f -m tmp-tlm1 tmp-tlm2 tmp-tlm3 # generate .lz4 to decompress
- $(CAT) tmp-tlm1 tmp-tlm2 tmp-tlm3 > tmp-tlm-concat1 # create concatenated reference
- $(RM) tmp-tlm1 tmp-tlm2 tmp-tlm3
- $(LZ4) -d -m tmp-tlm1.lz4 tmp-tlm2.lz4 tmp-tlm3.lz4 -c > tmp-tlm-concat2
- $(LZ4) -d -l -m tmp-tlm1.lz4 tmp-tlm2.lz4 tmp-tlm3.lz4 -c > tmp-tlm-concat2 # -l mustn't impact option -d
- test ! -f tmp-tlm1 # must not create file artefact
- $(CMP) tmp-tlm-concat1 tmp-tlm-concat2 # must be equivalent
+ $(RM) $(FPREFIX)-concat1 $(FPREFIX)-concat2
+ $(LZ4) -l -f -m $(FPREFIX)1 $(FPREFIX)2 $(FPREFIX)3 # generate .lz4 to decompress
+ $(CAT) $(FPREFIX)1 $(FPREFIX)2 $(FPREFIX)3 > $(FPREFIX)-concat1 # create concatenated reference
+ $(RM) $(FPREFIX)1 $(FPREFIX)2 $(FPREFIX)3
+ $(LZ4) -d -m $(FPREFIX)1.lz4 $(FPREFIX)2.lz4 $(FPREFIX)3.lz4 -c > $(FPREFIX)-concat2
+ $(LZ4) -d -l -m $(FPREFIX)1.lz4 $(FPREFIX)2.lz4 $(FPREFIX)3.lz4 -c > $(FPREFIX)-concat2 # -l mustn't impact option -d
+ test ! -f $(FPREFIX)1 # must not create file artefact
+ $(CMP) $(FPREFIX)-concat1 $(FPREFIX)-concat2 # must be equivalent
# # # compress multiple files, one of which is absent (must fail)
- ! $(LZ4) -f -l -m tmp-tlm-concat1 notHere-legacy tmp-tlm-concat2 # must fail : notHere-legacy not present
- @$(RM) tmp-tlm*
+ ! $(LZ4) -f -l -m $(FPREFIX)-concat1 notHere-legacy $(FPREFIX)-concat2 # must fail : notHere-legacy not present
+ @$(RM) $(FPREFIX)*
+SKIPFILE = goldenSamples/skip.bin
+test-lz4-skippable: FPREFIX = tmp-lsk
+test-lz4-skippable: lz4 datagen
+ @echo "\n ---- test lz4 with skippable frames ----"
+ $(LZ4) -dc $(SKIPFILE)
+ $(LZ4) -dc < $(SKIPFILE)
+ cat $(SKIPFILE) | $(LZ4) -dc
+ echo "Hello from Valid Frame!\n" | $(LZ4) -c > $(FPREFIX).lz4
+ cat $(SKIPFILE) $(FPREFIX).lz4 $(SKIPFILE) | $(LZ4) -dc
+ $(RM) $(FPREFIX)*
+
+test-lz4-basic: FPREFIX = tmp-tlb
test-lz4-basic: lz4 datagen unlz4 lz4cat
@echo "\n ---- test lz4 basic compression/decompression ----"
$(DATAGEN) -g0 | $(LZ4) -v | $(LZ4) -t
$(DATAGEN) -g16KB | $(LZ4) -9 | $(LZ4) -t
- $(DATAGEN) -g20KB > tmp-tlb-dg20k
- $(LZ4) < tmp-tlb-dg20k | $(LZ4) -d > tmp-tlb-dec
- $(DIFF) -q tmp-tlb-dg20k tmp-tlb-dec
- $(LZ4) --no-frame-crc < tmp-tlb-dg20k | $(LZ4) -d > tmp-tlb-dec
- $(DIFF) -q tmp-tlb-dg20k tmp-tlb-dec
+ $(DATAGEN) -g20KB > $(FPREFIX)-dg20k
+ $(LZ4) < $(FPREFIX)-dg20k | $(LZ4) -d > $(FPREFIX)-dec
+ $(DIFF) -q $(FPREFIX)-dg20k $(FPREFIX)-dec
+ $(LZ4) --no-frame-crc < $(FPREFIX)-dg20k | $(LZ4) -d > $(FPREFIX)-dec
+ $(DIFF) -q $(FPREFIX)-dg20k $(FPREFIX)-dec
$(DATAGEN) | $(LZ4) -BI | $(LZ4) -t
+ $(DATAGEN) | $(LZ4) --no-crc | $(LZ4) -t
$(DATAGEN) -g6M -P99 | $(LZ4) -9BD | $(LZ4) -t
$(DATAGEN) -g17M | $(LZ4) -9v | $(LZ4) -qt
$(DATAGEN) -g33M | $(LZ4) --no-frame-crc | $(LZ4) -t
- $(DATAGEN) -g256MB | $(LZ4) -vqB4D | $(LZ4) -t
- @echo "hello world" > tmp-tlb-hw
- $(LZ4) --rm -f tmp-tlb-hw tmp-tlb-hw.lz4
- test ! -f tmp-tlb-hw # must fail (--rm)
- test -f tmp-tlb-hw.lz4
- $(PRGDIR)/lz4cat tmp-tlb-hw.lz4 # must display hello world
- test -f tmp-tlb-hw.lz4
- $(PRGDIR)/unlz4 --rm tmp-tlb-hw.lz4 tmp-tlb-hw
- test -f tmp-tlb-hw
- test ! -f tmp-tlb-hw.lz4 # must fail (--rm)
- test ! -f tmp-tlb-hw.lz4.lz4 # must fail (unlz4)
- $(PRGDIR)/lz4cat tmp-tlb-hw # pass-through mode
- test -f tmp-tlb-hw
- test ! -f tmp-tlb-hw.lz4 # must fail (lz4cat)
- $(LZ4) tmp-tlb-hw tmp-tlb-hw.lz4 # creates tmp-tlb-hw.lz4
- $(PRGDIR)/lz4cat < tmp-tlb-hw.lz4 > tmp-tlb3 # checks lz4cat works with stdin (#285)
- $(DIFF) -q tmp-tlb-hw tmp-tlb3
- $(PRGDIR)/lz4cat < tmp-tlb-hw > tmp-tlb2 # checks lz4cat works in pass-through mode
- $(DIFF) -q tmp-tlb-hw tmp-tlb2
- cp tmp-tlb-hw ./-d
+ $(DATAGEN) -g256MB | $(LZ4) -vqB4D | $(LZ4) -t --no-crc
+ @echo "hello world" > $(FPREFIX)-hw
+ $(LZ4) --rm -f $(FPREFIX)-hw $(FPREFIX)-hw.lz4
+ test ! -f $(FPREFIX)-hw # must fail (--rm)
+ test -f $(FPREFIX)-hw.lz4
+ $(PRGDIR)/lz4cat $(FPREFIX)-hw.lz4 | $(GREP) "hello world"
+ $(PRGDIR)/unlz4 --rm $(FPREFIX)-hw.lz4 $(FPREFIX)-hw
+ test -f $(FPREFIX)-hw
+ test ! -f $(FPREFIX)-hw.lz4 # must fail (--rm)
+ test ! -f $(FPREFIX)-hw.lz4.lz4 # must fail (unlz4)
+ $(PRGDIR)/lz4cat $(FPREFIX)-hw # pass-through mode
+ test -f $(FPREFIX)-hw
+ test ! -f $(FPREFIX)-hw.lz4 # must fail (lz4cat)
+ $(LZ4) $(FPREFIX)-hw $(FPREFIX)-hw.lz4 # creates $(FPREFIX)-hw.lz4
+ $(PRGDIR)/lz4cat < $(FPREFIX)-hw.lz4 > $(FPREFIX)3 # checks lz4cat works with stdin (#285)
+ $(DIFF) -q $(FPREFIX)-hw $(FPREFIX)3
+ $(PRGDIR)/lz4cat < $(FPREFIX)-hw > $(FPREFIX)2 # checks lz4cat works in pass-through mode
+ $(DIFF) -q $(FPREFIX)-hw $(FPREFIX)2
+ cp $(FPREFIX)-hw ./-d
$(LZ4) --rm -- -d -d.lz4 # compresses ./d into ./-d.lz4
test -f ./-d.lz4
test ! -f ./-d
mv ./-d.lz4 ./-z
- $(LZ4) -d --rm -- -z tmp-tlb4 # uncompresses ./-z into tmp-tlb4
+ $(LZ4) -d --rm -- -z $(FPREFIX)4 # uncompresses ./-z into $(FPREFIX)4
test ! -f ./-z
- $(DIFF) -q tmp-tlb-hw tmp-tlb4
- $(LZ4) -f tmp-tlb-hw
- $(LZ4) --list tmp-tlb-hw.lz4 # test --list on valid single-frame file
- $(CAT) tmp-tlb-hw >> tmp-tlb-hw.lz4
- $(LZ4) -f tmp-tlb-hw.lz4 # uncompress valid frame followed by invalid data
- $(LZ4) -BX tmp-tlb-hw -c -q | $(LZ4) -tv # test block checksum
+ $(DIFF) -q $(FPREFIX)-hw $(FPREFIX)4
+ ! $(LZ4) $(FPREFIX)2 $(FPREFIX)3 $(FPREFIX)4 # must fail: refuse to handle 3+ file names
+ $(LZ4) -f $(FPREFIX)-hw # create $(FPREFIX)-hw.lz4, for next tests
+ $(LZ4) --list $(FPREFIX)-hw.lz4 # test --list on valid single-frame file
+ $(LZ4) --list < $(FPREFIX)-hw.lz4 # test --list from stdin (file only)
+ $(CAT) $(FPREFIX)-hw >> $(FPREFIX)-hw.lz4
+ ! $(LZ4) -f $(FPREFIX)-hw.lz4 # uncompress valid frame followed by invalid data (must fail now)
+ $(LZ4) -BX $(FPREFIX)-hw -c -q | $(LZ4) -tv # test block checksum
# $(DATAGEN) -g20KB generates the same file every single time
# cannot save output of $(DATAGEN) -g20KB as input file to lz4 because the following shell commands are run before $(DATAGEN) -g20KB
test "$(shell $(DATAGEN) -g20KB | $(LZ4) -c --fast | wc -c)" -lt "$(shell $(DATAGEN) -g20KB | $(LZ4) -c --fast=9 | wc -c)" # -1 vs -9
test "$(shell $(DATAGEN) -g20KB | $(LZ4) -c -1 | wc -c)" -lt "$(shell $(DATAGEN) -g20KB| $(LZ4) -c --fast=1 | wc -c)" # 1 vs -1
test "$(shell $(DATAGEN) -g20KB | $(LZ4) -c --fast=1 | wc -c)" -eq "$(shell $(DATAGEN) -g20KB| $(LZ4) -c --fast| wc -c)" # checks default fast compression is -1
- ! $(LZ4) -c --fast=0 tmp-tlb-dg20K # lz4 should fail when fast=0
- ! $(LZ4) -c --fast=-1 tmp-tlb-dg20K # lz4 should fail when fast=-1
+ ! $(LZ4) -c --fast=0 $(FPREFIX)-dg20K # lz4 should fail when fast=0
+ ! $(LZ4) -c --fast=-1 $(FPREFIX)-dg20K # lz4 should fail when fast=-1
# High --fast values can result in out-of-bound dereferences #876
$(DATAGEN) -g1M | $(LZ4) -c --fast=999999999 > /dev/null
# Test for #596
- @echo "TEST" > tmp-tlb-test
- $(LZ4) -m tmp-tlb-test
- $(LZ4) tmp-tlb-test.lz4 tmp-tlb-test2
- $(DIFF) -q tmp-tlb-test tmp-tlb-test2
- @$(RM) tmp-tlb*
+ @echo "TEST" > $(FPREFIX)-test
+ $(LZ4) -m $(FPREFIX)-test
+ $(LZ4) $(FPREFIX)-test.lz4 $(FPREFIX)-test2
+ $(DIFF) -q $(FPREFIX)-test $(FPREFIX)-test2
+ @$(RM) $(FPREFIX)*
-
+test-lz4-dict: FPREFIX = tmp-dict
test-lz4-dict: lz4 datagen
@echo "\n ---- test lz4 compression/decompression with dictionary ----"
- $(DATAGEN) -g16KB > tmp-dict
- $(DATAGEN) -g32KB > tmp-dict-sample-32k
- < tmp-dict-sample-32k $(LZ4) -D tmp-dict | $(LZ4) -dD tmp-dict | diff - tmp-dict-sample-32k
- $(DATAGEN) -g128MB > tmp-dict-sample-128m
- < tmp-dict-sample-128m $(LZ4) -D tmp-dict | $(LZ4) -dD tmp-dict | diff - tmp-dict-sample-128m
- touch tmp-dict-sample-0
- < tmp-dict-sample-0 $(LZ4) -D tmp-dict | $(LZ4) -dD tmp-dict | diff - tmp-dict-sample-0
+ $(DATAGEN) -g16KB > $(FPREFIX)
+ $(DATAGEN) -g32KB > $(FPREFIX)-sample-32k
+ < $(FPREFIX)-sample-32k $(LZ4) -D $(FPREFIX) | $(LZ4) -dD $(FPREFIX) | diff - $(FPREFIX)-sample-32k
+ $(DATAGEN) -g128MB > $(FPREFIX)-sample-128m
+ < $(FPREFIX)-sample-128m $(LZ4) -D $(FPREFIX) | $(LZ4) -dD $(FPREFIX) | diff - $(FPREFIX)-sample-128m
+ touch $(FPREFIX)-sample-0
+ < $(FPREFIX)-sample-0 $(LZ4) -D $(FPREFIX) | $(LZ4) -dD $(FPREFIX) | diff - $(FPREFIX)-sample-0
- < tmp-dict-sample-32k $(LZ4) -D tmp-dict-sample-0 | $(LZ4) -dD tmp-dict-sample-0 | diff - tmp-dict-sample-32k
- < tmp-dict-sample-0 $(LZ4) -D tmp-dict-sample-0 | $(LZ4) -dD tmp-dict-sample-0 | diff - tmp-dict-sample-0
+ < $(FPREFIX)-sample-32k $(LZ4) -D $(FPREFIX)-sample-0 | $(LZ4) -dD $(FPREFIX)-sample-0 | diff - $(FPREFIX)-sample-32k
+ < $(FPREFIX)-sample-0 $(LZ4) -D $(FPREFIX)-sample-0 | $(LZ4) -dD $(FPREFIX)-sample-0 | diff - $(FPREFIX)-sample-0
@echo "\n ---- test lz4 dictionary loading ----"
- $(DATAGEN) -g128KB > tmp-dict-data-128KB
+ $(DATAGEN) -g128KB > $(FPREFIX)-data-128KB
set -e; \
for l in 0 1 4 128 32767 32768 32769 65535 65536 65537 98303 98304 98305 131071 131072 131073; do \
- $(DATAGEN) -g$$l > tmp-dict-$$l; \
- $(DD) if=tmp-dict-$$l of=tmp-dict-$$l-tail bs=1 count=65536 skip=$$((l > 65536 ? l - 65536 : 0)); \
- < tmp-dict-$$l $(LZ4) -D stdin tmp-dict-data-128KB -c | $(LZ4) -dD tmp-dict-$$l-tail | $(DIFF) - tmp-dict-data-128KB; \
- < tmp-dict-$$l-tail $(LZ4) -D stdin tmp-dict-data-128KB -c | $(LZ4) -dD tmp-dict-$$l | $(DIFF) - tmp-dict-data-128KB; \
+ $(DATAGEN) -g$$l > $(FPREFIX)-$$l; \
+ $(DD) if=$(FPREFIX)-$$l of=$(FPREFIX)-$$l-tail bs=1 count=65536 skip=$$((l > 65536 ? l - 65536 : 0)); \
+ < $(FPREFIX)-$$l $(LZ4) -D stdin $(FPREFIX)-data-128KB -c | $(LZ4) -dD $(FPREFIX)-$$l-tail | $(DIFF) - $(FPREFIX)-data-128KB; \
+ < $(FPREFIX)-$$l-tail $(LZ4) -D stdin $(FPREFIX)-data-128KB -c | $(LZ4) -dD $(FPREFIX)-$$l | $(DIFF) - $(FPREFIX)-data-128KB; \
done
+ @$(RM) $(FPREFIX)*
- @$(RM) tmp-dict*
+test-lz4hc-hugefile: lz4 datagen
+ @echo "\n ---- test HC compression/decompression of huge files ----"
+ $(DATAGEN) -g4200MB | $(LZ4) -v3BD | $(LZ4) -qt
-test-lz4-hugefile: lz4 datagen
+test-lz4-fast-hugefile: FPREFIX = tmp-lfh
+test-lz4-fast-hugefile: lz4 datagen
@echo "\n ---- test huge files compression/decompression ----"
- ./datagen -g6GB | $(LZ4) -vB5D | $(LZ4) -qt
- ./datagen -g4500MB | $(LZ4) -v3BD | $(LZ4) -qt
+ $(DATAGEN) -g6GB | $(LZ4) -vB5D | $(LZ4) -qt
# test large file size [2-4] GB
- @$(DATAGEN) -g3G -P100 | $(LZ4) -vv | $(LZ4) --decompress --force --sparse - tmphf1
- @ls -ls tmphf1
- @$(DATAGEN) -g3G -P100 | $(LZ4) --quiet --content-size | $(LZ4) --verbose --decompress --force --sparse - tmphf2
- @ls -ls tmphf2
- $(DIFF) -s tmphf1 tmphf2
- @$(RM) tmphf*
+ @$(DATAGEN) -g3G -P100 | $(LZ4) -vv | $(LZ4) --decompress --force --sparse - $(FPREFIX)1
+ @ls -ls $(FPREFIX)1
+ @$(DATAGEN) -g3G -P100 | $(LZ4) --quiet --content-size | $(LZ4) --verbose --decompress --force --sparse - $(FPREFIX)2
+ @ls -ls $(FPREFIX)2
+ $(DIFF) -s $(FPREFIX)1 $(FPREFIX)2
+ @$(RM) $(FPREFIX)*
+test-lz4-hugefile: test-lz4-fast-hugefile test-lz4hc-hugefile
+
+test-lz4-testmode: FPREFIX = tmp-ltm
test-lz4-testmode: lz4 datagen
@echo "\n ---- bench mode ----"
$(LZ4) -bi0
+ $(DATAGEN) > $(FPREFIX)
+ $(LZ4) -f $(FPREFIX) -c > $(FPREFIX).lz4
+ $(LZ4) -bdi0 $(FPREFIX).lz4 # test benchmark decode-only mode
+ $(LZ4) -bdi0 --no-crc $(FPREFIX).lz4 # test benchmark decode-only mode
@echo "\n ---- test mode ----"
! $(DATAGEN) | $(LZ4) -t
! $(DATAGEN) | $(LZ4) -tf
@echo "\n ---- pass-through mode ----"
- @echo "Why hello there " > tmp-tlt2.lz4
- ! $(LZ4) -f tmp-tlt2.lz4 > $(VOID)
+ @echo "Why hello there " > $(FPREFIX)2.lz4
+ ! $(LZ4) -f $(FPREFIX)2.lz4 > $(VOID)
! $(DATAGEN) | $(LZ4) -dc > $(VOID)
! $(DATAGEN) | $(LZ4) -df > $(VOID)
$(DATAGEN) | $(LZ4) -dcf > $(VOID)
- @echo "Hello World !" > tmp-tlt1
- $(LZ4) -dcf tmp-tlt1
- @echo "from underground..." > tmp-tlt2
- $(LZ4) -dcfm tmp-tlt1 tmp-tlt2
- @echo "\n ---- non-existing source ----"
+ @echo "Hello World !" > $(FPREFIX)1
+ $(LZ4) -dcf $(FPREFIX)1
+ @echo "from underground..." > $(FPREFIX)2
+ $(LZ4) -dcfm $(FPREFIX)1 $(FPREFIX)2
+ @echo "\n ---- non-existing source (must fail cleanly) ----"
! $(LZ4) file-does-not-exist
! $(LZ4) -f file-does-not-exist
! $(LZ4) -t file-does-not-exist
! $(LZ4) -fm file1-dne file2-dne
- @$(RM) tmp-tlt tmp-tlt1 tmp-tlt2 tmp-tlt2.lz4
+ @$(RM) $(FPREFIX)*
test-lz4-opt-parser: lz4 datagen
@echo "\n ---- test opt-parser ----"
@@ -448,6 +511,7 @@
$(DATAGEN) -g256K | $(LZ4) -12B4D | $(LZ4) -t
$(DATAGEN) -g512K -P25 | $(LZ4) -12BD | $(LZ4) -t
$(DATAGEN) -g1M | $(LZ4) -12B5 | $(LZ4) -t
+ $(DATAGEN) -g1M -s2 | $(LZ4) -12B4D | $(LZ4) -t
$(DATAGEN) -g2M -P99 | $(LZ4) -11B4D | $(LZ4) -t
$(DATAGEN) -g4M | $(LZ4) -11vq | $(LZ4) -qt
$(DATAGEN) -g8M | $(LZ4) -11B4 | $(LZ4) -t
@@ -457,15 +521,15 @@
test-lz4-essentials : lz4 datagen test-lz4-basic test-lz4-multiple test-lz4-multiple-legacy \
test-lz4-frame-concatenation test-lz4-testmode \
test-lz4-contentSize test-lz4-dict
- @$(RM) tmp*
test-lz4: lz4 datagen test-lz4-essentials test-lz4-opt-parser \
- test-lz4-sparse test-lz4-hugefile test-lz4-dict
- @$(RM) tmp*
+ test-lz4-sparse test-lz4-hugefile test-lz4-dict \
+ test-lz4-skippable
+test-lz4c: LZ4C = $(LZ4)c
test-lz4c: lz4c datagen
@echo "\n ---- test lz4c variant ----"
- $(DATAGEN) -g256MB | $(LZ4)c -l -v | $(LZ4)c -t
+ $(DATAGEN) -g256MB | $(LZ4C) -l -v | $(LZ4C) -t
test-lz4c32: CFLAGS+=-m32
test-lz4c32: test-lz4
@@ -514,31 +578,41 @@
test-frametest32: CFLAGS += -m32
test-frametest32: test-frametest
+VALGRIND = valgrind --leak-check=yes --error-exitcode=1
+test-mem: FPREFIX = tmp-tvm
test-mem: lz4 datagen fuzzer frametest fullbench
@echo "\n ---- valgrind tests : memory analyzer ----"
- valgrind --leak-check=yes --error-exitcode=1 $(DATAGEN) -g50M > $(VOID)
- $(DATAGEN) -g16KB > ftmdg16K
- valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -9 -BD -f ftmdg16K $(VOID)
- $(DATAGEN) -g16KB -s2 > ftmdg16K2
- $(DATAGEN) -g16KB -s3 > ftmdg16K3
- valgrind --leak-check=yes --error-exitcode=1 $(LZ4) --force --multiple ftmdg16K ftmdg16K2 ftmdg16K3
- $(DATAGEN) -g7MB > ftmdg7M
- valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -9 -B5D -f ftmdg7M ftmdg16K2
- valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -t ftmdg16K2
- valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -bi1 ftmdg7M
- valgrind --leak-check=yes --error-exitcode=1 ./fullbench -i1 ftmdg7M ftmdg16K2
- valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -B4D -f -vq ftmdg7M $(VOID)
- valgrind --leak-check=yes --error-exitcode=1 $(LZ4) --list -m ftm*.lz4
- valgrind --leak-check=yes --error-exitcode=1 $(LZ4) --list -m -v ftm*.lz4
- $(RM) ftm*
- valgrind --leak-check=yes --error-exitcode=1 ./fuzzer -i64 -t1
- valgrind --leak-check=yes --error-exitcode=1 ./frametest -i256
+ $(VALGRIND) $(DATAGEN) -g50M > $(VOID)
+ $(DATAGEN) -g16KB > $(FPREFIX)dg16K
+ $(VALGRIND) $(LZ4) -9 -BD -f $(FPREFIX)dg16K $(VOID)
+ $(DATAGEN) -g16KB -s2 > $(FPREFIX)dg16K2
+ $(DATAGEN) -g16KB -s3 > $(FPREFIX)dg16K3
+ $(VALGRIND) $(LZ4) --force --multiple $(FPREFIX)dg16K $(FPREFIX)dg16K2 $(FPREFIX)dg16K3
+ $(DATAGEN) -g7MB > $(FPREFIX)dg7M
+ $(VALGRIND) $(LZ4) -9 -B5D -f $(FPREFIX)dg7M $(FPREFIX)dg16K2
+ $(VALGRIND) $(LZ4) -t $(FPREFIX)dg16K2
+ $(VALGRIND) $(LZ4) -bi1 $(FPREFIX)dg7M
+ $(VALGRIND) ./fullbench -i1 $(FPREFIX)dg7M $(FPREFIX)dg16K2
+ $(VALGRIND) $(LZ4) -B4D -f -vq $(FPREFIX)dg7M $(VOID)
+ $(VALGRIND) $(LZ4) --list -m $(FPREFIX)*.lz4
+ $(VALGRIND) $(LZ4) --list -m -v $(FPREFIX)*.lz4
+ $(RM) $(FPREFIX)*
+ $(VALGRIND) ./fuzzer -i64 -t1
+ $(VALGRIND) ./frametest -i256
test-mem32: lz4c32 datagen
# unfortunately, valgrind doesn't seem to work with non-native binary...
-test-decompress-partial : decompress-partial
+test-decompress-partial : decompress-partial decompress-partial-usingDict
@echo "\n ---- test decompress-partial ----"
./decompress-partial$(EXT)
+ @echo "\n ---- test decompress-partial-usingDict ----"
+ ./decompress-partial-usingDict$(EXT)
+
+test-freestanding: freestanding
+ @echo "\n ---- test freestanding ----"
+ ./freestanding$(EXT)
+ -strace ./freestanding$(EXT)
+ -ltrace ./freestanding$(EXT)
endif
diff --git a/tests/README.md b/tests/README.md
index 75b7b9f..65437de 100644
--- a/tests/README.md
+++ b/tests/README.md
@@ -13,7 +13,7 @@
#### `test-lz4-versions.py` - script for testing lz4 interoperability between versions
This script creates `versionsTest` directory to which lz4 repository is cloned.
-Then all taged (released) versions of lz4 are compiled.
+Then all tagged (released) versions of lz4 are compiled.
In the following step interoperability between lz4 versions is checked.
@@ -25,7 +25,7 @@
If a new commit is found it is compiled and a speed benchmark for this commit is performed.
The results of the speed benchmark are compared to the previous results.
If compression or decompression speed for one of lz4 levels is lower than `lowerLimit` (an optional parameter, default 0.98) the speed benchmark is restarted.
-If second results are also lower than `lowerLimit` the warning e-mail is send to recipients from the list (the `emails` parameter).
+If second results are also lower than `lowerLimit` the warning e-mail is sent to recipients from the list (the `emails` parameter).
Additional remarks:
- To be sure that speed results are accurate the script should be run on a "stable" target system with no other jobs running in parallel
@@ -37,7 +37,7 @@
The example usage with two test files, one e-mail address, and with an additional message:
```
./test-lz4-speed.py "silesia.tar calgary.tar" "email@gmail.com" --message "tested on my laptop" --sleepTime 60
-```
+```
To run the script in background please use:
```
diff --git a/tests/abiTest.c b/tests/abiTest.c
new file mode 100644
index 0000000..e46004a
--- /dev/null
+++ b/tests/abiTest.c
@@ -0,0 +1,218 @@
+/*
+ * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree),
+ * meaning you may select, at your option, one of the above-listed licenses.
+ */
+
+/*
+ * abiTest :
+ * ensure ABI stability expectations are not broken by a new version
+**/
+
+
+/*===========================================
+* Dependencies
+*==========================================*/
+#include <stddef.h> /* size_t */
+#include <stdlib.h> /* malloc, free, exit */
+#include <stdio.h> /* fprintf */
+#include <string.h> /* strcmp */
+#include <assert.h>
+#include <sys/types.h> /* stat */
+#include <sys/stat.h> /* stat */
+#include "xxhash.h"
+
+#include "lz4.h"
+#include "lz4frame.h"
+
+
+/*===========================================
+* Macros
+*==========================================*/
+#define MIN(a,b) ( (a) < (b) ? (a) : (b) )
+
+#define MSG(...) fprintf(stderr, __VA_ARGS__)
+
+#define CONTROL_MSG(c, ...) { \
+ if ((c)) { \
+ MSG(__VA_ARGS__); \
+ MSG(" \n"); \
+ abort(); \
+ } \
+}
+
+
+static size_t checkBuffers(const void* buff1, const void* buff2, size_t buffSize)
+{
+ const char* const ip1 = (const char*)buff1;
+ const char* const ip2 = (const char*)buff2;
+ size_t pos;
+
+ for (pos=0; pos<buffSize; pos++)
+ if (ip1[pos]!=ip2[pos])
+ break;
+
+ return pos;
+}
+
+
+LZ4_stream_t LZ4_cState;
+LZ4_streamDecode_t LZ4_dState;
+
+/** roundTripTest() :
+ * Compresses `srcBuff` into `compressedBuff`,
+ * then decompresses `compressedBuff` into `resultBuff`.
+ * If clevel==0, compression level is derived from srcBuff's content head bytes.
+ * This function abort() if it detects any round-trip error.
+ * Therefore, if it returns, round trip is considered successfully validated.
+ * Note : `compressedBuffCapacity` should be `>= LZ4_compressBound(srcSize)`
+ * for compression to be guaranteed to work */
+static void roundTripTest(void* resultBuff, size_t resultBuffCapacity,
+ void* compressedBuff, size_t compressedBuffCapacity,
+ const void* srcBuff, size_t srcSize)
+{
+ int const acceleration = 1;
+ // Note : can't use LZ4_initStream(), because it's only present since v1.9.0
+ memset(&LZ4_cState, 0, sizeof(LZ4_cState));
+ { int const cSize = LZ4_compress_fast_continue(&LZ4_cState, (const char*)srcBuff, (char*)compressedBuff, (int)srcSize, (int)compressedBuffCapacity, acceleration);
+ CONTROL_MSG(cSize == 0, "Compression error !");
+ { int const dInit = LZ4_setStreamDecode(&LZ4_dState, NULL, 0);
+ CONTROL_MSG(dInit == 0, "LZ4_setStreamDecode error !");
+ }
+ { int const dSize = LZ4_decompress_safe_continue (&LZ4_dState, (const char*)compressedBuff, (char*)resultBuff, cSize, (int)resultBuffCapacity);
+ CONTROL_MSG(dSize < 0, "Decompression detected an error !");
+ CONTROL_MSG(dSize != (int)srcSize, "Decompression corruption error : wrong decompressed size !");
+ } }
+
+ /* check potential content corruption error */
+ assert(resultBuffCapacity >= srcSize);
+ { size_t const errorPos = checkBuffers(srcBuff, resultBuff, srcSize);
+ CONTROL_MSG(errorPos != srcSize,
+ "Silent decoding corruption, at pos %u !!!",
+ (unsigned)errorPos);
+ }
+}
+
+static void roundTripCheck(const void* srcBuff, size_t srcSize)
+{
+ size_t const cBuffSize = LZ4_COMPRESSBOUND(srcSize);
+ void* const cBuff = malloc(cBuffSize);
+ void* const rBuff = malloc(cBuffSize);
+
+ if (!cBuff || !rBuff) {
+ fprintf(stderr, "not enough memory ! \n");
+ exit(1);
+ }
+
+ roundTripTest(rBuff, cBuffSize,
+ cBuff, cBuffSize,
+ srcBuff, srcSize);
+
+ free(rBuff);
+ free(cBuff);
+}
+
+
+static size_t getFileSize(const char* infilename)
+{
+ int r;
+#if defined(_MSC_VER)
+ struct _stat64 statbuf;
+ r = _stat64(infilename, &statbuf);
+ if (r || !(statbuf.st_mode & S_IFREG)) return 0; /* No good... */
+#else
+ struct stat statbuf;
+ r = stat(infilename, &statbuf);
+ if (r || !S_ISREG(statbuf.st_mode)) return 0; /* No good... */
+#endif
+ return (size_t)statbuf.st_size;
+}
+
+
+static int isDirectory(const char* infilename)
+{
+ int r;
+#if defined(_MSC_VER)
+ struct _stat64 statbuf;
+ r = _stat64(infilename, &statbuf);
+ if (!r && (statbuf.st_mode & _S_IFDIR)) return 1;
+#else
+ struct stat statbuf;
+ r = stat(infilename, &statbuf);
+ if (!r && S_ISDIR(statbuf.st_mode)) return 1;
+#endif
+ return 0;
+}
+
+
+/** loadFile() :
+ * requirement : `buffer` size >= `fileSize` */
+static void loadFile(void* buffer, const char* fileName, size_t fileSize)
+{
+ FILE* const f = fopen(fileName, "rb");
+ if (isDirectory(fileName)) {
+ MSG("Ignoring %s directory \n", fileName);
+ exit(2);
+ }
+ if (f==NULL) {
+ MSG("Impossible to open %s \n", fileName);
+ exit(3);
+ }
+ { size_t const readSize = fread(buffer, 1, fileSize, f);
+ if (readSize != fileSize) {
+ MSG("Error reading %s \n", fileName);
+ exit(5);
+ } }
+ fclose(f);
+}
+
+
+static void fileCheck(const char* fileName)
+{
+ size_t const fileSize = getFileSize(fileName);
+ void* const buffer = malloc(fileSize + !fileSize /* avoid 0 */);
+ if (!buffer) {
+ MSG("not enough memory \n");
+ exit(4);
+ }
+ loadFile(buffer, fileName, fileSize);
+ roundTripCheck(buffer, fileSize);
+ free (buffer);
+}
+
+
+int bad_usage(const char* exeName)
+{
+ MSG(" \n");
+ MSG("bad usage: \n");
+ MSG(" \n");
+ MSG("%s [Options] fileName \n", exeName);
+ MSG(" \n");
+ MSG("Options: \n");
+ MSG("-# : use #=[0-9] compression level (default:0 == random) \n");
+ return 1;
+}
+
+
+int main(int argCount, const char** argv)
+{
+ const char* const exeName = argv[0];
+ int argNb = 1;
+ // Note : LZ4_VERSION_STRING requires >= v1.7.3+
+ MSG("abiTest, built binary based on API %s \n", LZ4_VERSION_STRING);
+ // Note : LZ4_versionString() requires >= v1.7.5+
+ MSG("currently linked to dll %s \n", LZ4_versionString());
+
+ assert(argCount >= 1);
+ if (argCount < 2) return bad_usage(exeName);
+
+ if (argNb >= argCount) return bad_usage(exeName);
+
+ fileCheck(argv[argNb]);
+ MSG("no pb detected \n");
+ return 0;
+}
diff --git a/tests/checkFrame.c b/tests/checkFrame.c
index f9a1c14..946805f 100644
--- a/tests/checkFrame.c
+++ b/tests/checkFrame.c
@@ -1,6 +1,6 @@
/*
checkFrame - verify frame headers
- Copyright (C) Yann Collet 2014-present
+ Copyright (C) Yann Collet 2014-2020
GPL v2 License
@@ -153,7 +153,7 @@
if (LZ4F_isError(nextToLoad))
EXM_THROW(22, "Error getting frame info: %s",
LZ4F_getErrorName(nextToLoad));
- if (frameInfo.blockSizeID != bsid)
+ if (frameInfo.blockSizeID != (LZ4F_blockSizeID_t) bsid)
EXM_THROW(23, "Block size ID %u != expected %u",
frameInfo.blockSizeID, bsid);
pos += remaining;
diff --git a/tests/checkTag.c b/tests/checkTag.c
index 4a33415..5e5a034 100644
--- a/tests/checkTag.c
+++ b/tests/checkTag.c
@@ -1,6 +1,6 @@
/*
checkTag.c - Version validation tool for LZ4
- Copyright (C) Yann Collet 2018 - present
+ Copyright (C) Yann Collet 2018-2020
GPL v2 License
diff --git a/tests/check_liblz4_version.sh b/tests/check_liblz4_version.sh
new file mode 100755
index 0000000..9304204
--- /dev/null
+++ b/tests/check_liblz4_version.sh
@@ -0,0 +1,6 @@
+#!/usr/bin/env sh
+set -e
+
+# written as a script shell, because pipe management in python is horrible
+ldd $1 | grep liblz4
+
diff --git a/tests/datagencli.c b/tests/datagencli.c
index c985197..ccb27df 100644
--- a/tests/datagencli.c
+++ b/tests/datagencli.c
@@ -1,7 +1,7 @@
/*
datagencli.c
compressible data command line generator
- Copyright (C) Yann Collet 2012-2016
+ Copyright (C) Yann Collet 2012-2020
GPL v2 License
@@ -34,6 +34,14 @@
/**************************************
+* Compiler specific
+**************************************/
+#ifdef _MSC_VER /* Visual Studio */
+#define strtoull _strtoui64 /* https://docs.microsoft.com/en-us/cpp/c-runtime-library/reference/strtoui64-wcstoui64-strtoui64-l-wcstoui64-l */
+#endif
+
+
+/**************************************
* Constants
**************************************/
#define KB *(1 <<10)
@@ -103,13 +111,7 @@
return usage(programName);
case 'g':
argument++;
- size=0;
- while ((*argument>='0') && (*argument<='9'))
- {
- size *= 10;
- size += *argument - '0';
- argument++;
- }
+ size = strtoull(argument, &argument, 10);
if (*argument=='K') { size <<= 10; argument++; }
if (*argument=='M') { size <<= 20; argument++; }
if (*argument=='G') { size <<= 30; argument++; }
@@ -117,35 +119,16 @@
break;
case 's':
argument++;
- seed=0;
- while ((*argument>='0') && (*argument<='9'))
- {
- seed *= 10;
- seed += *argument - '0';
- argument++;
- }
+ seed = (U32) strtoul(argument, &argument, 10);
break;
case 'P':
argument++;
- proba=0.0;
- while ((*argument>='0') && (*argument<='9'))
- {
- proba *= 10;
- proba += *argument - '0';
- argument++;
- }
- if (proba>100.) proba=100.;
+ proba = (double) strtoull(argument, &argument, 10);
proba /= 100.;
break;
case 'L': /* hidden argument : Literal distribution probability */
argument++;
- litProba=0.;
- while ((*argument>='0') && (*argument<='9'))
- {
- litProba *= 10;
- litProba += *argument - '0';
- argument++;
- }
+ litProba = (double) strtoull(argument, &argument, 10);
if (litProba>100.) litProba=100.;
litProba /= 100.;
break;
diff --git a/tests/decompress-partial-usingDict.c b/tests/decompress-partial-usingDict.c
new file mode 100644
index 0000000..8b85106
--- /dev/null
+++ b/tests/decompress-partial-usingDict.c
@@ -0,0 +1,103 @@
+#include <stdlib.h>
+#include <string.h>
+#include <stdio.h>
+#include <assert.h>
+#include "lz4.h"
+
+const char source[] =
+ "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod\n"
+ "tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim\n"
+ "veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea\n"
+ "commodo consequat. Duis aute irure dolor in reprehenderit in voluptate\n"
+ "velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat\n"
+ "cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id\n"
+ "est laborum.\n"
+ "\n"
+ "Sed ut perspiciatis unde omnis iste natus error sit voluptatem accusantium\n"
+ "doloremque laudantium, totam rem aperiam, eaque ipsa quae ab illo inventore\n"
+ "veritatis et quasi architecto beatae vitae dicta sunt explicabo. Nemo enim\n"
+ "ipsam voluptatem quia voluptas sit aspernatur aut odit aut fugit, sed quia\n"
+ "consequuntur magni dolores eos qui ratione voluptatem sequi nesciunt. Neque\n"
+ "porro quisquam est, qui dolorem ipsum quia dolor sit amet, consectetur,\n"
+ "adipisci velit, sed quia non numquam eius modi tempora incidunt ut labore\n"
+ "et dolore magnam aliquam quaerat voluptatem. Ut enim ad minima veniam, quis\n"
+ "nostrum exercitationem ullam corporis suscipit laboriosam, nisi ut aliquid\n"
+ "ex ea commodi consequatur? Quis autem vel eum iure reprehenderit qui in ea\n"
+ "voluptate velit esse quam nihil molestiae consequatur, vel illum qui\n"
+ "dolorem eum fugiat quo voluptas nulla pariatur?\n";
+
+#define BUFFER_SIZE 2048
+
+int main(void)
+{
+ int srcLen = (int)strlen(source);
+ size_t const smallSize = 1024;
+ size_t const largeSize = 64 * 1024 - 1;
+ char cmpBuffer[BUFFER_SIZE];
+ char* const buffer = (char*)malloc(BUFFER_SIZE + largeSize);
+ char* outBuffer = buffer + largeSize;
+ char* const dict = (char*)malloc(largeSize);
+ char* const largeDict = dict;
+ char* const smallDict = dict + largeSize - smallSize;
+ int i;
+ int cmpSize;
+
+ printf("starting test decompress-partial-usingDict : \n");
+ assert(buffer != NULL);
+ assert(dict != NULL);
+
+ cmpSize = LZ4_compress_default(source, cmpBuffer, srcLen, BUFFER_SIZE);
+
+ for (i = cmpSize; i < cmpSize + 10; ++i) {
+ int result = LZ4_decompress_safe_partial_usingDict(cmpBuffer, outBuffer, i, srcLen, BUFFER_SIZE, NULL, 0);
+ if ( (result < 0)
+ || (result != srcLen)
+ || memcmp(source, outBuffer, (size_t)srcLen) ) {
+ printf("test decompress-partial-usingDict with no dict error \n");
+ return -1;
+ }
+ }
+
+ for (i = cmpSize; i < cmpSize + 10; ++i) {
+ int result = LZ4_decompress_safe_partial_usingDict(cmpBuffer, outBuffer, i, srcLen, BUFFER_SIZE, outBuffer - smallSize, smallSize);
+ if ( (result < 0)
+ || (result != srcLen)
+ || memcmp(source, outBuffer, (size_t)srcLen) ) {
+ printf("test decompress-partial-usingDict with small prefix error \n");
+ return -1;
+ }
+ }
+
+ for (i = cmpSize; i < cmpSize + 10; ++i) {
+ int result = LZ4_decompress_safe_partial_usingDict(cmpBuffer, outBuffer, i, srcLen, BUFFER_SIZE, buffer, largeSize);
+ if ( (result < 0)
+ || (result != srcLen)
+ || memcmp(source, outBuffer, (size_t)srcLen) ) {
+ printf("test decompress-partial-usingDict with large prefix error \n");
+ return -1;
+ }
+ }
+
+ for (i = cmpSize; i < cmpSize + 10; ++i) {
+ int result = LZ4_decompress_safe_partial_usingDict(cmpBuffer, outBuffer, i, srcLen, BUFFER_SIZE, smallDict, smallSize);
+ if ( (result < 0)
+ || (result != srcLen)
+ || memcmp(source, outBuffer, (size_t)srcLen) ) {
+ printf("test decompress-partial-usingDict with small external dict error \n");
+ return -1;
+ }
+ }
+
+ for (i = cmpSize; i < cmpSize + 10; ++i) {
+ int result = LZ4_decompress_safe_partial_usingDict(cmpBuffer, outBuffer, i, srcLen, BUFFER_SIZE, largeDict, largeSize);
+ if ( (result < 0)
+ || (result != srcLen)
+ || memcmp(source, outBuffer, (size_t)srcLen) ) {
+ printf("test decompress-partial-usingDict with large external dict error \n");
+ return -1;
+ }
+ }
+
+ printf("test decompress-partial-usingDict OK \n");
+ return 0;
+}
diff --git a/tests/frametest.c b/tests/frametest.c
index e613cbf..3301955 100644
--- a/tests/frametest.c
+++ b/tests/frametest.c
@@ -1,6 +1,6 @@
/*
frameTest - test tool for lz4frame
- Copyright (C) Yann Collet 2014-2016
+ Copyright (C) Yann Collet 2014-2020
GPL v2 License
@@ -51,10 +51,10 @@
#include "xxhash.h" /* XXH64 */
-/* unoptimized version; solves endianess & alignment issues */
+/* unoptimized version; solves endianness & alignment issues */
static void FUZ_writeLE32 (void* dstVoidPtr, U32 value32)
{
- BYTE* dstPtr = (BYTE*)dstVoidPtr;
+ BYTE* const dstPtr = (BYTE*)dstVoidPtr;
dstPtr[0] = (BYTE) value32;
dstPtr[1] = (BYTE)(value32 >> 8);
dstPtr[2] = (BYTE)(value32 >> 16);
@@ -65,8 +65,6 @@
/*-************************************
* Constants
**************************************/
-#define LZ4F_MAGIC_SKIPPABLE_START 0x184D2A50U
-
#define KB *(1U<<10)
#define MB *(1U<<20)
#define GB *(1U<<30)
@@ -104,12 +102,63 @@
#define MIN(a,b) ( (a) < (b) ? (a) : (b) )
#define MAX(a,b) ( (a) > (b) ? (a) : (b) )
+typedef struct {
+ int nbAllocs;
+} Test_alloc_state;
+static Test_alloc_state g_testAllocState = { 0 };
+
+static void* dummy_malloc(void* state, size_t s)
+{
+ Test_alloc_state* const t = (Test_alloc_state*)state;
+ void* const p = malloc(s);
+ if (p==NULL) return NULL;
+ assert(t != NULL);
+ t->nbAllocs += 1;
+ DISPLAYLEVEL(6, "Allocating %zu bytes at address %p \n", s, p);
+ DISPLAYLEVEL(5, "nb allocated memory segments : %i \n", t->nbAllocs);
+ return p;
+}
+
+static void* dummy_calloc(void* state, size_t s)
+{
+ Test_alloc_state* const t = (Test_alloc_state*)state;
+ void* const p = calloc(1, s);
+ if (p==NULL) return NULL;
+ assert(t != NULL);
+ t->nbAllocs += 1;
+ DISPLAYLEVEL(6, "Allocating and zeroing %zu bytes at address %p \n", s, p);
+ DISPLAYLEVEL(5, "nb allocated memory segments : %i \n", t->nbAllocs);
+ return p;
+}
+
+static void dummy_free(void* state, void* p)
+{
+ Test_alloc_state* const t = (Test_alloc_state*)state;
+ if (p==NULL) {
+ DISPLAYLEVEL(5, "free() on NULL \n");
+ return;
+ }
+ DISPLAYLEVEL(6, "freeing memory at address %p \n", p);
+ free(p);
+ assert(t != NULL);
+ t->nbAllocs -= 1;
+ DISPLAYLEVEL(5, "nb of allocated memory segments after this free : %i \n", t->nbAllocs);
+ assert(t->nbAllocs >= 0);
+}
+
+static const LZ4F_CustomMem lz4f_cmem_test = {
+ dummy_malloc,
+ dummy_calloc,
+ dummy_free,
+ &g_testAllocState
+};
+
+
static clock_t FUZ_GetClockSpan(clock_t clockStart)
{
return clock() - clockStart; /* works even if overflow; max span ~ 30 mn */
}
-
#define FUZ_rotl32(x,r) ((x << r) | (x >> (32 - r)))
unsigned int FUZ_rand(unsigned int* src)
{
@@ -121,7 +170,6 @@
return rand32 >> 5;
}
-
#define FUZ_RAND15BITS (FUZ_rand(seed) & 0x7FFF)
#define FUZ_RANDLENGTH ( (FUZ_rand(seed) & 3) ? (FUZ_rand(seed) % 15) : (FUZ_rand(seed) % 510) + 15)
static void FUZ_fillCompressibleNoiseBuffer(void* buffer, size_t bufferSize, double proba, U32* seed)
@@ -515,7 +563,9 @@
/* dictID tests */
{ size_t cErr;
U32 const dictID = 0x99;
- CHECK( LZ4F_createCompressionContext(&cctx, LZ4F_VERSION) );
+ /* test advanced variant with custom allocator functions */
+ cctx = LZ4F_createCompressionContext_advanced(lz4f_cmem_test, LZ4F_VERSION);
+ if (cctx==NULL) goto _output_error;
DISPLAYLEVEL(3, "insert a dictID : ");
memset(&prefs.frameInfo, 0, sizeof(prefs.frameInfo));
@@ -535,17 +585,25 @@
}
/* Dictionary compression test */
- { size_t const dictSize = 63 KB;
- size_t const dstCapacity = LZ4F_compressFrameBound(dictSize, NULL);
+ { size_t const dictSize = 7 KB; /* small enough for LZ4_MEMORY_USAGE == 10 */
+ size_t const srcSize = 65 KB; /* must be > 64 KB to avoid short-size optimizations */
+ size_t const dstCapacity = LZ4F_compressFrameBound(srcSize, NULL);
size_t cSizeNoDict, cSizeWithDict;
LZ4F_CDict* const cdict = LZ4F_createCDict(CNBuffer, dictSize);
if (cdict == NULL) goto _output_error;
CHECK( LZ4F_createCompressionContext(&cctx, LZ4F_VERSION) );
+ DISPLAYLEVEL(3, "Testing LZ4F_createCDict_advanced : ");
+ { LZ4F_CDict* const cda = LZ4F_createCDict_advanced(lz4f_cmem_test, CNBuffer, dictSize);
+ if (cda == NULL) goto _output_error;
+ LZ4F_freeCDict(cda);
+ }
+ DISPLAYLEVEL(3, "OK \n");
+
DISPLAYLEVEL(3, "LZ4F_compressFrame_usingCDict, with NULL dict : ");
CHECK_V(cSizeNoDict,
LZ4F_compressFrame_usingCDict(cctx, compressedBuffer, dstCapacity,
- CNBuffer, dictSize,
+ CNBuffer, srcSize,
NULL, NULL) );
DISPLAYLEVEL(3, "%u bytes \n", (unsigned)cSizeNoDict);
@@ -554,16 +612,19 @@
DISPLAYLEVEL(3, "LZ4F_compressFrame_usingCDict, with dict : ");
CHECK_V(cSizeWithDict,
LZ4F_compressFrame_usingCDict(cctx, compressedBuffer, dstCapacity,
- CNBuffer, dictSize,
+ CNBuffer, srcSize,
cdict, NULL) );
DISPLAYLEVEL(3, "compressed %u bytes into %u bytes \n",
- (unsigned)dictSize, (unsigned)cSizeWithDict);
- if ((LZ4_DISTANCE_MAX > dictSize) && (cSizeWithDict >= cSizeNoDict)) goto _output_error; /* must be more efficient */
- crcOrig = XXH64(CNBuffer, dictSize, 0);
+ (unsigned)srcSize, (unsigned)cSizeWithDict);
+ if (cSizeWithDict > cSizeNoDict) {
+ DISPLAYLEVEL(3, "cSizeWithDict (%zu) should have been more compact than cSizeNoDict(%zu) \n", cSizeWithDict, cSizeNoDict);
+ goto _output_error; /* must be more efficient */
+ }
+ crcOrig = XXH64(CNBuffer, srcSize, 0);
DISPLAYLEVEL(3, "LZ4F_decompress_usingDict : ");
{ LZ4F_dctx* dctx;
- size_t decodedSize = COMPRESSIBLE_NOISE_LENGTH;
+ size_t decodedSize = srcSize;
size_t compressedSize = cSizeWithDict;
CHECK( LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION) );
CHECK( LZ4F_decompress_usingDict(dctx,
@@ -572,7 +633,7 @@
CNBuffer, dictSize,
NULL) );
if (compressedSize != cSizeWithDict) goto _output_error;
- if (decodedSize != dictSize) goto _output_error;
+ if (decodedSize != srcSize) goto _output_error;
{ U64 const crcDest = XXH64(decodedBuffer, decodedSize, 0);
if (crcDest != crcOrig) goto _output_error; }
DISPLAYLEVEL(3, "Regenerated %u bytes \n", (U32)decodedSize);
@@ -682,20 +743,20 @@
{ size_t result;
unsigned blockSizeID;
for (blockSizeID = 4; blockSizeID < 8; ++blockSizeID) {
- result = LZ4F_getBlockSize(blockSizeID);
+ result = LZ4F_getBlockSize((LZ4F_blockSizeID_t)blockSizeID);
CHECK(result);
DISPLAYLEVEL(3, "Returned block size of %u bytes for blockID %u \n",
(unsigned)result, blockSizeID);
}
/* Test an invalid input that's too large */
- result = LZ4F_getBlockSize(8);
+ result = LZ4F_getBlockSize((LZ4F_blockSizeID_t)8);
if(!LZ4F_isError(result) ||
LZ4F_getErrorCode(result) != LZ4F_ERROR_maxBlockSize_invalid)
goto _output_error;
/* Test an invalid input that's too small */
- result = LZ4F_getBlockSize(3);
+ result = LZ4F_getBlockSize((LZ4F_blockSizeID_t)3);
if(!LZ4F_isError(result) ||
LZ4F_getErrorCode(result) != LZ4F_ERROR_maxBlockSize_invalid)
goto _output_error;
@@ -845,6 +906,7 @@
memset(&dOptions, 0, sizeof(dOptions));
dOptions.stableDst = FUZ_rand(randState) & 1;
if (o_scenario == o_overwrite) dOptions.stableDst = 0; /* overwrite mode */
+ dOptions.skipChecksums = FUZ_rand(randState) & 127;
if (sentinelTest) op[oSizeMax] = mark;
DISPLAYLEVEL(7, "dstCapacity=%u, presentedInput=%u \n", (unsigned)oSize, (unsigned)iSize);
@@ -940,7 +1002,7 @@
clock_t const startClock = clock();
clock_t const clockDuration = duration_s * CLOCKS_PER_SEC;
- /* Create buffers */
+ /* Create states & buffers */
{ size_t const creationStatus = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION);
CHECK(LZ4F_isError(creationStatus), "Allocation failed (error %i)", (int)creationStatus); }
{ size_t const creationStatus = LZ4F_createDecompressionContext(&dCtxNoise, LZ4F_VERSION);
@@ -1011,18 +1073,35 @@
while (ip < iend) {
unsigned const nbBitsSeg = FUZ_rand(&randState) % maxBits;
size_t const sampleMax = (FUZ_rand(&randState) & ((1<<nbBitsSeg)-1)) + 1;
- size_t const iSize = MIN(sampleMax, (size_t)(iend-ip));
+ size_t iSize = MIN(sampleMax, (size_t)(iend-ip));
size_t const oSize = LZ4F_compressBound(iSize, prefsPtr);
- size_t flushedSize;
cOptions.stableSrc = ((FUZ_rand(&randState) & 3) == 1);
DISPLAYLEVEL(6, "Sending %u bytes to compress (stableSrc:%u) \n",
(unsigned)iSize, cOptions.stableSrc);
- flushedSize = LZ4F_compressUpdate(cCtx, op, oSize, ip, iSize, &cOptions);
- CHECK(LZ4F_isError(flushedSize), "Compression failed (error %i : %s)",
+#if 1
+ /* insert uncompressed segment */
+ if ( (iSize>0)
+ && !neverFlush /* do not mess with compressBound when neverFlush is set */
+ && prefsPtr != NULL /* prefs are set */
+ && prefs.frameInfo.blockMode == LZ4F_blockIndependent /* uncompressedUpdate is only valid with blockMode==independent */
+ && (FUZ_rand(&randState) & 15) == 1 ) {
+ size_t const uSize = FUZ_rand(&randState) % iSize;
+ size_t const flushedSize = LZ4F_uncompressedUpdate(cCtx, op, (size_t)(oend-op), ip, uSize, &cOptions);
+ CHECK(LZ4F_isError(flushedSize), "Insert uncompressed data failed (error %i : %s)",
(int)flushedSize, LZ4F_getErrorName(flushedSize));
- op += flushedSize;
- ip += iSize;
+ op += flushedSize;
+ ip += uSize;
+ iSize -= uSize;
+ }
+#endif
+
+ { size_t const flushedSize = LZ4F_compressUpdate(cCtx, op, oSize, ip, iSize, &cOptions);
+ CHECK(LZ4F_isError(flushedSize), "Compression failed (error %i : %s)",
+ (int)flushedSize, LZ4F_getErrorName(flushedSize));
+ op += flushedSize;
+ ip += iSize;
+ }
{ unsigned const forceFlush = neverFlush ? 0 : ((FUZ_rand(&randState) & 3) == 1);
if (forceFlush) {
@@ -1036,11 +1115,8 @@
op[3] = 0x80; /* 0x80000000U in little-endian format */
op += 4;
if ((prefsPtr!= NULL) && prefsPtr->frameInfo.blockChecksumFlag) {
- U32 const bc32 = XXH32(op, 0, 0);
- op[0] = (BYTE)bc32; /* little endian format */
- op[1] = (BYTE)(bc32>>8);
- op[2] = (BYTE)(bc32>>16);
- op[3] = (BYTE)(bc32>>24);
+ /* add block checksum (even for empty blocks) */
+ FUZ_writeLE32(op, XXH32(op, 0, 0));
op += 4;
} } } }
} /* while (ip<iend) */
diff --git a/tests/freestanding.c b/tests/freestanding.c
new file mode 100644
index 0000000..ceff4c5
--- /dev/null
+++ b/tests/freestanding.c
@@ -0,0 +1,239 @@
+// Basic test for LZ4_FREESTANDING
+
+// $ gcc -ffreestanding -nostdlib freestanding.c && ./a.out || echo $?
+
+// $ strace ./a.out
+// execve("./a.out", ["./a.out"], 0x7fffaf5fa580 /* 22 vars */) = 0
+// brk(NULL) = 0x56536f4fe000
+// arch_prctl(0x3001 /* ARCH_??? */, 0x7fffc9e74950) = -1 EINVAL (Invalid argument)
+// mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fd5c9c2b000
+// access("/etc/ld.so.preload", R_OK) = -1 ENOENT (No such file or directory)
+// arch_prctl(ARCH_SET_FS, 0x7fd5c9c2bc40) = 0
+// set_tid_address(0x7fd5c9c2bf10) = 381
+// set_robust_list(0x7fd5c9c2bf20, 24) = 0
+// rseq(0x7fd5c9c2c5e0, 0x20, 0, 0x53053053) = 0
+// mprotect(0x56536ea63000, 4096, PROT_READ) = 0
+// exit(0) = ?
+// +++ exited with 0 +++
+
+// $ ltrace ./a.out
+// +++ exited (status 0) +++
+
+#include <stddef.h>
+#include <stdint.h>
+
+#if defined(__cplusplus)
+# define EXTERN_C extern "C"
+#else
+# define EXTERN_C
+#endif
+
+
+#if !defined(__x86_64__) || !defined(__linux__)
+EXTERN_C void _start(void) { }
+int main(int argc, char** argv) { return 0; }
+#else
+
+static void MY_exit(int exitCode);
+static void MY_abort(void);
+EXTERN_C void *memmove(void *dst, const void *src, size_t n);
+EXTERN_C void *memcpy(void * __restrict__ dst, const void * __restrict__ src, size_t n);
+EXTERN_C void *memset(void *s, int c, size_t n);
+EXTERN_C int memcmp(const void *s1, const void *s2, size_t n);
+
+// LZ4/HC basic freestanding setup
+#define LZ4_FREESTANDING 1
+#define LZ4_memmove(dst, src, size) memmove((dst),(src),(size))
+#define LZ4_memcpy(dst, src, size) memcpy((dst),(src),(size))
+#define LZ4_memset(p,v,s) memset((p),(v),(s))
+
+#include "../lib/lz4.c"
+#include "../lib/lz4hc.c"
+
+// Test for LZ4
+static void test_lz4(const uint8_t* srcData, int srcSize) {
+ // Compress
+ static uint8_t compressBuffer[1024 * 1024];
+ const int compressedSize = LZ4_compress_default(
+ (const char*) srcData,
+ (char*) compressBuffer,
+ srcSize,
+ sizeof(compressBuffer)
+ );
+ if (compressedSize <= 0) {
+ MY_exit(__LINE__);
+ }
+
+ // Decompress
+ static uint8_t decompressBuffer[1024 * 1024];
+ const int decompressedSize = LZ4_decompress_safe(
+ (const char*) compressBuffer,
+ (char*) decompressBuffer,
+ compressedSize,
+ sizeof(decompressBuffer)
+ );
+ if (decompressedSize <= 0) {
+ MY_exit(__LINE__);
+ }
+
+ // Verify
+ if (decompressedSize != srcSize) {
+ MY_exit(__LINE__);
+ }
+ if (memcmp(srcData, decompressBuffer, srcSize) != 0) {
+ MY_exit(__LINE__);
+ }
+}
+
+
+// Test for LZ4HC
+static void test_lz4hc(const uint8_t* srcData, int srcSize) {
+ // Compress
+ static uint8_t compressBuffer[1024 * 1024];
+ const int compressedSize = LZ4_compress_HC(
+ (const char*) srcData,
+ (char*) compressBuffer,
+ srcSize,
+ sizeof(compressBuffer),
+ LZ4HC_CLEVEL_DEFAULT
+ );
+ if (compressedSize <= 0) {
+ MY_exit(__LINE__);
+ }
+
+ // Decompress
+ static uint8_t decompressBuffer[1024 * 1024];
+ const int decompressedSize = LZ4_decompress_safe(
+ (const char*) compressBuffer,
+ (char*) decompressBuffer,
+ compressedSize,
+ sizeof(decompressBuffer)
+ );
+ if (decompressedSize <= 0) {
+ MY_exit(__LINE__);
+ }
+
+ // Verify
+ if (decompressedSize != srcSize) {
+ MY_exit(__LINE__);
+ }
+ if (memcmp(srcData, decompressBuffer, srcSize) != 0) {
+ MY_exit(__LINE__);
+ }
+}
+
+
+static void test(void) {
+ // First 256 bytes of lz4/README.md
+ static const uint8_t README_md[] = {
+ 0x4c, 0x5a, 0x34, 0x20, 0x2d, 0x20, 0x45, 0x78, 0x74, 0x72, 0x65, 0x6d, 0x65, 0x6c, 0x79, 0x20,
+ 0x66, 0x61, 0x73, 0x74, 0x20, 0x63, 0x6f, 0x6d, 0x70, 0x72, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e,
+ 0x0a, 0x3d, 0x3d, 0x3d, 0x3d, 0x3d, 0x3d, 0x3d, 0x3d, 0x3d, 0x3d, 0x3d, 0x3d, 0x3d, 0x3d, 0x3d,
+ 0x3d, 0x3d, 0x3d, 0x3d, 0x3d, 0x3d, 0x3d, 0x3d, 0x3d, 0x3d, 0x3d, 0x3d, 0x3d, 0x3d, 0x3d, 0x3d,
+ 0x3d, 0x0a, 0x0a, 0x4c, 0x5a, 0x34, 0x20, 0x69, 0x73, 0x20, 0x6c, 0x6f, 0x73, 0x73, 0x6c, 0x65,
+ 0x73, 0x73, 0x20, 0x63, 0x6f, 0x6d, 0x70, 0x72, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x20, 0x61,
+ 0x6c, 0x67, 0x6f, 0x72, 0x69, 0x74, 0x68, 0x6d, 0x2c, 0x0a, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64,
+ 0x69, 0x6e, 0x67, 0x20, 0x63, 0x6f, 0x6d, 0x70, 0x72, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x20,
+ 0x73, 0x70, 0x65, 0x65, 0x64, 0x20, 0x3e, 0x20, 0x35, 0x30, 0x30, 0x20, 0x4d, 0x42, 0x2f, 0x73,
+ 0x20, 0x70, 0x65, 0x72, 0x20, 0x63, 0x6f, 0x72, 0x65, 0x2c, 0x0a, 0x73, 0x63, 0x61, 0x6c, 0x61,
+ 0x62, 0x6c, 0x65, 0x20, 0x77, 0x69, 0x74, 0x68, 0x20, 0x6d, 0x75, 0x6c, 0x74, 0x69, 0x2d, 0x63,
+ 0x6f, 0x72, 0x65, 0x73, 0x20, 0x43, 0x50, 0x55, 0x2e, 0x0a, 0x49, 0x74, 0x20, 0x66, 0x65, 0x61,
+ 0x74, 0x75, 0x72, 0x65, 0x73, 0x20, 0x61, 0x6e, 0x20, 0x65, 0x78, 0x74, 0x72, 0x65, 0x6d, 0x65,
+ 0x6c, 0x79, 0x20, 0x66, 0x61, 0x73, 0x74, 0x20, 0x64, 0x65, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x2c,
+ 0x0a, 0x77, 0x69, 0x74, 0x68, 0x20, 0x73, 0x70, 0x65, 0x65, 0x64, 0x20, 0x69, 0x6e, 0x20, 0x6d,
+ 0x75, 0x6c, 0x74, 0x69, 0x70, 0x6c, 0x65, 0x20, 0x47, 0x42, 0x2f, 0x73, 0x20, 0x70, 0x65, 0x72,
+ };
+
+ static const uint8_t* srcData = README_md;
+ static const int srcSize = (int) sizeof(README_md);
+ test_lz4 (srcData, srcSize);
+ test_lz4hc(srcData, srcSize);
+}
+
+
+// low level syscall
+#define SYS_exit (60)
+
+static __inline long os_syscall1(long n, long a1) {
+ register long rax __asm__ ("rax") = n;
+ register long rdi __asm__ ("rdi") = a1;
+ __asm__ __volatile__ ("syscall" : "+r"(rax) : "r"(rdi) : "rcx", "r11", "memory");
+ return rax;
+}
+
+static void MY_exit(int exitCode) {
+ (void) os_syscall1(SYS_exit, exitCode);
+ __builtin_unreachable(); // suppress "warning: 'noreturn' function does return"
+}
+
+static void MY_abort(void) {
+ MY_exit(-1);
+}
+
+// https://refspecs.linuxbase.org/LSB_3.0.0/LSB-Core-generic/LSB-Core-generic/baselib---assert-fail-1.html
+void __assert_fail(const char * assertion, const char * file, unsigned int line, const char * function) {
+ MY_abort();
+}
+
+
+// GCC requires memcpy, memmove, memset and memcmp.
+// https://gcc.gnu.org/onlinedocs/gcc/Standards.html
+// > GCC requires the freestanding environment provide memcpy, memmove, memset and memcmp.
+EXTERN_C void *memmove(void *dst, const void *src, size_t n) {
+ uint8_t* d = (uint8_t*) dst;
+ const uint8_t* s = (const uint8_t*) src;
+
+ if (d > s) {
+ d += n;
+ s += n;
+ while (n--) {
+ *--d = *--s;
+ }
+ } else {
+ while (n--) {
+ *d++ = *s++;
+ }
+ }
+ return dst;
+}
+
+EXTERN_C void *memcpy(void * __restrict__ dst, const void * __restrict__ src, size_t n) {
+ return memmove(dst, src, n);
+}
+
+EXTERN_C void *memset(void *s, int c, size_t n) {
+ uint8_t* p = (uint8_t*) s;
+ while (n--) {
+ *p++ = (uint8_t) c;
+ }
+ return s;
+}
+
+EXTERN_C int memcmp(const void *s1, const void *s2, size_t n) {
+ const uint8_t* p1 = (const uint8_t*) s1;
+ const uint8_t* p2 = (const uint8_t*) s2;
+ while (n--) {
+ const uint8_t c1 = *p1++;
+ const uint8_t c2 = *p2++;
+ if (c1 < c2) {
+ return -1;
+ } else if (c1 > c2) {
+ return 1;
+ }
+ }
+ return 0;
+}
+
+
+//
+EXTERN_C void _start(void) {
+ test();
+ MY_exit(0);
+}
+
+int main(int argc, char** argv) {
+ test();
+ MY_exit(0);
+ return 0;
+}
+#endif
diff --git a/tests/fullbench.c b/tests/fullbench.c
index cb9b684..9c13996 100644
--- a/tests/fullbench.c
+++ b/tests/fullbench.c
@@ -1,6 +1,6 @@
/*
bench.c - Demo program to benchmark open-source compression algorithm
- Copyright (C) Yann Collet 2012-2016
+ Copyright (C) Yann Collet 2012-2020
GPL v2 License
@@ -312,6 +312,13 @@
return outSize;
}
+static int local_LZ4_decompress_safe_partial_usingDict(const char* in, char* out, int inSize, int outSize)
+{
+ int result = LZ4_decompress_safe_partial_usingDict(in, out, inSize, outSize - 5, outSize, out - 65536, 65536);
+ if (result < 0) return result;
+ return outSize;
+}
+
#ifndef LZ4_DLL_IMPORT
#if defined (__cplusplus)
extern "C" {
@@ -325,12 +332,30 @@
static int local_LZ4_decompress_safe_forceExtDict(const char* in, char* out, int inSize, int outSize)
{
- (void)inSize;
LZ4_decompress_safe_forceExtDict(in, out, inSize, outSize, out - 65536, 65536);
return outSize;
}
#endif
+#ifndef LZ4_DLL_IMPORT
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+extern int LZ4_decompress_safe_partial_forceExtDict(const char* in, char* out, int inSize, int targetOutputSize, int dstCapacity, const void* dict, size_t dictSize);
+
+#if defined (__cplusplus)
+}
+#endif
+
+static int local_LZ4_decompress_safe_partial_forceExtDict(const char* in, char* out, int inSize, int outSize)
+{
+ int result = LZ4_decompress_safe_partial_forceExtDict(in, out, inSize, outSize - 5, outSize, out - 65536, 65536);
+ if (result < 0) return result;
+ return outSize;
+}
+#endif
+
static int local_LZ4_decompress_safe_partial(const char* in, char* out, int inSize, int outSize)
{
int result = LZ4_decompress_safe_partial(in, out, inSize, outSize - 5, outSize);
@@ -372,7 +397,8 @@
size_t outRemaining = maxOutSize - outPos;
for (;;) {
- size_t const sizeHint = LZ4F_decompress(g_dCtx, dst+outPos, &outRemaining, src+inPos, &inSize, NULL);
+ size_t const sizeHint =
+ LZ4F_decompress(g_dCtx, dst+outPos, &outRemaining, src+inPos, &inSize, NULL);
assert(!LZ4F_isError(sizeHint));
inPos += inSize;
@@ -657,15 +683,17 @@
case 5: decompressionFunction = local_LZ4_decompress_safe_withPrefix64k; dName = "LZ4_decompress_safe_withPrefix64k"; break;
case 6: decompressionFunction = local_LZ4_decompress_safe_usingDict; dName = "LZ4_decompress_safe_usingDict"; break;
case 7: decompressionFunction = local_LZ4_decompress_safe_partial; dName = "LZ4_decompress_safe_partial"; checkResult = 0; break;
+ case 8: decompressionFunction = local_LZ4_decompress_safe_partial_usingDict; dName = "LZ4_decompress_safe_partial_usingDict"; checkResult = 0; break;
#ifndef LZ4_DLL_IMPORT
- case 8: decompressionFunction = local_LZ4_decompress_safe_forceExtDict; dName = "LZ4_decompress_safe_forceExtDict"; break;
+ case 9: decompressionFunction = local_LZ4_decompress_safe_partial_forceExtDict; dName = "LZ4_decompress_safe_partial_forceExtDict"; checkResult = 0; break;
+ case 10: decompressionFunction = local_LZ4_decompress_safe_forceExtDict; dName = "LZ4_decompress_safe_forceExtDict"; break;
#endif
- case 10:
case 11:
case 12:
- if (dAlgNb == 10) { decompressionFunction = local_LZ4F_decompress; dName = "LZ4F_decompress"; } /* can be skipped */
- if (dAlgNb == 11) { decompressionFunction = local_LZ4F_decompress_followHint; dName = "LZ4F_decompress_followHint"; } /* can be skipped */
- if (dAlgNb == 12) { decompressionFunction = local_LZ4F_decompress_noHint; dName = "LZ4F_decompress_noHint"; } /* can be skipped */
+ case 13:
+ if (dAlgNb == 11) { decompressionFunction = local_LZ4F_decompress; dName = "LZ4F_decompress"; } /* can be skipped */
+ if (dAlgNb == 12) { decompressionFunction = local_LZ4F_decompress_followHint; dName = "LZ4F_decompress_followHint"; } /* can be skipped */
+ if (dAlgNb == 13) { decompressionFunction = local_LZ4F_decompress_noHint; dName = "LZ4F_decompress_noHint"; } /* can be skipped */
/* prepare compressed data using frame format */
{ size_t const fcsize = LZ4F_compressFrame(compressed_buff, (size_t)compressedBuffSize, orig_buff, benchedSize, NULL);
assert(!LZ4F_isError(fcsize));
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index a824813..341a2b0 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -1,6 +1,6 @@
/*
fuzzer.c - Fuzzer test tool for LZ4
- Copyright (C) Yann Collet 2012-2017
+ Copyright (C) Yann Collet 2012-2020
GPL v2 License
@@ -30,6 +30,7 @@
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
# pragma warning(disable : 4146) /* disable: C4146: minus unsigned expression */
# pragma warning(disable : 4310) /* disable: C4310: constant char value > 127 */
+# pragma warning(disable : 26451) /* disable: C26451: Arithmetic overflow */
#endif
@@ -494,7 +495,14 @@
{ char* const cBuffer_exact = (char*)malloc((size_t)compressedSize);
assert(cBuffer_exact != NULL);
assert(compressedSize <= (int)compressedBufferSize);
+#if defined(_MSC_VER) && (_MSC_VER <= 1933) /* MSVC 2022 ver 17.3 or earlier */
+# pragma warning(push)
+# pragma warning(disable : 6385) /* lz4\tests\fuzzer.c(497): warning C6385: Reading invalid data from 'compressedBuffer'. */
+#endif
memcpy(cBuffer_exact, compressedBuffer, compressedSize);
+#if defined(_MSC_VER) && (_MSC_VER <= 1933) /* MSVC 2022 ver 17.3 or earlier */
+# pragma warning(pop)
+#endif
/* Test decoding with output size exactly correct => must work */
FUZ_DISPLAYTEST("LZ4_decompress_fast() with exact output buffer");
@@ -571,7 +579,7 @@
for (;;) {
/* keep some original src */
{ U32 const nbBits = FUZ_rand(&randState) % maxNbBits;
- size_t const mask = (1<<nbBits) - 1;
+ size_t const mask = (1ULL <<nbBits) - 1;
size_t const skipLength = FUZ_rand(&randState) & mask;
pos += skipLength;
}
@@ -579,7 +587,7 @@
/* add noise */
{ U32 const nbBitsCodes = FUZ_rand(&randState) % maxNbBits;
U32 const nbBits = nbBitsCodes ? nbBitsCodes-1 : 0;
- size_t const mask = (1<<nbBits) - 1;
+ size_t const mask = (1ULL <<nbBits) - 1;
size_t const rNoiseLength = (FUZ_rand(&randState) & mask) + 1;
size_t const noiseLength = MIN(rNoiseLength, (size_t)compressedSize-pos);
size_t const noiseStart = FUZ_rand(&randState) % (COMPRESSIBLE_NOISE_LENGTH - noiseLength);
@@ -630,6 +638,46 @@
FUZ_CHECKTEST(memcmp(block, decodedBuffer, (size_t)targetSize), "LZ4_decompress_safe_partial: corruption detected in regenerated data");
}
+ /* Partial decompression using dictionary. */
+ FUZ_DISPLAYTEST("test LZ4_decompress_safe_partial_usingDict using no dict");
+ { size_t const missingOutBytes = FUZ_rand(&randState) % (unsigned)blockSize;
+ int const targetSize = (int)((size_t)blockSize - missingOutBytes);
+ size_t const extraneousInBytes = FUZ_rand(&randState) % 2;
+ int const inCSize = (int)((size_t)compressedSize + extraneousInBytes);
+ char const sentinel = decodedBuffer[targetSize] = block[targetSize] ^ 0x5A;
+ int const decResult = LZ4_decompress_safe_partial_usingDict(compressedBuffer, decodedBuffer, inCSize, targetSize, blockSize, NULL, 0);
+ FUZ_CHECKTEST(decResult<0, "LZ4_decompress_safe_partial_usingDict failed despite valid input data (error:%i)", decResult);
+ FUZ_CHECKTEST(decResult != targetSize, "LZ4_decompress_safe_partial_usingDict did not regenerated required amount of data (%i < %i <= %i)", decResult, targetSize, blockSize);
+ FUZ_CHECKTEST(decodedBuffer[targetSize] != sentinel, "LZ4_decompress_safe_partial_usingDict overwrite beyond requested size (though %i <= %i <= %i)", decResult, targetSize, blockSize);
+ FUZ_CHECKTEST(memcmp(block, decodedBuffer, (size_t)targetSize), "LZ4_decompress_safe_partial_usingDict: corruption detected in regenerated data");
+ }
+
+ FUZ_DISPLAYTEST("test LZ4_decompress_safe_partial_usingDict() using prefix as dict");
+ { size_t const missingOutBytes = FUZ_rand(&randState) % (unsigned)blockSize;
+ int const targetSize = (int)((size_t)blockSize - missingOutBytes);
+ size_t const extraneousInBytes = FUZ_rand(&randState) % 2;
+ int const inCSize = (int)((size_t)compressedSize + extraneousInBytes);
+ char const sentinel = decodedBuffer[targetSize] = block[targetSize] ^ 0x5A;
+ int const decResult = LZ4_decompress_safe_partial_usingDict(compressedBuffer, decodedBuffer, inCSize, targetSize, blockSize, decodedBuffer, dictSize);
+ FUZ_CHECKTEST(decResult<0, "LZ4_decompress_safe_partial_usingDict failed despite valid input data (error:%i)", decResult);
+ FUZ_CHECKTEST(decResult != targetSize, "LZ4_decompress_safe_partial_usingDict did not regenerated required amount of data (%i < %i <= %i)", decResult, targetSize, blockSize);
+ FUZ_CHECKTEST(decodedBuffer[targetSize] != sentinel, "LZ4_decompress_safe_partial_usingDict overwrite beyond requested size (though %i <= %i <= %i)", decResult, targetSize, blockSize);
+ FUZ_CHECKTEST(memcmp(block, decodedBuffer, (size_t)targetSize), "LZ4_decompress_safe_partial_usingDict: corruption detected in regenerated data");
+ }
+
+ FUZ_DISPLAYTEST("test LZ4_decompress_safe_partial_usingDict() using external dict");
+ { size_t const missingOutBytes = FUZ_rand(&randState) % (unsigned)blockSize;
+ int const targetSize = (int)((size_t)blockSize - missingOutBytes);
+ size_t const extraneousInBytes = FUZ_rand(&randState) % 2;
+ int const inCSize = (int)((size_t)compressedSize + extraneousInBytes);
+ char const sentinel = decodedBuffer[targetSize] = block[targetSize] ^ 0x5A;
+ int const decResult = LZ4_decompress_safe_partial_usingDict(compressedBuffer, decodedBuffer, inCSize, targetSize, blockSize, dict, dictSize);
+ FUZ_CHECKTEST(decResult<0, "LZ4_decompress_safe_partial_usingDict failed despite valid input data (error:%i)", decResult);
+ FUZ_CHECKTEST(decResult != targetSize, "LZ4_decompress_safe_partial_usingDict did not regenerated required amount of data (%i < %i <= %i)", decResult, targetSize, blockSize);
+ FUZ_CHECKTEST(decodedBuffer[targetSize] != sentinel, "LZ4_decompress_safe_partial_usingDict overwrite beyond requested size (though %i <= %i <= %i)", decResult, targetSize, blockSize);
+ FUZ_CHECKTEST(memcmp(block, decodedBuffer, (size_t)targetSize), "LZ4_decompress_safe_partial_usingDict: corruption detected in regenerated data");
+ }
+
/* Test Compression with limited output size */
/* Test compression with output size being exactly what's necessary (should work) */
@@ -1136,12 +1184,12 @@
assert(shc != NULL);
memset(shc, 0, sizeof(*shc));
DISPLAYLEVEL(4, "state1(%p) state2(%p) state3(%p) LZ4_stream_t size(0x%x): ",
- &(shc->state1), &(shc->state2), &(shc->state3), (unsigned)sizeof(LZ4_stream_t));
- FUZ_CHECKTEST( LZ4_initStream(&(shc->state1), sizeof(shc->state1)) == NULL, "state1 (%p) failed init", &(shc->state1) );
- FUZ_CHECKTEST( LZ4_initStream(&(shc->state2), sizeof(shc->state2)) == NULL, "state2 (%p) failed init", &(shc->state2) );
- FUZ_CHECKTEST( LZ4_initStream(&(shc->state3), sizeof(shc->state3)) == NULL, "state3 (%p) failed init", &(shc->state3) );
+ (void*)&(shc->state1), (void*)&(shc->state2), (void*)&(shc->state3), (unsigned)sizeof(LZ4_stream_t));
+ FUZ_CHECKTEST( LZ4_initStream(&(shc->state1), sizeof(shc->state1)) == NULL, "state1 (%p) failed init", (void*)&(shc->state1) );
+ FUZ_CHECKTEST( LZ4_initStream(&(shc->state2), sizeof(shc->state2)) == NULL, "state2 (%p) failed init", (void*)&(shc->state2) );
+ FUZ_CHECKTEST( LZ4_initStream(&(shc->state3), sizeof(shc->state3)) == NULL, "state3 (%p) failed init", (void*)&(shc->state3) );
FUZ_CHECKTEST( LZ4_initStream((char*)&(shc->state1) + 1, sizeof(shc->state1)) != NULL,
- "hc1+1 (%p) init must fail, due to bad alignment", (char*)&(shc->state1) + 1 );
+ "hc1+1 (%p) init must fail, due to bad alignment", (void*)((char*)&(shc->state1) + 1) );
free(shc);
}
DISPLAYLEVEL(3, "all inits OK \n");
@@ -1246,12 +1294,13 @@
assert(shc != NULL);
memset(shc, 0, sizeof(*shc));
DISPLAYLEVEL(4, "hc1(%p) hc2(%p) hc3(%p) size(0x%x): ",
- &(shc->hc1), &(shc->hc2), &(shc->hc3), (unsigned)sizeof(LZ4_streamHC_t));
- FUZ_CHECKTEST( LZ4_initStreamHC(&(shc->hc1), sizeof(shc->hc1)) == NULL, "hc1 (%p) failed init", &(shc->hc1) );
- FUZ_CHECKTEST( LZ4_initStreamHC(&(shc->hc2), sizeof(shc->hc2)) == NULL, "hc2 (%p) failed init", &(shc->hc2) );
- FUZ_CHECKTEST( LZ4_initStreamHC(&(shc->hc3), sizeof(shc->hc3)) == NULL, "hc3 (%p) failed init", &(shc->hc3) );
+ (void*)&(shc->hc1), (void*)&(shc->hc2), (void*)&(shc->hc3),
+ (unsigned)sizeof(LZ4_streamHC_t));
+ FUZ_CHECKTEST( LZ4_initStreamHC(&(shc->hc1), sizeof(shc->hc1)) == NULL, "hc1 (%p) failed init", (void*)&(shc->hc1) );
+ FUZ_CHECKTEST( LZ4_initStreamHC(&(shc->hc2), sizeof(shc->hc2)) == NULL, "hc2 (%p) failed init", (void*)&(shc->hc2) );
+ FUZ_CHECKTEST( LZ4_initStreamHC(&(shc->hc3), sizeof(shc->hc3)) == NULL, "hc3 (%p) failed init", (void*)&(shc->hc3) );
FUZ_CHECKTEST( LZ4_initStreamHC((char*)&(shc->hc1) + 1, sizeof(shc->hc1)) != NULL,
- "hc1+1 (%p) init must fail, due to bad alignment", (char*)&(shc->hc1) + 1 );
+ "hc1+1 (%p) init must fail, due to bad alignment", (void*)((char*)&(shc->hc1) + 1) );
free(shc);
}
DISPLAYLEVEL(3, "all inits OK \n");
diff --git a/tests/goldenSamples/skip.bin b/tests/goldenSamples/skip.bin
new file mode 100644
index 0000000..1a8b9d5
--- /dev/null
+++ b/tests/goldenSamples/skip.bin
Binary files differ
diff --git a/tests/roundTripTest.c b/tests/roundTripTest.c
index 2d34451..3e9d6ed 100644
--- a/tests/roundTripTest.c
+++ b/tests/roundTripTest.c
@@ -1,11 +1,11 @@
/*
- * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
- * in the COPYING file in the root directory of this source tree).
- * You may select, at your option, one of the above-listed licenses.
+ * in the COPYING file in the root directory of this source tree),
+ * meaning you may select, at your option, one of the above-listed licenses.
*/
/*
@@ -104,7 +104,7 @@
int clevel)
{
int const proposed_clevel = clevel ? clevel : select_clevel(srcBuff, srcSize);
- int const selected_clevel = proposed_clevel < 0 ? -proposed_clevel : proposed_clevel; /* if level < 0, it becomes an accelearion value */
+ int const selected_clevel = proposed_clevel < 0 ? -proposed_clevel : proposed_clevel; /* if level < 0, it becomes an acceleration value */
compressFn compress = selected_clevel >= LZ4HC_CLEVEL_MIN ? LZ4_compress_HC : LZ4_compress_fast;
int const cSize = compress((const char*)srcBuff, (char*)compressedBuff, (int)srcSize, (int)compressedBuffCapacity, selected_clevel);
CONTROL_MSG(cSize == 0, "Compression error !");
@@ -126,7 +126,7 @@
static void roundTripCheck(const void* srcBuff, size_t srcSize, int clevel)
{
- size_t const cBuffSize = LZ4_compressBound((int)srcSize);
+ size_t const cBuffSize = LZ4_COMPRESSBOUND(srcSize);
void* const cBuff = malloc(cBuffSize);
void* const rBuff = malloc(cBuffSize);
diff --git a/tests/test-lz4-abi.py b/tests/test-lz4-abi.py
new file mode 100644
index 0000000..0c8fd05
--- /dev/null
+++ b/tests/test-lz4-abi.py
@@ -0,0 +1,173 @@
+#!/usr/bin/env python3
+"""Test LZ4 interoperability between versions"""
+
+#
+# Copyright (C) 2011-present, Takayuki Matsuoka
+# All rights reserved.
+# GPL v2 License
+#
+
+import glob
+import subprocess
+import filecmp
+import os
+import shutil
+import sys
+import hashlib
+
+repo_url = 'https://github.com/lz4/lz4.git'
+tmp_dir_name = 'tests/abiTests'
+env_flags = ' ' # '-j MOREFLAGS="-g -O0 -fsanitize=address"'
+make_cmd = 'make'
+git_cmd = 'git'
+test_dat_src = ['README.md']
+head = 'v999'
+
+def proc(cmd_args, pipe=True, env=False):
+ if env == False:
+ env = os.environ.copy()
+ if pipe:
+ s = subprocess.Popen(cmd_args,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ env = env)
+ else:
+ s = subprocess.Popen(cmd_args, env = env)
+ r = s.communicate()
+ if s.poll() != 0:
+ print(' s.poll() = ', s.poll())
+ sys.exit(1)
+ return r
+
+def make(args, pipe=True, env=False):
+ if env == False:
+ env = os.environ.copy()
+ # we want the address sanitizer for abi tests
+ env["MOREFLAGS"] = "-fsanitize=address"
+ return proc([make_cmd] + ['-j'] + ['V=1'] + args, pipe, env)
+
+def git(args, pipe=True):
+ return proc([git_cmd] + args, pipe)
+
+def get_git_tags():
+ # Only start from first v1.7.x format release
+ stdout, stderr = git(['tag', '-l', 'v[1-9].[0-9].[0-9]'])
+ tags = stdout.decode('utf-8').split()
+ return tags
+
+# https://stackoverflow.com/a/19711609/2132223
+def sha1_of_file(filepath):
+ with open(filepath, 'rb') as f:
+ return hashlib.sha1(f.read()).hexdigest()
+
+if __name__ == '__main__':
+ error_code = 0
+ base_dir = os.getcwd() + '/..' # /path/to/lz4
+ tmp_dir = base_dir + '/' + tmp_dir_name # /path/to/lz4/tests/versionsTest
+ clone_dir = tmp_dir + '/' + 'lz4' # /path/to/lz4/tests/versionsTest/lz4
+ lib_dir = base_dir + '/lib' # /path/to/lz4/lib
+ test_dir = base_dir + '/tests'
+ os.makedirs(tmp_dir, exist_ok=True)
+
+ # since Travis clones limited depth, we should clone full repository
+ if not os.path.isdir(clone_dir):
+ git(['clone', repo_url, clone_dir])
+
+ # Retrieve all release tags
+ print('Retrieve release tags >= v1.7.5 :')
+ os.chdir(clone_dir)
+ tags = [head] + get_git_tags()
+ tags = [x for x in tags if (x >= 'v1.7.5')]
+ print(tags)
+
+ # loop across architectures
+ for march in ['-m64', '-m32', '-mx32']:
+ print(' ')
+ print('=====================================')
+ print('Testing architecture ' + march);
+ print('=====================================')
+
+ # Build all versions of liblz4
+ # note : naming scheme only works on Linux
+ for tag in tags:
+ print('building library ', tag)
+ os.chdir(base_dir)
+ # if not os.path.isfile(dst_liblz4) or tag == head:
+ if tag != head:
+ r_dir = '{}/{}'.format(tmp_dir, tag) # /path/to/lz4/test/lz4test/<TAG>
+ #print('r_dir = ', r_dir) # for debug
+ os.makedirs(r_dir, exist_ok=True)
+ os.chdir(clone_dir)
+ git(['--work-tree=' + r_dir, 'checkout', tag, '--', '.'])
+ os.chdir(r_dir + '/lib') # /path/to/lz4/lz4test/<TAG>/lib
+ else:
+ # print('lib_dir = {}', lib_dir) # for debug
+ os.chdir(lib_dir)
+ make(['clean'])
+ build_env = os.environ.copy()
+ build_env["CFLAGS"] = march
+ build_env["MOREFLAGS"] = "-fsanitize=address"
+ make(['liblz4'], env=build_env)
+
+ print(' ')
+ print('******************************')
+ print('Round trip expecting current ABI but linking to older Dynamic Library version')
+ print('******************************')
+ os.chdir(test_dir)
+ # Start with matching version : should be no problem
+ build_env = os.environ.copy()
+ build_env["CFLAGS"] = march
+ build_env["LDFLAGS"] = "-L../lib"
+ build_env["LDLIBS"] = "-llz4"
+ # we use asan to detect any out-of-bound read or write
+ build_env["MOREFLAGS"] = "-fsanitize=address"
+ if os.path.isfile('abiTest'):
+ os.remove('abiTest')
+ make(['abiTest'], env=build_env, pipe=False)
+
+ for tag in tags:
+ print('linking to lib tag = ', tag)
+ run_env = os.environ.copy()
+ if tag == head:
+ run_env["LD_LIBRARY_PATH"] = '../lib'
+ else:
+ run_env["LD_LIBRARY_PATH"] = 'abiTests/{}/lib'.format(tag)
+ # check we are linking to the right library version at run time
+ proc(['./check_liblz4_version.sh'] + ['./abiTest'], pipe=False, env=run_env)
+ # now run with mismatched library version
+ proc(['./abiTest'] + test_dat_src, pipe=False, env=run_env)
+
+ print(' ')
+ print('******************************')
+ print('Round trip using current Dynamic Library expecting older ABI version')
+ print('******************************')
+
+ for tag in tags:
+ print(' ')
+ print('building using older lib ', tag)
+ build_env = os.environ.copy()
+ if tag != head:
+ build_env["CPPFLAGS"] = '-IabiTests/{}/lib'.format(tag)
+ build_env["LDFLAGS"] = '-LabiTests/{}/lib'.format(tag)
+ else:
+ build_env["CPPFLAGS"] = '-I../lib'
+ build_env["LDFLAGS"] = '-L../lib'
+ build_env["LDLIBS"] = "-llz4"
+ build_env["CFLAGS"] = march
+ build_env["MOREFLAGS"] = "-fsanitize=address"
+ os.remove('abiTest')
+ make(['abiTest'], pipe=False, env=build_env)
+
+ print('run with CURRENT library version (head)')
+ run_env = os.environ.copy()
+ run_env["LD_LIBRARY_PATH"] = '../lib'
+ # check we are linking to the right library version at run time
+ proc(['./check_liblz4_version.sh'] + ['./abiTest'], pipe=False, env=run_env)
+ # now run with mismatched library version
+ proc(['./abiTest'] + test_dat_src, pipe=False, env=run_env)
+
+
+ if error_code != 0:
+ print('ERROR')
+
+ sys.exit(error_code)
diff --git a/tests/test-lz4-list.py b/tests/test-lz4-list.py
index ce89757..fe11682 100644
--- a/tests/test-lz4-list.py
+++ b/tests/test-lz4-list.py
@@ -1,16 +1,17 @@
-#! /usr/bin/env python3
+#!/usr/bin/env python3
import subprocess
import time
import glob
import os
import tempfile
import unittest
+import sys
SIZES = [3, 11] # Always 2 sizes
MIB = 1048576
-LZ4 = os.path.dirname(os.path.realpath(__file__)) + "/../lz4"
+LZ4 = os.path.abspath(os.path.dirname(os.path.realpath(__file__)) + "/../lz4")
if not os.path.exists(LZ4):
- LZ4 = os.path.dirname(os.path.realpath(__file__)) + "/../programs/lz4"
+ LZ4 = os.path.abspath(os.path.dirname(os.path.realpath(__file__)) + "/../programs/lz4")
TEMP = tempfile.gettempdir()
@@ -19,26 +20,29 @@
self.line = line_in
splitlines = line_in.split()
if len(splitlines) != 7:
- errout("Unexpected line: {}".format(line_in))
+ errout(f"Unexpected line: {line_in}")
self.frames, self.type, self.block, self.compressed, self.uncompressed, self.ratio, self.filename = splitlines
self.exp_unc_size = 0
# Get real file sizes
if "concat-all" in self.filename or "2f--content-size" in self.filename:
for i in SIZES:
- self.exp_unc_size += os.path.getsize("{}/test_list_{}M".format(TEMP, i))
+ self.exp_unc_size += os.path.getsize(f"{TEMP}/test_list_{i}M")
else:
uncompressed_filename = self.filename.split("-")[0]
- self.exp_unc_size += os.path.getsize("{}/{}".format(TEMP, uncompressed_filename))
- self.exp_comp_size = os.path.getsize("{}/{}".format(TEMP, self.filename))
+ self.exp_unc_size += os.path.getsize(f"{TEMP}/{uncompressed_filename}")
+ self.exp_comp_size = os.path.getsize(f"{TEMP}/{self.filename}")
class TestNonVerbose(unittest.TestCase):
@classmethod
def setUpClass(self):
self.nvinfo_list = []
- for i, line in enumerate(execute("{} --list -m {}/test_list_*.lz4".format(LZ4, TEMP), print_output=True)):
- if i > 0:
- self.nvinfo_list.append(NVerboseFileInfo(line))
+ test_list_files = glob.glob(f"{TEMP}/test_list_*.lz4")
+ # One of the files has 2 frames so duplicate it in this list to map each frame 1 to a single file
+ for i, filename in enumerate(test_list_files):
+ for i, line in enumerate(execute(f"{LZ4} --list -m {filename}", print_output=True)):
+ if i > 0:
+ self.nvinfo_list.append(NVerboseFileInfo(line))
def test_frames(self):
all_concat_frames = 0
@@ -80,7 +84,7 @@
def test_ratio(self):
for nvinfo in self.nvinfo_list:
if "--content-size" in nvinfo.filename:
- self.assertEqual(nvinfo.ratio, "{:.2f}%".format(float(nvinfo.exp_comp_size) / float(nvinfo.exp_unc_size) * 100), nvinfo.line)
+ self.assertEqual(nvinfo.ratio, f"{float(nvinfo.exp_comp_size) / float(nvinfo.exp_unc_size) * 100:.2f}%", nvinfo.line)
def test_uncompressed_size(self):
for nvinfo in self.nvinfo_list:
@@ -112,17 +116,19 @@
# we're only really interested in testing the output of the concat-all file.
self.vinfo_list = []
start = end = 0
- output = execute("{} --list -m -v {}/test_list_concat-all.lz4 {}/test_list_*M-lz4f-2f--content-size.lz4".format(LZ4, TEMP, TEMP), print_output=True)
- for i, line in enumerate(output):
- if line.startswith("test_list"):
- if start != 0 and end != 0:
- self.vinfo_list.append(VerboseFileInfo(output[start:end]))
- start = i
- if not line:
- end = i
+ test_list_SM_lz4f = glob.glob(f"{TEMP}/test_list_*M-lz4f-2f--content-size.lz4")
+ for i, filename in enumerate(test_list_SM_lz4f):
+ output = execute(f"{LZ4} --list -m -v {TEMP}/test_list_concat-all.lz4 {filename}", print_output=True)
+ for i, line in enumerate(output):
+ if line.startswith("test_list"):
+ if start != 0 and end != 0:
+ self.vinfo_list.append(VerboseFileInfo(output[start:end]))
+ start = i
+ if not line:
+ end = i
self.vinfo_list.append(VerboseFileInfo(output[start:end]))
# Populate file_frame_map as a reference of the expected info
- concat_file_list = glob.glob("/tmp/test_list_[!concat]*.lz4")
+ concat_file_list = glob.glob(f"{TEMP}/test_list_[!concat]*.lz4")
# One of the files has 2 frames so duplicate it in this list to map each frame 1 to a single file
for i, filename in enumerate(concat_file_list):
if "2f--content-size" in filename:
@@ -130,11 +136,11 @@
break
self.cvinfo = self.vinfo_list[0]
self.cvinfo.file_frame_map = concat_file_list
- self.cvinfo.compressed_size = os.path.getsize("{}/test_list_concat-all.lz4".format(TEMP))
+ self.cvinfo.compressed_size = os.path.getsize(f"{TEMP}/test_list_concat-all.lz4")
def test_filename(self):
for i, vinfo in enumerate(self.vinfo_list):
- self.assertRegex(vinfo.filename, "^test_list_.*({}/{})".format(i + 1, len(self.vinfo_list)))
+ self.assertRegex(vinfo.filename, f"^test_list_.*({i + 1}/{len(self.vinfo_list)})".format(i + 1, len(self.vinfo_list)))
def test_frame_number(self):
for vinfo in self.vinfo_list:
@@ -169,7 +175,7 @@
expected_size = os.path.getsize(self.cvinfo.file_frame_map[i])
self.assertEqual(self.cvinfo.frame_list[i]["compressed"], str(expected_size), self.cvinfo.frame_list[i]["line"])
total += int(self.cvinfo.frame_list[i]["compressed"])
- self.assertEqual(total, self.cvinfo.compressed_size, "Expected total sum ({}) to match {} filesize".format(total, self.cvinfo.filename))
+ self.assertEqual(total, self.cvinfo.compressed_size, f"Expected total sum ({total}) to match {self.cvinfo.filename} filesize")
def test_uncompressed(self):
for i, frame_info in enumerate(self.cvinfo.frame_list):
@@ -182,7 +188,7 @@
for i, frame_info in enumerate(self.cvinfo.frame_list):
if "--content-size" in self.cvinfo.file_frame_map[i]:
self.assertEqual(self.cvinfo.frame_list[i]['ratio'],
- "{:.2f}%".format(float(self.cvinfo.frame_list[i]['compressed']) / float(self.cvinfo.frame_list[i]['uncompressed']) * 100),
+ f"{float(self.cvinfo.frame_list[i]['compressed']) / float(self.cvinfo.frame_list[i]['uncompressed']) * 100:.2f}%",
self.cvinfo.frame_list[i]["line"])
@@ -191,7 +197,7 @@
if size < 1024.0:
break
size /= 1024.0
- return "{:.2f}{}".format(size, unit)
+ return f"{size:.2f}{unit}"
def log(text):
@@ -203,12 +209,12 @@
exit(err)
-def execute(command, print_command=True, print_output=False, print_error=True, param_shell=True):
+def execute(command, print_command=True, print_output=False, print_error=True):
if os.environ.get('QEMU_SYS'):
- command = "{} {}".format(os.environ['QEMU_SYS'], command)
+ command = f"{os.environ['QEMU_SYS']} {command}"
if print_command:
log("> " + command)
- popen = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=param_shell)
+ popen = subprocess.Popen(command.split(" "), stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdout_lines, stderr_lines = popen.communicate()
stderr_lines = stderr_lines.decode("utf-8")
stdout_lines = stdout_lines.decode("utf-8")
@@ -220,14 +226,14 @@
if popen.returncode is not None and popen.returncode != 0:
if stderr_lines and not print_output and print_error:
print(stderr_lines)
- errout("Failed to run: {}\n".format(command, stdout_lines + stderr_lines))
+ errout(f"Failed to run: {command}, {stdout_lines + stderr_lines}\n")
return (stdout_lines + stderr_lines).splitlines()
def cleanup(silent=False):
- for f in glob.glob("{}/test_list*".format(TEMP)):
+ for f in glob.glob(f"{TEMP}/test_list*"):
if not silent:
- log("Deleting {}".format(f))
+ log(f"Deleting {f}")
os.unlink(f)
@@ -243,33 +249,33 @@
# file format ~ test_list<frametype>-<no_frames>f<create-args>.lz4 ~
# Generate LZ4Frames
for i in SIZES:
- filename = "{}/test_list_{}M".format(TEMP, i)
- log("Generating {}".format(filename))
+ filename = f"{TEMP}/test_list_{i}M"
+ log(f"Generating {filename}")
datagen(filename, i * MIB)
for j in ["--content-size", "-BI", "-BD", "-BX", "--no-frame-crc"]:
- lz4file = "{}-lz4f-1f{}.lz4".format(filename, j)
- execute("{} {} {} {}".format(LZ4, j, filename, lz4file))
+ lz4file = f"{filename}-lz4f-1f{j}.lz4"
+ execute(f"{LZ4} {j} {filename} {lz4file}")
# Generate skippable frames
- lz4file = "{}-skip-1f.lz4".format(filename)
+ lz4file = f"{filename}-skip-1f.lz4"
skipsize = i * 1024
skipbytes = bytes([80, 42, 77, 24]) + skipsize.to_bytes(4, byteorder='little', signed=False)
with open(lz4file, 'wb') as f:
f.write(skipbytes)
f.write(os.urandom(skipsize))
# Generate legacy frames
- lz4file = "{}-legc-1f.lz4".format(filename)
- execute("{} -l {} {}".format(LZ4, filename, lz4file))
+ lz4file = f"{filename}-legc-1f.lz4"
+ execute(f"{LZ4} -l {filename} {lz4file}")
# Concatenate --content-size files
- file_list = glob.glob("{}/test_list_*-lz4f-1f--content-size.lz4".format(TEMP))
- with open("{}/test_list_{}M-lz4f-2f--content-size.lz4".format(TEMP, sum(SIZES)), 'ab') as outfile:
+ file_list = glob.glob(f"{TEMP}/test_list_*-lz4f-1f--content-size.lz4")
+ with open(f"{TEMP}/test_list_{sum(SIZES)}M-lz4f-2f--content-size.lz4", 'ab') as outfile:
for fname in file_list:
with open(fname, 'rb') as infile:
outfile.write(infile.read())
# Concatenate all files
- file_list = glob.glob("{}/test_list_*.lz4".format(TEMP))
- with open("{}/test_list_concat-all.lz4".format(TEMP), 'ab') as outfile:
+ file_list = glob.glob(f"{TEMP}/test_list_*.lz4")
+ with open(f"{TEMP}/test_list_concat-all.lz4", 'ab') as outfile:
for fname in file_list:
with open(fname, 'rb') as infile:
outfile.write(infile.read())
@@ -278,5 +284,6 @@
if __name__ == '__main__':
cleanup()
generate_files()
- unittest.main(verbosity=2, exit=False)
+ ret = unittest.main(verbosity=2, exit=False)
cleanup(silent=True)
+ sys.exit(not ret.result.wasSuccessful())
diff --git a/tests/test-lz4-speed.py b/tests/test-lz4-speed.py
index ca8f010..658939c 100644
--- a/tests/test-lz4-speed.py
+++ b/tests/test-lz4-speed.py
@@ -1,7 +1,7 @@
-#! /usr/bin/env python3
+#!/usr/bin/env python3
#
-# Copyright (c) 2016-present, Przemyslaw Skibinski, Yann Collet, Facebook, Inc.
+# Copyright (c) 2016-2020, Przemyslaw Skibinski, Yann Collet, Facebook, Inc.
# All rights reserved.
#
# This source code is licensed under the BSD-style license found in the
@@ -152,7 +152,7 @@
% (os.getloadavg()[0], args.maxLoadAvg, sleepTime))
time.sleep(sleepTime)
start_load = str(os.getloadavg())
- result = execute('programs/%s -rqi5b1e%s %s' % (executableName, args.lastCLevel, testFilePath), print_output=True)
+ result = execute('programs/%s -rqi5b1e%s %s' % (executableName, args.lastCLevel, testFilePath), print_output=True)
end_load = str(os.getloadavg())
linesExpected = args.lastCLevel + 1
if len(result) != linesExpected:
diff --git a/tests/test_custom_block_sizes.sh b/tests/test_custom_block_sizes.sh
index aba6733..2f6591f 100755
--- a/tests/test_custom_block_sizes.sh
+++ b/tests/test_custom_block_sizes.sh
@@ -1,4 +1,4 @@
-#/usr/bin/env sh
+#!/usr/bin/env sh
set -e
LZ4=../lz4
@@ -62,7 +62,7 @@
done
rm $TMPFILE.lz4 $TMPFILE1 $TMPFILE1.lz4 $TMPFILE2 $TMPFILE2.lz4
-if [ "$failures" == "" ]
+if [ "$failures" = "" ]
then
echo ---- All tests passed
exit 0
diff --git a/tests/test_install.sh b/tests/test_install.sh
index 122bac5..ad0d4eb 100755
--- a/tests/test_install.sh
+++ b/tests/test_install.sh
@@ -1,4 +1,4 @@
-#/usr/bin/env sh
+#!/usr/bin/env sh
set -e
@@ -6,7 +6,7 @@
unamestr=$(uname)
if [ "$unamestr" = 'Linux' ]; then
make="make -C $lz4_root"
-elif [ "$unamestr" = 'FreeBSD' -o "$unamestr" = 'OpenBSD' ]; then
+elif [ "$unamestr" = 'FreeBSD' ] || [ "$unamestr" = 'OpenBSD' ]; then
make="gmake -C $lz4_root"
fi
diff --git a/tests/unicode_lint.sh b/tests/unicode_lint.sh
new file mode 100644
index 0000000..cd65d2d
--- /dev/null
+++ b/tests/unicode_lint.sh
@@ -0,0 +1,48 @@
+#!/bin/bash
+
+# `unicode_lint.sh' determines whether source files under the ./lib/, ./tests/ and ./programs/ directories
+# contain Unicode characters, and fails if any do.
+#
+# See https://github.com/lz4/lz4/issues/1018
+
+echo "Ensure no unicode character is present in source files *.{c,h}"
+pass=true
+
+# Scan ./lib/ for Unicode in source (*.c, *.h) files
+echo "Scanning lib/"
+result=$(
+ find ./lib/ -regex '.*\.\(c\|h\)$' -exec grep -P -n "[^\x00-\x7F]" {} \; -exec echo "{}: FAIL" \;
+)
+if [[ $result ]]; then
+ echo "$result"
+ pass=false
+fi
+
+# Scan ./programs/ for Unicode in source (*.c, *.h) files
+echo "Scanning programs/"
+result=$(
+ find ./programs/ -regex '.*\.\(c\|h\)$' -exec grep -P -n "[^\x00-\x7F]" {} \; -exec echo "{}: FAIL" \;
+)
+if [[ $result ]]; then
+ echo "$result"
+ pass=false
+fi
+
+# Scan ./tests/ for Unicode in source (*.c, *.h) files
+echo "Scanning tests/"
+result=$(
+ find ./tests/ -regex '.*\.\(c\|h\)$' -exec grep -P -n "[^\x00-\x7F]" {} \; -exec echo "{}: FAIL" \;
+)
+if [[ $result ]]; then
+ echo "$result"
+ pass=false
+fi
+
+if [ "$pass" = true ]; then
+ echo "All tests successful: no unicode character detected"
+ echo "Result: PASS"
+ exit 0
+else
+ echo "Result: FAIL"
+ exit 1
+fi
