Merge topic 'reorder_compiler_detection_error' 52d1b4ad05 CompilerId: Console error output has first try last Acked-by: Kitware Robot <kwrobot@kitware.com> Acked-by: buildbot <buildbot@kitware.com> Merge-request: !10141
diff --git a/Help/command/file.rst b/Help/command/file.rst index 890bdf4..9abe4cc 100644 --- a/Help/command/file.rst +++ b/Help/command/file.rst
@@ -37,7 +37,7 @@ `Filesystem`_ file({`GLOB`_ | `GLOB_RECURSE`_} <out-var> [...] <globbing-expr>...) - file(`MAKE_DIRECTORY`_ <directories>...) + file(`MAKE_DIRECTORY`_ <directories>... [...]) file({`REMOVE`_ | `REMOVE_RECURSE`_ } <files>...) file(`RENAME`_ <oldname> <newname> [...]) file(`COPY_FILE`_ <oldname> <newname> [...])
diff --git a/Help/guide/tutorial/A Basic Starting Point.rst b/Help/guide/tutorial/A Basic Starting Point.rst index 2325e9e..cca1fc3 100644 --- a/Help/guide/tutorial/A Basic Starting Point.rst +++ b/Help/guide/tutorial/A Basic Starting Point.rst
@@ -319,6 +319,7 @@ * ``CMakeLists.txt`` * ``tutorial.cxx`` +* ``TutorialConfig.h.in`` Getting Started ---------------
diff --git a/Help/manual/cmake-developer.7.rst b/Help/manual/cmake-developer.7.rst index dc11956..ef92ec6 100644 --- a/Help/manual/cmake-developer.7.rst +++ b/Help/manual/cmake-developer.7.rst
@@ -470,7 +470,6 @@ include(FindPackageHandleStandardArgs) find_package_handle_standard_args(Foo - FOUND_VAR Foo_FOUND REQUIRED_VARS Foo_LIBRARY Foo_INCLUDE_DIR
diff --git a/Help/release/dev/FindBISON.rst b/Help/release/dev/FindBISON.rst new file mode 100644 index 0000000..1005869 --- /dev/null +++ b/Help/release/dev/FindBISON.rst
@@ -0,0 +1,6 @@ +FindBISON +--------- + +* The :module:`FindBISON` module :command:`bison_target` command has a new + ``OPTIONS`` option to add Bison command-line options as a + :ref:`semicolon-separated list <CMake Language Lists>`.
diff --git a/Help/release/dev/FindFLEX.rst b/Help/release/dev/FindFLEX.rst new file mode 100644 index 0000000..6faec23 --- /dev/null +++ b/Help/release/dev/FindFLEX.rst
@@ -0,0 +1,6 @@ +FindFLEX +-------- + +* The :module:`FindFLEX` module :command:`flex_target` command has a new + ``OPTIONS`` option to add Flex command-line options as a + :ref:`semicolon-separated list <CMake Language Lists>`.
diff --git a/Modules/FeatureSummary.cmake b/Modules/FeatureSummary.cmake index 80a2972..3a4e5cd 100644 --- a/Modules/FeatureSummary.cmake +++ b/Modules/FeatureSummary.cmake
@@ -1,6 +1,8 @@ # Distributed under the OSI-approved BSD 3-Clause License. See accompanying # file Copyright.txt or https://cmake.org/licensing for details. +include_guard(GLOBAL) + #[=======================================================================[.rst: FeatureSummary --------------
diff --git a/Modules/FindBISON.cmake b/Modules/FindBISON.cmake index fca054d..a2c37cf 100644 --- a/Modules/FindBISON.cmake +++ b/Modules/FindBISON.cmake
@@ -28,6 +28,7 @@ .. code-block:: cmake bison_target(<Name> <YaccInput> <CodeOutput> + [OPTIONS <options>...] [COMPILE_FLAGS <string>] [DEFINES_FILE <file>] [VERBOSE [<file>]] @@ -45,9 +46,18 @@ The options are: +``OPTIONS <options>...`` + .. versionadded:: 3.32 + + A :ref:`semicolon-separated list <CMake Language Lists>` of options added to + the ``bison`` command line. + ``COMPILE_FLAGS <string>`` + .. deprecated:: 3.32 + Space-separated bison options added to the ``bison`` command line. A :ref:`;-list <CMake Language Lists>` will not work. + This option is deprecated in favor of ``OPTIONS <options>...``. ``DEFINES_FILE <file>`` .. versionadded:: 3.4 @@ -85,9 +95,17 @@ All files generated by ``bison`` including the source, the header and the report. -``BISON_<Name>_COMPILE_FLAGS`` +``BISON_<Name>_OPTIONS`` + .. versionadded:: 3.32 + Options used in the ``bison`` command line. +``BISON_<Name>_COMPILE_FLAGS`` + .. deprecated:: 3.32 + + Options used in the ``bison`` command line. This variable is deprecated in + favor of ``BISON_<Name>_OPTIONS`` variable. + Examples ^^^^^^^^ @@ -97,6 +115,29 @@ bison_target(MyParser parser.y ${CMAKE_CURRENT_BINARY_DIR}/parser.cpp DEFINES_FILE ${CMAKE_CURRENT_BINARY_DIR}/parser.h) add_executable(Foo main.cpp ${BISON_MyParser_OUTPUTS}) + +Adding additional command-line options to the ``bison`` executable can be passed +as a list. For example, adding the ``-Wall`` option to report all warnings, and +``--no-lines`` (``-l``) to not generate ``#line`` directives. + +.. code-block:: cmake + + find_package(BISON) + + if(BISON_FOUND) + bison_target(MyParser parser.y parser.cpp OPTIONS -Wall --no-lines) + endif() + +Generator expressions can be used in ``OPTIONS <options...``. For example, to +add the ``--debug`` (``-t``) option only for the ``Debug`` build type: + +.. code-block:: cmake + + find_package(BISON) + + if(BISON_FOUND) + bison_target(MyParser parser.y parser.cpp OPTIONS $<$<CONFIG:Debug>:-t>) + endif() #]=======================================================================] find_program(BISON_EXECUTABLE NAMES bison win-bison win_bison DOC "path to the bison executable") @@ -236,6 +277,7 @@ REPORT_FILE ) set(BISON_TARGET_PARAM_MULTI_VALUE_KEYWORDS + OPTIONS VERBOSE ) cmake_parse_arguments( @@ -254,6 +296,11 @@ else() BISON_TARGET_option_extraopts("${BISON_TARGET_ARG_COMPILE_FLAGS}") + + if(BISON_TARGET_ARG_OPTIONS) + list(APPEND BISON_TARGET_cmdopt ${BISON_TARGET_ARG_OPTIONS}) + endif() + BISON_TARGET_option_defines("${BisonOutput}" "${BISON_TARGET_ARG_DEFINES_FILE}") BISON_TARGET_option_report_file("${BisonOutput}" "${BISON_TARGET_ARG_REPORT_FILE}") if(NOT "${BISON_TARGET_ARG_VERBOSE}" STREQUAL "") @@ -283,7 +330,27 @@ endif() unset(_BISON_CMP0088) + # Bison cannot create output directories. Create any missing determined + # directories where the files will be generated if they don't exist yet. + set(_BisonMakeDirectoryCommand "") + foreach(output IN LISTS BISON_TARGET_outputs) + cmake_path(GET output PARENT_PATH dir) + if(dir) + list(APPEND _BisonMakeDirectoryCommand ${dir}) + endif() + unset(dir) + endforeach() + if(_BisonMakeDirectoryCommand) + list(REMOVE_DUPLICATES _BisonMakeDirectoryCommand) + list( + PREPEND + _BisonMakeDirectoryCommand + COMMAND ${CMAKE_COMMAND} -E make_directory + ) + endif() + add_custom_command(OUTPUT ${BISON_TARGET_outputs} + ${_BisonMakeDirectoryCommand} COMMAND ${BISON_EXECUTABLE} ${BISON_TARGET_cmdopt} -o ${BisonOutput} ${_BisonInput} VERBATIM DEPENDS ${_BisonInput} @@ -297,12 +364,13 @@ set(BISON_${Name}_DEFINED TRUE) set(BISON_${Name}_INPUT ${_BisonInput}) set(BISON_${Name}_OUTPUTS ${BISON_TARGET_outputs} ${BISON_TARGET_extraoutputs}) + set(BISON_${Name}_OPTIONS ${BISON_TARGET_cmdopt}) set(BISON_${Name}_COMPILE_FLAGS ${BISON_TARGET_cmdopt}) set(BISON_${Name}_OUTPUT_SOURCE "${BisonOutput}") set(BISON_${Name}_OUTPUT_HEADER "${BISON_TARGET_output_header}") unset(_BisonInput) - + unset(_BisonMakeDirectoryCommand) endif() endmacro() #
diff --git a/Modules/FindBacktrace.cmake b/Modules/FindBacktrace.cmake index 4d3186c..abd0117 100644 --- a/Modules/FindBacktrace.cmake +++ b/Modules/FindBacktrace.cmake
@@ -98,7 +98,7 @@ set(Backtrace_LIBRARIES ${Backtrace_LIBRARY}) set(Backtrace_HEADER "${_Backtrace_HEADER_TRY}" CACHE STRING "Header providing backtrace(3) facility") -find_package_handle_standard_args(Backtrace FOUND_VAR Backtrace_FOUND REQUIRED_VARS ${_Backtrace_STD_ARGS}) +find_package_handle_standard_args(Backtrace REQUIRED_VARS ${_Backtrace_STD_ARGS}) mark_as_advanced(Backtrace_HEADER Backtrace_INCLUDE_DIR Backtrace_LIBRARY) if(Backtrace_FOUND AND NOT TARGET Backtrace::Backtrace)
diff --git a/Modules/FindBoost.cmake b/Modules/FindBoost.cmake index aed1854..39ddfbe 100644 --- a/Modules/FindBoost.cmake +++ b/Modules/FindBoost.cmake
@@ -1396,7 +1396,7 @@ set(_Boost_TIMER_DEPENDENCIES chrono) set(_Boost_WAVE_DEPENDENCIES filesystem serialization thread chrono atomic) set(_Boost_WSERIALIZATION_DEPENDENCIES serialization) - else() + elseif(Boost_VERSION_STRING VERSION_LESS 1.87.0) set(_Boost_CONTRACT_DEPENDENCIES thread chrono) set(_Boost_COROUTINE_DEPENDENCIES context) set(_Boost_FIBER_DEPENDENCIES context) @@ -1410,7 +1410,21 @@ set(_Boost_THREAD_DEPENDENCIES chrono atomic) set(_Boost_WAVE_DEPENDENCIES filesystem serialization thread chrono atomic) set(_Boost_WSERIALIZATION_DEPENDENCIES serialization) - if(Boost_VERSION_STRING VERSION_GREATER_EQUAL 1.87.0 AND NOT Boost_NO_WARN_NEW_VERSIONS) + else() + set(_Boost_CONTRACT_DEPENDENCIES thread chrono) + set(_Boost_COROUTINE_DEPENDENCIES context) + set(_Boost_FIBER_DEPENDENCIES context) + set(_Boost_IOSTREAMS_DEPENDENCIES regex) + set(_Boost_JSON_DEPENDENCIES container) + set(_Boost_LOG_DEPENDENCIES log_setup filesystem thread regex atomic) + set(_Boost_MATH_DEPENDENCIES math_c99 math_c99f math_c99l math_tr1 math_tr1f math_tr1l) + set(_Boost_MPI_DEPENDENCIES serialization) + set(_Boost_MPI_PYTHON_DEPENDENCIES python${component_python_version} mpi serialization) + set(_Boost_NUMPY_DEPENDENCIES python${component_python_version}) + set(_Boost_THREAD_DEPENDENCIES chrono atomic) + set(_Boost_WAVE_DEPENDENCIES filesystem serialization thread chrono atomic) + set(_Boost_WSERIALIZATION_DEPENDENCIES serialization) + if(Boost_VERSION_STRING VERSION_GREATER_EQUAL 1.88.0 AND NOT Boost_NO_WARN_NEW_VERSIONS) message(WARNING "New Boost version may have incorrect or missing dependencies and imported targets") endif() endif() @@ -1685,7 +1699,7 @@ # _Boost_COMPONENT_HEADERS. See the instructions at the top of # _Boost_COMPONENT_DEPENDENCIES. set(_Boost_KNOWN_VERSIONS ${Boost_ADDITIONAL_VERSIONS} - "1.86.0" "1.86" "1.85.0" "1.85" "1.84.0" "1.84" + "1.87.0" "1.87" "1.86.0" "1.86" "1.85.0" "1.85" "1.84.0" "1.84" "1.83.0" "1.83" "1.82.0" "1.82" "1.81.0" "1.81" "1.80.0" "1.80" "1.79.0" "1.79" "1.78.0" "1.78" "1.77.0" "1.77" "1.76.0" "1.76" "1.75.0" "1.75" "1.74.0" "1.74" "1.73.0" "1.73" "1.72.0" "1.72" "1.71.0" "1.71" "1.70.0" "1.70" "1.69.0" "1.69"
diff --git a/Modules/FindFLEX.cmake b/Modules/FindFLEX.cmake index 009e6b0..9cbe721 100644 --- a/Modules/FindFLEX.cmake +++ b/Modules/FindFLEX.cmake
@@ -35,6 +35,7 @@ .. code-block:: cmake flex_target(<Name> <FlexInput> <FlexOutput> + [OPTIONS <options>...] [COMPILE_FLAGS <string>] [DEFINES_FILE <string>] ) @@ -45,9 +46,18 @@ The options are: +``OPTIONS <options>...`` + .. versionadded:: 3.32 + + A :ref:`semicolon-separated list <CMake Language Lists>` of flex options added + to the ``flex`` command line. + ``COMPILE_FLAGS <string>`` + .. deprecated:: 3.32 + Space-separated flex options added to the ``flex`` command line. A :ref:`;-list <CMake Language Lists>` will not work. + This option is deprecated in favor of ``OPTIONS <options>...``. ``DEFINES_FILE <string>`` .. versionadded:: 3.5 @@ -73,6 +83,11 @@ ``FLEX_<Name>_OUTPUT_HEADER`` The header flex output, if any. +``FLEX_<Name>_OPTIONS`` + .. versionadded:: 3.32 + + Options used in the ``flex`` command line. + Flex scanners often use tokens defined by Bison: the code generated by Flex depends of the header generated by Bison. This module also defines a macro: @@ -106,6 +121,29 @@ ${FLEX_MyScanner_OUTPUTS} ) target_link_libraries(Foo ${FLEX_LIBRARIES}) + +Adding additional command-line options to the ``flex`` executable can be passed +as a list. For example, adding the ``--warn`` option to report warnings, and the +``--noline`` (``-L``) to not generate ``#line`` directives. + +.. code-block:: cmake + + find_package(FLEX) + + if(FLEX_FOUND) + flex_target(MyScanner lexer.l lexer.cpp OPTIONS --warn --noline) + endif() + +Generator expressions can be used in ``OPTIONS <options...``. For example, to +add the ``--debug`` (``-d``) option only for the ``Debug`` build type: + +.. code-block:: cmake + + find_package(FLEX) + + if(FLEX_FOUND) + flex_target(MyScanner lexer.l lexer.cpp OPTIONS $<$<CONFIG:Debug>:--debug>) + endif() #]=======================================================================] find_program(FLEX_EXECUTABLE NAMES flex win-flex win_flex DOC "path to the flex executable") @@ -157,20 +195,35 @@ COMPILE_FLAGS DEFINES_FILE ) - set(FLEX_TARGET_PARAM_MULTI_VALUE_KEYWORDS) + set(FLEX_TARGET_PARAM_MULTI_VALUE_KEYWORDS OPTIONS) cmake_parse_arguments( FLEX_TARGET_ARG "${FLEX_TARGET_PARAM_OPTIONS}" "${FLEX_TARGET_PARAM_ONE_VALUE_KEYWORDS}" - "${FLEX_TARGET_MULTI_VALUE_KEYWORDS}" + "${FLEX_TARGET_PARAM_MULTI_VALUE_KEYWORDS}" ${ARGN} ) - set(FLEX_TARGET_usage "FLEX_TARGET(<Name> <Input> <Output> [COMPILE_FLAGS <string>] [DEFINES_FILE <string>]") + string( + JOIN "\n" FLEX_TARGET_usage + "Usage:" + " flex_target(" + " <Name>" + " <Input>" + " <Output>" + " [OPTIONS <options>...]" + " [COMPILE_FLAGS <string>]" + " [DEFINES_FILE <string>]" + " )" + ) if(NOT "${FLEX_TARGET_ARG_UNPARSED_ARGUMENTS}" STREQUAL "") - message(SEND_ERROR ${FLEX_TARGET_usage}) + message( + SEND_ERROR + "Unrecognized arguments: ${FLEX_TARGET_ARG_UNPARSED_ARGUMENTS}\n" + "${FLEX_TARGET_usage}" + ) else() cmake_policy(GET CMP0098 _flex_CMP0098 @@ -199,6 +252,10 @@ separate_arguments(_flex_EXE_OPTS) endif() + if(FLEX_TARGET_ARG_OPTIONS) + list(APPEND _flex_EXE_OPTS ${FLEX_TARGET_ARG_OPTIONS}) + endif() + set(_flex_OUTPUT_HEADER "") if(NOT "${FLEX_TARGET_ARG_DEFINES_FILE}" STREQUAL "") set(_flex_OUTPUT_HEADER "${FLEX_TARGET_ARG_DEFINES_FILE}") @@ -211,29 +268,51 @@ list(APPEND _flex_EXE_OPTS --header-file=${_flex_OUTPUT_HEADER_ABS}) endif() + # Flex cannot create output directories. Create any missing determined + # directories where the files will be generated if they don't exist yet. + set(_flex_MAKE_DIRECTORY_COMMAND "") + foreach(output IN LISTS _flex_TARGET_OUTPUTS) + cmake_path(GET output PARENT_PATH dir) + if(dir) + list(APPEND _flex_MAKE_DIRECTORY_COMMAND ${dir}) + endif() + unset(dir) + endforeach() + if(_flex_MAKE_DIRECTORY_COMMAND) + list(REMOVE_DUPLICATES _flex_MAKE_DIRECTORY_COMMAND) + list( + PREPEND + _flex_MAKE_DIRECTORY_COMMAND + COMMAND ${CMAKE_COMMAND} -E make_directory + ) + endif() + get_filename_component(_flex_EXE_NAME_WE "${FLEX_EXECUTABLE}" NAME_WE) add_custom_command(OUTPUT ${_flex_TARGET_OUTPUTS} + ${_flex_MAKE_DIRECTORY_COMMAND} COMMAND ${FLEX_EXECUTABLE} ${_flex_EXE_OPTS} -o${_flex_OUTPUT} ${_flex_INPUT} VERBATIM DEPENDS ${_flex_INPUT} COMMENT "[FLEX][${Name}] Building scanner with ${_flex_EXE_NAME_WE} ${FLEX_VERSION}" - WORKING_DIRECTORY ${_flex_WORKING_DIR}) + WORKING_DIRECTORY ${_flex_WORKING_DIR} + COMMAND_EXPAND_LISTS) set(FLEX_${Name}_DEFINED TRUE) set(FLEX_${Name}_OUTPUTS ${_flex_TARGET_OUTPUTS}) set(FLEX_${Name}_INPUT ${_flex_INPUT}) + set(FLEX_${Name}_OPTIONS ${_flex_EXE_OPTS}) set(FLEX_${Name}_COMPILE_FLAGS ${_flex_EXE_OPTS}) set(FLEX_${Name}_OUTPUT_HEADER ${_flex_OUTPUT_HEADER}) unset(_flex_EXE_NAME_WE) unset(_flex_EXE_OPTS) unset(_flex_INPUT) + unset(_flex_MAKE_DIRECTORY_COMMAND) unset(_flex_OUTPUT) unset(_flex_OUTPUT_HEADER) unset(_flex_OUTPUT_HEADER_ABS) unset(_flex_TARGET_OUTPUTS) unset(_flex_WORKING_DIR) - endif() endmacro() #============================================================
diff --git a/Modules/FindFontconfig.cmake b/Modules/FindFontconfig.cmake index 1cd84c1..8b8066b 100644 --- a/Modules/FindFontconfig.cmake +++ b/Modules/FindFontconfig.cmake
@@ -80,8 +80,6 @@ include(FindPackageHandleStandardArgs) find_package_handle_standard_args(Fontconfig - FOUND_VAR - Fontconfig_FOUND REQUIRED_VARS Fontconfig_LIBRARY Fontconfig_INCLUDE_DIR
diff --git a/Modules/FindGSL.cmake b/Modules/FindGSL.cmake index 3bb9272..dc784cd 100644 --- a/Modules/FindGSL.cmake +++ b/Modules/FindGSL.cmake
@@ -153,8 +153,6 @@ # handle the QUIETLY and REQUIRED arguments and set GSL_FOUND to TRUE if all # listed variables are TRUE find_package_handle_standard_args( GSL - FOUND_VAR - GSL_FOUND REQUIRED_VARS GSL_INCLUDE_DIR GSL_LIBRARY
diff --git a/Modules/FindIce.cmake b/Modules/FindIce.cmake index 543e10c..f2b4d5a 100644 --- a/Modules/FindIce.cmake +++ b/Modules/FindIce.cmake
@@ -542,7 +542,6 @@ include(${CMAKE_CURRENT_LIST_DIR}/FindPackageHandleStandardArgs.cmake) FIND_PACKAGE_HANDLE_STANDARD_ARGS(Ice - FOUND_VAR Ice_FOUND REQUIRED_VARS Ice_SLICE2CPP_EXECUTABLE Ice_INCLUDE_DIR Ice_SLICE_DIR
diff --git a/Modules/FindImageMagick.cmake b/Modules/FindImageMagick.cmake index 809fec7..7b0671f 100644 --- a/Modules/FindImageMagick.cmake +++ b/Modules/FindImageMagick.cmake
@@ -327,8 +327,6 @@ REQUIRED_VARS ${ImageMagick_REQUIRED_VARS} VERSION_VAR ImageMagick_VERSION_STRING ) -# Maintain consistency with all other variables. -set(ImageMagick_FOUND ${IMAGEMAGICK_FOUND}) #--------------------------------------------------------------------- # DEPRECATED: Setting variables for backward compatibility.
diff --git a/Modules/FindIntl.cmake b/Modules/FindIntl.cmake index 04f5a23..e2c5a83 100644 --- a/Modules/FindIntl.cmake +++ b/Modules/FindIntl.cmake
@@ -162,7 +162,6 @@ include(${CMAKE_CURRENT_LIST_DIR}/FindPackageHandleStandardArgs.cmake) FIND_PACKAGE_HANDLE_STANDARD_ARGS(Intl - FOUND_VAR Intl_FOUND REQUIRED_VARS ${_Intl_REQUIRED_VARS} VERSION_VAR Intl_VERSION FAIL_MESSAGE "Failed to find Gettext libintl")
diff --git a/Modules/FindLTTngUST.cmake b/Modules/FindLTTngUST.cmake index 34b2e82..32c6a16 100644 --- a/Modules/FindLTTngUST.cmake +++ b/Modules/FindLTTngUST.cmake
@@ -98,7 +98,7 @@ endif() include(${CMAKE_CURRENT_LIST_DIR}/FindPackageHandleStandardArgs.cmake) -find_package_handle_standard_args(LTTngUST FOUND_VAR LTTNGUST_FOUND +find_package_handle_standard_args(LTTngUST REQUIRED_VARS LTTNGUST_LIBRARIES LTTNGUST_INCLUDE_DIRS VERSION_VAR LTTNGUST_VERSION_STRING)
diff --git a/Modules/FindLibinput.cmake b/Modules/FindLibinput.cmake index 291d828..8ed0e4c 100644 --- a/Modules/FindLibinput.cmake +++ b/Modules/FindLibinput.cmake
@@ -60,8 +60,6 @@ include(FindPackageHandleStandardArgs) find_package_handle_standard_args(Libinput - FOUND_VAR - Libinput_FOUND REQUIRED_VARS Libinput_LIBRARY Libinput_INCLUDE_DIR
diff --git a/Modules/FindMatlab.cmake b/Modules/FindMatlab.cmake index c94ef72..35f8b4e 100644 --- a/Modules/FindMatlab.cmake +++ b/Modules/FindMatlab.cmake
@@ -2079,7 +2079,6 @@ find_package_handle_standard_args( Matlab - FOUND_VAR Matlab_FOUND REQUIRED_VARS ${_matlab_required_variables} VERSION_VAR Matlab_VERSION HANDLE_VERSION_RANGE
diff --git a/Modules/FindOpenCL.cmake b/Modules/FindOpenCL.cmake index 55be667..b3fa1cc 100644 --- a/Modules/FindOpenCL.cmake +++ b/Modules/FindOpenCL.cmake
@@ -176,7 +176,6 @@ include(${CMAKE_CURRENT_LIST_DIR}/FindPackageHandleStandardArgs.cmake) find_package_handle_standard_args( OpenCL - FOUND_VAR OpenCL_FOUND REQUIRED_VARS OpenCL_LIBRARY OpenCL_INCLUDE_DIR VERSION_VAR OpenCL_VERSION_STRING)
diff --git a/Modules/FindOpenMP.cmake b/Modules/FindOpenMP.cmake index 061ba2f..674b161 100644 --- a/Modules/FindOpenMP.cmake +++ b/Modules/FindOpenMP.cmake
@@ -709,8 +709,6 @@ VERSION_VAR ${_OpenMP_MIN_VERSION} HANDLE_COMPONENTS) -set(OPENMP_FOUND ${OpenMP_FOUND}) - if(CMAKE_Fortran_COMPILER_LOADED AND OpenMP_Fortran_FOUND) if(NOT DEFINED OpenMP_Fortran_HAVE_OMPLIB_MODULE) set(OpenMP_Fortran_HAVE_OMPLIB_MODULE FALSE CACHE BOOL INTERNAL "")
diff --git a/Modules/FindOpenSP.cmake b/Modules/FindOpenSP.cmake index 4255aef..5cd9570 100644 --- a/Modules/FindOpenSP.cmake +++ b/Modules/FindOpenSP.cmake
@@ -122,7 +122,6 @@ include(FindPackageHandleStandardArgs) find_package_handle_standard_args(OpenSP - FOUND_VAR OpenSP_FOUND REQUIRED_VARS OpenSP_LIBRARY OpenSP_INCLUDE_DIR VERSION_VAR OpenSP_VERSION )
diff --git a/Modules/FindPackageHandleStandardArgs.cmake b/Modules/FindPackageHandleStandardArgs.cmake index ffa187b..3360cb2 100644 --- a/Modules/FindPackageHandleStandardArgs.cmake +++ b/Modules/FindPackageHandleStandardArgs.cmake
@@ -56,7 +56,8 @@ Specifies either ``<PackageName>_FOUND`` or ``<PACKAGENAME>_FOUND`` as the result variable. This exists only for compatibility with older versions of CMake and is now ignored. - Result variables of both names are always set for compatibility. + Result variables of both names are now always set for compatibility + also with or without this option. ``REQUIRED_VARS <required-var>...`` Specify the variables which are required for this package. @@ -225,7 +226,7 @@ set(__msg "${_msg}") if(FPHSA_REASON_FAILURE_MESSAGE) string(APPEND __msg "\n Reason given by package: ${FPHSA_REASON_FAILURE_MESSAGE}\n") - elseif(NOT DEFINED PROJECT_NAME) + elseif(NOT DEFINED PROJECT_NAME AND NOT CMAKE_SCRIPT_MODE_FILE) string(APPEND __msg "\n" "Hint: The project() command has not yet been called. It sets up system-specific search paths.") endif() @@ -467,13 +468,12 @@ if(FPHSA_FOUND_VAR) set(_FOUND_VAR_UPPER ${_NAME_UPPER}_FOUND) set(_FOUND_VAR_MIXED ${_NAME}_FOUND) - if(FPHSA_FOUND_VAR STREQUAL _FOUND_VAR_MIXED OR FPHSA_FOUND_VAR STREQUAL _FOUND_VAR_UPPER) - set(_FOUND_VAR ${FPHSA_FOUND_VAR}) - else() + if( + NOT FPHSA_FOUND_VAR STREQUAL _FOUND_VAR_MIXED + AND NOT FPHSA_FOUND_VAR STREQUAL _FOUND_VAR_UPPER + ) message(FATAL_ERROR "The argument for FOUND_VAR is \"${FPHSA_FOUND_VAR}\", but only \"${_FOUND_VAR_MIXED}\" and \"${_FOUND_VAR_UPPER}\" are valid names.") endif() - else() - set(_FOUND_VAR ${_NAME_UPPER}_FOUND) endif() # collect all variables which were not found, so they can be printed, so the
diff --git a/Modules/FindPostgreSQL.cmake b/Modules/FindPostgreSQL.cmake index 1984e18..229f364 100644 --- a/Modules/FindPostgreSQL.cmake +++ b/Modules/FindPostgreSQL.cmake
@@ -274,7 +274,6 @@ REQUIRED_VARS PostgreSQL_LIBRARY PostgreSQL_INCLUDE_DIR HANDLE_COMPONENTS VERSION_VAR PostgreSQL_VERSION_STRING) -set(PostgreSQL_FOUND ${POSTGRESQL_FOUND}) function(__postgresql_import_library _target _var _config) if(_config)
diff --git a/Modules/FindQt4.cmake b/Modules/FindQt4.cmake index 6d764bd..91b83c8 100644 --- a/Modules/FindQt4.cmake +++ b/Modules/FindQt4.cmake
@@ -1325,7 +1325,7 @@ # explicitly. set(FPHSA_NAME_MISMATCHED 1) endif () - FIND_PACKAGE_HANDLE_STANDARD_ARGS(Qt4 FOUND_VAR Qt4_FOUND + FIND_PACKAGE_HANDLE_STANDARD_ARGS(Qt4 REQUIRED_VARS ${_QT4_FOUND_REQUIRED_VARS} VERSION_VAR QTVERSION ) @@ -1341,5 +1341,4 @@ set (QT_MOC_EXE ${QT_MOC_EXECUTABLE} ) set (QT_UIC_EXE ${QT_UIC_EXECUTABLE} ) set( QT_QT_LIBRARY "") -set(QT4_FOUND ${Qt4_FOUND}) set(QT_FOUND ${Qt4_FOUND})
diff --git a/Modules/FindSubversion.cmake b/Modules/FindSubversion.cmake index 7a9c440..7535854 100644 --- a/Modules/FindSubversion.cmake +++ b/Modules/FindSubversion.cmake
@@ -165,5 +165,4 @@ VERSION_VAR Subversion_VERSION_SVN ) # for compatibility -set(Subversion_FOUND ${SUBVERSION_FOUND}) set(Subversion_SVN_FOUND ${SUBVERSION_FOUND})
diff --git a/Modules/FindXCTest.cmake b/Modules/FindXCTest.cmake index 92ee25d..d198ad9 100644 --- a/Modules/FindXCTest.cmake +++ b/Modules/FindXCTest.cmake
@@ -103,7 +103,6 @@ include(${CMAKE_CURRENT_LIST_DIR}/FindPackageHandleStandardArgs.cmake) find_package_handle_standard_args(XCTest - FOUND_VAR XCTest_FOUND REQUIRED_VARS XCTest_LIBRARY XCTest_INCLUDE_DIR XCTest_EXECUTABLE) if(XCTest_FOUND)
diff --git a/Modules/FindXalanC.cmake b/Modules/FindXalanC.cmake index 338d0c0..6d4801e 100644 --- a/Modules/FindXalanC.cmake +++ b/Modules/FindXalanC.cmake
@@ -115,7 +115,6 @@ include(${CMAKE_CURRENT_LIST_DIR}/FindPackageHandleStandardArgs.cmake) FIND_PACKAGE_HANDLE_STANDARD_ARGS(XalanC - FOUND_VAR XalanC_FOUND REQUIRED_VARS XalanC_LIBRARY XalanC_INCLUDE_DIR XalanC_VERSION
diff --git a/Modules/FindXercesC.cmake b/Modules/FindXercesC.cmake index ee6a401..8eaed02 100644 --- a/Modules/FindXercesC.cmake +++ b/Modules/FindXercesC.cmake
@@ -113,7 +113,6 @@ include(${CMAKE_CURRENT_LIST_DIR}/FindPackageHandleStandardArgs.cmake) FIND_PACKAGE_HANDLE_STANDARD_ARGS(XercesC - FOUND_VAR XercesC_FOUND REQUIRED_VARS XercesC_LIBRARY XercesC_INCLUDE_DIR XercesC_VERSION
diff --git a/Source/CMakeVersion.cmake b/Source/CMakeVersion.cmake index aa9e8a4..37663ac 100644 --- a/Source/CMakeVersion.cmake +++ b/Source/CMakeVersion.cmake
@@ -1,7 +1,7 @@ # CMake version number components. set(CMake_VERSION_MAJOR 3) set(CMake_VERSION_MINOR 31) -set(CMake_VERSION_PATCH 20250103) +set(CMake_VERSION_PATCH 20250109) #set(CMake_VERSION_RC 0) set(CMake_VERSION_IS_DIRTY 0)
diff --git a/Source/Modules/FindJsonCpp.cmake b/Source/Modules/FindJsonCpp.cmake index 1951b61..74fa181 100644 --- a/Source/Modules/FindJsonCpp.cmake +++ b/Source/Modules/FindJsonCpp.cmake
@@ -85,11 +85,9 @@ #----------------------------------------------------------------------------- include(${CMAKE_CURRENT_LIST_DIR}/../../Modules/FindPackageHandleStandardArgs.cmake) FIND_PACKAGE_HANDLE_STANDARD_ARGS(JsonCpp - FOUND_VAR JsonCpp_FOUND REQUIRED_VARS JsonCpp_LIBRARY JsonCpp_INCLUDE_DIR VERSION_VAR JsonCpp_VERSION_STRING ) -set(JSONCPP_FOUND ${JsonCpp_FOUND}) #----------------------------------------------------------------------------- # Provide documented result variables and targets.
diff --git a/Source/Modules/FindLibRHash.cmake b/Source/Modules/FindLibRHash.cmake index 86c6189..c23a028 100644 --- a/Source/Modules/FindLibRHash.cmake +++ b/Source/Modules/FindLibRHash.cmake
@@ -53,10 +53,8 @@ #----------------------------------------------------------------------------- include(${CMAKE_CURRENT_LIST_DIR}/../../Modules/FindPackageHandleStandardArgs.cmake) FIND_PACKAGE_HANDLE_STANDARD_ARGS(LibRHash - FOUND_VAR LibRHash_FOUND REQUIRED_VARS LibRHash_LIBRARY LibRHash_INCLUDE_DIR ) -set(LIBRHASH_FOUND ${LibRHash_FOUND}) #----------------------------------------------------------------------------- # Provide documented result variables and targets.
diff --git a/Source/Modules/FindLibUUID.cmake b/Source/Modules/FindLibUUID.cmake index ca5b61d..28bbec2 100644 --- a/Source/Modules/FindLibUUID.cmake +++ b/Source/Modules/FindLibUUID.cmake
@@ -72,10 +72,8 @@ #----------------------------------------------------------------------------- include(${CMAKE_CURRENT_LIST_DIR}/../../Modules/FindPackageHandleStandardArgs.cmake) FIND_PACKAGE_HANDLE_STANDARD_ARGS(LibUUID - FOUND_VAR LibUUID_FOUND REQUIRED_VARS LibUUID_LIBRARY LibUUID_INCLUDE_DIR ) -set(LIBUUID_FOUND ${LibUUID_FOUND}) #----------------------------------------------------------------------------- # Provide documented result variables and targets.
diff --git a/Source/Modules/FindLibUV.cmake b/Source/Modules/FindLibUV.cmake index 0554d62..9baeb82 100644 --- a/Source/Modules/FindLibUV.cmake +++ b/Source/Modules/FindLibUV.cmake
@@ -102,11 +102,9 @@ #----------------------------------------------------------------------------- include(${CMAKE_CURRENT_LIST_DIR}/../../Modules/FindPackageHandleStandardArgs.cmake) FIND_PACKAGE_HANDLE_STANDARD_ARGS(LibUV - FOUND_VAR LibUV_FOUND REQUIRED_VARS LibUV_LIBRARY LibUV_INCLUDE_DIR VERSION_VAR LibUV_VERSION ) -set(LIBUV_FOUND ${LibUV_FOUND}) #----------------------------------------------------------------------------- # Provide documented result variables and targets.
diff --git a/Source/cmGlobalUnixMakefileGenerator3.cxx b/Source/cmGlobalUnixMakefileGenerator3.cxx index 32c2a20..671dae2 100644 --- a/Source/cmGlobalUnixMakefileGenerator3.cxx +++ b/Source/cmGlobalUnixMakefileGenerator3.cxx
@@ -951,6 +951,9 @@ if (!this->GlobalSettingIsOn("CMAKE_SUPPRESS_REGENERATION")) { lg->AppendEcho(commands, "... depend"); } + if (this->CheckCMP0171()) { + lg->AppendEcho(commands, "... codegen"); + } // Keep track of targets already listed. std::set<std::string> emittedTargets;
diff --git a/Source/cmVisualStudio10TargetGenerator.cxx b/Source/cmVisualStudio10TargetGenerator.cxx index 45ad128..0df5a64 100644 --- a/Source/cmVisualStudio10TargetGenerator.cxx +++ b/Source/cmVisualStudio10TargetGenerator.cxx
@@ -1013,6 +1013,7 @@ this->WriteXamlFilesGroup(e0); this->WriteDotNetReferences(e0); this->WritePackageReferences(e0); + this->WriteImports(e0); this->WriteProjectReferences(e0); }
diff --git a/Utilities/Scripts/update-liblzma.bash b/Utilities/Scripts/update-liblzma.bash index a5a7175..8bfde77 100755 --- a/Utilities/Scripts/update-liblzma.bash +++ b/Utilities/Scripts/update-liblzma.bash
@@ -8,7 +8,7 @@ readonly ownership="liblzma upstream <xz-devel@tukaani.org>" readonly subtree="Utilities/cmliblzma" readonly repo="https://git.tukaani.org/xz.git" -readonly tag="v5.2.5" +readonly tag="v5.6.3" readonly shortlog=false readonly paths=" COPYING @@ -33,6 +33,7 @@ rm liblzma/*/Makefile.* rm liblzma/liblzma.map rm liblzma/validate_map.sh + echo "* -whitespace" > .gitattributes popd }
diff --git a/Utilities/cmliblzma/CMakeLists.txt b/Utilities/cmliblzma/CMakeLists.txt index 3ba3ce9..6ac02f0 100644 --- a/Utilities/cmliblzma/CMakeLists.txt +++ b/Utilities/cmliblzma/CMakeLists.txt
@@ -176,6 +176,11 @@ ADD_LIBRARY(cmliblzma STATIC ${LZMA_SRCS}) +# Disable inline assembly in a case where it does not compile. +if(CMAKE_C_COMPILER_ID STREQUAL "AppleClang" AND CMAKE_C_FLAGS MATCHES "-ftrapv") + set_property(SOURCE liblzma/lzma/lzma_decoder.c PROPERTY COMPILE_DEFINITIONS LZMA_RANGE_DECODER_CONFIG=0) +endif() + IF(CMAKE_C_COMPILER_ID STREQUAL "XL") # Disable the XL compiler optimizer because it causes crashes # and other bad behavior in liblzma code.
diff --git a/Utilities/cmliblzma/COPYING b/Utilities/cmliblzma/COPYING index 20e60d5..aed2153 100644 --- a/Utilities/cmliblzma/COPYING +++ b/Utilities/cmliblzma/COPYING
@@ -3,63 +3,81 @@ ================== Different licenses apply to different files in this package. Here - is a rough summary of which licenses apply to which parts of this - package (but check the individual files to be sure!): + is a summary of which licenses apply to which parts of this package: - - liblzma is in the public domain. + - liblzma is under the BSD Zero Clause License (0BSD). - - xz, xzdec, and lzmadec command line tools are in the public - domain unless GNU getopt_long had to be compiled and linked - in from the lib directory. The getopt_long code is under - GNU LGPLv2.1+. + - The command line tools xz, xzdec, lzmadec, and lzmainfo are + under 0BSD except that, on systems that don't have a usable + getopt_long, GNU getopt_long is compiled and linked in from the + 'lib' directory. The getopt_long code is under GNU LGPLv2.1+. - The scripts to grep, diff, and view compressed files have been - adapted from gzip. These scripts and their documentation are - under GNU GPLv2+. + adapted from GNU gzip. These scripts (xzgrep, xzdiff, xzless, + and xzmore) are under GNU GPLv2+. The man pages of the scripts + are under 0BSD; they aren't based on the man pages of GNU gzip. - - All the documentation in the doc directory and most of the - XZ Utils specific documentation files in other directories - are in the public domain. + - Most of the XZ Utils specific documentation that is in + plain text files (like README, INSTALL, PACKAGERS, NEWS, + and ChangeLog) are under 0BSD unless stated otherwise in + the file itself. The files xz-file-format.txt and + lzma-file-format.xt are in the public domain but may + be distributed under the terms of 0BSD too. - - Translated messages are in the public domain. + - Translated messages and man pages are under 0BSD except that + some old translations are in the public domain. - - The build system contains public domain files, and files that - are under GNU GPLv2+ or GNU GPLv3+. None of these files end up - in the binaries being built. + - Test files and test code in the 'tests' directory, and + debugging utilities in the 'debug' directory are under + the BSD Zero Clause License (0BSD). - - Test files and test code in the tests directory, and debugging - utilities in the debug directory are in the public domain. + - The GNU Autotools based build system contains files that are + under GNU GPLv2+, GNU GPLv3+, and a few permissive licenses. + These files don't affect the licensing of the binaries being + built. - - The extra directory may contain public domain files, and files - that are under various free software licenses. + - The 'extra' directory contains files that are under various + free software licenses. These aren't built or installed as + part of XZ Utils. - You can do whatever you want with the files that have been put into - the public domain. If you find public domain legally problematic, - take the previous sentence as a license grant. If you still find - the lack of copyright legally problematic, you have too many - lawyers. + For the files under the BSD Zero Clause License (0BSD), if + a copyright notice is needed, the following is sufficient: - As usual, this software is provided "as is", without any warranty. + Copyright (C) The XZ Utils authors and contributors - If you copy significant amounts of public domain code from XZ Utils + If you copy significant amounts of 0BSD-licensed code from XZ Utils into your project, acknowledging this somewhere in your software is polite (especially if it is proprietary, non-free software), but - naturally it is not legally required. Here is an example of a good - notice to put into "about box" or into documentation: + it is not legally required by the license terms. Here is an example + of a good notice to put into "about box" or into documentation: This software includes code from XZ Utils <https://tukaani.org/xz/>. The following license texts are included in the following files: + - COPYING.0BSD: BSD Zero Clause License - COPYING.LGPLv2.1: GNU Lesser General Public License version 2.1 - COPYING.GPLv2: GNU General Public License version 2 - COPYING.GPLv3: GNU General Public License version 3 - Note that the toolchain (compiler, linker etc.) may add some code - pieces that are copyrighted. Thus, it is possible that e.g. liblzma - binary wouldn't actually be in the public domain in its entirety - even though it contains no copyrighted code from the XZ Utils source - package. + A note about old XZ Utils releases: - If you have questions, don't hesitate to ask the author(s) for more - information. + XZ Utils releases 5.4.6 and older and 5.5.1alpha have a + significant amount of code put into the public domain and + that obviously remains so. The switch from public domain to + 0BSD for newer releases was made in Febrary 2024 because + public domain has (real or perceived) legal ambiguities in + some jurisdictions. + + There is very little *practical* difference between public + domain and 0BSD. The main difference likely is that one + shouldn't claim that 0BSD-licensed code is in the public + domain; 0BSD-licensed code is copyrighted but available under + an extremely permissive license. Neither 0BSD nor public domain + require retaining or reproducing author, copyright holder, or + license notices when distributing the software. (Compare to, + for example, BSD 2-Clause "Simplified" License which does have + such requirements.) + + If you have questions, don't hesitate to ask for more information. + The contact information is in the README file.
diff --git a/Utilities/cmliblzma/common/common_w32res.rc b/Utilities/cmliblzma/common/common_w32res.rc index a70de34..8114ee3 100644 --- a/Utilities/cmliblzma/common/common_w32res.rc +++ b/Utilities/cmliblzma/common/common_w32res.rc
@@ -1,12 +1,13 @@ +/* SPDX-License-Identifier: 0BSD */ + /* * Author: Lasse Collin - * - * This file has been put into the public domain. - * You can do whatever you want with this file. */ #include <winresrc.h> -#include "config.h" +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif #define LZMA_H_INTERNAL #define LZMA_H_INTERNAL_RC #include "lzma/version.h" @@ -21,14 +22,15 @@ #define MY_PRODUCT PACKAGE_NAME " <" PACKAGE_URL ">" LANGUAGE LANG_ENGLISH, SUBLANG_ENGLISH_US + VS_VERSION_INFO VERSIONINFO - FILEVERSION MY_VERSION - PRODUCTVERSION MY_VERSION - FILEFLAGSMASK VS_FFI_FILEFLAGSMASK - FILEFLAGS 0 - FILEOS VOS_NT_WINDOWS32 - FILETYPE MY_TYPE - FILESUBTYPE 0x0L +FILEVERSION MY_VERSION +PRODUCTVERSION MY_VERSION +FILEFLAGSMASK VS_FFI_FILEFLAGSMASK +FILEFLAGS 0 +FILEOS VOS_NT_WINDOWS32 +FILETYPE MY_TYPE +FILESUBTYPE 0x0L BEGIN BLOCK "StringFileInfo" BEGIN @@ -48,3 +50,8 @@ VALUE "Translation", 0x409, 1200 END END + +/* Omit the manifest on Cygwin and MSYS2 (both define __CYGWIN__). */ +#if MY_TYPE == VFT_APP && !defined(__CYGWIN__) +CREATEPROCESS_MANIFEST_RESOURCE_ID RT_MANIFEST "w32_application.manifest" +#endif
diff --git a/Utilities/cmliblzma/common/mythread.h b/Utilities/cmliblzma/common/mythread.h index be22654..10ea2d4 100644 --- a/Utilities/cmliblzma/common/mythread.h +++ b/Utilities/cmliblzma/common/mythread.h
@@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file mythread.h @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef MYTHREAD_H @@ -79,7 +78,7 @@ } while (0) -#if !(defined(_WIN32) && !defined(__CYGWIN__)) +#if !(defined(_WIN32) && !defined(__CYGWIN__)) && !defined(__wasm__) // Use sigprocmask() to set the signal mask in single-threaded programs. #include <signal.h> @@ -100,12 +99,37 @@ // Using pthreads // //////////////////// -#include <sys/time.h> #include <pthread.h> #include <signal.h> #include <time.h> #include <errno.h> +// If clock_gettime() isn't available, use gettimeofday() from <sys/time.h> +// as a fallback. gettimeofday() is in SUSv2 and thus is supported on all +// relevant POSIX systems. +#ifndef HAVE_CLOCK_GETTIME +# include <sys/time.h> +#endif + +// MinGW-w64 with winpthreads: +// +// NOTE: Typical builds with MinGW-w64 don't use this code (MYTHREAD_POSIX). +// Instead, native Windows threading APIs are used (MYTHREAD_VISTA or +// MYTHREAD_WIN95). +// +// MinGW-w64 has _sigset_t (an integer type) in <sys/types.h>. +// If _POSIX was #defined, the header would add the alias sigset_t too. +// Let's keep this working even without _POSIX. +// +// There are no functions that actually do something with sigset_t +// because signals barely exist on Windows. The sigfillset macro below +// is just to silence warnings. There is no sigfillset() in MinGW-w64. +#ifdef __MINGW32__ +# include <sys/types.h> +# define sigset_t _sigset_t +# define sigfillset(set_ptr) do { *(set_ptr) = 0; } while (0) +#endif + #define MYTHREAD_RET_TYPE void * #define MYTHREAD_RET_VALUE NULL @@ -134,11 +158,13 @@ // Use pthread_sigmask() to set the signal mask in multi-threaded programs. // Do nothing on OpenVMS since it lacks pthread_sigmask(). +// Do nothing on MinGW-w64 too to silence warnings (its pthread_sigmask() +// is #defined to 0 so it's a no-op). static inline void mythread_sigmask(int how, const sigset_t *restrict set, sigset_t *restrict oset) { -#ifdef __VMS +#if defined(__VMS) || defined(__MINGW32__) (void)how; (void)set; (void)oset; @@ -174,7 +200,7 @@ } -// Initiatlizes a mutex. Returns zero on success and non-zero on error. +// Initializes a mutex. Returns zero on success and non-zero on error. static inline int mythread_mutex_init(mythread_mutex *mutex) { @@ -219,8 +245,8 @@ mythread_cond_init(mythread_cond *mycond) { #ifdef HAVE_CLOCK_GETTIME - // NOTE: HAVE_DECL_CLOCK_MONOTONIC is always defined to 0 or 1. -# if defined(HAVE_PTHREAD_CONDATTR_SETCLOCK) && HAVE_DECL_CLOCK_MONOTONIC +# if defined(HAVE_PTHREAD_CONDATTR_SETCLOCK) && \ + defined(HAVE_CLOCK_MONOTONIC) struct timespec ts; pthread_condattr_t condattr; @@ -294,8 +320,8 @@ mythread_condtime_set(mythread_condtime *condtime, const mythread_cond *cond, uint32_t timeout_ms) { - condtime->tv_sec = timeout_ms / 1000; - condtime->tv_nsec = (timeout_ms % 1000) * 1000000; + condtime->tv_sec = (time_t)(timeout_ms / 1000); + condtime->tv_nsec = (long)((timeout_ms % 1000) * 1000000); #ifdef HAVE_CLOCK_GETTIME struct timespec now; @@ -370,10 +396,11 @@ BOOL pending_; \ if (!InitOnceBeginInitialize(&once_, 0, &pending_, NULL)) \ abort(); \ - if (pending_) \ + if (pending_) { \ func(); \ - if (!InitOnceComplete(&once, 0, NULL)) \ - abort(); \ + if (!InitOnceComplete(&once_, 0, NULL)) \ + abort(); \ + } \ } while (0) #endif
diff --git a/Utilities/cmliblzma/common/sysdefs.h b/Utilities/cmliblzma/common/sysdefs.h index 6e3495e..3cbdf1e 100644 --- a/Utilities/cmliblzma/common/sysdefs.h +++ b/Utilities/cmliblzma/common/sysdefs.h
@@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file sysdefs.h @@ -8,9 +10,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef LZMA_SYSDEFS_H @@ -29,7 +28,15 @@ #include "config.h" -// Get standard-compliant stdio functions under MinGW and MinGW-w64. +// This #define ensures that C99 and POSIX compliant stdio functions are +// available with MinGW-w64 (both 32-bit and 64-bit). Modern MinGW-w64 adds +// this automatically, for example, when the compiler is in C99 (or later) +// mode when building against msvcrt.dll. It still doesn't hurt to be explicit +// that we always want this and #define this unconditionally. +// +// With Universal CRT (UCRT) this is less important because UCRT contains +// C99-compatible stdio functions. It's still nice to #define this as UCRT +// doesn't support the POSIX thousand separator flag in printf (like "%'u"). #ifdef __MINGW32__ # define __USE_MINGW_ANSI_STDIO 1 #endif @@ -138,7 +145,7 @@ #include <stdlib.h> #include <assert.h> -// Pre-C99 systems lack stdbool.h. All the code in LZMA Utils must be written +// Pre-C99 systems lack stdbool.h. All the code in XZ Utils must be written // so that it works with fake bool type, for example: // // bool foo = (flags & 0x100) != 0; @@ -160,21 +167,21 @@ # define __bool_true_false_are_defined 1 #endif -// string.h should be enough but let's include strings.h and memory.h too if -// they exists, since that shouldn't do any harm, but may improve portability. #include <string.h> -#ifdef HAVE_STRINGS_H -# include <strings.h> -#endif - -#ifdef HAVE_MEMORY_H -# include <memory.h> -#endif - -// As of MSVC 2013 and LCC <= 1.21, inline and restrict are supported with -// non-standard keywords. -#if (defined(_WIN32) && defined(_MSC_VER)) || (defined(__EDG__) && defined(__LCC__)) +// Visual Studio 2013 update 2 supports only __inline, not inline. +// MSVC v19.0 / VS 2015 and newer support both. +// +// MSVC v19.27 (VS 2019 version 16.7) added support for restrict. +// Older ones support only __restrict. +#ifdef _MSC_VER +# if _MSC_VER < 1900 && !defined(inline) +# define inline __inline +# endif +# if _MSC_VER < 1927 && !defined(restrict) +# define restrict __restrict +# endif +#elif defined(__EDG__) && defined(__LCC__) # ifndef inline # define inline __inline # endif
diff --git a/Utilities/cmliblzma/common/tuklib_common.h b/Utilities/cmliblzma/common/tuklib_common.h index 31fbab5..7554dfc 100644 --- a/Utilities/cmliblzma/common/tuklib_common.h +++ b/Utilities/cmliblzma/common/tuklib_common.h
@@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file tuklib_common.h @@ -5,16 +7,13 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef TUKLIB_COMMON_H #define TUKLIB_COMMON_H // The config file may be replaced by a package-specific file. -// It should include at least stddef.h, inttypes.h, and limits.h. +// It should include at least stddef.h, stdbool.h, inttypes.h, and limits.h. #include "tuklib_config.h" // TUKLIB_SYMBOL_PREFIX is prefixed to all symbols exported by @@ -57,8 +56,28 @@ # define TUKLIB_GNUC_REQ(major, minor) 0 #endif -#if TUKLIB_GNUC_REQ(2, 5) +// tuklib_attr_noreturn attribute is used to mark functions as non-returning. +// We cannot use "noreturn" as the macro name because then C23 code that +// uses [[noreturn]] would break as it would expand to [[ [[noreturn]] ]]. +// +// tuklib_attr_noreturn must be used at the beginning of function declaration +// to work in all cases. The [[noreturn]] syntax is the most limiting, it +// must be even before any GNU C's __attribute__ keywords: +// +// tuklib_attr_noreturn +// __attribute__((nonnull(1))) +// extern void foo(const char *s); +// +// FIXME: Update __STDC_VERSION__ for the final C23 version. 202000 is used +// by GCC 13 and Clang 15 with -std=c2x. +#if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 202000 +# define tuklib_attr_noreturn [[noreturn]] +#elif defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112 +# define tuklib_attr_noreturn _Noreturn +#elif TUKLIB_GNUC_REQ(2, 5) # define tuklib_attr_noreturn __attribute__((__noreturn__)) +#elif defined(_MSC_VER) +# define tuklib_attr_noreturn __declspec(noreturn) #else # define tuklib_attr_noreturn #endif
diff --git a/Utilities/cmliblzma/common/tuklib_config.h b/Utilities/cmliblzma/common/tuklib_config.h index 549cb24..b27251d 100644 --- a/Utilities/cmliblzma/common/tuklib_config.h +++ b/Utilities/cmliblzma/common/tuklib_config.h
@@ -1,7 +1,12 @@ +// SPDX-License-Identifier: 0BSD + +// If config.h isn't available, assume that the headers required by +// tuklib_common.h are available. This is required by crc32_tablegen.c. #ifdef HAVE_CONFIG_H # include "sysdefs.h" #else # include <stddef.h> +# include <stdbool.h> # include <inttypes.h> # include <limits.h> #endif
diff --git a/Utilities/cmliblzma/common/tuklib_cpucores.c b/Utilities/cmliblzma/common/tuklib_cpucores.c index cc968dd..c4a781a 100644 --- a/Utilities/cmliblzma/common/tuklib_cpucores.c +++ b/Utilities/cmliblzma/common/tuklib_cpucores.c
@@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file tuklib_cpucores.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "tuklib_cpucores.h" @@ -72,7 +71,16 @@ } #elif defined(TUKLIB_CPUCORES_SYSCTL) + // On OpenBSD HW_NCPUONLINE tells the number of processor cores that + // are online so it is preferred over HW_NCPU which also counts cores + // that aren't currently available. The number of cores online is + // often less than HW_NCPU because OpenBSD disables simultaneous + // multi-threading (SMT) by default. +# ifdef HW_NCPUONLINE + int name[2] = { CTL_HW, HW_NCPUONLINE }; +# else int name[2] = { CTL_HW, HW_NCPU }; +# endif int cpus; size_t cpus_size = sizeof(cpus); if (sysctl(name, 2, &cpus, &cpus_size, NULL, 0) != -1
diff --git a/Utilities/cmliblzma/common/tuklib_cpucores.h b/Utilities/cmliblzma/common/tuklib_cpucores.h index be1ce1c..edff939 100644 --- a/Utilities/cmliblzma/common/tuklib_cpucores.h +++ b/Utilities/cmliblzma/common/tuklib_cpucores.h
@@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file tuklib_cpucores.h @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef TUKLIB_CPUCORES_H
diff --git a/Utilities/cmliblzma/common/tuklib_integer.h b/Utilities/cmliblzma/common/tuklib_integer.h index 4e61357..ea6af84 100644 --- a/Utilities/cmliblzma/common/tuklib_integer.h +++ b/Utilities/cmliblzma/common/tuklib_integer.h
@@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file tuklib_integer.h @@ -14,11 +16,11 @@ /// /// Endianness-converting integer operations (these can be macros!) /// (XX = 16, 32, or 64; Y = b or l): -/// - Byte swapping: bswapXX(num) +/// - Byte swapping: byteswapXX(num) /// - Byte order conversions to/from native (byteswaps if Y isn't /// the native endianness): convXXYe(num) -/// - Unaligned reads (16/32-bit only): readXXYe(ptr) -/// - Unaligned writes (16/32-bit only): writeXXYe(ptr, num) +/// - Unaligned reads: readXXYe(ptr) +/// - Unaligned writes: writeXXYe(ptr, num) /// - Aligned reads: aligned_readXXYe(ptr) /// - Aligned writes: aligned_writeXXYe(ptr, num) /// @@ -37,9 +39,6 @@ // Authors: Lasse Collin // Joachim Henke // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef TUKLIB_INTEGER_H @@ -52,6 +51,12 @@ // and such functions. #if defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 1500) # include <immintrin.h> +// Only include <intrin.h> when it is needed. GCC and Clang can both +// use __builtin's, so we only need Windows instrincs when using MSVC. +// GCC and Clang can set _MSC_VER on Windows, so we need to exclude these +// cases explicitly. +#elif defined(_MSC_VER) && !TUKLIB_GNUC_REQ(3, 4) && !defined(__clang__) +# include <intrin.h> #endif @@ -61,38 +66,47 @@ #if defined(HAVE___BUILTIN_BSWAPXX) // GCC >= 4.8 and Clang -# define bswap16(n) __builtin_bswap16(n) -# define bswap32(n) __builtin_bswap32(n) -# define bswap64(n) __builtin_bswap64(n) +# define byteswap16(num) __builtin_bswap16(num) +# define byteswap32(num) __builtin_bswap32(num) +# define byteswap64(num) __builtin_bswap64(num) #elif defined(HAVE_BYTESWAP_H) // glibc, uClibc, dietlibc # include <byteswap.h> # ifdef HAVE_BSWAP_16 -# define bswap16(num) bswap_16(num) +# define byteswap16(num) bswap_16(num) # endif # ifdef HAVE_BSWAP_32 -# define bswap32(num) bswap_32(num) +# define byteswap32(num) bswap_32(num) # endif # ifdef HAVE_BSWAP_64 -# define bswap64(num) bswap_64(num) +# define byteswap64(num) bswap_64(num) # endif #elif defined(HAVE_SYS_ENDIAN_H) // *BSDs and Darwin # include <sys/endian.h> +# ifdef __OpenBSD__ +# define byteswap16(num) swap16(num) +# define byteswap32(num) swap32(num) +# define byteswap64(num) swap64(num) +# else +# define byteswap16(num) bswap16(num) +# define byteswap32(num) bswap32(num) +# define byteswap64(num) bswap64(num) +# endif #elif defined(HAVE_SYS_BYTEORDER_H) // Solaris # include <sys/byteorder.h> # ifdef BSWAP_16 -# define bswap16(num) BSWAP_16(num) +# define byteswap16(num) BSWAP_16(num) # endif # ifdef BSWAP_32 -# define bswap32(num) BSWAP_32(num) +# define byteswap32(num) BSWAP_32(num) # endif # ifdef BSWAP_64 -# define bswap64(num) BSWAP_64(num) +# define byteswap64(num) BSWAP_64(num) # endif # ifdef BE_16 # define conv16be(num) BE_16(num) @@ -114,15 +128,15 @@ # endif #endif -#ifndef bswap16 -# define bswap16(n) (uint16_t)( \ +#ifndef byteswap16 +# define byteswap16(n) (uint16_t)( \ (((n) & 0x00FFU) << 8) \ | (((n) & 0xFF00U) >> 8) \ ) #endif -#ifndef bswap32 -# define bswap32(n) (uint32_t)( \ +#ifndef byteswap32 +# define byteswap32(n) (uint32_t)( \ (((n) & UINT32_C(0x000000FF)) << 24) \ | (((n) & UINT32_C(0x0000FF00)) << 8) \ | (((n) & UINT32_C(0x00FF0000)) >> 8) \ @@ -130,8 +144,8 @@ ) #endif -#ifndef bswap64 -# define bswap64(n) (uint64_t)( \ +#ifndef byteswap64 +# define byteswap64(n) (uint64_t)( \ (((n) & UINT64_C(0x00000000000000FF)) << 56) \ | (((n) & UINT64_C(0x000000000000FF00)) << 40) \ | (((n) & UINT64_C(0x0000000000FF0000)) << 24) \ @@ -155,23 +169,23 @@ # define conv64be(num) ((uint64_t)(num)) # endif # ifndef conv16le -# define conv16le(num) bswap16(num) +# define conv16le(num) byteswap16(num) # endif # ifndef conv32le -# define conv32le(num) bswap32(num) +# define conv32le(num) byteswap32(num) # endif # ifndef conv64le -# define conv64le(num) bswap64(num) +# define conv64le(num) byteswap64(num) # endif #else # ifndef conv16be -# define conv16be(num) bswap16(num) +# define conv16be(num) byteswap16(num) # endif # ifndef conv32be -# define conv32be(num) bswap32(num) +# define conv32be(num) byteswap32(num) # endif # ifndef conv64be -# define conv64be(num) bswap64(num) +# define conv64be(num) byteswap64(num) # endif # ifndef conv16le # define conv16le(num) ((uint16_t)(num)) @@ -189,6 +203,9 @@ // Unaligned reads and writes // //////////////////////////////// +// No-strict-align archs like x86-64 +// --------------------------------- +// // The traditional way of casting e.g. *(const uint16_t *)uint8_pointer // is bad even if the uint8_pointer is properly aligned because this kind // of casts break strict aliasing rules and result in undefined behavior. @@ -203,12 +220,115 @@ // build time. A third method, casting to a packed struct, would also be // an option but isn't provided to keep things simpler (it's already a mess). // Hopefully this is flexible enough in practice. +// +// Some compilers on x86-64 like Clang >= 10 and GCC >= 5.1 detect that +// +// buf[0] | (buf[1] << 8) +// +// reads a 16-bit value and can emit a single 16-bit load and produce +// identical code than with the memcpy() method. In other cases Clang and GCC +// produce either the same or better code with memcpy(). For example, Clang 9 +// on x86-64 can detect 32-bit load but not 16-bit load. +// +// MSVC uses unaligned access with the memcpy() method but emits byte-by-byte +// code for "buf[0] | (buf[1] << 8)". +// +// Conclusion: The memcpy() method is the best choice when unaligned access +// is supported. +// +// Strict-align archs like SPARC +// ----------------------------- +// +// GCC versions from around 4.x to to at least 13.2.0 produce worse code +// from the memcpy() method than from simple byte-by-byte shift-or code +// when reading a 32-bit integer: +// +// (1) It may be constructed on stack using four 8-bit loads, +// four 8-bit stores to stack, and finally one 32-bit load from stack. +// +// (2) Especially with -Os, an actual memcpy() call may be emitted. +// +// This is true on at least on ARM, ARM64, SPARC, SPARC64, MIPS64EL, and +// RISC-V. Of these, ARM, ARM64, and RISC-V support unaligned access in +// some processors but not all so this is relevant only in the case when +// GCC assumes that unaligned is not supported or -mstrict-align or +// -mno-unaligned-access is used. +// +// For Clang it makes little difference. ARM64 with -O2 -mstrict-align +// was one the very few with a minor difference: the memcpy() version +// was one instruction longer. +// +// Conclusion: At least in case of GCC and Clang, byte-by-byte code is +// the best choice for strict-align archs to do unaligned access. +// +// See also: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=111502 +// +// Thanks to <https://godbolt.org/> it was easy to test different compilers. +// The following is for little endian targets: +/* +#include <stdint.h> +#include <string.h> + +uint32_t bytes16(const uint8_t *b) +{ + return (uint32_t)b[0] + | ((uint32_t)b[1] << 8); +} + +uint32_t copy16(const uint8_t *b) +{ + uint16_t v; + memcpy(&v, b, sizeof(v)); + return v; +} + +uint32_t bytes32(const uint8_t *b) +{ + return (uint32_t)b[0] + | ((uint32_t)b[1] << 8) + | ((uint32_t)b[2] << 16) + | ((uint32_t)b[3] << 24); +} + +uint32_t copy32(const uint8_t *b) +{ + uint32_t v; + memcpy(&v, b, sizeof(v)); + return v; +} + +void wbytes16(uint8_t *b, uint16_t v) +{ + b[0] = (uint8_t)v; + b[1] = (uint8_t)(v >> 8); +} + +void wcopy16(uint8_t *b, uint16_t v) +{ + memcpy(b, &v, sizeof(v)); +} + +void wbytes32(uint8_t *b, uint32_t v) +{ + b[0] = (uint8_t)v; + b[1] = (uint8_t)(v >> 8); + b[2] = (uint8_t)(v >> 16); + b[3] = (uint8_t)(v >> 24); +} + +void wcopy32(uint8_t *b, uint32_t v) +{ + memcpy(b, &v, sizeof(v)); +} +*/ + + +#ifdef TUKLIB_FAST_UNALIGNED_ACCESS static inline uint16_t read16ne(const uint8_t *buf) { -#if defined(TUKLIB_FAST_UNALIGNED_ACCESS) \ - && defined(TUKLIB_USE_UNSAFE_TYPE_PUNNING) +#ifdef TUKLIB_USE_UNSAFE_TYPE_PUNNING return *(const uint16_t *)buf; #else uint16_t num; @@ -221,8 +341,7 @@ static inline uint32_t read32ne(const uint8_t *buf) { -#if defined(TUKLIB_FAST_UNALIGNED_ACCESS) \ - && defined(TUKLIB_USE_UNSAFE_TYPE_PUNNING) +#ifdef TUKLIB_USE_UNSAFE_TYPE_PUNNING return *(const uint32_t *)buf; #else uint32_t num; @@ -235,8 +354,7 @@ static inline uint64_t read64ne(const uint8_t *buf) { -#if defined(TUKLIB_FAST_UNALIGNED_ACCESS) \ - && defined(TUKLIB_USE_UNSAFE_TYPE_PUNNING) +#ifdef TUKLIB_USE_UNSAFE_TYPE_PUNNING return *(const uint64_t *)buf; #else uint64_t num; @@ -249,8 +367,7 @@ static inline void write16ne(uint8_t *buf, uint16_t num) { -#if defined(TUKLIB_FAST_UNALIGNED_ACCESS) \ - && defined(TUKLIB_USE_UNSAFE_TYPE_PUNNING) +#ifdef TUKLIB_USE_UNSAFE_TYPE_PUNNING *(uint16_t *)buf = num; #else memcpy(buf, &num, sizeof(num)); @@ -262,8 +379,7 @@ static inline void write32ne(uint8_t *buf, uint32_t num) { -#if defined(TUKLIB_FAST_UNALIGNED_ACCESS) \ - && defined(TUKLIB_USE_UNSAFE_TYPE_PUNNING) +#ifdef TUKLIB_USE_UNSAFE_TYPE_PUNNING *(uint32_t *)buf = num; #else memcpy(buf, &num, sizeof(num)); @@ -275,8 +391,7 @@ static inline void write64ne(uint8_t *buf, uint64_t num) { -#if defined(TUKLIB_FAST_UNALIGNED_ACCESS) \ - && defined(TUKLIB_USE_UNSAFE_TYPE_PUNNING) +#ifdef TUKLIB_USE_UNSAFE_TYPE_PUNNING *(uint64_t *)buf = num; #else memcpy(buf, &num, sizeof(num)); @@ -288,58 +403,48 @@ static inline uint16_t read16be(const uint8_t *buf) { -#if defined(WORDS_BIGENDIAN) || defined(TUKLIB_FAST_UNALIGNED_ACCESS) uint16_t num = read16ne(buf); return conv16be(num); -#else - uint16_t num = ((uint16_t)buf[0] << 8) | (uint16_t)buf[1]; - return num; -#endif } static inline uint16_t read16le(const uint8_t *buf) { -#if !defined(WORDS_BIGENDIAN) || defined(TUKLIB_FAST_UNALIGNED_ACCESS) uint16_t num = read16ne(buf); return conv16le(num); -#else - uint16_t num = ((uint16_t)buf[0]) | ((uint16_t)buf[1] << 8); - return num; -#endif } static inline uint32_t read32be(const uint8_t *buf) { -#if defined(WORDS_BIGENDIAN) || defined(TUKLIB_FAST_UNALIGNED_ACCESS) uint32_t num = read32ne(buf); return conv32be(num); -#else - uint32_t num = (uint32_t)buf[0] << 24; - num |= (uint32_t)buf[1] << 16; - num |= (uint32_t)buf[2] << 8; - num |= (uint32_t)buf[3]; - return num; -#endif } static inline uint32_t read32le(const uint8_t *buf) { -#if !defined(WORDS_BIGENDIAN) || defined(TUKLIB_FAST_UNALIGNED_ACCESS) uint32_t num = read32ne(buf); return conv32le(num); -#else - uint32_t num = (uint32_t)buf[0]; - num |= (uint32_t)buf[1] << 8; - num |= (uint32_t)buf[2] << 16; - num |= (uint32_t)buf[3] << 24; - return num; -#endif +} + + +static inline uint64_t +read64be(const uint8_t *buf) +{ + uint64_t num = read64ne(buf); + return conv64be(num); +} + + +static inline uint64_t +read64le(const uint8_t *buf) +{ + uint64_t num = read64ne(buf); + return conv64le(num); } @@ -347,18 +452,100 @@ // to optimize byte swapping of constants when using glibc's or *BSD's // byte swapping macros. The actual write is done in an inline function // to make type checking of the buf pointer possible. -#if defined(WORDS_BIGENDIAN) || defined(TUKLIB_FAST_UNALIGNED_ACCESS) -# define write16be(buf, num) write16ne(buf, conv16be(num)) -# define write32be(buf, num) write32ne(buf, conv32be(num)) -#endif +#define write16be(buf, num) write16ne(buf, conv16be(num)) +#define write32be(buf, num) write32ne(buf, conv32be(num)) +#define write64be(buf, num) write64ne(buf, conv64be(num)) +#define write16le(buf, num) write16ne(buf, conv16le(num)) +#define write32le(buf, num) write32ne(buf, conv32le(num)) +#define write64le(buf, num) write64ne(buf, conv64le(num)) -#if !defined(WORDS_BIGENDIAN) || defined(TUKLIB_FAST_UNALIGNED_ACCESS) -# define write16le(buf, num) write16ne(buf, conv16le(num)) -# define write32le(buf, num) write32ne(buf, conv32le(num)) +#else + +#ifdef WORDS_BIGENDIAN +# define read16ne read16be +# define read32ne read32be +# define read64ne read64be +# define write16ne write16be +# define write32ne write32be +# define write64ne write64be +#else +# define read16ne read16le +# define read32ne read32le +# define read64ne read64le +# define write16ne write16le +# define write32ne write32le +# define write64ne write64le #endif -#ifndef write16be +static inline uint16_t +read16be(const uint8_t *buf) +{ + uint16_t num = ((uint16_t)buf[0] << 8) | (uint16_t)buf[1]; + return num; +} + + +static inline uint16_t +read16le(const uint8_t *buf) +{ + uint16_t num = ((uint16_t)buf[0]) | ((uint16_t)buf[1] << 8); + return num; +} + + +static inline uint32_t +read32be(const uint8_t *buf) +{ + uint32_t num = (uint32_t)buf[0] << 24; + num |= (uint32_t)buf[1] << 16; + num |= (uint32_t)buf[2] << 8; + num |= (uint32_t)buf[3]; + return num; +} + + +static inline uint32_t +read32le(const uint8_t *buf) +{ + uint32_t num = (uint32_t)buf[0]; + num |= (uint32_t)buf[1] << 8; + num |= (uint32_t)buf[2] << 16; + num |= (uint32_t)buf[3] << 24; + return num; +} + + +static inline uint64_t +read64be(const uint8_t *buf) +{ + uint64_t num = (uint64_t)buf[0] << 56; + num |= (uint64_t)buf[1] << 48; + num |= (uint64_t)buf[2] << 40; + num |= (uint64_t)buf[3] << 32; + num |= (uint64_t)buf[4] << 24; + num |= (uint64_t)buf[5] << 16; + num |= (uint64_t)buf[6] << 8; + num |= (uint64_t)buf[7]; + return num; +} + + +static inline uint64_t +read64le(const uint8_t *buf) +{ + uint64_t num = (uint64_t)buf[0]; + num |= (uint64_t)buf[1] << 8; + num |= (uint64_t)buf[2] << 16; + num |= (uint64_t)buf[3] << 24; + num |= (uint64_t)buf[4] << 32; + num |= (uint64_t)buf[5] << 40; + num |= (uint64_t)buf[6] << 48; + num |= (uint64_t)buf[7] << 56; + return num; +} + + static inline void write16be(uint8_t *buf, uint16_t num) { @@ -366,10 +553,8 @@ buf[1] = (uint8_t)num; return; } -#endif -#ifndef write16le static inline void write16le(uint8_t *buf, uint16_t num) { @@ -377,10 +562,8 @@ buf[1] = (uint8_t)(num >> 8); return; } -#endif -#ifndef write32be static inline void write32be(uint8_t *buf, uint32_t num) { @@ -390,10 +573,8 @@ buf[3] = (uint8_t)num; return; } -#endif -#ifndef write32le static inline void write32le(uint8_t *buf, uint32_t num) { @@ -403,6 +584,37 @@ buf[3] = (uint8_t)(num >> 24); return; } + + +static inline void +write64be(uint8_t *buf, uint64_t num) +{ + buf[0] = (uint8_t)(num >> 56); + buf[1] = (uint8_t)(num >> 48); + buf[2] = (uint8_t)(num >> 40); + buf[3] = (uint8_t)(num >> 32); + buf[4] = (uint8_t)(num >> 24); + buf[5] = (uint8_t)(num >> 16); + buf[6] = (uint8_t)(num >> 8); + buf[7] = (uint8_t)num; + return; +} + + +static inline void +write64le(uint8_t *buf, uint64_t num) +{ + buf[0] = (uint8_t)num; + buf[1] = (uint8_t)(num >> 8); + buf[2] = (uint8_t)(num >> 16); + buf[3] = (uint8_t)(num >> 24); + buf[4] = (uint8_t)(num >> 32); + buf[5] = (uint8_t)(num >> 40); + buf[6] = (uint8_t)(num >> 48); + buf[7] = (uint8_t)(num >> 56); + return; +} + #endif @@ -421,7 +633,7 @@ // aligned but some compilers have language extensions to do that. With // such language extensions the memcpy() method gives excellent results. // -// What to do on a strict-align system when no known language extentensions +// What to do on a strict-align system when no known language extensions // are available? Falling back to byte-by-byte access would be safe but ruin // optimizations that have been made specifically with aligned access in mind. // As a compromise, aligned reads will fall back to non-compliant type punning @@ -588,7 +800,7 @@ #if defined(__INTEL_COMPILER) return _bit_scan_reverse(n); -#elif TUKLIB_GNUC_REQ(3, 4) && UINT_MAX == UINT32_MAX +#elif (TUKLIB_GNUC_REQ(3, 4) || defined(__clang__)) && UINT_MAX == UINT32_MAX // GCC >= 3.4 has __builtin_clz(), which gives good results on // multiple architectures. On x86, __builtin_clz() ^ 31U becomes // either plain BSR (so the XOR gets optimized away) or LZCNT and @@ -637,7 +849,7 @@ #if defined(__INTEL_COMPILER) return _bit_scan_reverse(n) ^ 31U; -#elif TUKLIB_GNUC_REQ(3, 4) && UINT_MAX == UINT32_MAX +#elif (TUKLIB_GNUC_REQ(3, 4) || defined(__clang__)) && UINT_MAX == UINT32_MAX return (uint32_t)__builtin_clz(n); #elif defined(__GNUC__) && (defined(__i386__) || defined(__x86_64__)) @@ -684,7 +896,7 @@ #if defined(__INTEL_COMPILER) return _bit_scan_forward(n); -#elif TUKLIB_GNUC_REQ(3, 4) && UINT_MAX >= UINT32_MAX +#elif (TUKLIB_GNUC_REQ(3, 4) || defined(__clang__)) && UINT_MAX >= UINT32_MAX return (uint32_t)__builtin_ctz(n); #elif defined(__GNUC__) && (defined(__i386__) || defined(__x86_64__))
diff --git a/Utilities/cmliblzma/liblzma/api/lzma.h b/Utilities/cmliblzma/liblzma/api/lzma.h index 122dab8..6ca6e50 100644 --- a/Utilities/cmliblzma/liblzma/api/lzma.h +++ b/Utilities/cmliblzma/liblzma/api/lzma.h
@@ -1,30 +1,30 @@ +/* SPDX-License-Identifier: 0BSD */ + /** * \file api/lzma.h * \brief The public API of liblzma data compression library + * \mainpage * - * liblzma is a public domain general-purpose data compression library with - * a zlib-like API. The native file format is .xz, but also the old .lzma - * format and raw (no headers) streams are supported. Multiple compression - * algorithms (filters) are supported. Currently LZMA2 is the primary filter. + * liblzma is a general-purpose data compression library with a zlib-like API. + * The native file format is .xz, but also the old .lzma format and raw (no + * headers) streams are supported. Multiple compression algorithms (filters) + * are supported. Currently LZMA2 is the primary filter. * - * liblzma is part of XZ Utils <http://tukaani.org/xz/>. XZ Utils includes - * a gzip-like command line tool named xz and some other tools. XZ Utils - * is developed and maintained by Lasse Collin. + * liblzma is part of XZ Utils <https://tukaani.org/xz/>. XZ Utils + * includes a gzip-like command line tool named xz and some other tools. + * XZ Utils is developed and maintained by Lasse Collin. * - * Major parts of liblzma are based on Igor Pavlov's public domain LZMA SDK - * <http://7-zip.org/sdk.html>. + * Major parts of liblzma are based on code written by Igor Pavlov, + * specifically the LZMA SDK <https://7-zip.org/sdk.html>. * - * The SHA-256 implementation is based on the public domain code found from - * 7-Zip <http://7-zip.org/>, which has a modified version of the public - * domain SHA-256 code found from Crypto++ <http://www.cryptopp.com/>. - * The SHA-256 code in Crypto++ was written by Kevin Springle and Wei Dai. + * The SHA-256 implementation in liblzma is based on code written by + * Wei Dai in Crypto++ Library <https://www.cryptopp.com/>. + * + * liblzma is distributed under the BSD Zero Clause License (0BSD). */ /* * Author: Lasse Collin - * - * This file has been put into the public domain. - * You can do whatever you want with this file. */ #ifndef LZMA_H @@ -181,11 +181,11 @@ * against static liblzma on them, don't worry about LZMA_API_STATIC. That * is, most developers will never need to use LZMA_API_STATIC. * - * The GCC variants are a special case on Windows (Cygwin and MinGW). + * The GCC variants are a special case on Windows (Cygwin and MinGW-w64). * We rely on GCC doing the right thing with its auto-import feature, * and thus don't use __declspec(dllimport). This way developers don't * need to worry about LZMA_API_STATIC. Also the calling convention is - * omitted on Cygwin but not on MinGW. + * omitted on Cygwin but not on MinGW-w64. */ #ifndef LZMA_API_IMPORT # if !defined(LZMA_API_STATIC) && defined(_WIN32) && !defined(__GNUC__) @@ -219,7 +219,8 @@ */ #ifndef lzma_nothrow # if defined(__cplusplus) -# if __cplusplus >= 201103L +# if __cplusplus >= 201103L || (defined(_MSVC_LANG) \ + && _MSVC_LANG >= 201103L) # define lzma_nothrow noexcept # else # define lzma_nothrow throw()
diff --git a/Utilities/cmliblzma/liblzma/api/lzma/base.h b/Utilities/cmliblzma/liblzma/api/lzma/base.h index a6005ac..590e1d2 100644 --- a/Utilities/cmliblzma/liblzma/api/lzma/base.h +++ b/Utilities/cmliblzma/liblzma/api/lzma/base.h
@@ -1,15 +1,13 @@ +/* SPDX-License-Identifier: 0BSD */ + /** * \file lzma/base.h * \brief Data types and functions used in many places in liblzma API + * \note Never include this file directly. Use <lzma.h> instead. */ /* * Author: Lasse Collin - * - * This file has been put into the public domain. - * You can do whatever you want with this file. - * - * See ../lzma.h for information about liblzma as a whole. */ #ifndef LZMA_H_INTERNAL @@ -22,8 +20,8 @@ * * This is here because C89 doesn't have stdbool.h. To set a value for * variables having type lzma_bool, you can use - * - C99's `true' and `false' from stdbool.h; - * - C++'s internal `true' and `false'; or + * - C99's 'true' and 'false' from stdbool.h; + * - C++'s internal 'true' and 'false'; or * - integers one (true) and zero (false). */ typedef unsigned char lzma_bool; @@ -138,13 +136,19 @@ */ LZMA_MEMLIMIT_ERROR = 6, - /** + /**< * \brief Memory usage limit was reached * * Decoder would need more memory than allowed by the * specified memory usage limit. To continue decoding, * the memory usage limit has to be increased with * lzma_memlimit_set(). + * + * liblzma 5.2.6 and earlier had a bug in single-threaded .xz + * decoder (lzma_stream_decoder()) which made it impossible + * to continue decoding after LZMA_MEMLIMIT_ERROR even if + * the limit was increased using lzma_memlimit_set(). + * Other decoders worked correctly. */ LZMA_FORMAT_ERROR = 7, @@ -234,17 +238,47 @@ * can be a sign of a bug in liblzma. See the documentation * how to report bugs. */ + + LZMA_SEEK_NEEDED = 12, + /**< + * \brief Request to change the input file position + * + * Some coders can do random access in the input file. The + * initialization functions of these coders take the file size + * as an argument. No other coders can return LZMA_SEEK_NEEDED. + * + * When this value is returned, the application must seek to + * the file position given in lzma_stream.seek_pos. This value + * is guaranteed to never exceed the file size that was + * specified at the coder initialization. + * + * After seeking the application should read new input and + * pass it normally via lzma_stream.next_in and .avail_in. + */ + + /* + * These enumerations may be used internally by liblzma + * but they will never be returned to applications. + */ + LZMA_RET_INTERNAL1 = 101, + LZMA_RET_INTERNAL2 = 102, + LZMA_RET_INTERNAL3 = 103, + LZMA_RET_INTERNAL4 = 104, + LZMA_RET_INTERNAL5 = 105, + LZMA_RET_INTERNAL6 = 106, + LZMA_RET_INTERNAL7 = 107, + LZMA_RET_INTERNAL8 = 108 } lzma_ret; /** - * \brief The `action' argument for lzma_code() + * \brief The 'action' argument for lzma_code() * * After the first use of LZMA_SYNC_FLUSH, LZMA_FULL_FLUSH, LZMA_FULL_BARRIER, - * or LZMA_FINISH, the same `action' must is used until lzma_code() returns + * or LZMA_FINISH, the same 'action' must be used until lzma_code() returns * LZMA_STREAM_END. Also, the amount of input (that is, strm->avail_in) must * not be modified by the application until lzma_code() returns - * LZMA_STREAM_END. Changing the `action' or modifying the amount of input + * LZMA_STREAM_END. Changing the 'action' or modifying the amount of input * will make lzma_code() return LZMA_PROG_ERROR. */ typedef enum { @@ -358,8 +392,8 @@ * Single-threaded mode only: liblzma doesn't make an internal copy of * lzma_allocator. Thus, it is OK to change these function pointers in * the middle of the coding process, but obviously it must be done - * carefully to make sure that the replacement `free' can deallocate - * memory allocated by the earlier `alloc' function(s). + * carefully to make sure that the replacement 'free' can deallocate + * memory allocated by the earlier 'alloc' function(s). * * Multithreaded mode: liblzma might internally store pointers to the * lzma_allocator given via the lzma_stream structure. The application @@ -387,7 +421,7 @@ * liblzma never sets this to zero. * * \return Pointer to the beginning of a memory block of - * `size' bytes, or NULL if allocation fails + * 'size' bytes, or NULL if allocation fails * for some reason. When allocation fails, functions * of liblzma return LZMA_MEM_ERROR. * @@ -447,7 +481,7 @@ * * The lzma_stream structure is used for * - passing pointers to input and output buffers to liblzma; - * - defining custom memory hander functions; and + * - defining custom memory handler functions; and * - holding a pointer to coder-specific internal data structures. * * Typical usage: @@ -510,15 +544,44 @@ * you should not touch these, because the names of these variables * may change. */ + + /** \private Reserved member. */ void *reserved_ptr1; + + /** \private Reserved member. */ void *reserved_ptr2; + + /** \private Reserved member. */ void *reserved_ptr3; + + /** \private Reserved member. */ void *reserved_ptr4; - uint64_t reserved_int1; + + /** + * \brief New seek input position for LZMA_SEEK_NEEDED + * + * When lzma_code() returns LZMA_SEEK_NEEDED, the new input position + * needed by liblzma will be available seek_pos. The value is + * guaranteed to not exceed the file size that was specified when + * this lzma_stream was initialized. + * + * In all other situations the value of this variable is undefined. + */ + uint64_t seek_pos; + + /** \private Reserved member. */ uint64_t reserved_int2; + + /** \private Reserved member. */ size_t reserved_int3; + + /** \private Reserved member. */ size_t reserved_int4; + + /** \private Reserved member. */ lzma_reserved_enum reserved_enum1; + + /** \private Reserved member. */ lzma_reserved_enum reserved_enum2; } lzma_stream; @@ -558,7 +621,15 @@ * to and get output from liblzma. * * See the description of the coder-specific initialization function to find - * out what `action' values are supported by the coder. + * out what 'action' values are supported by the coder. + * + * \param strm Pointer to lzma_stream that is at least initialized + * with LZMA_STREAM_INIT. + * \param action Action for this function to take. Must be a valid + * lzma_action enum value. + * + * \return Any valid lzma_ret. See the lzma_ret enum description for more + * information. */ extern LZMA_API(lzma_ret) lzma_code(lzma_stream *strm, lzma_action action) lzma_nothrow lzma_attr_warn_unused_result; @@ -567,15 +638,15 @@ /** * \brief Free memory allocated for the coder data structures * - * \param strm Pointer to lzma_stream that is at least initialized - * with LZMA_STREAM_INIT. - * * After lzma_end(strm), strm->internal is guaranteed to be NULL. No other * members of the lzma_stream structure are touched. * * \note zlib indicates an error if application end()s unfinished * stream structure. liblzma doesn't do this, and assumes that * application knows what it is doing. + * + * \param strm Pointer to lzma_stream that is at least initialized + * with LZMA_STREAM_INIT. */ extern LZMA_API(void) lzma_end(lzma_stream *strm) lzma_nothrow; @@ -594,6 +665,11 @@ * mode by taking into account the progress made by each thread. In * single-threaded mode *progress_in and *progress_out are set to * strm->total_in and strm->total_out, respectively. + * + * \param strm Pointer to lzma_stream that is at least + * initialized with LZMA_STREAM_INIT. + * \param[out] progress_in Pointer to the number of input bytes processed. + * \param[out] progress_out Pointer to the number of output bytes processed. */ extern LZMA_API(void) lzma_get_progress(lzma_stream *strm, uint64_t *progress_in, uint64_t *progress_out) lzma_nothrow; @@ -612,6 +688,9 @@ * this may give misleading information if decoding .xz Streams that have * multiple Blocks, because each Block can have different memory requirements. * + * \param strm Pointer to lzma_stream that is at least initialized + * with LZMA_STREAM_INIT. + * * \return How much memory is currently allocated for the filter * decoders. If no filter chain is currently allocated, * some non-zero value is still returned, which is less than @@ -631,6 +710,9 @@ * This function is supported only when *strm has been initialized with * a function that takes a memlimit argument. * + * \param strm Pointer to lzma_stream that is at least initialized + * with LZMA_STREAM_INIT. + * * \return On success, the current memory usage limit is returned * (always non-zero). On error, zero is returned. */ @@ -649,7 +731,13 @@ * return LZMA_OK. Later versions treat 0 as if 1 had been specified (so * lzma_memlimit_get() will return 1 even if you specify 0 here). * - * \return - LZMA_OK: New memory usage limit successfully set. + * liblzma 5.2.6 and earlier had a bug in single-threaded .xz decoder + * (lzma_stream_decoder()) which made it impossible to continue decoding + * after LZMA_MEMLIMIT_ERROR even if the limit was increased using + * lzma_memlimit_set(). Other decoders worked correctly. + * + * \return Possible lzma_ret values: + * - LZMA_OK: New memory usage limit successfully set. * - LZMA_MEMLIMIT_ERROR: The new limit is too small. * The limit was not changed. * - LZMA_PROG_ERROR: Invalid arguments, e.g. *strm doesn't
diff --git a/Utilities/cmliblzma/liblzma/api/lzma/bcj.h b/Utilities/cmliblzma/liblzma/api/lzma/bcj.h index 8e37538..7f6611f 100644 --- a/Utilities/cmliblzma/liblzma/api/lzma/bcj.h +++ b/Utilities/cmliblzma/liblzma/api/lzma/bcj.h
@@ -1,15 +1,13 @@ +/* SPDX-License-Identifier: 0BSD */ + /** * \file lzma/bcj.h * \brief Branch/Call/Jump conversion filters + * \note Never include this file directly. Use <lzma.h> instead. */ /* * Author: Lasse Collin - * - * This file has been put into the public domain. - * You can do whatever you want with this file. - * - * See ../lzma.h for information about liblzma as a whole. */ #ifndef LZMA_H_INTERNAL @@ -19,35 +17,45 @@ /* Filter IDs for lzma_filter.id */ +/** + * \brief Filter for x86 binaries + */ #define LZMA_FILTER_X86 LZMA_VLI_C(0x04) - /**< - * Filter for x86 binaries - */ +/** + * \brief Filter for Big endian PowerPC binaries + */ #define LZMA_FILTER_POWERPC LZMA_VLI_C(0x05) - /**< - * Filter for Big endian PowerPC binaries - */ +/** + * \brief Filter for IA-64 (Itanium) binaries + */ #define LZMA_FILTER_IA64 LZMA_VLI_C(0x06) - /**< - * Filter for IA-64 (Itanium) binaries. - */ +/** + * \brief Filter for ARM binaries + */ #define LZMA_FILTER_ARM LZMA_VLI_C(0x07) - /**< - * Filter for ARM binaries. - */ +/** + * \brief Filter for ARM-Thumb binaries + */ #define LZMA_FILTER_ARMTHUMB LZMA_VLI_C(0x08) - /**< - * Filter for ARM-Thumb binaries. - */ +/** + * \brief Filter for SPARC binaries + */ #define LZMA_FILTER_SPARC LZMA_VLI_C(0x09) - /**< - * Filter for SPARC binaries. - */ + +/** + * \brief Filter for ARM64 binaries + */ +#define LZMA_FILTER_ARM64 LZMA_VLI_C(0x0A) + +/** + * \brief Filter for RISC-V binaries + */ +#define LZMA_FILTER_RISCV LZMA_VLI_C(0x0B) /**
diff --git a/Utilities/cmliblzma/liblzma/api/lzma/block.h b/Utilities/cmliblzma/liblzma/api/lzma/block.h index 962f387..05b77e5 100644 --- a/Utilities/cmliblzma/liblzma/api/lzma/block.h +++ b/Utilities/cmliblzma/liblzma/api/lzma/block.h
@@ -1,15 +1,13 @@ +/* SPDX-License-Identifier: 0BSD */ + /** * \file lzma/block.h * \brief .xz Block handling + * \note Never include this file directly. Use <lzma.h> instead. */ /* * Author: Lasse Collin - * - * This file has been put into the public domain. - * You can do whatever you want with this file. - * - * See ../lzma.h for information about liblzma as a whole. */ #ifndef LZMA_H_INTERNAL @@ -32,19 +30,28 @@ * \brief Block format version * * To prevent API and ABI breakages when new features are needed, - * a version number is used to indicate which fields in this + * a version number is used to indicate which members in this * structure are in use: * - liblzma >= 5.0.0: version = 0 is supported. * - liblzma >= 5.1.4beta: Support for version = 1 was added, - * which adds the ignore_check field. + * which adds the ignore_check member. * * If version is greater than one, most Block related functions * will return LZMA_OPTIONS_ERROR (lzma_block_header_decode() works * with any version value). * * Read by: - * - All functions that take pointer to lzma_block as argument, - * including lzma_block_header_decode(). + * - lzma_block_header_size() + * - lzma_block_header_encode() + * - lzma_block_header_decode() + * - lzma_block_compressed_size() + * - lzma_block_unpadded_size() + * - lzma_block_total_size() + * - lzma_block_encoder() + * - lzma_block_decoder() + * - lzma_block_buffer_encode() + * - lzma_block_uncomp_encode() + * - lzma_block_buffer_decode() * * Written by: * - lzma_block_header_decode() @@ -52,7 +59,7 @@ uint32_t version; /** - * \brief Size of the Block Header field + * \brief Size of the Block Header field in bytes * * This is always a multiple of four. * @@ -68,6 +75,7 @@ * Written by: * - lzma_block_header_size() * - lzma_block_buffer_encode() + * - lzma_block_uncomp_encode() */ uint32_t header_size; # define LZMA_BLOCK_HEADER_SIZE_MIN 8 @@ -143,6 +151,7 @@ * - lzma_block_encoder() * - lzma_block_decoder() * - lzma_block_buffer_encode() + * - lzma_block_uncomp_encode() * - lzma_block_buffer_decode() */ lzma_vli compressed_size; @@ -167,6 +176,7 @@ * - lzma_block_encoder() * - lzma_block_decoder() * - lzma_block_buffer_encode() + * - lzma_block_uncomp_encode() * - lzma_block_buffer_decode() */ lzma_vli uncompressed_size; @@ -212,6 +222,7 @@ * - lzma_block_encoder() * - lzma_block_decoder() * - lzma_block_buffer_encode() + * - lzma_block_uncomp_encode() * - lzma_block_buffer_decode() */ uint8_t raw_check[LZMA_CHECK_SIZE_MAX]; @@ -223,26 +234,56 @@ * with the currently supported options, so it is safe to leave these * uninitialized. */ + + /** \private Reserved member. */ void *reserved_ptr1; + + /** \private Reserved member. */ void *reserved_ptr2; + + /** \private Reserved member. */ void *reserved_ptr3; + + /** \private Reserved member. */ uint32_t reserved_int1; + + /** \private Reserved member. */ uint32_t reserved_int2; + + /** \private Reserved member. */ lzma_vli reserved_int3; + + /** \private Reserved member. */ lzma_vli reserved_int4; + + /** \private Reserved member. */ lzma_vli reserved_int5; + + /** \private Reserved member. */ lzma_vli reserved_int6; + + /** \private Reserved member. */ lzma_vli reserved_int7; + + /** \private Reserved member. */ lzma_vli reserved_int8; + + /** \private Reserved member. */ lzma_reserved_enum reserved_enum1; + + /** \private Reserved member. */ lzma_reserved_enum reserved_enum2; + + /** \private Reserved member. */ lzma_reserved_enum reserved_enum3; + + /** \private Reserved member. */ lzma_reserved_enum reserved_enum4; /** * \brief A flag to Block decoder to not verify the Check field * - * This field is supported by liblzma >= 5.1.4beta if .version >= 1. + * This member is supported by liblzma >= 5.1.4beta if .version >= 1. * * If this is set to true, the integrity check won't be calculated * and verified. Unless you know what you are doing, you should @@ -260,12 +301,25 @@ */ lzma_bool ignore_check; + /** \private Reserved member. */ lzma_bool reserved_bool2; + + /** \private Reserved member. */ lzma_bool reserved_bool3; + + /** \private Reserved member. */ lzma_bool reserved_bool4; + + /** \private Reserved member. */ lzma_bool reserved_bool5; + + /** \private Reserved member. */ lzma_bool reserved_bool6; + + /** \private Reserved member. */ lzma_bool reserved_bool7; + + /** \private Reserved member. */ lzma_bool reserved_bool8; } lzma_block; @@ -280,7 +334,8 @@ * Note that if the first byte is 0x00, it indicates beginning of Index; use * this macro only when the byte is not 0x00. * - * There is no encoding macro, because Block Header encoder is enough for that. + * There is no encoding macro because lzma_block_header_size() and + * lzma_block_header_encode() should be used. */ #define lzma_block_header_size_decode(b) (((uint32_t)(b) + 1) * 4) @@ -294,17 +349,20 @@ * four and doesn't exceed LZMA_BLOCK_HEADER_SIZE_MAX. Increasing header_size * just means that lzma_block_header_encode() will add Header Padding. * - * \return - LZMA_OK: Size calculated successfully and stored to - * block->header_size. - * - LZMA_OPTIONS_ERROR: Unsupported version, filters or - * filter options. - * - LZMA_PROG_ERROR: Invalid values like compressed_size == 0. - * * \note This doesn't check that all the options are valid i.e. this * may return LZMA_OK even if lzma_block_header_encode() or * lzma_block_encoder() would fail. If you want to validate the * filter chain, consider using lzma_memlimit_encoder() which as * a side-effect validates the filter chain. + * + * \param block Block options + * + * \return Possible lzma_ret values: + * - LZMA_OK: Size calculated successfully and stored to + * block->header_size. + * - LZMA_OPTIONS_ERROR: Unsupported version, filters or + * filter options. + * - LZMA_PROG_ERROR: Invalid values like compressed_size == 0. */ extern LZMA_API(lzma_ret) lzma_block_header_size(lzma_block *block) lzma_nothrow lzma_attr_warn_unused_result; @@ -318,11 +376,12 @@ * lzma_block_header_size() is used, the Block Header will be padded to the * specified size. * - * \param out Beginning of the output buffer. This must be - * at least block->header_size bytes. * \param block Block options to be encoded. + * \param[out] out Beginning of the output buffer. This must be + * at least block->header_size bytes. * - * \return - LZMA_OK: Encoding was successful. block->header_size + * \return Possible lzma_ret values: + * - LZMA_OK: Encoding was successful. block->header_size * bytes were written to output buffer. * - LZMA_OPTIONS_ERROR: Invalid or unsupported options. * - LZMA_PROG_ERROR: Invalid arguments, for example @@ -354,14 +413,15 @@ * block->filters must have been allocated, but they don't need to be * initialized (possible existing filter options are not freed). * - * \param block Destination for Block options. + * \param[out] block Destination for Block options * \param allocator lzma_allocator for custom allocator functions. * Set to NULL to use malloc() (and also free() * if an error occurs). * \param in Beginning of the input buffer. This must be * at least block->header_size bytes. * - * \return - LZMA_OK: Decoding was successful. block->header_size + * \return Possible lzma_ret values: + * - LZMA_OK: Decoding was successful. block->header_size * bytes were read from the input buffer. * - LZMA_OPTIONS_ERROR: The Block Header specifies some * unsupported options such as unsupported filters. This can @@ -398,7 +458,12 @@ * field so that it can properly validate Compressed Size if it * was present in Block Header. * - * \return - LZMA_OK: block->compressed_size was set successfully. + * \param block Block options: block->header_size must + * already be set with lzma_block_header_size(). + * \param unpadded_size Unpadded Size from the Index field in bytes + * + * \return Possible lzma_ret values: + * - LZMA_OK: block->compressed_size was set successfully. * - LZMA_DATA_ERROR: unpadded_size is too small compared to * block->header_size and lzma_check_size(block->check). * - LZMA_PROG_ERROR: Some values are invalid. For example, @@ -419,6 +484,9 @@ * Compressed Size, and size of the Check field. This is where this function * is needed. * + * \param block Block options: block->header_size must already be + * set with lzma_block_header_size(). + * * \return Unpadded Size on success, or zero on error. */ extern LZMA_API(lzma_vli) lzma_block_unpadded_size(const lzma_block *block) @@ -431,6 +499,9 @@ * This is equivalent to lzma_block_unpadded_size() except that the returned * value includes the size of the Block Padding field. * + * \param block Block options: block->header_size must already be + * set with lzma_block_header_size(). + * * \return On success, total encoded size of the Block. On error, * zero is returned. */ @@ -444,7 +515,17 @@ * Valid actions for lzma_code() are LZMA_RUN, LZMA_SYNC_FLUSH (only if the * filter chain supports it), and LZMA_FINISH. * - * \return - LZMA_OK: All good, continue with lzma_code(). + * The Block encoder encodes the Block Data, Block Padding, and Check value. + * It does NOT encode the Block Header which can be encoded with + * lzma_block_header_encode(). + * + * \param strm Pointer to lzma_stream that is at least initialized + * with LZMA_STREAM_INIT. + * \param block Block options: block->version, block->check, + * and block->filters must have been initialized. + * + * \return Possible lzma_ret values: + * - LZMA_OK: All good, continue with lzma_code(). * - LZMA_MEM_ERROR * - LZMA_OPTIONS_ERROR * - LZMA_UNSUPPORTED_CHECK: block->check specifies a Check ID @@ -463,10 +544,16 @@ * Valid actions for lzma_code() are LZMA_RUN and LZMA_FINISH. Using * LZMA_FINISH is not required. It is supported only for convenience. * - * \return - LZMA_OK: All good, continue with lzma_code(). - * - LZMA_UNSUPPORTED_CHECK: Initialization was successful, but - * the given Check ID is not supported, thus Check will be - * ignored. + * The Block decoder decodes the Block Data, Block Padding, and Check value. + * It does NOT decode the Block Header which can be decoded with + * lzma_block_header_decode(). + * + * \param strm Pointer to lzma_stream that is at least initialized + * with LZMA_STREAM_INIT. + * \param block Block options + * + * \return Possible lzma_ret values: + * - LZMA_OK: All good, continue with lzma_code(). * - LZMA_PROG_ERROR * - LZMA_MEM_ERROR */ @@ -480,6 +567,11 @@ * * This is equivalent to lzma_stream_buffer_bound() but for .xz Blocks. * See the documentation of lzma_stream_buffer_bound(). + * + * \param uncompressed_size Size of the data to be encoded with the + * single-call Block encoder. + * + * \return Maximum output size in bytes for single-call Block encoding. */ extern LZMA_API(size_t) lzma_block_buffer_bound(size_t uncompressed_size) lzma_nothrow; @@ -508,13 +600,14 @@ * Set to NULL to use malloc() and free(). * \param in Beginning of the input buffer * \param in_size Size of the input buffer - * \param out Beginning of the output buffer - * \param out_pos The next byte will be written to out[*out_pos]. + * \param[out] out Beginning of the output buffer + * \param[out] out_pos The next byte will be written to out[*out_pos]. * *out_pos is updated only if encoding succeeds. * \param out_size Size of the out buffer; the first byte into * which no data is written to is out[out_size]. * - * \return - LZMA_OK: Encoding was successful. + * \return Possible lzma_ret values: + * - LZMA_OK: Encoding was successful. * - LZMA_BUF_ERROR: Not enough output buffer space. * - LZMA_UNSUPPORTED_CHECK * - LZMA_OPTIONS_ERROR @@ -540,6 +633,25 @@ * Since the data won't be compressed, this function ignores block->filters. * This function doesn't take lzma_allocator because this function doesn't * allocate any memory from the heap. + * + * \param block Block options: block->version, block->check, + * and block->filters must have been initialized. + * \param in Beginning of the input buffer + * \param in_size Size of the input buffer + * \param[out] out Beginning of the output buffer + * \param[out] out_pos The next byte will be written to out[*out_pos]. + * *out_pos is updated only if encoding succeeds. + * \param out_size Size of the out buffer; the first byte into + * which no data is written to is out[out_size]. + * + * \return Possible lzma_ret values: + * - LZMA_OK: Encoding was successful. + * - LZMA_BUF_ERROR: Not enough output buffer space. + * - LZMA_UNSUPPORTED_CHECK + * - LZMA_OPTIONS_ERROR + * - LZMA_MEM_ERROR + * - LZMA_DATA_ERROR + * - LZMA_PROG_ERROR */ extern LZMA_API(lzma_ret) lzma_block_uncomp_encode(lzma_block *block, const uint8_t *in, size_t in_size, @@ -553,7 +665,7 @@ * This is single-call equivalent of lzma_block_decoder(), and requires that * the caller has already decoded Block Header and checked its memory usage. * - * \param block Block options just like with lzma_block_decoder(). + * \param block Block options * \param allocator lzma_allocator for custom allocator functions. * Set to NULL to use malloc() and free(). * \param in Beginning of the input buffer @@ -561,13 +673,14 @@ * *in_pos is updated only if decoding succeeds. * \param in_size Size of the input buffer; the first byte that * won't be read is in[in_size]. - * \param out Beginning of the output buffer - * \param out_pos The next byte will be written to out[*out_pos]. + * \param[out] out Beginning of the output buffer + * \param[out] out_pos The next byte will be written to out[*out_pos]. * *out_pos is updated only if encoding succeeds. * \param out_size Size of the out buffer; the first byte into * which no data is written to is out[out_size]. * - * \return - LZMA_OK: Decoding was successful. + * \return Possible lzma_ret values: + * - LZMA_OK: Decoding was successful. * - LZMA_OPTIONS_ERROR * - LZMA_DATA_ERROR * - LZMA_MEM_ERROR
diff --git a/Utilities/cmliblzma/liblzma/api/lzma/check.h b/Utilities/cmliblzma/liblzma/api/lzma/check.h index 6a243db..e7a50ed 100644 --- a/Utilities/cmliblzma/liblzma/api/lzma/check.h +++ b/Utilities/cmliblzma/liblzma/api/lzma/check.h
@@ -1,15 +1,13 @@ +/* SPDX-License-Identifier: 0BSD */ + /** * \file lzma/check.h * \brief Integrity checks + * \note Never include this file directly. Use <lzma.h> instead. */ /* * Author: Lasse Collin - * - * This file has been put into the public domain. - * You can do whatever you want with this file. - * - * See ../lzma.h for information about liblzma as a whole. */ #ifndef LZMA_H_INTERNAL @@ -71,12 +69,17 @@ /** * \brief Test if the given Check ID is supported * - * Return true if the given Check ID is supported by this liblzma build. - * Otherwise false is returned. It is safe to call this with a value that - * is not in the range [0, 15]; in that case the return value is always false. + * LZMA_CHECK_NONE and LZMA_CHECK_CRC32 are always supported (even if + * liblzma is built with limited features). * - * You can assume that LZMA_CHECK_NONE and LZMA_CHECK_CRC32 are always - * supported (even if liblzma is built with limited features). + * \note It is safe to call this with a value that is not in the + * range [0, 15]; in that case the return value is always false. + * + * \param check Check ID + * + * \return lzma_bool: + * - true if Check ID is supported by this liblzma build. + * - false otherwise. */ extern LZMA_API(lzma_bool) lzma_check_is_supported(lzma_check check) lzma_nothrow lzma_attr_const; @@ -90,7 +93,10 @@ * the Check field with the specified Check ID. The values are: * { 0, 4, 4, 4, 8, 8, 8, 16, 16, 16, 32, 32, 32, 64, 64, 64 } * - * If the argument is not in the range [0, 15], UINT32_MAX is returned. + * \param check Check ID + * + * \return Size of the Check field in bytes. If the argument is not in + * the range [0, 15], UINT32_MAX is returned. */ extern LZMA_API(uint32_t) lzma_check_size(lzma_check check) lzma_nothrow lzma_attr_const; @@ -126,25 +132,32 @@ * * Calculate CRC64 using the polynomial from the ECMA-182 standard. * - * This function is used similarly to lzma_crc32(). See its documentation. + * This function is used similarly to lzma_crc32(). + * + * \param buf Pointer to the input buffer + * \param size Size of the input buffer + * \param crc Previously returned CRC value. This is used to + * calculate the CRC of a big buffer in smaller chunks. + * Set to zero when starting a new calculation. + * + * \return Updated CRC value, which can be passed to this function + * again to continue CRC calculation. */ extern LZMA_API(uint64_t) lzma_crc64( const uint8_t *buf, size_t size, uint64_t crc) lzma_nothrow lzma_attr_pure; -/* - * SHA-256 functions are currently not exported to public API. - * Contact Lasse Collin if you think it should be. - */ - - /** * \brief Get the type of the integrity check * * This function can be called only immediately after lzma_code() has * returned LZMA_NO_CHECK, LZMA_UNSUPPORTED_CHECK, or LZMA_GET_CHECK. * Calling this function in any other situation has undefined behavior. + * + * \param strm Pointer to lzma_stream meeting the above conditions. + * + * \return Check ID in the lzma_stream, or undefined if called improperly. */ extern LZMA_API(lzma_check) lzma_get_check(const lzma_stream *strm) lzma_nothrow;
diff --git a/Utilities/cmliblzma/liblzma/api/lzma/container.h b/Utilities/cmliblzma/liblzma/api/lzma/container.h index 9fbf4df..ee5d77e 100644 --- a/Utilities/cmliblzma/liblzma/api/lzma/container.h +++ b/Utilities/cmliblzma/liblzma/api/lzma/container.h
@@ -1,15 +1,13 @@ +/* SPDX-License-Identifier: 0BSD */ + /** * \file lzma/container.h * \brief File formats + * \note Never include this file directly. Use <lzma.h> instead. */ /* * Author: Lasse Collin - * - * This file has been put into the public domain. - * You can do whatever you want with this file. - * - * See ../lzma.h for information about liblzma as a whole. */ #ifndef LZMA_H_INTERNAL @@ -51,7 +49,7 @@ * * This flag modifies the preset to make the encoding significantly slower * while improving the compression ratio only marginally. This is useful - * when you don't mind wasting time to get as small result as possible. + * when you don't mind spending time to get as small result as possible. * * This flag doesn't affect the memory usage requirements of the decoder (at * least not significantly). The memory usage of the encoder may be increased @@ -69,7 +67,15 @@ * * Set this to zero if no flags are wanted. * - * No flags are currently supported. + * Encoder: No flags are currently supported. + * + * Decoder: Bitwise-or of zero or more of the decoder flags: + * - LZMA_TELL_NO_CHECK + * - LZMA_TELL_UNSUPPORTED_CHECK + * - LZMA_TELL_ANY_CHECK + * - LZMA_IGNORE_CHECK + * - LZMA_CONCATENATED + * - LZMA_FAIL_FAST */ uint32_t flags; @@ -79,7 +85,7 @@ uint32_t threads; /** - * \brief Maximum uncompressed size of a Block + * \brief Encoder only: Maximum uncompressed size of a Block * * The encoder will start a new .xz Block every block_size bytes. * Using LZMA_FULL_FLUSH or LZMA_FULL_BARRIER with lzma_code() @@ -106,7 +112,7 @@ /** * \brief Timeout to allow lzma_code() to return early * - * Multithreading can make liblzma to consume input and produce + * Multithreading can make liblzma consume input and produce * output in a very bursty way: it may first read a lot of input * to fill internal buffers, then no input or output occurs for * a while. @@ -123,19 +129,18 @@ * LZMA_OK. Reasonable values are 100 ms or more. The xz command * line tool uses 300 ms. * - * If long blocking times are fine for you, set timeout to a special - * value of 0, which will disable the timeout mechanism and will make + * If long blocking times are acceptable, set timeout to a special + * value of 0. This will disable the timeout mechanism and will make * lzma_code() block until all the input is consumed or the output * buffer has been filled. * * \note Even with a timeout, lzma_code() might sometimes take - * somewhat long time to return. No timing guarantees - * are made. + * a long time to return. No timing guarantees are made. */ uint32_t timeout; /** - * \brief Compression preset (level and possible flags) + * \brief Encoder only: Compression preset * * The preset is set just like with lzma_easy_encoder(). * The preset is ignored if filters below is non-NULL. @@ -143,7 +148,7 @@ uint32_t preset; /** - * \brief Filter chain (alternative to a preset) + * \brief Encoder only: Filter chain (alternative to a preset) * * If this is NULL, the preset above is used. Otherwise the preset * is ignored and the filter chain specified here is used. @@ -151,7 +156,7 @@ const lzma_filter *filters; /** - * \brief Integrity check type + * \brief Encoder only: Integrity check type * * See check.h for available checks. The xz command line tool * defaults to LZMA_CHECK_CRC64, which is a good choice if you @@ -166,20 +171,86 @@ * with the currently supported options, so it is safe to leave these * uninitialized. */ + /** \private Reserved member. */ lzma_reserved_enum reserved_enum1; + + /** \private Reserved member. */ lzma_reserved_enum reserved_enum2; + + /** \private Reserved member. */ lzma_reserved_enum reserved_enum3; + + /** \private Reserved member. */ uint32_t reserved_int1; + + /** \private Reserved member. */ uint32_t reserved_int2; + + /** \private Reserved member. */ uint32_t reserved_int3; + + /** \private Reserved member. */ uint32_t reserved_int4; - uint64_t reserved_int5; - uint64_t reserved_int6; + + /** + * \brief Memory usage limit to reduce the number of threads + * + * Encoder: Ignored. + * + * Decoder: + * + * If the number of threads has been set so high that more than + * memlimit_threading bytes of memory would be needed, the number + * of threads will be reduced so that the memory usage will not exceed + * memlimit_threading bytes. However, if memlimit_threading cannot + * be met even in single-threaded mode, then decoding will continue + * in single-threaded mode and memlimit_threading may be exceeded + * even by a large amount. That is, memlimit_threading will never make + * lzma_code() return LZMA_MEMLIMIT_ERROR. To truly cap the memory + * usage, see memlimit_stop below. + * + * Setting memlimit_threading to UINT64_MAX or a similar huge value + * means that liblzma is allowed to keep the whole compressed file + * and the whole uncompressed file in memory in addition to the memory + * needed by the decompressor data structures used by each thread! + * In other words, a reasonable value limit must be set here or it + * will cause problems sooner or later. If you have no idea what + * a reasonable value could be, try lzma_physmem() / 4 as a starting + * point. Setting this limit will never prevent decompression of + * a file; this will only reduce the number of threads. + * + * If memlimit_threading is greater than memlimit_stop, then the value + * of memlimit_stop will be used for both. + */ + uint64_t memlimit_threading; + + /** + * \brief Memory usage limit that should never be exceeded + * + * Encoder: Ignored. + * + * Decoder: If decompressing will need more than this amount of + * memory even in the single-threaded mode, then lzma_code() will + * return LZMA_MEMLIMIT_ERROR. + */ + uint64_t memlimit_stop; + + /** \private Reserved member. */ uint64_t reserved_int7; + + /** \private Reserved member. */ uint64_t reserved_int8; + + /** \private Reserved member. */ void *reserved_ptr1; + + /** \private Reserved member. */ void *reserved_ptr2; + + /** \private Reserved member. */ void *reserved_ptr3; + + /** \private Reserved member. */ void *reserved_ptr4; } lzma_mt; @@ -193,8 +264,7 @@ * \param preset Compression preset (level and possible flags) * * \return Number of bytes of memory required for the given - * preset when encoding. If an error occurs, for example - * due to unsupported preset, UINT64_MAX is returned. + * preset when encoding or UINT64_MAX on error. */ extern LZMA_API(uint64_t) lzma_easy_encoder_memusage(uint32_t preset) lzma_nothrow lzma_attr_pure; @@ -208,9 +278,8 @@ * \param preset Compression preset (level and possible flags) * * \return Number of bytes of memory required to decompress a file - * that was compressed using the given preset. If an error - * occurs, for example due to unsupported preset, UINT64_MAX - * is returned. + * that was compressed using the given preset or UINT64_MAX + * on error. */ extern LZMA_API(uint64_t) lzma_easy_decoder_memusage(uint32_t preset) lzma_nothrow lzma_attr_pure; @@ -220,7 +289,16 @@ * \brief Initialize .xz Stream encoder using a preset number * * This function is intended for those who just want to use the basic features - * if liblzma (that is, most developers out there). + * of liblzma (that is, most developers out there). + * + * If initialization fails (return value is not LZMA_OK), all the memory + * allocated for *strm by liblzma is always freed. Thus, there is no need + * to call lzma_end() after failed initialization. + * + * If initialization succeeds, use lzma_code() to do the actual encoding. + * Valid values for 'action' (the second argument of lzma_code()) are + * LZMA_RUN, LZMA_SYNC_FLUSH, LZMA_FULL_FLUSH, and LZMA_FINISH. In future, + * there may be compression levels or flags that don't support LZMA_SYNC_FLUSH. * * \param strm Pointer to lzma_stream that is at least initialized * with LZMA_STREAM_INIT. @@ -228,7 +306,7 @@ * number and zero or more flags. Usually flags aren't * used, so preset is simply a number [0, 9] which match * the options -0 ... -9 of the xz command line tool. - * Additional flags can be be set using bitwise-or with + * Additional flags can be set using bitwise-or with * the preset level number, e.g. 6 | LZMA_PRESET_EXTREME. * \param check Integrity check type to use. See check.h for available * checks. The xz command line tool defaults to @@ -236,7 +314,8 @@ * unsure. LZMA_CHECK_CRC32 is good too as long as the * uncompressed file is not many gigabytes. * - * \return - LZMA_OK: Initialization succeeded. Use lzma_code() to + * \return Possible lzma_ret values: + * - LZMA_OK: Initialization succeeded. Use lzma_code() to * encode your data. * - LZMA_MEM_ERROR: Memory allocation failed. * - LZMA_OPTIONS_ERROR: The given compression preset is not @@ -245,15 +324,6 @@ * supported by this liblzma build. * - LZMA_PROG_ERROR: One or more of the parameters have values * that will never be valid. For example, strm == NULL. - * - * If initialization fails (return value is not LZMA_OK), all the memory - * allocated for *strm by liblzma is always freed. Thus, there is no need - * to call lzma_end() after failed initialization. - * - * If initialization succeeds, use lzma_code() to do the actual encoding. - * Valid values for `action' (the second argument of lzma_code()) are - * LZMA_RUN, LZMA_SYNC_FLUSH, LZMA_FULL_FLUSH, and LZMA_FINISH. In future, - * there may be compression levels or flags that don't support LZMA_SYNC_FLUSH. */ extern LZMA_API(lzma_ret) lzma_easy_encoder( lzma_stream *strm, uint32_t preset, lzma_check check) @@ -274,13 +344,14 @@ * Set to NULL to use malloc() and free(). * \param in Beginning of the input buffer * \param in_size Size of the input buffer - * \param out Beginning of the output buffer - * \param out_pos The next byte will be written to out[*out_pos]. + * \param[out] out Beginning of the output buffer + * \param[out] out_pos The next byte will be written to out[*out_pos]. * *out_pos is updated only if encoding succeeds. * \param out_size Size of the out buffer; the first byte into * which no data is written to is out[out_size]. * - * \return - LZMA_OK: Encoding was successful. + * \return Possible lzma_ret values: + * - LZMA_OK: Encoding was successful. * - LZMA_BUF_ERROR: Not enough output buffer space. * - LZMA_UNSUPPORTED_CHECK * - LZMA_OPTIONS_ERROR @@ -298,14 +369,16 @@ /** * \brief Initialize .xz Stream encoder using a custom filter chain * - * \param strm Pointer to properly prepared lzma_stream - * \param filters Array of filters. This must be terminated with - * filters[n].id = LZMA_VLI_UNKNOWN. See filter.h for - * more information. + * \param strm Pointer to lzma_stream that is at least initialized + * with LZMA_STREAM_INIT. + * \param filters Array of filters terminated with + * .id == LZMA_VLI_UNKNOWN. See filters.h for more + * information. * \param check Type of the integrity check to calculate from * uncompressed data. * - * \return - LZMA_OK: Initialization was successful. + * \return Possible lzma_ret values: + * - LZMA_OK: Initialization was successful. * - LZMA_MEM_ERROR * - LZMA_UNSUPPORTED_CHECK * - LZMA_OPTIONS_ERROR @@ -345,10 +418,12 @@ * LZMA_FULL_BARRIER, and LZMA_FINISH. Support for LZMA_SYNC_FLUSH might be * added in the future. * - * \param strm Pointer to properly prepared lzma_stream + * \param strm Pointer to lzma_stream that is at least initialized + * with LZMA_STREAM_INIT. * \param options Pointer to multithreaded compression options * - * \return - LZMA_OK + * \return Possible lzma_ret values: + * - LZMA_OK * - LZMA_MEM_ERROR * - LZMA_UNSUPPORTED_CHECK * - LZMA_OPTIONS_ERROR @@ -360,6 +435,34 @@ /** + * \brief Calculate recommended Block size for multithreaded .xz encoder + * + * This calculates a recommended Block size for multithreaded encoding given + * a filter chain. This is used internally by lzma_stream_encoder_mt() to + * determine the Block size if the block_size member is not set to the + * special value of 0 in the lzma_mt options struct. + * + * If one wishes to change the filters between Blocks, this function is + * helpful to set the block_size member of the lzma_mt struct before calling + * lzma_stream_encoder_mt(). Since the block_size member represents the + * maximum possible Block size for the multithreaded .xz encoder, one can + * use this function to find the maximum recommended Block size based on + * all planned filter chains. Otherwise, the multithreaded encoder will + * base its maximum Block size on the first filter chain used (if the + * block_size member is not set), which may unnecessarily limit the Block + * size for a later filter chain. + * + * \param filters Array of filters terminated with + * .id == LZMA_VLI_UNKNOWN. + * + * \return Recommended Block size in bytes, or UINT64_MAX if + * an error occurred. + */ +extern LZMA_API(uint64_t) lzma_mt_block_size(const lzma_filter *filters) + lzma_nothrow; + + +/** * \brief Initialize .lzma encoder (legacy file format) * * The .lzma format is sometimes called the LZMA_Alone format, which is the @@ -374,7 +477,12 @@ * No kind of flushing is supported, because the file format doesn't make * it possible. * - * \return - LZMA_OK + * \param strm Pointer to lzma_stream that is at least initialized + * with LZMA_STREAM_INIT. + * \param options Pointer to encoder options + * + * \return Possible lzma_ret values: + * - LZMA_OK * - LZMA_MEM_ERROR * - LZMA_OPTIONS_ERROR * - LZMA_PROG_ERROR @@ -387,7 +495,7 @@ /** * \brief Calculate output buffer size for single-call Stream encoder * - * When trying to compress uncompressible data, the encoded size will be + * When trying to compress incompressible data, the encoded size will be * slightly bigger than the input data. This function calculates how much * output buffer space is required to be sure that lzma_stream_buffer_encode() * doesn't return LZMA_BUF_ERROR. @@ -403,8 +511,13 @@ * \note The limit calculated by this function applies only to * single-call encoding. Multi-call encoding may (and probably * will) have larger maximum expansion when encoding - * uncompressible data. Currently there is no function to + * incompressible data. Currently there is no function to * calculate the maximum expansion of multi-call encoding. + * + * \param uncompressed_size Size in bytes of the uncompressed + * input data + * + * \return Maximum number of bytes needed to store the compressed data. */ extern LZMA_API(size_t) lzma_stream_buffer_bound(size_t uncompressed_size) lzma_nothrow; @@ -413,22 +526,23 @@ /** * \brief Single-call .xz Stream encoder * - * \param filters Array of filters. This must be terminated with - * filters[n].id = LZMA_VLI_UNKNOWN. See filter.h - * for more information. + * \param filters Array of filters terminated with + * .id == LZMA_VLI_UNKNOWN. See filters.h for more + * information. * \param check Type of the integrity check to calculate from * uncompressed data. * \param allocator lzma_allocator for custom allocator functions. * Set to NULL to use malloc() and free(). * \param in Beginning of the input buffer * \param in_size Size of the input buffer - * \param out Beginning of the output buffer - * \param out_pos The next byte will be written to out[*out_pos]. + * \param[out] out Beginning of the output buffer + * \param[out] out_pos The next byte will be written to out[*out_pos]. * *out_pos is updated only if encoding succeeds. * \param out_size Size of the out buffer; the first byte into * which no data is written to is out[out_size]. * - * \return - LZMA_OK: Encoding was successful. + * \return Possible lzma_ret values: + * - LZMA_OK: Encoding was successful. * - LZMA_BUF_ERROR: Not enough output buffer space. * - LZMA_UNSUPPORTED_CHECK * - LZMA_OPTIONS_ERROR @@ -444,6 +558,66 @@ lzma_nothrow lzma_attr_warn_unused_result; +/** + * \brief MicroLZMA encoder + * + * The MicroLZMA format is a raw LZMA stream whose first byte (always 0x00) + * has been replaced with bitwise-negation of the LZMA properties (lc/lp/pb). + * This encoding ensures that the first byte of MicroLZMA stream is never + * 0x00. There is no end of payload marker and thus the uncompressed size + * must be stored separately. For the best error detection the dictionary + * size should be stored separately as well but alternatively one may use + * the uncompressed size as the dictionary size when decoding. + * + * With the MicroLZMA encoder, lzma_code() behaves slightly unusually. + * The action argument must be LZMA_FINISH and the return value will never be + * LZMA_OK. Thus the encoding is always done with a single lzma_code() after + * the initialization. The benefit of the combination of initialization + * function and lzma_code() is that memory allocations can be re-used for + * better performance. + * + * lzma_code() will try to encode as much input as is possible to fit into + * the given output buffer. If not all input can be encoded, the stream will + * be finished without encoding all the input. The caller must check both + * input and output buffer usage after lzma_code() (total_in and total_out + * in lzma_stream can be convenient). Often lzma_code() can fill the output + * buffer completely if there is a lot of input, but sometimes a few bytes + * may remain unused because the next LZMA symbol would require more space. + * + * lzma_stream.avail_out must be at least 6. Otherwise LZMA_PROG_ERROR + * will be returned. + * + * The LZMA dictionary should be reasonably low to speed up the encoder + * re-initialization. A good value is bigger than the resulting + * uncompressed size of most of the output chunks. For example, if output + * size is 4 KiB, dictionary size of 32 KiB or 64 KiB is good. If the + * data compresses extremely well, even 128 KiB may be useful. + * + * The MicroLZMA format and this encoder variant were made with the EROFS + * file system in mind. This format may be convenient in other embedded + * uses too where many small streams are needed. XZ Embedded includes a + * decoder for this format. + * + * \param strm Pointer to lzma_stream that is at least initialized + * with LZMA_STREAM_INIT. + * \param options Pointer to encoder options + * + * \return Possible lzma_ret values: + * - LZMA_STREAM_END: All good. Check the amounts of input used + * and output produced. Store the amount of input used + * (uncompressed size) as it needs to be known to decompress + * the data. + * - LZMA_OPTIONS_ERROR + * - LZMA_MEM_ERROR + * - LZMA_PROG_ERROR: In addition to the generic reasons for this + * error code, this may also be returned if there isn't enough + * output space (6 bytes) to create a valid MicroLZMA stream. + */ +extern LZMA_API(lzma_ret) lzma_microlzma_encoder( + lzma_stream *strm, const lzma_options_lzma *options) + lzma_nothrow; + + /************ * Decoding * ************/ @@ -501,24 +675,54 @@ /** * This flag enables decoding of concatenated files with file formats that * allow concatenating compressed files as is. From the formats currently - * supported by liblzma, only the .xz format allows concatenated files. - * Concatenated files are not allowed with the legacy .lzma format. + * supported by liblzma, only the .xz and .lz formats allow concatenated + * files. Concatenated files are not allowed with the legacy .lzma format. * - * This flag also affects the usage of the `action' argument for lzma_code(). + * This flag also affects the usage of the 'action' argument for lzma_code(). * When LZMA_CONCATENATED is used, lzma_code() won't return LZMA_STREAM_END - * unless LZMA_FINISH is used as `action'. Thus, the application has to set + * unless LZMA_FINISH is used as 'action'. Thus, the application has to set * LZMA_FINISH in the same way as it does when encoding. * * If LZMA_CONCATENATED is not used, the decoders still accept LZMA_FINISH - * as `action' for lzma_code(), but the usage of LZMA_FINISH isn't required. + * as 'action' for lzma_code(), but the usage of LZMA_FINISH isn't required. */ #define LZMA_CONCATENATED UINT32_C(0x08) /** + * This flag makes the threaded decoder report errors (like LZMA_DATA_ERROR) + * as soon as they are detected. This saves time when the application has no + * interest in a partially decompressed truncated or corrupt file. Note that + * due to timing randomness, if the same truncated or corrupt input is + * decompressed multiple times with this flag, a different amount of output + * may be produced by different runs, and even the error code might vary. + * + * When using LZMA_FAIL_FAST, it is recommended to use LZMA_FINISH to tell + * the decoder when no more input will be coming because it can help fast + * detection and reporting of truncated files. Note that in this situation + * truncated files might be diagnosed with LZMA_DATA_ERROR instead of + * LZMA_OK or LZMA_BUF_ERROR! + * + * Without this flag the threaded decoder will provide as much output as + * possible at first and then report the pending error. This default behavior + * matches the single-threaded decoder and provides repeatable behavior + * with truncated or corrupt input. There are a few special cases where the + * behavior can still differ like memory allocation failures (LZMA_MEM_ERROR). + * + * Single-threaded decoders currently ignore this flag. + * + * Support for this flag was added in liblzma 5.3.3alpha. Note that in older + * versions this flag isn't supported (LZMA_OPTIONS_ERROR) even by functions + * that ignore this flag in newer liblzma versions. + */ +#define LZMA_FAIL_FAST UINT32_C(0x20) + + +/** * \brief Initialize .xz Stream decoder * - * \param strm Pointer to properly prepared lzma_stream + * \param strm Pointer to lzma_stream that is at least initialized + * with LZMA_STREAM_INIT. * \param memlimit Memory usage limit as bytes. Use UINT64_MAX * to effectively disable the limiter. liblzma * 5.2.3 and earlier don't allow 0 here and return @@ -526,9 +730,11 @@ * had been specified. * \param flags Bitwise-or of zero or more of the decoder flags: * LZMA_TELL_NO_CHECK, LZMA_TELL_UNSUPPORTED_CHECK, - * LZMA_TELL_ANY_CHECK, LZMA_CONCATENATED + * LZMA_TELL_ANY_CHECK, LZMA_IGNORE_CHECK, + * LZMA_CONCATENATED, LZMA_FAIL_FAST * - * \return - LZMA_OK: Initialization was successful. + * \return Possible lzma_ret values: + * - LZMA_OK: Initialization was successful. * - LZMA_MEM_ERROR: Cannot allocate memory. * - LZMA_OPTIONS_ERROR: Unsupported flags * - LZMA_PROG_ERROR @@ -539,21 +745,67 @@ /** - * \brief Decode .xz Streams and .lzma files with autodetection + * \brief Initialize multithreaded .xz Stream decoder * - * This decoder autodetects between the .xz and .lzma file formats, and - * calls lzma_stream_decoder() or lzma_alone_decoder() once the type - * of the input file has been detected. + * The decoder can decode multiple Blocks in parallel. This requires that each + * Block Header contains the Compressed Size and Uncompressed size fields + * which are added by the multi-threaded encoder, see lzma_stream_encoder_mt(). * - * \param strm Pointer to properly prepared lzma_stream + * A Stream with one Block will only utilize one thread. A Stream with multiple + * Blocks but without size information in Block Headers will be processed in + * single-threaded mode in the same way as done by lzma_stream_decoder(). + * Concatenated Streams are processed one Stream at a time; no inter-Stream + * parallelization is done. + * + * This function behaves like lzma_stream_decoder() when options->threads == 1 + * and options->memlimit_threading <= 1. + * + * \param strm Pointer to lzma_stream that is at least initialized + * with LZMA_STREAM_INIT. + * \param options Pointer to multithreaded compression options + * + * \return Possible lzma_ret values: + * - LZMA_OK: Initialization was successful. + * - LZMA_MEM_ERROR: Cannot allocate memory. + * - LZMA_MEMLIMIT_ERROR: Memory usage limit was reached. + * - LZMA_OPTIONS_ERROR: Unsupported flags. + * - LZMA_PROG_ERROR + */ +extern LZMA_API(lzma_ret) lzma_stream_decoder_mt( + lzma_stream *strm, const lzma_mt *options) + lzma_nothrow lzma_attr_warn_unused_result; + + +/** + * \brief Decode .xz, .lzma, and .lz (lzip) files with autodetection + * + * This decoder autodetects between the .xz, .lzma, and .lz file formats, + * and calls lzma_stream_decoder(), lzma_alone_decoder(), or + * lzma_lzip_decoder() once the type of the input file has been detected. + * + * Support for .lz was added in 5.4.0. + * + * If the flag LZMA_CONCATENATED is used and the input is a .lzma file: + * For historical reasons concatenated .lzma files aren't supported. + * If there is trailing data after one .lzma stream, lzma_code() will + * return LZMA_DATA_ERROR. (lzma_alone_decoder() doesn't have such a check + * as it doesn't support any decoder flags. It will return LZMA_STREAM_END + * after one .lzma stream.) + * + * \param strm Pointer to lzma_stream that is at least initialized + * with LZMA_STREAM_INIT. * \param memlimit Memory usage limit as bytes. Use UINT64_MAX * to effectively disable the limiter. liblzma * 5.2.3 and earlier don't allow 0 here and return * LZMA_PROG_ERROR; later versions treat 0 as if 1 * had been specified. - * \param flags Bitwise-or of flags, or zero for no flags. + * \param flags Bitwise-or of zero or more of the decoder flags: + * LZMA_TELL_NO_CHECK, LZMA_TELL_UNSUPPORTED_CHECK, + * LZMA_TELL_ANY_CHECK, LZMA_IGNORE_CHECK, + * LZMA_CONCATENATED, LZMA_FAIL_FAST * - * \return - LZMA_OK: Initialization was successful. + * \return Possible lzma_ret values: + * - LZMA_OK: Initialization was successful. * - LZMA_MEM_ERROR: Cannot allocate memory. * - LZMA_OPTIONS_ERROR: Unsupported flags * - LZMA_PROG_ERROR @@ -566,18 +818,20 @@ /** * \brief Initialize .lzma decoder (legacy file format) * - * \param strm Pointer to properly prepared lzma_stream + * Valid 'action' arguments to lzma_code() are LZMA_RUN and LZMA_FINISH. + * There is no need to use LZMA_FINISH, but it's allowed because it may + * simplify certain types of applications. + * + * \param strm Pointer to lzma_stream that is at least initialized + * with LZMA_STREAM_INIT. * \param memlimit Memory usage limit as bytes. Use UINT64_MAX * to effectively disable the limiter. liblzma * 5.2.3 and earlier don't allow 0 here and return * LZMA_PROG_ERROR; later versions treat 0 as if 1 * had been specified. * - * Valid `action' arguments to lzma_code() are LZMA_RUN and LZMA_FINISH. - * There is no need to use LZMA_FINISH, but it's allowed because it may - * simplify certain types of applications. - * - * \return - LZMA_OK + * \return Possible lzma_ret values: + * - LZMA_OK * - LZMA_MEM_ERROR * - LZMA_PROG_ERROR */ @@ -587,6 +841,66 @@ /** + * \brief Initialize .lz (lzip) decoder (a foreign file format) + * + * This decoder supports the .lz format version 0 and the unextended .lz + * format version 1: + * + * - Files in the format version 0 were produced by lzip 1.3 and older. + * Such files aren't common but may be found from file archives + * as a few source packages were released in this format. People + * might have old personal files in this format too. Decompression + * support for the format version 0 was removed in lzip 1.18. + * + * - lzip 1.3 added decompression support for .lz format version 1 files. + * Compression support was added in lzip 1.4. In lzip 1.6 the .lz format + * version 1 was extended to support the Sync Flush marker. This extension + * is not supported by liblzma. lzma_code() will return LZMA_DATA_ERROR + * at the location of the Sync Flush marker. In practice files with + * the Sync Flush marker are very rare and thus liblzma can decompress + * almost all .lz files. + * + * Just like with lzma_stream_decoder() for .xz files, LZMA_CONCATENATED + * should be used when decompressing normal standalone .lz files. + * + * The .lz format allows putting non-.lz data at the end of a file after at + * least one valid .lz member. That is, one can append custom data at the end + * of a .lz file and the decoder is required to ignore it. In liblzma this + * is relevant only when LZMA_CONCATENATED is used. In that case lzma_code() + * will return LZMA_STREAM_END and leave lzma_stream.next_in pointing to + * the first byte of the non-.lz data. An exception to this is if the first + * 1-3 bytes of the non-.lz data are identical to the .lz magic bytes + * (0x4C, 0x5A, 0x49, 0x50; "LZIP" in US-ASCII). In such a case the 1-3 bytes + * will have been ignored by lzma_code(). If one wishes to locate the non-.lz + * data reliably, one must ensure that the first byte isn't 0x4C. Actually + * one should ensure that none of the first four bytes of trailing data are + * equal to the magic bytes because lzip >= 1.20 requires it by default. + * + * \param strm Pointer to lzma_stream that is at least initialized + * with LZMA_STREAM_INIT. + * \param memlimit Memory usage limit as bytes. Use UINT64_MAX + * to effectively disable the limiter. + * \param flags Bitwise-or of flags, or zero for no flags. + * All decoder flags listed above are supported + * although only LZMA_CONCATENATED and (in very rare + * cases) LZMA_IGNORE_CHECK are actually useful. + * LZMA_TELL_NO_CHECK, LZMA_TELL_UNSUPPORTED_CHECK, + * and LZMA_FAIL_FAST do nothing. LZMA_TELL_ANY_CHECK + * is supported for consistency only as CRC32 is + * always used in the .lz format. + * + * \return Possible lzma_ret values: + * - LZMA_OK: Initialization was successful. + * - LZMA_MEM_ERROR: Cannot allocate memory. + * - LZMA_OPTIONS_ERROR: Unsupported flags + * - LZMA_PROG_ERROR + */ +extern LZMA_API(lzma_ret) lzma_lzip_decoder( + lzma_stream *strm, uint64_t memlimit, uint32_t flags) + lzma_nothrow lzma_attr_warn_unused_result; + + +/** * \brief Single-call .xz Stream decoder * * \param memlimit Pointer to how much memory the decoder is allowed @@ -595,7 +909,8 @@ * returned. * \param flags Bitwise-or of zero or more of the decoder flags: * LZMA_TELL_NO_CHECK, LZMA_TELL_UNSUPPORTED_CHECK, - * LZMA_CONCATENATED. Note that LZMA_TELL_ANY_CHECK + * LZMA_IGNORE_CHECK, LZMA_CONCATENATED, + * LZMA_FAIL_FAST. Note that LZMA_TELL_ANY_CHECK * is not allowed and will return LZMA_PROG_ERROR. * \param allocator lzma_allocator for custom allocator functions. * Set to NULL to use malloc() and free(). @@ -604,13 +919,14 @@ * *in_pos is updated only if decoding succeeds. * \param in_size Size of the input buffer; the first byte that * won't be read is in[in_size]. - * \param out Beginning of the output buffer - * \param out_pos The next byte will be written to out[*out_pos]. + * \param[out] out Beginning of the output buffer + * \param[out] out_pos The next byte will be written to out[*out_pos]. * *out_pos is updated only if decoding succeeds. * \param out_size Size of the out buffer; the first byte into * which no data is written to is out[out_size]. * - * \return - LZMA_OK: Decoding was successful. + * \return Possible lzma_ret values: + * - LZMA_OK: Decoding was successful. * - LZMA_FORMAT_ERROR * - LZMA_OPTIONS_ERROR * - LZMA_DATA_ERROR @@ -630,3 +946,50 @@ const uint8_t *in, size_t *in_pos, size_t in_size, uint8_t *out, size_t *out_pos, size_t out_size) lzma_nothrow lzma_attr_warn_unused_result; + + +/** + * \brief MicroLZMA decoder + * + * See lzma_microlzma_encoder() for more information. + * + * The lzma_code() usage with this decoder is completely normal. The + * special behavior of lzma_code() applies to lzma_microlzma_encoder() only. + * + * \param strm Pointer to lzma_stream that is at least initialized + * with LZMA_STREAM_INIT. + * \param comp_size Compressed size of the MicroLZMA stream. + * The caller must somehow know this exactly. + * \param uncomp_size Uncompressed size of the MicroLZMA stream. + * If the exact uncompressed size isn't known, this + * can be set to a value that is at most as big as + * the exact uncompressed size would be, but then the + * next argument uncomp_size_is_exact must be false. + * \param uncomp_size_is_exact + * If true, uncomp_size must be exactly correct. + * This will improve error detection at the end of + * the stream. If the exact uncompressed size isn't + * known, this must be false. uncomp_size must still + * be at most as big as the exact uncompressed size + * is. Setting this to false when the exact size is + * known will work but error detection at the end of + * the stream will be weaker. + * \param dict_size LZMA dictionary size that was used when + * compressing the data. It is OK to use a bigger + * value too but liblzma will then allocate more + * memory than would actually be required and error + * detection will be slightly worse. (Note that with + * the implementation in XZ Embedded it doesn't + * affect the memory usage if one specifies bigger + * dictionary than actually required.) + * + * \return Possible lzma_ret values: + * - LZMA_OK + * - LZMA_MEM_ERROR + * - LZMA_OPTIONS_ERROR + * - LZMA_PROG_ERROR + */ +extern LZMA_API(lzma_ret) lzma_microlzma_decoder( + lzma_stream *strm, uint64_t comp_size, + uint64_t uncomp_size, lzma_bool uncomp_size_is_exact, + uint32_t dict_size) lzma_nothrow;
diff --git a/Utilities/cmliblzma/liblzma/api/lzma/delta.h b/Utilities/cmliblzma/liblzma/api/lzma/delta.h index 592fc4f..5ebacef 100644 --- a/Utilities/cmliblzma/liblzma/api/lzma/delta.h +++ b/Utilities/cmliblzma/liblzma/api/lzma/delta.h
@@ -1,15 +1,13 @@ +/* SPDX-License-Identifier: 0BSD */ + /** * \file lzma/delta.h * \brief Delta filter + * \note Never include this file directly. Use <lzma.h> instead. */ /* * Author: Lasse Collin - * - * This file has been put into the public domain. - * You can do whatever you want with this file. - * - * See ../lzma.h for information about liblzma as a whole. */ #ifndef LZMA_H_INTERNAL @@ -57,7 +55,15 @@ * - 24-bit RGB image data: distance = 3 bytes */ uint32_t dist; + + /** + * \brief Minimum value for lzma_options_delta.dist. + */ # define LZMA_DELTA_DIST_MIN 1 + + /** + * \brief Maximum value for lzma_options_delta.dist. + */ # define LZMA_DELTA_DIST_MAX 256 /* @@ -67,11 +73,23 @@ * when type is LZMA_DELTA_TYPE_BYTE, so it is safe to leave these * uninitialized. */ + + /** \private Reserved member. */ uint32_t reserved_int1; + + /** \private Reserved member. */ uint32_t reserved_int2; + + /** \private Reserved member. */ uint32_t reserved_int3; + + /** \private Reserved member. */ uint32_t reserved_int4; + + /** \private Reserved member. */ void *reserved_ptr1; + + /** \private Reserved member. */ void *reserved_ptr2; } lzma_options_delta;
diff --git a/Utilities/cmliblzma/liblzma/api/lzma/filter.h b/Utilities/cmliblzma/liblzma/api/lzma/filter.h index 8c85931..e86809c 100644 --- a/Utilities/cmliblzma/liblzma/api/lzma/filter.h +++ b/Utilities/cmliblzma/liblzma/api/lzma/filter.h
@@ -1,15 +1,13 @@ +/* SPDX-License-Identifier: 0BSD */ + /** * \file lzma/filter.h * \brief Common filter related types and functions + * \note Never include this file directly. Use <lzma.h> instead. */ /* * Author: Lasse Collin - * - * This file has been put into the public domain. - * You can do whatever you want with this file. - * - * See ../lzma.h for information about liblzma as a whole. */ #ifndef LZMA_H_INTERNAL @@ -29,7 +27,7 @@ /** * \brief Filter options * - * This structure is used to pass Filter ID and a pointer filter's + * This structure is used to pass a Filter ID and a pointer to the filter's * options to liblzma. A few functions work with a single lzma_filter * structure, while most functions expect a filter chain. * @@ -37,14 +35,14 @@ * The array is terminated with .id = LZMA_VLI_UNKNOWN. Thus, the filter * array must have LZMA_FILTERS_MAX + 1 elements (that is, five) to * be able to hold any arbitrary filter chain. This is important when - * using lzma_block_header_decode() from block.h, because too small - * array would make liblzma write past the end of the filters array. + * using lzma_block_header_decode() from block.h, because a filter array + * that is too small would make liblzma write past the end of the array. */ typedef struct { /** * \brief Filter ID * - * Use constants whose name begin with `LZMA_FILTER_' to specify + * Use constants whose name begin with 'LZMA_FILTER_' to specify * different filters. In an array of lzma_filter structures, use * LZMA_VLI_UNKNOWN to indicate end of filters. * @@ -68,12 +66,12 @@ /** * \brief Test if the given Filter ID is supported for encoding * - * Return true if the give Filter ID is supported for encoding by this - * liblzma build. Otherwise false is returned. + * \param id Filter ID * - * There is no way to list which filters are available in this particular - * liblzma version and build. It would be useless, because the application - * couldn't know what kind of options the filter would need. + * \return lzma_bool: + * - true if the Filter ID is supported for encoding by this + * liblzma build. + * - false otherwise. */ extern LZMA_API(lzma_bool) lzma_filter_encoder_is_supported(lzma_vli id) lzma_nothrow lzma_attr_const; @@ -82,8 +80,12 @@ /** * \brief Test if the given Filter ID is supported for decoding * - * Return true if the give Filter ID is supported for decoding by this - * liblzma build. Otherwise false is returned. + * \param id Filter ID + * + * \return lzma_bool: + * - true if the Filter ID is supported for decoding by this + * liblzma build. + * - false otherwise. */ extern LZMA_API(lzma_bool) lzma_filter_decoder_is_supported(lzma_vli id) lzma_nothrow lzma_attr_const; @@ -108,9 +110,18 @@ * need to be initialized by the caller in any way. * * If an error occurs, memory possibly already allocated by this function - * is always freed. + * is always freed. liblzma versions older than 5.2.7 may modify the dest + * array and leave its contents in an undefined state if an error occurs. + * liblzma 5.2.7 and newer only modify the dest array when returning LZMA_OK. * - * \return - LZMA_OK + * \param src Array of filters terminated with + * .id == LZMA_VLI_UNKNOWN. + * \param[out] dest Destination filter array + * \param allocator lzma_allocator for custom allocator functions. + * Set to NULL to use malloc() and free(). + * + * \return Possible lzma_ret values: + * - LZMA_OK * - LZMA_MEM_ERROR * - LZMA_OPTIONS_ERROR: Unsupported Filter ID and its options * is not NULL. @@ -118,7 +129,34 @@ */ extern LZMA_API(lzma_ret) lzma_filters_copy( const lzma_filter *src, lzma_filter *dest, - const lzma_allocator *allocator) lzma_nothrow; + const lzma_allocator *allocator) + lzma_nothrow lzma_attr_warn_unused_result; + + +/** + * \brief Free the options in the array of lzma_filter structures + * + * This frees the filter chain options. The filters array itself is not freed. + * + * The filters array must have at most LZMA_FILTERS_MAX + 1 elements + * including the terminating element which must have .id = LZMA_VLI_UNKNOWN. + * For all elements before the terminating element: + * - options will be freed using the given lzma_allocator or, + * if allocator is NULL, using free(). + * - options will be set to NULL. + * - id will be set to LZMA_VLI_UNKNOWN. + * + * If filters is NULL, this does nothing. Again, this never frees the + * filters array itself. + * + * \param filters Array of filters terminated with + * .id == LZMA_VLI_UNKNOWN. + * \param allocator lzma_allocator for custom allocator functions. + * Set to NULL to use malloc() and free(). + */ +extern LZMA_API(void) lzma_filters_free( + lzma_filter *filters, const lzma_allocator *allocator) + lzma_nothrow; /** @@ -132,9 +170,7 @@ * .id == LZMA_VLI_UNKNOWN. * * \return Number of bytes of memory required for the given - * filter chain when encoding. If an error occurs, - * for example due to unsupported filter chain, - * UINT64_MAX is returned. + * filter chain when encoding or UINT64_MAX on error. */ extern LZMA_API(uint64_t) lzma_raw_encoder_memusage(const lzma_filter *filters) lzma_nothrow lzma_attr_pure; @@ -151,9 +187,7 @@ * .id == LZMA_VLI_UNKNOWN. * * \return Number of bytes of memory required for the given - * filter chain when decoding. If an error occurs, - * for example due to unsupported filter chain, - * UINT64_MAX is returned. + * filter chain when decoding or UINT64_MAX on error. */ extern LZMA_API(uint64_t) lzma_raw_decoder_memusage(const lzma_filter *filters) lzma_nothrow lzma_attr_pure; @@ -164,14 +198,16 @@ * * This function may be useful when implementing custom file formats. * - * \param strm Pointer to properly prepared lzma_stream - * \param filters Array of lzma_filter structures. The end of the - * array must be marked with .id = LZMA_VLI_UNKNOWN. - * - * The `action' with lzma_code() can be LZMA_RUN, LZMA_SYNC_FLUSH (if the + * The 'action' with lzma_code() can be LZMA_RUN, LZMA_SYNC_FLUSH (if the * filter chain supports it), or LZMA_FINISH. * - * \return - LZMA_OK + * \param strm Pointer to lzma_stream that is at least + * initialized with LZMA_STREAM_INIT. + * \param filters Array of filters terminated with + * .id == LZMA_VLI_UNKNOWN. + * + * \return Possible lzma_ret values: + * - LZMA_OK * - LZMA_MEM_ERROR * - LZMA_OPTIONS_ERROR * - LZMA_PROG_ERROR @@ -186,10 +222,16 @@ * * The initialization of raw decoder goes similarly to raw encoder. * - * The `action' with lzma_code() can be LZMA_RUN or LZMA_FINISH. Using + * The 'action' with lzma_code() can be LZMA_RUN or LZMA_FINISH. Using * LZMA_FINISH is not required, it is supported just for convenience. * - * \return - LZMA_OK + * \param strm Pointer to lzma_stream that is at least + * initialized with LZMA_STREAM_INIT. + * \param filters Array of filters terminated with + * .id == LZMA_VLI_UNKNOWN. + * + * \return Possible lzma_ret values: + * - LZMA_OK * - LZMA_MEM_ERROR * - LZMA_OPTIONS_ERROR * - LZMA_PROG_ERROR @@ -202,24 +244,36 @@ /** * \brief Update the filter chain in the encoder * - * This function is for advanced users only. This function has two slightly - * different purposes: + * This function may be called after lzma_code() has returned LZMA_STREAM_END + * when LZMA_FULL_BARRIER, LZMA_FULL_FLUSH, or LZMA_SYNC_FLUSH was used: * - * - After LZMA_FULL_FLUSH when using Stream encoder: Set a new filter - * chain, which will be used starting from the next Block. + * - After LZMA_FULL_BARRIER or LZMA_FULL_FLUSH: Single-threaded .xz Stream + * encoder (lzma_stream_encoder()) and (since liblzma 5.4.0) multi-threaded + * Stream encoder (lzma_stream_encoder_mt()) allow setting a new filter + * chain to be used for the next Block(s). * - * - After LZMA_SYNC_FLUSH using Raw, Block, or Stream encoder: Change - * the filter-specific options in the middle of encoding. The actual - * filters in the chain (Filter IDs) cannot be changed. In the future, - * it might become possible to change the filter options without - * using LZMA_SYNC_FLUSH. + * - After LZMA_SYNC_FLUSH: Raw encoder (lzma_raw_encoder()), + * Block encoder (lzma_block_encoder()), and single-threaded .xz Stream + * encoder (lzma_stream_encoder()) allow changing certain filter-specific + * options in the middle of encoding. The actual filters in the chain + * (Filter IDs) must not be changed! Currently only the lc, lp, and pb + * options of LZMA2 (not LZMA1) can be changed this way. * - * While rarely useful, this function may be called also when no data has - * been compressed yet. In that case, this function will behave as if - * LZMA_FULL_FLUSH (Stream encoder) or LZMA_SYNC_FLUSH (Raw or Block + * - In the future some filters might allow changing some of their options + * without any barrier or flushing but currently such filters don't exist. + * + * This function may also be called when no data has been compressed yet + * although this is rarely useful. In that case, this function will behave + * as if LZMA_FULL_FLUSH (Stream encoders) or LZMA_SYNC_FLUSH (Raw or Block * encoder) had been used right before calling this function. * - * \return - LZMA_OK + * \param strm Pointer to lzma_stream that is at least + * initialized with LZMA_STREAM_INIT. + * \param filters Array of filters terminated with + * .id == LZMA_VLI_UNKNOWN. + * + * \return Possible lzma_ret values: + * - LZMA_OK * - LZMA_MEM_ERROR * - LZMA_MEMLIMIT_ERROR * - LZMA_OPTIONS_ERROR @@ -232,29 +286,30 @@ /** * \brief Single-call raw encoder * - * \param filters Array of lzma_filter structures. The end of the - * array must be marked with .id = LZMA_VLI_UNKNOWN. + * \note There is no function to calculate how big output buffer + * would surely be big enough. (lzma_stream_buffer_bound() + * works only for lzma_stream_buffer_encode(); raw encoder + * won't necessarily meet that bound.) + * + * \param filters Array of filters terminated with + * .id == LZMA_VLI_UNKNOWN. * \param allocator lzma_allocator for custom allocator functions. * Set to NULL to use malloc() and free(). * \param in Beginning of the input buffer * \param in_size Size of the input buffer - * \param out Beginning of the output buffer - * \param out_pos The next byte will be written to out[*out_pos]. + * \param[out] out Beginning of the output buffer + * \param[out] out_pos The next byte will be written to out[*out_pos]. * *out_pos is updated only if encoding succeeds. * \param out_size Size of the out buffer; the first byte into * which no data is written to is out[out_size]. * - * \return - LZMA_OK: Encoding was successful. + * \return Possible lzma_ret values: + * - LZMA_OK: Encoding was successful. * - LZMA_BUF_ERROR: Not enough output buffer space. * - LZMA_OPTIONS_ERROR * - LZMA_MEM_ERROR * - LZMA_DATA_ERROR * - LZMA_PROG_ERROR - * - * \note There is no function to calculate how big output buffer - * would surely be big enough. (lzma_stream_buffer_bound() - * works only for lzma_stream_buffer_encode(); raw encoder - * won't necessarily meet that bound.) */ extern LZMA_API(lzma_ret) lzma_raw_buffer_encode( const lzma_filter *filters, const lzma_allocator *allocator, @@ -265,8 +320,8 @@ /** * \brief Single-call raw decoder * - * \param filters Array of lzma_filter structures. The end of the - * array must be marked with .id = LZMA_VLI_UNKNOWN. + * \param filters Array of filters terminated with + * .id == LZMA_VLI_UNKNOWN. * \param allocator lzma_allocator for custom allocator functions. * Set to NULL to use malloc() and free(). * \param in Beginning of the input buffer @@ -274,11 +329,19 @@ * *in_pos is updated only if decoding succeeds. * \param in_size Size of the input buffer; the first byte that * won't be read is in[in_size]. - * \param out Beginning of the output buffer - * \param out_pos The next byte will be written to out[*out_pos]. + * \param[out] out Beginning of the output buffer + * \param[out] out_pos The next byte will be written to out[*out_pos]. * *out_pos is updated only if encoding succeeds. * \param out_size Size of the out buffer; the first byte into * which no data is written to is out[out_size]. + * + * \return Possible lzma_ret values: + * - LZMA_OK: Decoding was successful. + * - LZMA_BUF_ERROR: Not enough output buffer space. + * - LZMA_OPTIONS_ERROR + * - LZMA_MEM_ERROR + * - LZMA_DATA_ERROR + * - LZMA_PROG_ERROR */ extern LZMA_API(lzma_ret) lzma_raw_buffer_decode( const lzma_filter *filters, const lzma_allocator *allocator, @@ -292,18 +355,19 @@ * This function may be useful when implementing custom file formats * using the raw encoder and decoder. * - * \param size Pointer to uint32_t to hold the size of the properties - * \param filter Filter ID and options (the size of the properties may - * vary depending on the options) - * - * \return - LZMA_OK - * - LZMA_OPTIONS_ERROR - * - LZMA_PROG_ERROR - * * \note This function validates the Filter ID, but does not * necessarily validate the options. Thus, it is possible * that this returns LZMA_OK while the following call to * lzma_properties_encode() returns LZMA_OPTIONS_ERROR. + * + * \param[out] size Pointer to uint32_t to hold the size of the properties + * \param filter Filter ID and options (the size of the properties may + * vary depending on the options) + * + * \return Possible lzma_ret values: + * - LZMA_OK + * - LZMA_OPTIONS_ERROR + * - LZMA_PROG_ERROR */ extern LZMA_API(lzma_ret) lzma_properties_size( uint32_t *size, const lzma_filter *filter) lzma_nothrow; @@ -312,15 +376,6 @@ /** * \brief Encode the Filter Properties field * - * \param filter Filter ID and options - * \param props Buffer to hold the encoded options. The size of - * buffer must have been already determined with - * lzma_properties_size(). - * - * \return - LZMA_OK - * - LZMA_OPTIONS_ERROR - * - LZMA_PROG_ERROR - * * \note Even this function won't validate more options than actually * necessary. Thus, it is possible that encoding the properties * succeeds but using the same options to initialize the encoder @@ -330,6 +385,15 @@ * of the Filter Properties field is zero, calling * lzma_properties_encode() is not required, but it * won't do any harm either. + * + * \param filter Filter ID and options + * \param[out] props Buffer to hold the encoded options. The size of + * the buffer must have been already determined with + * lzma_properties_size(). + * + * \return Possible lzma_ret values: + * - LZMA_OK + * - LZMA_PROG_ERROR */ extern LZMA_API(lzma_ret) lzma_properties_encode( const lzma_filter *filter, uint8_t *props) lzma_nothrow; @@ -345,15 +409,16 @@ * it's application's responsibility to free it when * appropriate. filter->options is set to NULL if * there are no properties or if an error occurs. - * \param allocator Custom memory allocator used to allocate the - * options. Set to NULL to use the default malloc(), + * \param allocator lzma_allocator for custom allocator functions. + * Set to NULL to use malloc() and free(). * and in case of an error, also free(). * \param props Input buffer containing the properties. * \param props_size Size of the properties. This must be the exact * size; giving too much or too little input will * return LZMA_OPTIONS_ERROR. * - * \return - LZMA_OK + * \return Possible lzma_ret values: + * - LZMA_OK * - LZMA_OPTIONS_ERROR * - LZMA_MEM_ERROR */ @@ -368,18 +433,19 @@ * Knowing the size of Filter Flags is useful to know when allocating * memory to hold the encoded Filter Flags. * - * \param size Pointer to integer to hold the calculated size + * \note If you need to calculate size of List of Filter Flags, + * you need to loop over every lzma_filter entry. + * + * \param[out] size Pointer to integer to hold the calculated size * \param filter Filter ID and associated options whose encoded * size is to be calculated * - * \return - LZMA_OK: *size set successfully. Note that this doesn't + * \return Possible lzma_ret values: + * - LZMA_OK: *size set successfully. Note that this doesn't * guarantee that filter->options is valid, thus * lzma_filter_flags_encode() may still fail. * - LZMA_OPTIONS_ERROR: Unknown Filter ID or unsupported options. * - LZMA_PROG_ERROR: Invalid options - * - * \note If you need to calculate size of List of Filter Flags, - * you need to loop over every lzma_filter entry. */ extern LZMA_API(lzma_ret) lzma_filter_flags_size( uint32_t *size, const lzma_filter *filter) @@ -393,12 +459,13 @@ * This is due to how this function is used internally by liblzma. * * \param filter Filter ID and options to be encoded - * \param out Beginning of the output buffer - * \param out_pos out[*out_pos] is the next write position. This + * \param[out] out Beginning of the output buffer + * \param[out] out_pos out[*out_pos] is the next write position. This * is updated by the encoder. * \param out_size out[out_size] is the first byte to not write. * - * \return - LZMA_OK: Encoding was successful. + * \return Possible lzma_ret values: + * - LZMA_OK: Encoding was successful. * - LZMA_OPTIONS_ERROR: Invalid or unsupported options. * - LZMA_PROG_ERROR: Invalid options or not enough output * buffer space (you should have checked it with @@ -413,14 +480,290 @@ * \brief Decode Filter Flags from given buffer * * The decoded result is stored into *filter. The old value of - * filter->options is not free()d. + * filter->options is not free()d. If anything other than LZMA_OK + * is returned, filter->options is set to NULL. * - * \return - LZMA_OK + * \param[out] filter Destination filter. The decoded Filter ID will + * be stored in filter->id. If options are needed + * they will be allocated and the pointer will be + * stored in filter->options. + * \param allocator lzma_allocator for custom allocator functions. + * Set to NULL to use malloc() and free(). + * \param in Beginning of the input buffer + * \param[out] in_pos The next byte will be read from in[*in_pos]. + * *in_pos is updated only if decoding succeeds. + * \param in_size Size of the input buffer; the first byte that + * won't be read is in[in_size]. + * + * \return Possible lzma_ret values: + * - LZMA_OK * - LZMA_OPTIONS_ERROR * - LZMA_MEM_ERROR + * - LZMA_DATA_ERROR * - LZMA_PROG_ERROR */ extern LZMA_API(lzma_ret) lzma_filter_flags_decode( lzma_filter *filter, const lzma_allocator *allocator, const uint8_t *in, size_t *in_pos, size_t in_size) lzma_nothrow lzma_attr_warn_unused_result; + + +/*********** + * Strings * + ***********/ + +/** + * \brief Allow or show all filters + * + * By default only the filters supported in the .xz format are accept by + * lzma_str_to_filters() or shown by lzma_str_list_filters(). + */ +#define LZMA_STR_ALL_FILTERS UINT32_C(0x01) + + +/** + * \brief Do not validate the filter chain in lzma_str_to_filters() + * + * By default lzma_str_to_filters() can return an error if the filter chain + * as a whole isn't usable in the .xz format or in the raw encoder or decoder. + * With this flag, this validation is skipped. This flag doesn't affect the + * handling of the individual filter options. To allow non-.xz filters also + * LZMA_STR_ALL_FILTERS is needed. + */ +#define LZMA_STR_NO_VALIDATION UINT32_C(0x02) + + +/** + * \brief Stringify encoder options + * + * Show the filter-specific options that the encoder will use. + * This may be useful for verbose diagnostic messages. + * + * Note that if options were decoded from .xz headers then the encoder options + * may be undefined. This flag shouldn't be used in such a situation. + */ +#define LZMA_STR_ENCODER UINT32_C(0x10) + + +/** + * \brief Stringify decoder options + * + * Show the filter-specific options that the decoder will use. + * This may be useful for showing what filter options were decoded + * from file headers. + */ +#define LZMA_STR_DECODER UINT32_C(0x20) + + +/** + * \brief Produce xz-compatible getopt_long() syntax + * + * That is, "delta:dist=2 lzma2:dict=4MiB,pb=1,lp=1" becomes + * "--delta=dist=2 --lzma2=dict=4MiB,pb=1,lp=1". + * + * This syntax is compatible with xz 5.0.0 as long as the filters and + * their options are supported too. + */ +#define LZMA_STR_GETOPT_LONG UINT32_C(0x40) + + +/** + * \brief Use two dashes "--" instead of a space to separate filters + * + * That is, "delta:dist=2 lzma2:pb=1,lp=1" becomes + * "delta:dist=2--lzma2:pb=1,lp=1". This looks slightly odd but this + * kind of strings should be usable on the command line without quoting. + * However, it is possible that future versions with new filter options + * might produce strings that require shell quoting anyway as the exact + * set of possible characters isn't frozen for now. + * + * It is guaranteed that the single quote (') will never be used in + * filter chain strings (even if LZMA_STR_NO_SPACES isn't used). + */ +#define LZMA_STR_NO_SPACES UINT32_C(0x80) + + +/** + * \brief Convert a string to a filter chain + * + * This tries to make it easier to write applications that allow users + * to set custom compression options. This only handles the filter + * configuration (including presets) but not the number of threads, + * block size, check type, or memory limits. + * + * The input string can be either a preset or a filter chain. Presets + * begin with a digit 0-9 and may be followed by zero or more flags + * which are lower-case letters. Currently only "e" is supported, matching + * LZMA_PRESET_EXTREME. For partial xz command line syntax compatibility, + * a preset string may start with a single dash "-". + * + * A filter chain consists of one or more "filtername:opt1=value1,opt2=value2" + * strings separated by one or more spaces. Leading and trailing spaces are + * ignored. All names and values must be lower-case. Extra commas in the + * option list are ignored. The order of filters is significant: when + * encoding, the uncompressed input data goes to the leftmost filter first. + * Normally "lzma2" is the last filter in the chain. + * + * If one wishes to avoid spaces, for example, to avoid shell quoting, + * it is possible to use two dashes "--" instead of spaces to separate + * the filters. + * + * For xz command line compatibility, each filter may be prefixed with + * two dashes "--" and the colon ":" separating the filter name from + * the options may be replaced with an equals sign "=". + * + * By default, only filters that can be used in the .xz format are accepted. + * To allow all filters (LZMA1) use the flag LZMA_STR_ALL_FILTERS. + * + * By default, very basic validation is done for the filter chain as a whole, + * for example, that LZMA2 is only used as the last filter in the chain. + * The validation isn't perfect though and it's possible that this function + * succeeds but using the filter chain for encoding or decoding will still + * result in LZMA_OPTIONS_ERROR. To disable this validation, use the flag + * LZMA_STR_NO_VALIDATION. + * + * The available filter names and their options are available via + * lzma_str_list_filters(). See the xz man page for the description + * of filter names and options. + * + * For command line applications, below is an example how an error message + * can be displayed. Note the use of an empty string for the field width. + * If "^" was used there it would create an off-by-one error except at + * the very beginning of the line. + * + * \code{.c} + * const char *str = ...; // From user + * lzma_filter filters[LZMA_FILTERS_MAX + 1]; + * int pos; + * const char *msg = lzma_str_to_filters(str, &pos, filters, 0, NULL); + * if (msg != NULL) { + * printf("%s: Error in XZ compression options:\n", argv[0]); + * printf("%s: %s\n", argv[0], str); + * printf("%s: %*s^\n", argv[0], errpos, ""); + * printf("%s: %s\n", argv[0], msg); + * } + * \endcode + * + * \param str User-supplied string describing a preset or + * a filter chain. If a default value is needed and + * you don't know what would be good, use "6" since + * that is the default preset in xz too. + * \param[out] error_pos If this isn't NULL, this value will be set on + * both success and on all errors. This tells the + * location of the error in the string. This is + * an int to make it straightforward to use this + * as printf() field width. The value is guaranteed + * to be in the range [0, INT_MAX] even if strlen(str) + * somehow was greater than INT_MAX. + * \param[out] filters An array of lzma_filter structures. There must + * be LZMA_FILTERS_MAX + 1 (that is, five) elements + * in the array. The old contents are ignored so it + * doesn't need to be initialized. This array is + * modified only if this function returns NULL. + * Once the allocated filter options are no longer + * needed, lzma_filters_free() can be used to free the + * options (it doesn't free the filters array itself). + * \param flags Bitwise-or of zero or more of the flags + * LZMA_STR_ALL_FILTERS and LZMA_STR_NO_VALIDATION. + * \param allocator lzma_allocator for custom allocator functions. + * Set to NULL to use malloc() and free(). + * + * \return On success, NULL is returned. On error, a statically-allocated + * error message is returned which together with the error_pos + * should give some idea what is wrong. + */ +extern LZMA_API(const char *) lzma_str_to_filters( + const char *str, int *error_pos, lzma_filter *filters, + uint32_t flags, const lzma_allocator *allocator) + lzma_nothrow lzma_attr_warn_unused_result; + + +/** + * \brief Convert a filter chain to a string + * + * Use cases: + * + * - Verbose output showing the full encoder options to the user + * (use LZMA_STR_ENCODER in flags) + * + * - Showing the filters and options that are required to decode a file + * (use LZMA_STR_DECODER in flags) + * + * - Showing the filter names without any options in informational messages + * where the technical details aren't important (no flags). In this case + * the .options in the filters array are ignored and may be NULL even if + * a filter has a mandatory options structure. + * + * Note that even if the filter chain was specified using a preset, + * the resulting filter chain isn't reversed to a preset. So if you + * specify "6" to lzma_str_to_filters() then lzma_str_from_filters() + * will produce a string containing "lzma2". + * + * \param[out] str On success *str will be set to point to an + * allocated string describing the given filter + * chain. Old value is ignored. On error *str is + * always set to NULL. + * \param filters Array of filters terminated with + * .id == LZMA_VLI_UNKNOWN. + * \param flags Bitwise-or of zero or more of the flags + * LZMA_STR_ENCODER, LZMA_STR_DECODER, + * LZMA_STR_GETOPT_LONG, and LZMA_STR_NO_SPACES. + * \param allocator lzma_allocator for custom allocator functions. + * Set to NULL to use malloc() and free(). + * + * \return Possible lzma_ret values: + * - LZMA_OK + * - LZMA_OPTIONS_ERROR: Empty filter chain + * (filters[0].id == LZMA_VLI_UNKNOWN) or the filter chain + * includes a Filter ID that is not supported by this function. + * - LZMA_MEM_ERROR + * - LZMA_PROG_ERROR + */ +extern LZMA_API(lzma_ret) lzma_str_from_filters( + char **str, const lzma_filter *filters, uint32_t flags, + const lzma_allocator *allocator) + lzma_nothrow lzma_attr_warn_unused_result; + + +/** + * \brief List available filters and/or their options (for help message) + * + * If a filter_id is given then only one line is created which contains the + * filter name. If LZMA_STR_ENCODER or LZMA_STR_DECODER is used then the + * options read by the encoder or decoder are printed on the same line. + * + * If filter_id is LZMA_VLI_UNKNOWN then all supported .xz-compatible filters + * are listed: + * + * - If neither LZMA_STR_ENCODER nor LZMA_STR_DECODER is used then + * the supported filter names are listed on a single line separated + * by spaces. + * + * - If LZMA_STR_ENCODER or LZMA_STR_DECODER is used then filters and + * the supported options are listed one filter per line. There won't + * be a newline after the last filter. + * + * - If LZMA_STR_ALL_FILTERS is used then the list will include also + * those filters that cannot be used in the .xz format (LZMA1). + * + * \param str On success *str will be set to point to an + * allocated string listing the filters and options. + * Old value is ignored. On error *str is always set + * to NULL. + * \param filter_id Filter ID or LZMA_VLI_UNKNOWN. + * \param flags Bitwise-or of zero or more of the flags + * LZMA_STR_ALL_FILTERS, LZMA_STR_ENCODER, + * LZMA_STR_DECODER, and LZMA_STR_GETOPT_LONG. + * \param allocator lzma_allocator for custom allocator functions. + * Set to NULL to use malloc() and free(). + * + * \return Possible lzma_ret values: + * - LZMA_OK + * - LZMA_OPTIONS_ERROR: Unsupported filter_id or flags + * - LZMA_MEM_ERROR + * - LZMA_PROG_ERROR + */ +extern LZMA_API(lzma_ret) lzma_str_list_filters( + char **str, lzma_vli filter_id, uint32_t flags, + const lzma_allocator *allocator) + lzma_nothrow lzma_attr_warn_unused_result;
diff --git a/Utilities/cmliblzma/liblzma/api/lzma/hardware.h b/Utilities/cmliblzma/liblzma/api/lzma/hardware.h index 47481f2..7a1a84f 100644 --- a/Utilities/cmliblzma/liblzma/api/lzma/hardware.h +++ b/Utilities/cmliblzma/liblzma/api/lzma/hardware.h
@@ -1,6 +1,9 @@ +/* SPDX-License-Identifier: 0BSD */ + /** * \file lzma/hardware.h * \brief Hardware information + * \note Never include this file directly. Use <lzma.h> instead. * * Since liblzma can consume a lot of system resources, it also provides * ways to limit the resource usage. Applications linking against liblzma @@ -22,11 +25,6 @@ /* * Author: Lasse Collin - * - * This file has been put into the public domain. - * You can do whatever you want with this file. - * - * See ../lzma.h for information about liblzma as a whole. */ #ifndef LZMA_H_INTERNAL @@ -57,7 +55,7 @@ * If the hardware supports more than one thread per CPU core, the number * of hardware threads is returned if that information is available. * - * \brief On success, the number of available CPU threads or cores is + * \return On success, the number of available CPU threads or cores is * returned. If this information isn't available or an error * occurs, zero is returned. */
diff --git a/Utilities/cmliblzma/liblzma/api/lzma/index.h b/Utilities/cmliblzma/liblzma/api/lzma/index.h index 3dac6fb..b17025e 100644 --- a/Utilities/cmliblzma/liblzma/api/lzma/index.h +++ b/Utilities/cmliblzma/liblzma/api/lzma/index.h
@@ -1,15 +1,13 @@ +/* SPDX-License-Identifier: 0BSD */ + /** * \file lzma/index.h * \brief Handling of .xz Index and related information + * \note Never include this file directly. Use <lzma.h> instead. */ /* * Author: Lasse Collin - * - * This file has been put into the public domain. - * You can do whatever you want with this file. - * - * See ../lzma.h for information about liblzma as a whole. */ #ifndef LZMA_H_INTERNAL @@ -50,8 +48,13 @@ */ const lzma_stream_flags *flags; + /** \private Reserved member. */ const void *reserved_ptr1; + + /** \private Reserved member. */ const void *reserved_ptr2; + + /** \private Reserved member. */ const void *reserved_ptr3; /** @@ -107,9 +110,17 @@ */ lzma_vli padding; + + /** \private Reserved member. */ lzma_vli reserved_vli1; + + /** \private Reserved member. */ lzma_vli reserved_vli2; + + /** \private Reserved member. */ lzma_vli reserved_vli3; + + /** \private Reserved member. */ lzma_vli reserved_vli4; } stream; @@ -196,25 +207,46 @@ */ lzma_vli total_size; + /** \private Reserved member. */ lzma_vli reserved_vli1; + + /** \private Reserved member. */ lzma_vli reserved_vli2; + + /** \private Reserved member. */ lzma_vli reserved_vli3; + + /** \private Reserved member. */ lzma_vli reserved_vli4; + /** \private Reserved member. */ const void *reserved_ptr1; + + /** \private Reserved member. */ const void *reserved_ptr2; + + /** \private Reserved member. */ const void *reserved_ptr3; + + /** \private Reserved member. */ const void *reserved_ptr4; } block; - /* + /** + * \private Internal data + * * Internal data which is used to store the state of the iterator. * The exact format may vary between liblzma versions, so don't * touch these in any way. */ union { + /** \private Internal member. */ const void *p; + + /** \private Internal member. */ size_t s; + + /** \private Internal member. */ lzma_vli v; } internal[6]; } lzma_index_iter; @@ -269,19 +301,46 @@ /** + * \brief Mask for return value from lzma_index_checks() for check none + * + * \note This and the other CHECK_MASK macros were added in 5.5.1alpha. + */ +#define LZMA_INDEX_CHECK_MASK_NONE (UINT32_C(1) << LZMA_CHECK_NONE) + +/** + * \brief Mask for return value from lzma_index_checks() for check CRC32 + */ +#define LZMA_INDEX_CHECK_MASK_CRC32 (UINT32_C(1) << LZMA_CHECK_CRC32) + +/** + * \brief Mask for return value from lzma_index_checks() for check CRC64 + */ +#define LZMA_INDEX_CHECK_MASK_CRC64 (UINT32_C(1) << LZMA_CHECK_CRC64) + +/** + * \brief Mask for return value from lzma_index_checks() for check SHA256 + */ +#define LZMA_INDEX_CHECK_MASK_SHA256 (UINT32_C(1) << LZMA_CHECK_SHA256) + +/** * \brief Calculate memory usage of lzma_index * * On disk, the size of the Index field depends on both the number of Records - * stored and how big values the Records store (due to variable-length integer + * stored and the size of the Records (due to variable-length integer * encoding). When the Index is kept in lzma_index structure, the memory usage * depends only on the number of Records/Blocks stored in the Index(es), and * in case of concatenated lzma_indexes, the number of Streams. The size in * RAM is almost always significantly bigger than in the encoded form on disk. * - * This function calculates an approximate amount of memory needed hold + * This function calculates an approximate amount of memory needed to hold * the given number of Streams and Blocks in lzma_index structure. This * value may vary between CPU architectures and also between liblzma versions * if the internal implementation is modified. + * + * \param streams Number of Streams + * \param blocks Number of Blocks + * + * \return Approximate memory in bytes needed in a lzma_index structure. */ extern LZMA_API(uint64_t) lzma_index_memusage( lzma_vli streams, lzma_vli blocks) lzma_nothrow; @@ -292,6 +351,10 @@ * * This is a shorthand for lzma_index_memusage(lzma_index_stream_count(i), * lzma_index_block_count(i)). + * + * \param i Pointer to lzma_index structure + * + * \return Approximate memory in bytes used by the lzma_index structure. */ extern LZMA_API(uint64_t) lzma_index_memused(const lzma_index *i) lzma_nothrow; @@ -300,6 +363,9 @@ /** * \brief Allocate and initialize a new lzma_index structure * + * \param allocator lzma_allocator for custom allocator functions. + * Set to NULL to use malloc() and free(). + * * \return On success, a pointer to an empty initialized lzma_index is * returned. If allocation fails, NULL is returned. */ @@ -311,6 +377,10 @@ * \brief Deallocate lzma_index * * If i is NULL, this does nothing. + * + * \param i Pointer to lzma_index structure to deallocate + * \param allocator lzma_allocator for custom allocator functions. + * Set to NULL to use malloc() and free(). */ extern LZMA_API(void) lzma_index_end( lzma_index *i, const lzma_allocator *allocator) lzma_nothrow; @@ -320,8 +390,9 @@ * \brief Add a new Block to lzma_index * * \param i Pointer to a lzma_index structure - * \param allocator Pointer to lzma_allocator, or NULL to - * use malloc() + * \param allocator lzma_allocator for custom allocator + * functions. Set to NULL to use malloc() + * and free(). * \param unpadded_size Unpadded Size of a Block. This can be * calculated with lzma_block_unpadded_size() * after encoding or decoding the Block. @@ -334,7 +405,8 @@ * lzma_index_append() it is possible to read the next Block with * an existing iterator. * - * \return - LZMA_OK + * \return Possible lzma_ret values: + * - LZMA_OK * - LZMA_MEM_ERROR * - LZMA_DATA_ERROR: Compressed or uncompressed size of the * Stream or size of the Index field would grow too big. @@ -354,11 +426,15 @@ * lzma_index, because to decode Blocks, knowing the integrity check type * is needed. * - * The given Stream Flags are copied into internal preallocated structure - * in the lzma_index, thus the caller doesn't need to keep the *stream_flags - * available after calling this function. + * \param i Pointer to lzma_index structure + * \param stream_flags Pointer to lzma_stream_flags structure. This + * is copied into the internal preallocated + * structure, so the caller doesn't need to keep + * the flags' data available after calling this + * function. * - * \return - LZMA_OK + * \return Possible lzma_ret values: + * - LZMA_OK * - LZMA_OPTIONS_ERROR: Unsupported stream_flags->version. * - LZMA_PROG_ERROR */ @@ -376,6 +452,11 @@ * showing the Check types to the user. * * The bitmask is 1 << check_id, e.g. CRC32 is 1 << 1 and SHA-256 is 1 << 10. + * These masks are defined for convenience as LZMA_INDEX_CHECK_MASK_XXX + * + * \param i Pointer to lzma_index structure + * + * \return Bitmask indicating which Check types are used in the lzma_index */ extern LZMA_API(uint32_t) lzma_index_checks(const lzma_index *i) lzma_nothrow lzma_attr_pure; @@ -390,7 +471,8 @@ * * By default, the amount of Stream Padding is assumed to be zero bytes. * - * \return - LZMA_OK + * \return Possible lzma_ret values: + * - LZMA_OK * - LZMA_DATA_ERROR: The file size would grow too big. * - LZMA_PROG_ERROR */ @@ -401,6 +483,10 @@ /** * \brief Get the number of Streams + * + * \param i Pointer to lzma_index structure + * + * \return Number of Streams in the lzma_index */ extern LZMA_API(lzma_vli) lzma_index_stream_count(const lzma_index *i) lzma_nothrow lzma_attr_pure; @@ -411,6 +497,10 @@ * * This returns the total number of Blocks in lzma_index. To get number * of Blocks in individual Streams, use lzma_index_iter. + * + * \param i Pointer to lzma_index structure + * + * \return Number of blocks in the lzma_index */ extern LZMA_API(lzma_vli) lzma_index_block_count(const lzma_index *i) lzma_nothrow lzma_attr_pure; @@ -420,6 +510,10 @@ * \brief Get the size of the Index field as bytes * * This is needed to verify the Backward Size field in the Stream Footer. + * + * \param i Pointer to lzma_index structure + * + * \return Size in bytes of the Index */ extern LZMA_API(lzma_vli) lzma_index_size(const lzma_index *i) lzma_nothrow lzma_attr_pure; @@ -431,6 +525,11 @@ * If multiple lzma_indexes have been combined, this works as if the Blocks * were in a single Stream. This is useful if you are going to combine * Blocks from multiple Streams into a single new Stream. + * + * \param i Pointer to lzma_index structure + * + * \return Size in bytes of the Stream (if all Blocks are combined + * into one Stream). */ extern LZMA_API(lzma_vli) lzma_index_stream_size(const lzma_index *i) lzma_nothrow lzma_attr_pure; @@ -441,6 +540,10 @@ * * This doesn't include the Stream Header, Stream Footer, Stream Padding, * or Index fields. + * + * \param i Pointer to lzma_index structure + * + * \return Size in bytes of all Blocks in the Stream(s) */ extern LZMA_API(lzma_vli) lzma_index_total_size(const lzma_index *i) lzma_nothrow lzma_attr_pure; @@ -453,6 +556,10 @@ * no Stream Padding, this function is identical to lzma_index_stream_size(). * If multiple lzma_indexes have been combined, this includes also the headers * of each separate Stream and the possible Stream Padding fields. + * + * \param i Pointer to lzma_index structure + * + * \return Total size of the .xz file in bytes */ extern LZMA_API(lzma_vli) lzma_index_file_size(const lzma_index *i) lzma_nothrow lzma_attr_pure; @@ -460,6 +567,10 @@ /** * \brief Get the uncompressed size of the file + * + * \param i Pointer to lzma_index structure + * + * \return Size in bytes of the uncompressed data in the file */ extern LZMA_API(lzma_vli) lzma_index_uncompressed_size(const lzma_index *i) lzma_nothrow lzma_attr_pure; @@ -468,9 +579,6 @@ /** * \brief Initialize an iterator * - * \param iter Pointer to a lzma_index_iter structure - * \param i lzma_index to which the iterator will be associated - * * This function associates the iterator with the given lzma_index, and calls * lzma_index_iter_rewind() on the iterator. * @@ -483,6 +591,9 @@ * * It is safe to make copies of an initialized lzma_index_iter, for example, * to easily restart reading at some particular position. + * + * \param iter Pointer to a lzma_index_iter structure + * \param i lzma_index to which the iterator will be associated */ extern LZMA_API(void) lzma_index_iter_init( lzma_index_iter *iter, const lzma_index *i) lzma_nothrow; @@ -493,6 +604,8 @@ * * Rewind the iterator so that next call to lzma_index_iter_next() will * return the first Block or Stream. + * + * \param iter Pointer to a lzma_index_iter structure */ extern LZMA_API(void) lzma_index_iter_rewind(lzma_index_iter *iter) lzma_nothrow; @@ -505,11 +618,11 @@ * \param mode Specify what kind of information the caller wants * to get. See lzma_index_iter_mode for details. * - * \return If next Block or Stream matching the mode was found, *iter - * is updated and this function returns false. If no Block or - * Stream matching the mode is found, *iter is not modified - * and this function returns true. If mode is set to an unknown - * value, *iter is not modified and this function returns true. + * \return lzma_bool: + * - true if no Block or Stream matching the mode is found. + * *iter is not updated (failure). + * - false if the next Block or Stream matching the mode was + * found. *iter is updated (success). */ extern LZMA_API(lzma_bool) lzma_index_iter_next( lzma_index_iter *iter, lzma_index_iter_mode mode) @@ -523,21 +636,26 @@ * the Index field(s) and use lzma_index_iter_locate() to do random-access * reading with granularity of Block size. * - * \param iter Iterator that was earlier initialized with - * lzma_index_iter_init(). - * \param target Uncompressed target offset which the caller would - * like to locate from the Stream - * * If the target is smaller than the uncompressed size of the Stream (can be * checked with lzma_index_uncompressed_size()): * - Information about the Stream and Block containing the requested * uncompressed offset is stored into *iter. * - Internal state of the iterator is adjusted so that * lzma_index_iter_next() can be used to read subsequent Blocks or Streams. - * - This function returns false. * - * If target is greater than the uncompressed size of the Stream, *iter - * is not modified, and this function returns true. + * If the target is greater than the uncompressed size of the Stream, *iter + * is not modified. + * + * \param iter Iterator that was earlier initialized with + * lzma_index_iter_init(). + * \param target Uncompressed target offset which the caller would + * like to locate from the Stream + * + * \return lzma_bool: + * - true if the target is greater than or equal to the + * uncompressed size of the Stream (failure) + * - false if the target is smaller than the uncompressed size + * of the Stream (success) */ extern LZMA_API(lzma_bool) lzma_index_iter_locate( lzma_index_iter *iter, lzma_vli target) lzma_nothrow; @@ -550,15 +668,16 @@ * multi-Stream .xz file, or when combining multiple Streams into single * Stream. * - * \param dest lzma_index after which src is appended + * \param[out] dest lzma_index after which src is appended * \param src lzma_index to be appended after dest. If this * function succeeds, the memory allocated for src * is freed or moved to be part of dest, and all * iterators pointing to src will become invalid. - * \param allocator Custom memory allocator; can be NULL to use - * malloc() and free(). + * \param allocator lzma_allocator for custom allocator functions. + * Set to NULL to use malloc() and free(). * - * \return - LZMA_OK: lzma_indexes were concatenated successfully. + * \return Possible lzma_ret values: + * - LZMA_OK: lzma_indexes were concatenated successfully. * src is now a dangling pointer. * - LZMA_DATA_ERROR: *dest would grow too big. * - LZMA_MEM_ERROR @@ -572,6 +691,10 @@ /** * \brief Duplicate lzma_index * + * \param i Pointer to lzma_index structure to be duplicated + * \param allocator lzma_allocator for custom allocator functions. + * Set to NULL to use malloc() and free(). + * * \return A copy of the lzma_index, or NULL if memory allocation failed. */ extern LZMA_API(lzma_index *) lzma_index_dup( @@ -585,10 +708,11 @@ * \param strm Pointer to properly prepared lzma_stream * \param i Pointer to lzma_index which should be encoded. * - * The valid `action' values for lzma_code() are LZMA_RUN and LZMA_FINISH. + * The valid 'action' values for lzma_code() are LZMA_RUN and LZMA_FINISH. * It is enough to use only one of them (you can choose freely). * - * \return - LZMA_OK: Initialization succeeded, continue with lzma_code(). + * \return Possible lzma_ret values: + * - LZMA_OK: Initialization succeeded, continue with lzma_code(). * - LZMA_MEM_ERROR * - LZMA_PROG_ERROR */ @@ -601,7 +725,7 @@ * \brief Initialize .xz Index decoder * * \param strm Pointer to properly prepared lzma_stream - * \param i The decoded Index will be made available via + * \param[out] i The decoded Index will be made available via * this pointer. Initially this function will * set *i to NULL (the old value is ignored). If * decoding succeeds (lzma_code() returns @@ -613,15 +737,16 @@ * don't allow 0 here and return LZMA_PROG_ERROR; * later versions treat 0 as if 1 had been specified. * - * Valid `action' arguments to lzma_code() are LZMA_RUN and LZMA_FINISH. + * Valid 'action' arguments to lzma_code() are LZMA_RUN and LZMA_FINISH. * There is no need to use LZMA_FINISH, but it's allowed because it may * simplify certain types of applications. * - * \return - LZMA_OK: Initialization succeeded, continue with lzma_code(). + * \return Possible lzma_ret values: + * - LZMA_OK: Initialization succeeded, continue with lzma_code(). * - LZMA_MEM_ERROR * - LZMA_PROG_ERROR * - * liblzma 5.2.3 and older list also LZMA_MEMLIMIT_ERROR here + * \note liblzma 5.2.3 and older list also LZMA_MEMLIMIT_ERROR here * but that error code has never been possible from this * initialization function. */ @@ -633,21 +758,23 @@ /** * \brief Single-call .xz Index encoder * + * \note This function doesn't take allocator argument since all + * the internal data is allocated on stack. + * * \param i lzma_index to be encoded - * \param out Beginning of the output buffer - * \param out_pos The next byte will be written to out[*out_pos]. + * \param[out] out Beginning of the output buffer + * \param[out] out_pos The next byte will be written to out[*out_pos]. * *out_pos is updated only if encoding succeeds. * \param out_size Size of the out buffer; the first byte into * which no data is written to is out[out_size]. * - * \return - LZMA_OK: Encoding was successful. + * \return Possible lzma_ret values: + * - LZMA_OK: Encoding was successful. * - LZMA_BUF_ERROR: Output buffer is too small. Use * lzma_index_size() to find out how much output * space is needed. * - LZMA_PROG_ERROR * - * \note This function doesn't take allocator argument since all - * the internal data is allocated on stack. */ extern LZMA_API(lzma_ret) lzma_index_buffer_encode(const lzma_index *i, uint8_t *out, size_t *out_pos, size_t out_size) lzma_nothrow; @@ -656,24 +783,26 @@ /** * \brief Single-call .xz Index decoder * - * \param i If decoding succeeds, *i will point to a new + * \param[out] i If decoding succeeds, *i will point to a new * lzma_index, which the application has to * later free with lzma_index_end(). If an error * occurs, *i will be NULL. The old value of *i * is always ignored and thus doesn't need to be * initialized by the caller. - * \param memlimit Pointer to how much memory the resulting + * \param[out] memlimit Pointer to how much memory the resulting * lzma_index is allowed to require. The value * pointed by this pointer is modified if and only * if LZMA_MEMLIMIT_ERROR is returned. - * \param allocator Pointer to lzma_allocator, or NULL to use malloc() + * \param allocator lzma_allocator for custom allocator functions. + * Set to NULL to use malloc() and free(). * \param in Beginning of the input buffer * \param in_pos The next byte will be read from in[*in_pos]. * *in_pos is updated only if decoding succeeds. * \param in_size Size of the input buffer; the first byte that * won't be read is in[in_size]. * - * \return - LZMA_OK: Decoding was successful. + * \return Possible lzma_ret values: + * - LZMA_OK: Decoding was successful. * - LZMA_MEM_ERROR * - LZMA_MEMLIMIT_ERROR: Memory usage limit was reached. * The minimum required memlimit value was stored to *memlimit. @@ -684,3 +813,70 @@ uint64_t *memlimit, const lzma_allocator *allocator, const uint8_t *in, size_t *in_pos, size_t in_size) lzma_nothrow; + + +/** + * \brief Initialize a .xz file information decoder + * + * This decoder decodes the Stream Header, Stream Footer, Index, and + * Stream Padding field(s) from the input .xz file and stores the resulting + * combined index in *dest_index. This information can be used to get the + * uncompressed file size with lzma_index_uncompressed_size(*dest_index) or, + * for example, to implement random access reading by locating the Blocks + * in the Streams. + * + * To get the required information from the .xz file, lzma_code() may ask + * the application to seek in the input file by returning LZMA_SEEK_NEEDED + * and having the target file position specified in lzma_stream.seek_pos. + * The number of seeks required depends on the input file and how big buffers + * the application provides. When possible, the decoder will seek backward + * and forward in the given buffer to avoid useless seek requests. Thus, if + * the application provides the whole file at once, no external seeking will + * be required (that is, lzma_code() won't return LZMA_SEEK_NEEDED). + * + * The value in lzma_stream.total_in can be used to estimate how much data + * liblzma had to read to get the file information. However, due to seeking + * and the way total_in is updated, the value of total_in will be somewhat + * inaccurate (a little too big). Thus, total_in is a good estimate but don't + * expect to see the same exact value for the same file if you change the + * input buffer size or switch to a different liblzma version. + * + * Valid 'action' arguments to lzma_code() are LZMA_RUN and LZMA_FINISH. + * You only need to use LZMA_RUN; LZMA_FINISH is only supported because it + * might be convenient for some applications. If you use LZMA_FINISH and if + * lzma_code() asks the application to seek, remember to reset 'action' back + * to LZMA_RUN unless you hit the end of the file again. + * + * Possible return values from lzma_code(): + * - LZMA_OK: All OK so far, more input needed + * - LZMA_SEEK_NEEDED: Provide more input starting from the absolute + * file position strm->seek_pos + * - LZMA_STREAM_END: Decoding was successful, *dest_index has been set + * - LZMA_FORMAT_ERROR: The input file is not in the .xz format (the + * expected magic bytes were not found from the beginning of the file) + * - LZMA_OPTIONS_ERROR: File looks valid but contains headers that aren't + * supported by this version of liblzma + * - LZMA_DATA_ERROR: File is corrupt + * - LZMA_BUF_ERROR + * - LZMA_MEM_ERROR + * - LZMA_MEMLIMIT_ERROR + * - LZMA_PROG_ERROR + * + * \param strm Pointer to a properly prepared lzma_stream + * \param[out] dest_index Pointer to a pointer where the decoder will put + * the decoded lzma_index. The old value + * of *dest_index is ignored (not freed). + * \param memlimit How much memory the resulting lzma_index is + * allowed to require. Use UINT64_MAX to + * effectively disable the limiter. + * \param file_size Size of the input .xz file + * + * \return Possible lzma_ret values: + * - LZMA_OK + * - LZMA_MEM_ERROR + * - LZMA_PROG_ERROR + */ +extern LZMA_API(lzma_ret) lzma_file_info_decoder( + lzma_stream *strm, lzma_index **dest_index, + uint64_t memlimit, uint64_t file_size) + lzma_nothrow;
diff --git a/Utilities/cmliblzma/liblzma/api/lzma/index_hash.h b/Utilities/cmliblzma/liblzma/api/lzma/index_hash.h index 9287f1d..68f9024 100644 --- a/Utilities/cmliblzma/liblzma/api/lzma/index_hash.h +++ b/Utilities/cmliblzma/liblzma/api/lzma/index_hash.h
@@ -1,6 +1,9 @@ +/* SPDX-License-Identifier: 0BSD */ + /** * \file lzma/index_hash.h * \brief Validate Index by using a hash function + * \note Never include this file directly. Use <lzma.h> instead. * * Hashing makes it possible to use constant amount of memory to validate * Index of arbitrary size. @@ -8,11 +11,6 @@ /* * Author: Lasse Collin - * - * This file has been put into the public domain. - * You can do whatever you want with this file. - * - * See ../lzma.h for information about liblzma as a whole. */ #ifndef LZMA_H_INTERNAL @@ -28,13 +26,21 @@ /** * \brief Allocate and initialize a new lzma_index_hash structure * - * If index_hash is NULL, a new lzma_index_hash structure is allocated, - * initialized, and a pointer to it returned. If allocation fails, NULL - * is returned. + * If index_hash is NULL, this function allocates and initializes a new + * lzma_index_hash structure and returns a pointer to it. If allocation + * fails, NULL is returned. * - * If index_hash is non-NULL, it is reinitialized and the same pointer - * returned. In this case, return value cannot be NULL or a different - * pointer than the index_hash that was given as an argument. + * If index_hash is non-NULL, this function reinitializes the lzma_index_hash + * structure and returns the same pointer. In this case, return value cannot + * be NULL or a different pointer than the index_hash that was given as + * an argument. + * + * \param index_hash Pointer to a lzma_index_hash structure or NULL. + * \param allocator lzma_allocator for custom allocator functions. + * Set to NULL to use malloc() and free(). + * + * \return Initialized lzma_index_hash structure on success or + * NULL on failure. */ extern LZMA_API(lzma_index_hash *) lzma_index_hash_init( lzma_index_hash *index_hash, const lzma_allocator *allocator) @@ -43,6 +49,10 @@ /** * \brief Deallocate lzma_index_hash structure + * + * \param index_hash Pointer to a lzma_index_hash structure to free. + * \param allocator lzma_allocator for custom allocator functions. + * Set to NULL to use malloc() and free(). */ extern LZMA_API(void) lzma_index_hash_end( lzma_index_hash *index_hash, const lzma_allocator *allocator) @@ -52,11 +62,12 @@ /** * \brief Add a new Record to an Index hash * - * \param index Pointer to a lzma_index_hash structure + * \param index_hash Pointer to a lzma_index_hash structure * \param unpadded_size Unpadded Size of a Block * \param uncompressed_size Uncompressed Size of a Block * - * \return - LZMA_OK + * \return Possible lzma_ret values: + * - LZMA_OK * - LZMA_DATA_ERROR: Compressed or uncompressed size of the * Stream or size of the Index field would grow too big. * - LZMA_PROG_ERROR: Invalid arguments or this function is being @@ -81,10 +92,11 @@ * * \param index_hash Pointer to a lzma_index_hash structure * \param in Pointer to the beginning of the input buffer - * \param in_pos in[*in_pos] is the next byte to process + * \param[out] in_pos in[*in_pos] is the next byte to process * \param in_size in[in_size] is the first byte not to process * - * \return - LZMA_OK: So far good, but more input is needed. + * \return Possible lzma_ret values: + * - LZMA_OK: So far good, but more input is needed. * - LZMA_STREAM_END: Index decoded successfully and it matches * the Records given with lzma_index_hash_append(). * - LZMA_DATA_ERROR: Index is corrupt or doesn't match the @@ -101,6 +113,10 @@ * \brief Get the size of the Index field as bytes * * This is needed to verify the Backward Size field in the Stream Footer. + * + * \param index_hash Pointer to a lzma_index_hash structure + * + * \return Size of the Index field in bytes. */ extern LZMA_API(lzma_vli) lzma_index_hash_size( const lzma_index_hash *index_hash)
diff --git a/Utilities/cmliblzma/liblzma/api/lzma/lzma12.h b/Utilities/cmliblzma/liblzma/api/lzma/lzma12.h index df5f23b..05f5b66 100644 --- a/Utilities/cmliblzma/liblzma/api/lzma/lzma12.h +++ b/Utilities/cmliblzma/liblzma/api/lzma/lzma12.h
@@ -1,15 +1,13 @@ +/* SPDX-License-Identifier: 0BSD */ + /** * \file lzma/lzma12.h * \brief LZMA1 and LZMA2 filters + * \note Never include this file directly. Use <lzma.h> instead. */ /* * Author: Lasse Collin - * - * This file has been put into the public domain. - * You can do whatever you want with this file. - * - * See ../lzma.h for information about liblzma as a whole. */ #ifndef LZMA_H_INTERNAL @@ -18,23 +16,46 @@ /** - * \brief LZMA1 Filter ID + * \brief LZMA1 Filter ID (for raw encoder/decoder only, not in .xz) * * LZMA1 is the very same thing as what was called just LZMA in LZMA Utils, * 7-Zip, and LZMA SDK. It's called LZMA1 here to prevent developers from * accidentally using LZMA when they actually want LZMA2. - * - * LZMA1 shouldn't be used for new applications unless you _really_ know - * what you are doing. LZMA2 is almost always a better choice. */ #define LZMA_FILTER_LZMA1 LZMA_VLI_C(0x4000000000000001) /** + * \brief LZMA1 Filter ID with extended options (for raw encoder/decoder) + * + * This is like LZMA_FILTER_LZMA1 but with this ID a few extra options + * are supported in the lzma_options_lzma structure: + * + * - A flag to tell the encoder if the end of payload marker (EOPM) alias + * end of stream (EOS) marker must be written at the end of the stream. + * In contrast, LZMA_FILTER_LZMA1 always writes the end marker. + * + * - Decoder needs to be told the uncompressed size of the stream + * or that it is unknown (using the special value UINT64_MAX). + * If the size is known, a flag can be set to allow the presence of + * the end marker anyway. In contrast, LZMA_FILTER_LZMA1 always + * behaves as if the uncompressed size was unknown. + * + * This allows handling file formats where LZMA1 streams are used but where + * the end marker isn't allowed or where it might not (always) be present. + * This extended LZMA1 functionality is provided as a Filter ID for raw + * encoder and decoder instead of adding new encoder and decoder initialization + * functions because this way it is possible to also use extra filters, + * for example, LZMA_FILTER_X86 in a filter chain with LZMA_FILTER_LZMA1EXT, + * which might be needed to handle some file formats. + */ +#define LZMA_FILTER_LZMA1EXT LZMA_VLI_C(0x4000000000000002) + +/** * \brief LZMA2 Filter ID * * Usually you want this instead of LZMA1. Compared to LZMA1, LZMA2 adds * support for LZMA_SYNC_FLUSH, uncompressed chunks (smaller expansion - * when trying to compress uncompressible data), possibility to change + * when trying to compress incompressible data), possibility to change * lc/lp/pb in the middle of encoding, and some other internal improvements. */ #define LZMA_FILTER_LZMA2 LZMA_VLI_C(0x21) @@ -114,16 +135,20 @@ /** * \brief Test if given match finder is supported * - * Return true if the given match finder is supported by this liblzma build. - * Otherwise false is returned. It is safe to call this with a value that - * isn't listed in lzma_match_finder enumeration; the return value will be - * false. + * It is safe to call this with a value that isn't listed in + * lzma_match_finder enumeration; the return value will be false. * * There is no way to list which match finders are available in this * particular liblzma version and build. It would be useless, because * a new match finder, which the application developer wasn't aware, * could require giving additional options to the encoder that the older * match finders don't need. + * + * \param match_finder Match finder ID + * + * \return lzma_bool: + * - true if the match finder is supported by this liblzma build. + * - false otherwise. */ extern LZMA_API(lzma_bool) lzma_mf_is_supported(lzma_match_finder match_finder) lzma_nothrow lzma_attr_const; @@ -158,14 +183,20 @@ /** * \brief Test if given compression mode is supported * - * Return true if the given compression mode is supported by this liblzma - * build. Otherwise false is returned. It is safe to call this with a value - * that isn't listed in lzma_mode enumeration; the return value will be false. + * It is safe to call this with a value that isn't listed in lzma_mode + * enumeration; the return value will be false. * * There is no way to list which modes are available in this particular * liblzma version and build. It would be useless, because a new compression * mode, which the application developer wasn't aware, could require giving * additional options to the encoder that the older modes don't need. + * + * \param mode Mode ID. + * + * \return lzma_bool: + * - true if the compression mode is supported by this liblzma + * build. + * - false otherwise. */ extern LZMA_API(lzma_bool) lzma_mode_is_supported(lzma_mode mode) lzma_nothrow lzma_attr_const; @@ -257,7 +288,7 @@ * \brief Number of literal context bits * * How many of the highest bits of the previous uncompressed - * eight-bit byte (also known as `literal') are taken into + * eight-bit byte (also known as 'literal') are taken into * account when predicting the bits of the next literal. * * E.g. in typical English text, an upper-case letter is @@ -374,6 +405,82 @@ */ uint32_t depth; + /** + * \brief For LZMA_FILTER_LZMA1EXT: Extended flags + * + * This is used only with LZMA_FILTER_LZMA1EXT. + * + * Currently only one flag is supported, LZMA_LZMA1EXT_ALLOW_EOPM: + * + * - Encoder: If the flag is set, then end marker is written just + * like it is with LZMA_FILTER_LZMA1. Without this flag the + * end marker isn't written and the application has to store + * the uncompressed size somewhere outside the compressed stream. + * To decompress streams without the end marker, the application + * has to set the correct uncompressed size in ext_size_low and + * ext_size_high. + * + * - Decoder: If the uncompressed size in ext_size_low and + * ext_size_high is set to the special value UINT64_MAX + * (indicating unknown uncompressed size) then this flag is + * ignored and the end marker must always be present, that is, + * the behavior is identical to LZMA_FILTER_LZMA1. + * + * Otherwise, if this flag isn't set, then the input stream + * must not have the end marker; if the end marker is detected + * then it will result in LZMA_DATA_ERROR. This is useful when + * it is known that the stream must not have the end marker and + * strict validation is wanted. + * + * If this flag is set, then it is autodetected if the end marker + * is present after the specified number of uncompressed bytes + * has been decompressed (ext_size_low and ext_size_high). The + * end marker isn't allowed in any other position. This behavior + * is useful when uncompressed size is known but the end marker + * may or may not be present. This is the case, for example, + * in .7z files (valid .7z files that have the end marker in + * LZMA1 streams are rare but they do exist). + */ + uint32_t ext_flags; +# define LZMA_LZMA1EXT_ALLOW_EOPM UINT32_C(0x01) + + /** + * \brief For LZMA_FILTER_LZMA1EXT: Uncompressed size (low bits) + * + * The 64-bit uncompressed size is needed for decompression with + * LZMA_FILTER_LZMA1EXT. The size is ignored by the encoder. + * + * The special value UINT64_MAX indicates that the uncompressed size + * is unknown and that the end of payload marker (also known as + * end of stream marker) must be present to indicate the end of + * the LZMA1 stream. Any other value indicates the expected + * uncompressed size of the LZMA1 stream. (If LZMA1 was used together + * with filters that change the size of the data then the uncompressed + * size of the LZMA1 stream could be different than the final + * uncompressed size of the filtered stream.) + * + * ext_size_low holds the least significant 32 bits of the + * uncompressed size. The most significant 32 bits must be set + * in ext_size_high. The macro lzma_ext_size_set(opt_lzma, u64size) + * can be used to set these members. + * + * The 64-bit uncompressed size is split into two uint32_t variables + * because there were no reserved uint64_t members and using the + * same options structure for LZMA_FILTER_LZMA1, LZMA_FILTER_LZMA1EXT, + * and LZMA_FILTER_LZMA2 was otherwise more convenient than having + * a new options structure for LZMA_FILTER_LZMA1EXT. (Replacing two + * uint32_t members with one uint64_t changes the ABI on some systems + * as the alignment of this struct can increase from 4 bytes to 8.) + */ + uint32_t ext_size_low; + + /** + * \brief For LZMA_FILTER_LZMA1EXT: Uncompressed size (high bits) + * + * This holds the most significant 32 bits of the uncompressed size. + */ + uint32_t ext_size_high; + /* * Reserved space to allow possible future extensions without * breaking the ABI. You should not touch these, because the names @@ -381,25 +488,57 @@ * with the currently supported options, so it is safe to leave these * uninitialized. */ - uint32_t reserved_int1; - uint32_t reserved_int2; - uint32_t reserved_int3; + + /** \private Reserved member. */ uint32_t reserved_int4; + + /** \private Reserved member. */ uint32_t reserved_int5; + + /** \private Reserved member. */ uint32_t reserved_int6; + + /** \private Reserved member. */ uint32_t reserved_int7; + + /** \private Reserved member. */ uint32_t reserved_int8; + + /** \private Reserved member. */ lzma_reserved_enum reserved_enum1; + + /** \private Reserved member. */ lzma_reserved_enum reserved_enum2; + + /** \private Reserved member. */ lzma_reserved_enum reserved_enum3; + + /** \private Reserved member. */ lzma_reserved_enum reserved_enum4; + + /** \private Reserved member. */ void *reserved_ptr1; + + /** \private Reserved member. */ void *reserved_ptr2; } lzma_options_lzma; /** + * \brief Macro to set the 64-bit uncompressed size in ext_size_* + * + * This might be convenient when decoding using LZMA_FILTER_LZMA1EXT. + * This isn't used with LZMA_FILTER_LZMA1 or LZMA_FILTER_LZMA2. + */ +#define lzma_set_ext_size(opt_lzma2, u64size) \ +do { \ + (opt_lzma2).ext_size_low = (uint32_t)(u64size); \ + (opt_lzma2).ext_size_high = (uint32_t)((uint64_t)(u64size) >> 32); \ +} while (0) + + +/** * \brief Set a compression preset to lzma_options_lzma structure * * 0 is the fastest and 9 is the slowest. These match the switches -0 .. -9 @@ -408,13 +547,22 @@ * The flags are defined in container.h, because the flags are used also * with lzma_easy_encoder(). * - * The preset values are subject to changes between liblzma versions. + * The preset levels are subject to changes between liblzma versions. * * This function is available only if LZMA1 or LZMA2 encoder has been enabled * when building liblzma. * - * \return On success, false is returned. If the preset is not - * supported, true is returned. + * If features (like certain match finders) have been disabled at build time, + * then the function may return success (false) even though the resulting + * LZMA1/LZMA2 options may not be usable for encoder initialization + * (LZMA_OPTIONS_ERROR). + * + * \param[out] options Pointer to LZMA1 or LZMA2 options to be filled + * \param preset Preset level bitwse-ORed with preset flags + * + * \return lzma_bool: + * - true if the preset is not supported (failure). + * - false otherwise (success). */ extern LZMA_API(lzma_bool) lzma_lzma_preset( lzma_options_lzma *options, uint32_t preset) lzma_nothrow;
diff --git a/Utilities/cmliblzma/liblzma/api/lzma/stream_flags.h b/Utilities/cmliblzma/liblzma/api/lzma/stream_flags.h index bbdd408..a33fe46 100644 --- a/Utilities/cmliblzma/liblzma/api/lzma/stream_flags.h +++ b/Utilities/cmliblzma/liblzma/api/lzma/stream_flags.h
@@ -1,15 +1,13 @@ +/* SPDX-License-Identifier: 0BSD */ + /** * \file lzma/stream_flags.h * \brief .xz Stream Header and Stream Footer encoder and decoder + * \note Never include this file directly. Use <lzma.h> instead. */ /* * Author: Lasse Collin - * - * This file has been put into the public domain. - * You can do whatever you want with this file. - * - * See ../lzma.h for information about liblzma as a whole. */ #ifndef LZMA_H_INTERNAL @@ -36,7 +34,7 @@ * * To prevent API and ABI breakages if new features are needed in * Stream Header or Stream Footer, a version number is used to - * indicate which fields in this structure are in use. For now, + * indicate which members in this structure are in use. For now, * version must always be zero. With non-zero version, the * lzma_stream_header_encode() and lzma_stream_footer_encode() * will return LZMA_OPTIONS_ERROR. @@ -67,7 +65,15 @@ * Footer have been decoded. */ lzma_vli backward_size; + + /** + * \brief Minimum value for lzma_stream_flags.backward_size + */ # define LZMA_BACKWARD_SIZE_MIN 4 + + /** + * \brief Maximum value for lzma_stream_flags.backward_size + */ # define LZMA_BACKWARD_SIZE_MAX (LZMA_VLI_C(1) << 34) /** @@ -87,19 +93,47 @@ * is just two bytes plus Backward Size of four bytes. But it's * nice to have the proper types when they are needed.) */ + + /** \private Reserved member. */ lzma_reserved_enum reserved_enum1; + + /** \private Reserved member. */ lzma_reserved_enum reserved_enum2; + + /** \private Reserved member. */ lzma_reserved_enum reserved_enum3; + + /** \private Reserved member. */ lzma_reserved_enum reserved_enum4; + + /** \private Reserved member. */ lzma_bool reserved_bool1; + + /** \private Reserved member. */ lzma_bool reserved_bool2; + + /** \private Reserved member. */ lzma_bool reserved_bool3; + + /** \private Reserved member. */ lzma_bool reserved_bool4; + + /** \private Reserved member. */ lzma_bool reserved_bool5; + + /** \private Reserved member. */ lzma_bool reserved_bool6; + + /** \private Reserved member. */ lzma_bool reserved_bool7; + + /** \private Reserved member. */ lzma_bool reserved_bool8; + + /** \private Reserved member. */ uint32_t reserved_int1; + + /** \private Reserved member. */ uint32_t reserved_int2; } lzma_stream_flags; @@ -111,10 +145,11 @@ * \param options Stream Header options to be encoded. * options->backward_size is ignored and doesn't * need to be initialized. - * \param out Beginning of the output buffer of + * \param[out] out Beginning of the output buffer of * LZMA_STREAM_HEADER_SIZE bytes. * - * \return - LZMA_OK: Encoding was successful. + * \return Possible lzma_ret values: + * - LZMA_OK: Encoding was successful. * - LZMA_OPTIONS_ERROR: options->version is not supported by * this liblzma version. * - LZMA_PROG_ERROR: Invalid options. @@ -128,10 +163,11 @@ * \brief Encode Stream Footer * * \param options Stream Footer options to be encoded. - * \param out Beginning of the output buffer of + * \param[out] out Beginning of the output buffer of * LZMA_STREAM_HEADER_SIZE bytes. * - * \return - LZMA_OK: Encoding was successful. + * \return Possible lzma_ret values: + * - LZMA_OK: Encoding was successful. * - LZMA_OPTIONS_ERROR: options->version is not supported by * this liblzma version. * - LZMA_PROG_ERROR: Invalid options. @@ -144,32 +180,33 @@ /** * \brief Decode Stream Header * - * \param options Target for the decoded Stream Header options. - * \param in Beginning of the input buffer of - * LZMA_STREAM_HEADER_SIZE bytes. - * * options->backward_size is always set to LZMA_VLI_UNKNOWN. This is to * help comparing Stream Flags from Stream Header and Stream Footer with * lzma_stream_flags_compare(). * - * \return - LZMA_OK: Decoding was successful. + * \note When decoding .xz files that contain multiple Streams, it may + * make sense to print "file format not recognized" only if + * decoding of the Stream Header of the \a first Stream gives + * LZMA_FORMAT_ERROR. If non-first Stream Header gives + * LZMA_FORMAT_ERROR, the message used for LZMA_DATA_ERROR is + * probably more appropriate. + * For example, the Stream decoder in liblzma uses + * LZMA_DATA_ERROR if LZMA_FORMAT_ERROR is returned by + * lzma_stream_header_decode() when decoding non-first Stream. + * + * \param[out] options Target for the decoded Stream Header options. + * \param in Beginning of the input buffer of + * LZMA_STREAM_HEADER_SIZE bytes. + * + * + * \return Possible lzma_ret values: + * - LZMA_OK: Decoding was successful. * - LZMA_FORMAT_ERROR: Magic bytes don't match, thus the given * buffer cannot be Stream Header. * - LZMA_DATA_ERROR: CRC32 doesn't match, thus the header * is corrupt. * - LZMA_OPTIONS_ERROR: Unsupported options are present * in the header. - * - * \note When decoding .xz files that contain multiple Streams, it may - * make sense to print "file format not recognized" only if - * decoding of the Stream Header of the _first_ Stream gives - * LZMA_FORMAT_ERROR. If non-first Stream Header gives - * LZMA_FORMAT_ERROR, the message used for LZMA_DATA_ERROR is - * probably more appropriate. - * - * For example, Stream decoder in liblzma uses LZMA_DATA_ERROR if - * LZMA_FORMAT_ERROR is returned by lzma_stream_header_decode() - * when decoding non-first Stream. */ extern LZMA_API(lzma_ret) lzma_stream_header_decode( lzma_stream_flags *options, const uint8_t *in) @@ -179,24 +216,25 @@ /** * \brief Decode Stream Footer * - * \param options Target for the decoded Stream Header options. + * \note If Stream Header was already decoded successfully, but + * decoding Stream Footer returns LZMA_FORMAT_ERROR, the + * application should probably report some other error message + * than "file format not recognized". The file likely + * is corrupt (possibly truncated). The Stream decoder in liblzma + * uses LZMA_DATA_ERROR in this situation. + * + * \param[out] options Target for the decoded Stream Footer options. * \param in Beginning of the input buffer of * LZMA_STREAM_HEADER_SIZE bytes. * - * \return - LZMA_OK: Decoding was successful. + * \return Possible lzma_ret values: + * - LZMA_OK: Decoding was successful. * - LZMA_FORMAT_ERROR: Magic bytes don't match, thus the given * buffer cannot be Stream Footer. * - LZMA_DATA_ERROR: CRC32 doesn't match, thus the Stream Footer * is corrupt. * - LZMA_OPTIONS_ERROR: Unsupported options are present * in Stream Footer. - * - * \note If Stream Header was already decoded successfully, but - * decoding Stream Footer returns LZMA_FORMAT_ERROR, the - * application should probably report some other error message - * than "file format not recognized", since the file more likely - * is corrupt (possibly truncated). Stream decoder in liblzma - * uses LZMA_DATA_ERROR in this situation. */ extern LZMA_API(lzma_ret) lzma_stream_footer_decode( lzma_stream_flags *options, const uint8_t *in) @@ -209,7 +247,11 @@ * backward_size values are compared only if both are not * LZMA_VLI_UNKNOWN. * - * \return - LZMA_OK: Both are equal. If either had backward_size set + * \param a Pointer to lzma_stream_flags structure + * \param b Pointer to lzma_stream_flags structure + * + * \return Possible lzma_ret values: + * - LZMA_OK: Both are equal. If either had backward_size set * to LZMA_VLI_UNKNOWN, backward_size values were not * compared or validated. * - LZMA_DATA_ERROR: The structures differ.
diff --git a/Utilities/cmliblzma/liblzma/api/lzma/version.h b/Utilities/cmliblzma/liblzma/api/lzma/version.h index 2bf3eae..e86c0ea 100644 --- a/Utilities/cmliblzma/liblzma/api/lzma/version.h +++ b/Utilities/cmliblzma/liblzma/api/lzma/version.h
@@ -1,15 +1,13 @@ +/* SPDX-License-Identifier: 0BSD */ + /** * \file lzma/version.h * \brief Version number + * \note Never include this file directly. Use <lzma.h> instead. */ /* * Author: Lasse Collin - * - * This file has been put into the public domain. - * You can do whatever you want with this file. - * - * See ../lzma.h for information about liblzma as a whole. */ #ifndef LZMA_H_INTERNAL @@ -17,14 +15,26 @@ #endif -/* - * Version number split into components - */ +/** \brief Major version number of the liblzma release. */ #define LZMA_VERSION_MAJOR 5 -#define LZMA_VERSION_MINOR 2 -#define LZMA_VERSION_PATCH 5 + +/** \brief Minor version number of the liblzma release. */ +#define LZMA_VERSION_MINOR 6 + +/** \brief Patch version number of the liblzma release. */ +#define LZMA_VERSION_PATCH 3 + +/** + * \brief Version stability marker + * + * This will always be one of three values: + * - LZMA_VERSION_STABILITY_ALPHA + * - LZMA_VERSION_STABILITY_BETA + * - LZMA_VERSION_STABILITY_STABLE + */ #define LZMA_VERSION_STABILITY LZMA_VERSION_STABILITY_STABLE +/** \brief Commit version number of the liblzma release */ #ifndef LZMA_VERSION_COMMIT # define LZMA_VERSION_COMMIT "" #endif @@ -95,15 +105,16 @@ LZMA_VERSION_COMMIT) -/* #ifndef is needed for use with windres (MinGW or Cygwin). */ +/* #ifndef is needed for use with windres (MinGW-w64 or Cygwin). */ #ifndef LZMA_H_INTERNAL_RC /** * \brief Run-time version number as an integer * - * Return the value of LZMA_VERSION macro at the compile time of liblzma. - * This allows the application to compare if it was built against the same, + * This allows an application to compare if it was built against the same, * older, or newer version of liblzma that is currently running. + * + * \return The value of LZMA_VERSION macro at the compile time of liblzma */ extern LZMA_API(uint32_t) lzma_version_number(void) lzma_nothrow lzma_attr_const; @@ -112,8 +123,10 @@ /** * \brief Run-time version as a string * - * This function may be useful if you want to display which version of - * liblzma your application is currently using. + * This function may be useful to display which version of liblzma an + * application is currently using. + * + * \return Run-time version of liblzma */ extern LZMA_API(const char *) lzma_version_string(void) lzma_nothrow lzma_attr_const;
diff --git a/Utilities/cmliblzma/liblzma/api/lzma/vli.h b/Utilities/cmliblzma/liblzma/api/lzma/vli.h index 1b7a952..6b04902 100644 --- a/Utilities/cmliblzma/liblzma/api/lzma/vli.h +++ b/Utilities/cmliblzma/liblzma/api/lzma/vli.h
@@ -1,6 +1,9 @@ +/* SPDX-License-Identifier: 0BSD */ + /** * \file lzma/vli.h * \brief Variable-length integer handling + * \note Never include this file directly. Use <lzma.h> instead. * * In the .xz format, most integers are encoded in a variable-length * representation, which is sometimes called little endian base-128 encoding. @@ -16,11 +19,6 @@ /* * Author: Lasse Collin - * - * This file has been put into the public domain. - * You can do whatever you want with this file. - * - * See ../lzma.h for information about liblzma as a whole. */ #ifndef LZMA_H_INTERNAL @@ -69,8 +67,8 @@ * This is useful to test that application has given acceptable values * for example in the uncompressed_size and compressed_size variables. * - * \return True if the integer is representable as VLI or if it - * indicates unknown value. + * \return True if the integer is representable as a VLI or if it + * indicates an unknown value. False otherwise. */ #define lzma_vli_is_valid(vli) \ ((vli) <= LZMA_VLI_MAX || (vli) == LZMA_VLI_UNKNOWN) @@ -86,12 +84,12 @@ * integer has been encoded. * * \param vli Integer to be encoded - * \param vli_pos How many VLI-encoded bytes have already been written + * \param[out] vli_pos How many VLI-encoded bytes have already been written * out. When starting to encode a new integer in * multi-call mode, *vli_pos must be set to zero. * To use single-call encoding, set vli_pos to NULL. - * \param out Beginning of the output buffer - * \param out_pos The next byte will be written to out[*out_pos]. + * \param[out] out Beginning of the output buffer + * \param[out] out_pos The next byte will be written to out[*out_pos]. * \param out_size Size of the out buffer; the first byte into * which no data is written to is out[out_size]. * @@ -121,15 +119,15 @@ * * Like lzma_vli_encode(), this function has single-call and multi-call modes. * - * \param vli Pointer to decoded integer. The decoder will + * \param[out] vli Pointer to decoded integer. The decoder will * initialize it to zero when *vli_pos == 0, so * application isn't required to initialize *vli. - * \param vli_pos How many bytes have already been decoded. When + * \param[out] vli_pos How many bytes have already been decoded. When * starting to decode a new integer in multi-call * mode, *vli_pos must be initialized to zero. To * use single-call decoding, set vli_pos to NULL. * \param in Beginning of the input buffer - * \param in_pos The next byte will be read from in[*in_pos]. + * \param[out] in_pos The next byte will be read from in[*in_pos]. * \param in_size Size of the input buffer; the first byte that * won't be read is in[in_size]. * @@ -159,6 +157,8 @@ /** * \brief Get the number of bytes required to encode a VLI * + * \param vli Integer whose encoded size is to be determined + * * \return Number of bytes on success (1-9). If vli isn't valid, * zero is returned. */
diff --git a/Utilities/cmliblzma/liblzma/check/check.c b/Utilities/cmliblzma/liblzma/check/check.c index 428ddae..7734ace 100644 --- a/Utilities/cmliblzma/liblzma/check/check.c +++ b/Utilities/cmliblzma/liblzma/check/check.c
@@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file check.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "check.h"
diff --git a/Utilities/cmliblzma/liblzma/check/check.h b/Utilities/cmliblzma/liblzma/check/check.h index 3007d88..f0eb117 100644 --- a/Utilities/cmliblzma/liblzma/check/check.h +++ b/Utilities/cmliblzma/liblzma/check/check.h
@@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file check.h @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef LZMA_CHECK_H @@ -99,19 +98,22 @@ /// lzma_crc32_table[0] is needed by LZ encoder so we need to keep /// the array two-dimensional. #ifdef HAVE_SMALL +lzma_attr_visibility_hidden extern uint32_t lzma_crc32_table[1][256]; + extern void lzma_crc32_init(void); + #else + +lzma_attr_visibility_hidden extern const uint32_t lzma_crc32_table[8][256]; + +lzma_attr_visibility_hidden extern const uint64_t lzma_crc64_table[4][256]; #endif /// \brief Initialize *check depending on type -/// -/// \return LZMA_OK on success. LZMA_UNSUPPORTED_CHECK if the type is not -/// supported by the current version or build of liblzma. -/// LZMA_PROG_ERROR if type > LZMA_CHECK_ID_MAX. extern void lzma_check_init(lzma_check_state *check, lzma_check type); /// Update the check state
diff --git a/Utilities/cmliblzma/liblzma/check/crc32_arm64.h b/Utilities/cmliblzma/liblzma/check/crc32_arm64.h new file mode 100644 index 0000000..39c1c63 --- /dev/null +++ b/Utilities/cmliblzma/liblzma/check/crc32_arm64.h
@@ -0,0 +1,122 @@ +// SPDX-License-Identifier: 0BSD + +/////////////////////////////////////////////////////////////////////////////// +// +/// \file crc32_arm64.h +/// \brief CRC32 calculation with ARM64 optimization +// +// Authors: Chenxi Mao +// Jia Tan +// Hans Jansen +// +/////////////////////////////////////////////////////////////////////////////// + +#ifndef LZMA_CRC32_ARM64_H +#define LZMA_CRC32_ARM64_H + +// MSVC always has the CRC intrinsics available when building for ARM64 +// there is no need to include any header files. +#ifndef _MSC_VER +# include <arm_acle.h> +#endif + +// If both versions are going to be built, we need runtime detection +// to check if the instructions are supported. +#if defined(CRC32_GENERIC) && defined(CRC32_ARCH_OPTIMIZED) +# if defined(HAVE_GETAUXVAL) || defined(HAVE_ELF_AUX_INFO) +# include <sys/auxv.h> +# elif defined(_WIN32) +# include <processthreadsapi.h> +# elif defined(__APPLE__) && defined(HAVE_SYSCTLBYNAME) +# include <sys/sysctl.h> +# endif +#endif + +// Some EDG-based compilers support ARM64 and define __GNUC__ +// (such as Nvidia's nvcc), but do not support function attributes. +// +// NOTE: Build systems check for this too, keep them in sync with this. +#if (defined(__GNUC__) || defined(__clang__)) && !defined(__EDG__) +# define crc_attr_target __attribute__((__target__("+crc"))) +#else +# define crc_attr_target +#endif + + +crc_attr_target +static uint32_t +crc32_arch_optimized(const uint8_t *buf, size_t size, uint32_t crc) +{ + crc = ~crc; + + // Align the input buffer because this was shown to be + // significantly faster than unaligned accesses. + const size_t align_amount = my_min(size, (0U - (uintptr_t)buf) & 7); + + for (const uint8_t *limit = buf + align_amount; buf < limit; ++buf) + crc = __crc32b(crc, *buf); + + size -= align_amount; + + // Process 8 bytes at a time. The end point is determined by + // ignoring the least significant three bits of size to ensure + // we do not process past the bounds of the buffer. This guarantees + // that limit is a multiple of 8 and is strictly less than size. + for (const uint8_t *limit = buf + (size & ~(size_t)7); + buf < limit; buf += 8) + crc = __crc32d(crc, aligned_read64le(buf)); + + // Process the remaining bytes that are not 8 byte aligned. + for (const uint8_t *limit = buf + (size & 7); buf < limit; ++buf) + crc = __crc32b(crc, *buf); + + return ~crc; +} + + +#if defined(CRC32_GENERIC) && defined(CRC32_ARCH_OPTIMIZED) +static inline bool +is_arch_extension_supported(void) +{ +#if defined(HAVE_GETAUXVAL) + return (getauxval(AT_HWCAP) & HWCAP_CRC32) != 0; + +#elif defined(HAVE_ELF_AUX_INFO) + unsigned long feature_flags; + + if (elf_aux_info(AT_HWCAP, &feature_flags, sizeof(feature_flags)) != 0) + return false; + + return (feature_flags & HWCAP_CRC32) != 0; + +#elif defined(_WIN32) + return IsProcessorFeaturePresent( + PF_ARM_V8_CRC32_INSTRUCTIONS_AVAILABLE); + +#elif defined(__APPLE__) && defined(HAVE_SYSCTLBYNAME) + int has_crc32 = 0; + size_t size = sizeof(has_crc32); + + // The sysctlbyname() function requires a string identifier for the + // CPU feature it tests. The Apple documentation lists the string + // "hw.optional.armv8_crc32", which can be found here: + // https://developer.apple.com/documentation/kernel/1387446-sysctlbyname/determining_instruction_set_characteristics#3915619 + if (sysctlbyname("hw.optional.armv8_crc32", &has_crc32, + &size, NULL, 0) != 0) + return false; + + return has_crc32; + +#else + // If a runtime detection method cannot be found, then this must + // be a compile time error. The checks in crc_common.h should ensure + // a runtime detection method is always found if this function is + // built. It would be possible to just return false here, but this + // is inefficient for binary size and runtime since only the generic + // method could ever be used. +# error Runtime detection method unavailable. +#endif +} +#endif + +#endif // LZMA_CRC32_ARM64_H
diff --git a/Utilities/cmliblzma/liblzma/check/crc32_fast.c b/Utilities/cmliblzma/liblzma/check/crc32_fast.c index eed7350..16dbb74 100644 --- a/Utilities/cmliblzma/liblzma/check/crc32_fast.c +++ b/Utilities/cmliblzma/liblzma/check/crc32_fast.c
@@ -1,35 +1,40 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file crc32.c /// \brief CRC32 calculation -/// -/// Calculate the CRC32 using the slice-by-eight algorithm. -/// It is explained in this document: -/// http://www.intel.com/technology/comms/perfnet/download/CRC_generators.pdf -/// The code in this file is not the same as in Intel's paper, but -/// the basic principle is identical. // -// Author: Lasse Collin -// -// This file has been put into the public domain. -// You can do whatever you want with this file. +// Authors: Lasse Collin +// Ilya Kurdyukov +// Hans Jansen // /////////////////////////////////////////////////////////////////////////////// #include "check.h" -#include "crc_macros.h" +#include "crc_common.h" + +#if defined(CRC_X86_CLMUL) +# define BUILDING_CRC32_CLMUL +# include "crc_x86_clmul.h" +#elif defined(CRC32_ARM64) +# include "crc32_arm64.h" +#endif -// If you make any changes, do some benchmarking! Seemingly unrelated -// changes can very easily ruin the performance (and very probably is -// very compiler dependent). -extern LZMA_API(uint32_t) -lzma_crc32(const uint8_t *buf, size_t size, uint32_t crc) +#ifdef CRC32_GENERIC + +/////////////////// +// Generic CRC32 // +/////////////////// + +static uint32_t +crc32_generic(const uint8_t *buf, size_t size, uint32_t crc) { crc = ~crc; #ifdef WORDS_BIGENDIAN - crc = bswap32(crc); + crc = byteswap32(crc); #endif if (size > 8) { @@ -75,8 +80,125 @@ crc = lzma_crc32_table[0][*buf++ ^ A(crc)] ^ S8(crc); #ifdef WORDS_BIGENDIAN - crc = bswap32(crc); + crc = byteswap32(crc); #endif return ~crc; } +#endif + + +#if defined(CRC32_GENERIC) && defined(CRC32_ARCH_OPTIMIZED) + +////////////////////////// +// Function dispatching // +////////////////////////// + +// If both the generic and arch-optimized implementations are built, then +// the function to use is selected at runtime because the system running +// the binary might not have the arch-specific instruction set extension(s) +// available. The dispatch methods in order of priority: +// +// 1. Constructor. This method uses __attribute__((__constructor__)) to +// set crc32_func at load time. This avoids extra computation (and any +// unlikely threading bugs) on the first call to lzma_crc32() to decide +// which implementation should be used. +// +// 2. First Call Resolution. On the very first call to lzma_crc32(), the +// call will be directed to crc32_dispatch() instead. This will set the +// appropriate implementation function and will not be called again. +// This method does not use any kind of locking but is safe because if +// multiple threads run the dispatcher simultaneously then they will all +// set crc32_func to the same value. + +typedef uint32_t (*crc32_func_type)( + const uint8_t *buf, size_t size, uint32_t crc); + +// This resolver is shared between all dispatch methods. +static crc32_func_type +crc32_resolve(void) +{ + return is_arch_extension_supported() + ? &crc32_arch_optimized : &crc32_generic; +} + + +#ifdef HAVE_FUNC_ATTRIBUTE_CONSTRUCTOR +// Constructor method. +# define CRC32_SET_FUNC_ATTR __attribute__((__constructor__)) +static crc32_func_type crc32_func; +#else +// First Call Resolution method. +# define CRC32_SET_FUNC_ATTR +static uint32_t crc32_dispatch(const uint8_t *buf, size_t size, uint32_t crc); +static crc32_func_type crc32_func = &crc32_dispatch; +#endif + +CRC32_SET_FUNC_ATTR +static void +crc32_set_func(void) +{ + crc32_func = crc32_resolve(); + return; +} + +#ifndef HAVE_FUNC_ATTRIBUTE_CONSTRUCTOR +static uint32_t +crc32_dispatch(const uint8_t *buf, size_t size, uint32_t crc) +{ + // When __attribute__((__constructor__)) isn't supported, set the + // function pointer without any locking. If multiple threads run + // the detection code in parallel, they will all end up setting + // the pointer to the same value. This avoids the use of + // mythread_once() on every call to lzma_crc32() but this likely + // isn't strictly standards compliant. Let's change it if it breaks. + crc32_set_func(); + return crc32_func(buf, size, crc); +} + +#endif +#endif + + +extern LZMA_API(uint32_t) +lzma_crc32(const uint8_t *buf, size_t size, uint32_t crc) +{ +#if defined(CRC32_GENERIC) && defined(CRC32_ARCH_OPTIMIZED) + // On x86-64, if CLMUL is available, it is the best for non-tiny + // inputs, being over twice as fast as the generic slice-by-four + // version. However, for size <= 16 it's different. In the extreme + // case of size == 1 the generic version can be five times faster. + // At size >= 8 the CLMUL starts to become reasonable. It + // varies depending on the alignment of buf too. + // + // The above doesn't include the overhead of mythread_once(). + // At least on x86-64 GNU/Linux, pthread_once() is very fast but + // it still makes lzma_crc32(buf, 1, crc) 50-100 % slower. When + // size reaches 12-16 bytes the overhead becomes negligible. + // + // So using the generic version for size <= 16 may give better + // performance with tiny inputs but if such inputs happen rarely + // it's not so obvious because then the lookup table of the + // generic version may not be in the processor cache. +#ifdef CRC_USE_GENERIC_FOR_SMALL_INPUTS + if (size <= 16) + return crc32_generic(buf, size, crc); +#endif + +/* +#ifndef HAVE_FUNC_ATTRIBUTE_CONSTRUCTOR + // See crc32_dispatch(). This would be the alternative which uses + // locking and doesn't use crc32_dispatch(). Note that on Windows + // this method needs Vista threads. + mythread_once(crc64_set_func); +#endif +*/ + return crc32_func(buf, size, crc); + +#elif defined(CRC32_ARCH_OPTIMIZED) + return crc32_arch_optimized(buf, size, crc); + +#else + return crc32_generic(buf, size, crc); +#endif +}
diff --git a/Utilities/cmliblzma/liblzma/check/crc32_small.c b/Utilities/cmliblzma/liblzma/check/crc32_small.c index 5f8a328..6a1bd66 100644 --- a/Utilities/cmliblzma/liblzma/check/crc32_small.c +++ b/Utilities/cmliblzma/liblzma/check/crc32_small.c
@@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file crc32_small.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "check.h" @@ -16,6 +15,9 @@ uint32_t lzma_crc32_table[1][256]; +#ifdef HAVE_FUNC_ATTRIBUTE_CONSTRUCTOR +__attribute__((__constructor__)) +#endif static void crc32_init(void) { @@ -37,18 +39,22 @@ } +#ifndef HAVE_FUNC_ATTRIBUTE_CONSTRUCTOR extern void lzma_crc32_init(void) { mythread_once(crc32_init); return; } +#endif extern LZMA_API(uint32_t) lzma_crc32(const uint8_t *buf, size_t size, uint32_t crc) { +#ifndef HAVE_FUNC_ATTRIBUTE_CONSTRUCTOR lzma_crc32_init(); +#endif crc = ~crc;
diff --git a/Utilities/cmliblzma/liblzma/check/crc32_table.c b/Utilities/cmliblzma/liblzma/check/crc32_table.c index b11762a..56413ee 100644 --- a/Utilities/cmliblzma/liblzma/check/crc32_table.c +++ b/Utilities/cmliblzma/liblzma/check/crc32_table.c
@@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file crc32_table.c @@ -5,18 +7,36 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "common.h" + +// FIXME: Compared to crc_common.h this has to check for __x86_64__ too +// so that in 32-bit builds crc32_x86.S won't break due to a missing table. +#if defined(HAVE_USABLE_CLMUL) && ((defined(__x86_64__) && defined(__SSSE3__) \ + && defined(__SSE4_1__) && defined(__PCLMUL__)) \ + || (defined(__e2k__) && __iset__ >= 6)) +# define NO_CRC32_TABLE + +#elif defined(HAVE_ARM64_CRC32) \ + && !defined(WORDS_BIGENDIAN) \ + && defined(__ARM_FEATURE_CRC32) +# define NO_CRC32_TABLE +#endif + + +#if !defined(HAVE_ENCODERS) && defined(NO_CRC32_TABLE) +// No table needed. Use a typedef to avoid an empty translation unit. +typedef void lzma_crc32_dummy; + +#else // Having the declaration here silences clang -Wmissing-variable-declarations. extern const uint32_t lzma_crc32_table[8][256]; -#ifdef WORDS_BIGENDIAN -# include "crc32_table_be.h" -#else -# include "crc32_table_le.h" +# ifdef WORDS_BIGENDIAN +# include "crc32_table_be.h" +# else +# include "crc32_table_le.h" +# endif #endif
diff --git a/Utilities/cmliblzma/liblzma/check/crc32_table_be.h b/Utilities/cmliblzma/liblzma/check/crc32_table_be.h index c483cb6..505c230 100644 --- a/Utilities/cmliblzma/liblzma/check/crc32_table_be.h +++ b/Utilities/cmliblzma/liblzma/check/crc32_table_be.h
@@ -1,4 +1,6 @@ -/* This file has been automatically generated by crc32_tablegen.c. */ +// SPDX-License-Identifier: 0BSD + +// This file has been generated by crc32_tablegen.c. const uint32_t lzma_crc32_table[8][256] = { {
diff --git a/Utilities/cmliblzma/liblzma/check/crc32_table_le.h b/Utilities/cmliblzma/liblzma/check/crc32_table_le.h index 25f4fc4..e89c21a 100644 --- a/Utilities/cmliblzma/liblzma/check/crc32_table_le.h +++ b/Utilities/cmliblzma/liblzma/check/crc32_table_le.h
@@ -1,4 +1,6 @@ -/* This file has been automatically generated by crc32_tablegen.c. */ +// SPDX-License-Identifier: 0BSD + +// This file has been generated by crc32_tablegen.c. const uint32_t lzma_crc32_table[8][256] = { {
diff --git a/Utilities/cmliblzma/liblzma/check/crc32_tablegen.c b/Utilities/cmliblzma/liblzma/check/crc32_tablegen.c index 31a4d27..b8cf459 100644 --- a/Utilities/cmliblzma/liblzma/check/crc32_tablegen.c +++ b/Utilities/cmliblzma/liblzma/check/crc32_tablegen.c
@@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file crc32_tablegen.c @@ -9,9 +11,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include <stdio.h> @@ -44,7 +43,7 @@ #ifdef WORDS_BIGENDIAN for (size_t s = 0; s < 8; ++s) for (size_t b = 0; b < 256; ++b) - crc32_table[s][b] = bswap32(crc32_table[s][b]); + crc32_table[s][b] = byteswap32(crc32_table[s][b]); #endif return; @@ -54,9 +53,11 @@ static void print_crc32_table(void) { - printf("/* This file has been automatically generated by " - "crc32_tablegen.c. */\n\n" - "const uint32_t lzma_crc32_table[8][256] = {\n\t{"); + // Split the SPDX string so that it won't accidentally match + // when tools search for the string. + printf("// SPDX" "-License-Identifier" ": 0BSD\n\n" + "// This file has been generated by crc32_tablegen.c.\n\n" + "const uint32_t lzma_crc32_table[8][256] = {\n\t{"); for (size_t s = 0; s < 8; ++s) { for (size_t b = 0; b < 256; ++b) { @@ -82,9 +83,11 @@ static void print_lz_table(void) { - printf("/* This file has been automatically generated by " - "crc32_tablegen.c. */\n\n" - "const uint32_t lzma_lz_hash_table[256] = {"); + // Split the SPDX string so that it won't accidentally match + // when tools search for the string. + printf("// SPDX" "-License-Identifier" ": 0BSD\n\n" + "// This file has been generated by crc32_tablegen.c.\n\n" + "const uint32_t lzma_lz_hash_table[256] = {"); for (size_t b = 0; b < 256; ++b) { if ((b % 4) == 0)
diff --git a/Utilities/cmliblzma/liblzma/check/crc32_x86.S b/Utilities/cmliblzma/liblzma/check/crc32_x86.S index 67f68a4..ddc3cee 100644 --- a/Utilities/cmliblzma/liblzma/check/crc32_x86.S +++ b/Utilities/cmliblzma/liblzma/check/crc32_x86.S
@@ -1,3 +1,5 @@ +/* SPDX-License-Identifier: 0BSD */ + /* * Speed-optimized CRC32 using slicing-by-eight algorithm * @@ -11,9 +13,6 @@ * Authors: Igor Pavlov (original version) * Lasse Collin (AT&T syntax, PIC support, better portability) * - * This file has been put into the public domain. - * You can do whatever you want with this file. - * * This code needs lzma_crc32_table, which can be created using the * following C code: @@ -51,6 +50,14 @@ * extern uint32_t lzma_crc32(const uint8_t *buf, size_t size, uint32_t crc); */ +/* When Intel CET is enabled, include <cet.h> in assembly code to mark + Intel CET support. */ +#ifdef __CET__ +# include <cet.h> +#else +# define _CET_ENDBR +#endif + /* * On some systems, the functions need to be prefixed. The prefix is * usually an underscore. @@ -83,6 +90,7 @@ ALIGN(4, 16) LZMA_CRC32: + _CET_ENDBR /* * Register usage: * %eax crc @@ -195,7 +203,7 @@ /* * Read the next four bytes, for which the CRC is calculated - * on the next interation of the loop. + * on the next iteration of the loop. */ movl 12(%esi), %ecx @@ -296,9 +304,9 @@ /* * This is needed to support non-executable stack. It's ugly to - * use __linux__ here, but I don't know a way to detect when + * use __FreeBSD__ and __linux__ here, but I don't know a way to detect when * we are using GNU assembler. */ -#if defined(__ELF__) && defined(__linux__) +#if defined(__ELF__) && (defined(__FreeBSD__) || defined(__linux__)) .section .note.GNU-stack,"",@progbits #endif
diff --git a/Utilities/cmliblzma/liblzma/check/crc64_fast.c b/Utilities/cmliblzma/liblzma/check/crc64_fast.c index 8af54cd..0ce83fe 100644 --- a/Utilities/cmliblzma/liblzma/check/crc64_fast.c +++ b/Utilities/cmliblzma/liblzma/check/crc64_fast.c
@@ -1,22 +1,29 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file crc64.c /// \brief CRC64 calculation -/// -/// Calculate the CRC64 using the slice-by-four algorithm. This is the same -/// idea that is used in crc32_fast.c, but for CRC64 we use only four tables -/// instead of eight to avoid increasing CPU cache usage. // -// Author: Lasse Collin -// -// This file has been put into the public domain. -// You can do whatever you want with this file. +// Authors: Lasse Collin +// Ilya Kurdyukov // /////////////////////////////////////////////////////////////////////////////// #include "check.h" -#include "crc_macros.h" +#include "crc_common.h" +#if defined(CRC_X86_CLMUL) +# define BUILDING_CRC64_CLMUL +# include "crc_x86_clmul.h" +#endif + + +#ifdef CRC64_GENERIC + +///////////////////////////////// +// Generic slice-by-four CRC64 // +///////////////////////////////// #ifdef WORDS_BIGENDIAN # define A1(x) ((x) >> 56) @@ -26,13 +33,13 @@ // See the comments in crc32_fast.c. They aren't duplicated here. -extern LZMA_API(uint64_t) -lzma_crc64(const uint8_t *buf, size_t size, uint64_t crc) +static uint64_t +crc64_generic(const uint8_t *buf, size_t size, uint64_t crc) { crc = ~crc; #ifdef WORDS_BIGENDIAN - crc = bswap64(crc); + crc = byteswap64(crc); #endif if (size > 4) { @@ -46,10 +53,11 @@ while (buf < limit) { #ifdef WORDS_BIGENDIAN - const uint32_t tmp = (crc >> 32) + const uint32_t tmp = (uint32_t)(crc >> 32) ^ aligned_read32ne(buf); #else - const uint32_t tmp = crc ^ aligned_read32ne(buf); + const uint32_t tmp = (uint32_t)crc + ^ aligned_read32ne(buf); #endif buf += 4; @@ -65,8 +73,84 @@ crc = lzma_crc64_table[0][*buf++ ^ A1(crc)] ^ S8(crc); #ifdef WORDS_BIGENDIAN - crc = bswap64(crc); + crc = byteswap64(crc); #endif return ~crc; } +#endif + + +#if defined(CRC64_GENERIC) && defined(CRC64_ARCH_OPTIMIZED) + +////////////////////////// +// Function dispatching // +////////////////////////// + +// If both the generic and arch-optimized implementations are usable, then +// the function that is used is selected at runtime. See crc32_fast.c. + +typedef uint64_t (*crc64_func_type)( + const uint8_t *buf, size_t size, uint64_t crc); + +static crc64_func_type +crc64_resolve(void) +{ + return is_arch_extension_supported() + ? &crc64_arch_optimized : &crc64_generic; +} + +#ifdef HAVE_FUNC_ATTRIBUTE_CONSTRUCTOR +# define CRC64_SET_FUNC_ATTR __attribute__((__constructor__)) +static crc64_func_type crc64_func; +#else +# define CRC64_SET_FUNC_ATTR +static uint64_t crc64_dispatch(const uint8_t *buf, size_t size, uint64_t crc); +static crc64_func_type crc64_func = &crc64_dispatch; +#endif + + +CRC64_SET_FUNC_ATTR +static void +crc64_set_func(void) +{ + crc64_func = crc64_resolve(); + return; +} + + +#ifndef HAVE_FUNC_ATTRIBUTE_CONSTRUCTOR +static uint64_t +crc64_dispatch(const uint8_t *buf, size_t size, uint64_t crc) +{ + crc64_set_func(); + return crc64_func(buf, size, crc); +} +#endif +#endif + + +extern LZMA_API(uint64_t) +lzma_crc64(const uint8_t *buf, size_t size, uint64_t crc) +{ +#if defined(CRC64_GENERIC) && defined(CRC64_ARCH_OPTIMIZED) + +#ifdef CRC_USE_GENERIC_FOR_SMALL_INPUTS + if (size <= 16) + return crc64_generic(buf, size, crc); +#endif + return crc64_func(buf, size, crc); + +#elif defined(CRC64_ARCH_OPTIMIZED) + // If arch-optimized version is used unconditionally without runtime + // CPU detection then omitting the generic version and its 8 KiB + // lookup table makes the library smaller. + // + // FIXME: Lookup table isn't currently omitted on 32-bit x86, + // see crc64_table.c. + return crc64_arch_optimized(buf, size, crc); + +#else + return crc64_generic(buf, size, crc); +#endif +}
diff --git a/Utilities/cmliblzma/liblzma/check/crc64_small.c b/Utilities/cmliblzma/liblzma/check/crc64_small.c index 55d7231..ee4ea26 100644 --- a/Utilities/cmliblzma/liblzma/check/crc64_small.c +++ b/Utilities/cmliblzma/liblzma/check/crc64_small.c
@@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file crc64_small.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "check.h" @@ -16,6 +15,9 @@ static uint64_t crc64_table[256]; +#ifdef HAVE_FUNC_ATTRIBUTE_CONSTRUCTOR +__attribute__((__constructor__)) +#endif static void crc64_init(void) { @@ -40,7 +42,9 @@ extern LZMA_API(uint64_t) lzma_crc64(const uint8_t *buf, size_t size, uint64_t crc) { +#ifndef HAVE_FUNC_ATTRIBUTE_CONSTRUCTOR mythread_once(crc64_init); +#endif crc = ~crc;
diff --git a/Utilities/cmliblzma/liblzma/check/crc64_table.c b/Utilities/cmliblzma/liblzma/check/crc64_table.c index 7560eb0..78e4275 100644 --- a/Utilities/cmliblzma/liblzma/check/crc64_table.c +++ b/Utilities/cmliblzma/liblzma/check/crc64_table.c
@@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file crc64_table.c @@ -5,18 +7,31 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "common.h" + +// FIXME: Compared to crc_common.h this has to check for __x86_64__ too +// so that in 32-bit builds crc64_x86.S won't break due to a missing table. +#if defined(HAVE_USABLE_CLMUL) && ((defined(__x86_64__) && defined(__SSSE3__) \ + && defined(__SSE4_1__) && defined(__PCLMUL__)) \ + || (defined(__e2k__) && __iset__ >= 6)) +# define NO_CRC64_TABLE +#endif + + +#ifdef NO_CRC64_TABLE +// No table needed. Use a typedef to avoid an empty translation unit. +typedef void lzma_crc64_dummy; + +#else // Having the declaration here silences clang -Wmissing-variable-declarations. extern const uint64_t lzma_crc64_table[4][256]; -#ifdef WORDS_BIGENDIAN -# include "crc64_table_be.h" -#else -# include "crc64_table_le.h" +# if defined(WORDS_BIGENDIAN) +# include "crc64_table_be.h" +# else +# include "crc64_table_le.h" +# endif #endif
diff --git a/Utilities/cmliblzma/liblzma/check/crc64_table_be.h b/Utilities/cmliblzma/liblzma/check/crc64_table_be.h index ea074f3..db76cc7 100644 --- a/Utilities/cmliblzma/liblzma/check/crc64_table_be.h +++ b/Utilities/cmliblzma/liblzma/check/crc64_table_be.h
@@ -1,4 +1,6 @@ -/* This file has been automatically generated by crc64_tablegen.c. */ +// SPDX-License-Identifier: 0BSD + +// This file has been generated by crc64_tablegen.c. const uint64_t lzma_crc64_table[4][256] = { {
diff --git a/Utilities/cmliblzma/liblzma/check/crc64_table_le.h b/Utilities/cmliblzma/liblzma/check/crc64_table_le.h index 1196b31..e40a8c8 100644 --- a/Utilities/cmliblzma/liblzma/check/crc64_table_le.h +++ b/Utilities/cmliblzma/liblzma/check/crc64_table_le.h
@@ -1,4 +1,6 @@ -/* This file has been automatically generated by crc64_tablegen.c. */ +// SPDX-License-Identifier: 0BSD + +// This file has been generated by crc64_tablegen.c. const uint64_t lzma_crc64_table[4][256] = { {
diff --git a/Utilities/cmliblzma/liblzma/check/crc64_tablegen.c b/Utilities/cmliblzma/liblzma/check/crc64_tablegen.c index fddaa7e..2035127 100644 --- a/Utilities/cmliblzma/liblzma/check/crc64_tablegen.c +++ b/Utilities/cmliblzma/liblzma/check/crc64_tablegen.c
@@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file crc64_tablegen.c @@ -8,9 +10,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include <stdio.h> @@ -43,7 +42,7 @@ #ifdef WORDS_BIGENDIAN for (size_t s = 0; s < 4; ++s) for (size_t b = 0; b < 256; ++b) - crc64_table[s][b] = bswap64(crc64_table[s][b]); + crc64_table[s][b] = byteswap64(crc64_table[s][b]); #endif return; @@ -53,9 +52,11 @@ static void print_crc64_table(void) { - printf("/* This file has been automatically generated by " - "crc64_tablegen.c. */\n\n" - "const uint64_t lzma_crc64_table[4][256] = {\n\t{"); + // Split the SPDX string so that it won't accidentally match + // when tools search for the string. + printf("// SPDX" "-License-Identifier" ": 0BSD\n\n" + "// This file has been generated by crc64_tablegen.c.\n\n" + "const uint64_t lzma_crc64_table[4][256] = {\n\t{"); for (size_t s = 0; s < 4; ++s) { for (size_t b = 0; b < 256; ++b) {
diff --git a/Utilities/cmliblzma/liblzma/check/crc64_x86.S b/Utilities/cmliblzma/liblzma/check/crc64_x86.S index f5bb84b..47f6081 100644 --- a/Utilities/cmliblzma/liblzma/check/crc64_x86.S +++ b/Utilities/cmliblzma/liblzma/check/crc64_x86.S
@@ -1,3 +1,5 @@ +/* SPDX-License-Identifier: 0BSD */ + /* * Speed-optimized CRC64 using slicing-by-four algorithm * @@ -7,9 +9,6 @@ * Authors: Igor Pavlov (original CRC32 assembly code) * Lasse Collin (CRC64 adaptation of the modified CRC32 code) * - * This file has been put into the public domain. - * You can do whatever you want with this file. - * * This code needs lzma_crc64_table, which can be created using the * following C code: @@ -41,6 +40,14 @@ * extern uint64_t lzma_crc64(const uint8_t *buf, size_t size, uint64_t crc); */ +/* When Intel CET is enabled, include <cet.h> in assembly code to mark + Intel CET support. */ +#ifdef __CET__ +# include <cet.h> +#else +# define _CET_ENDBR +#endif + /* * On some systems, the functions need to be prefixed. The prefix is * usually an underscore. @@ -73,6 +80,7 @@ ALIGN(4, 16) LZMA_CRC64: + _CET_ENDBR /* * Register usage: * %eax crc LSB @@ -279,9 +287,9 @@ /* * This is needed to support non-executable stack. It's ugly to - * use __linux__ here, but I don't know a way to detect when + * use __FreeBSD__ and __linux__ here, but I don't know a way to detect when * we are using GNU assembler. */ -#if defined(__ELF__) && defined(__linux__) +#if defined(__ELF__) && (defined(__FreeBSD__) || defined(__linux__)) .section .note.GNU-stack,"",@progbits #endif
diff --git a/Utilities/cmliblzma/liblzma/check/crc_common.h b/Utilities/cmliblzma/liblzma/check/crc_common.h new file mode 100644 index 0000000..c15d4c6 --- /dev/null +++ b/Utilities/cmliblzma/liblzma/check/crc_common.h
@@ -0,0 +1,137 @@ +// SPDX-License-Identifier: 0BSD + +/////////////////////////////////////////////////////////////////////////////// +// +/// \file crc_common.h +/// \brief Some functions and macros for CRC32 and CRC64 +// +// Authors: Lasse Collin +// Ilya Kurdyukov +// Hans Jansen +// Jia Tan +// +/////////////////////////////////////////////////////////////////////////////// + +#ifndef LZMA_CRC_COMMON_H +#define LZMA_CRC_COMMON_H + +#include "common.h" + + +#ifdef WORDS_BIGENDIAN +# define A(x) ((x) >> 24) +# define B(x) (((x) >> 16) & 0xFF) +# define C(x) (((x) >> 8) & 0xFF) +# define D(x) ((x) & 0xFF) + +# define S8(x) ((x) << 8) +# define S32(x) ((x) << 32) + +#else +# define A(x) ((x) & 0xFF) +# define B(x) (((x) >> 8) & 0xFF) +# define C(x) (((x) >> 16) & 0xFF) +# define D(x) ((x) >> 24) + +# define S8(x) ((x) >> 8) +# define S32(x) ((x) >> 32) +#endif + + +// CRC CLMUL code needs this because accessing input buffers that aren't +// aligned to the vector size will inherently trip the address sanitizer. +#if lzma_has_attribute(__no_sanitize_address__) +# define crc_attr_no_sanitize_address \ + __attribute__((__no_sanitize_address__)) +#else +# define crc_attr_no_sanitize_address +#endif + +// Keep this in sync with changes to crc32_arm64.h +#if defined(_WIN32) || defined(HAVE_GETAUXVAL) \ + || defined(HAVE_ELF_AUX_INFO) \ + || (defined(__APPLE__) && defined(HAVE_SYSCTLBYNAME)) +# define ARM64_RUNTIME_DETECTION 1 +#endif + + +#undef CRC32_GENERIC +#undef CRC64_GENERIC + +#undef CRC32_ARCH_OPTIMIZED +#undef CRC64_ARCH_OPTIMIZED + +// The x86 CLMUL is used for both CRC32 and CRC64. +#undef CRC_X86_CLMUL + +#undef CRC32_ARM64 +#undef CRC64_ARM64_CLMUL + +#undef CRC_USE_GENERIC_FOR_SMALL_INPUTS + +// ARM64 CRC32 instruction is only useful for CRC32. Currently, only +// little endian is supported since we were unable to test on a big +// endian machine. +// +// NOTE: Keep this and the next check in sync with the macro +// NO_CRC32_TABLE in crc32_table.c +#if defined(HAVE_ARM64_CRC32) && !defined(WORDS_BIGENDIAN) + // Allow ARM64 CRC32 instruction without a runtime check if + // __ARM_FEATURE_CRC32 is defined. GCC and Clang only define + // this if the proper compiler options are used. +# if defined(__ARM_FEATURE_CRC32) +# define CRC32_ARCH_OPTIMIZED 1 +# define CRC32_ARM64 1 +# elif defined(ARM64_RUNTIME_DETECTION) +# define CRC32_ARCH_OPTIMIZED 1 +# define CRC32_ARM64 1 +# define CRC32_GENERIC 1 +# endif +#endif + +#if defined(HAVE_USABLE_CLMUL) +// If CLMUL is allowed unconditionally in the compiler options then the +// generic version can be omitted. Note that this doesn't work with MSVC +// as I don't know how to detect the features here. +// +// NOTE: Keep this in sync with the NO_CRC32_TABLE macro in crc32_table.c +// and NO_CRC64_TABLE in crc64_table.c. +# if (defined(__SSSE3__) && defined(__SSE4_1__) && defined(__PCLMUL__)) \ + || (defined(__e2k__) && __iset__ >= 6) +# define CRC32_ARCH_OPTIMIZED 1 +# define CRC64_ARCH_OPTIMIZED 1 +# define CRC_X86_CLMUL 1 +# else +# define CRC32_GENERIC 1 +# define CRC64_GENERIC 1 +# define CRC32_ARCH_OPTIMIZED 1 +# define CRC64_ARCH_OPTIMIZED 1 +# define CRC_X86_CLMUL 1 + +/* + // The generic code is much faster with 1-8-byte inputs and + // has similar performance up to 16 bytes at least in + // microbenchmarks (it depends on input buffer alignment + // too). If both versions are built, this #define will use + // the generic version for inputs up to 16 bytes and CLMUL + // for bigger inputs. It saves a little in code size since + // the special cases for 0-16-byte inputs will be omitted + // from the CLMUL code. +# define CRC_USE_GENERIC_FOR_SMALL_INPUTS 1 +*/ +# endif +#endif + +// For CRC32 use the generic slice-by-eight implementation if no optimized +// version is available. +#if !defined(CRC32_ARCH_OPTIMIZED) && !defined(CRC32_GENERIC) +# define CRC32_GENERIC 1 +#endif + +// For CRC64 use the generic slice-by-four implementation if no optimized +// version is available. +#if !defined(CRC64_ARCH_OPTIMIZED) && !defined(CRC64_GENERIC) +# define CRC64_GENERIC 1 +#endif + +#endif
diff --git a/Utilities/cmliblzma/liblzma/check/crc_macros.h b/Utilities/cmliblzma/liblzma/check/crc_macros.h deleted file mode 100644 index a7c21b7..0000000 --- a/Utilities/cmliblzma/liblzma/check/crc_macros.h +++ /dev/null
@@ -1,30 +0,0 @@ -/////////////////////////////////////////////////////////////////////////////// -// -/// \file crc_macros.h -/// \brief Some endian-dependent macros for CRC32 and CRC64 -// -// Author: Lasse Collin -// -// This file has been put into the public domain. -// You can do whatever you want with this file. -// -/////////////////////////////////////////////////////////////////////////////// - -#ifdef WORDS_BIGENDIAN -# define A(x) ((x) >> 24) -# define B(x) (((x) >> 16) & 0xFF) -# define C(x) (((x) >> 8) & 0xFF) -# define D(x) ((x) & 0xFF) - -# define S8(x) ((x) << 8) -# define S32(x) ((x) << 32) - -#else -# define A(x) ((x) & 0xFF) -# define B(x) (((x) >> 8) & 0xFF) -# define C(x) (((x) >> 16) & 0xFF) -# define D(x) ((x) >> 24) - -# define S8(x) ((x) >> 8) -# define S32(x) ((x) >> 32) -#endif
diff --git a/Utilities/cmliblzma/liblzma/check/crc_x86_clmul.h b/Utilities/cmliblzma/liblzma/check/crc_x86_clmul.h new file mode 100644 index 0000000..50306e4 --- /dev/null +++ b/Utilities/cmliblzma/liblzma/check/crc_x86_clmul.h
@@ -0,0 +1,432 @@ +// SPDX-License-Identifier: 0BSD + +/////////////////////////////////////////////////////////////////////////////// +// +/// \file crc_x86_clmul.h +/// \brief CRC32 and CRC64 implementations using CLMUL instructions. +/// +/// The CRC32 and CRC64 implementations use 32/64-bit x86 SSSE3, SSE4.1, and +/// CLMUL instructions. This is compatible with Elbrus 2000 (E2K) too. +/// +/// They were derived from +/// https://www.researchgate.net/publication/263424619_Fast_CRC_computation +/// and the public domain code from https://github.com/rawrunprotected/crc +/// (URLs were checked on 2023-10-14). +/// +/// While this file has both CRC32 and CRC64 implementations, only one +/// should be built at a time to ensure that crc_simd_body() is inlined +/// even with compilers with which lzma_always_inline expands to plain inline. +/// The version to build is selected by defining BUILDING_CRC32_CLMUL or +/// BUILDING_CRC64_CLMUL before including this file. +/// +/// FIXME: Builds for 32-bit x86 use the assembly .S files by default +/// unless configured with --disable-assembler. Even then the lookup table +/// isn't omitted in crc64_table.c since it doesn't know that assembly +/// code has been disabled. +// +// Authors: Ilya Kurdyukov +// Hans Jansen +// Lasse Collin +// Jia Tan +// +/////////////////////////////////////////////////////////////////////////////// + +// This file must not be included more than once. +#ifdef LZMA_CRC_X86_CLMUL_H +# error crc_x86_clmul.h was included twice. +#endif +#define LZMA_CRC_X86_CLMUL_H + +#include <immintrin.h> + +#if defined(_MSC_VER) +# include <intrin.h> +#elif defined(HAVE_CPUID_H) +# include <cpuid.h> +#endif + + +// EDG-based compilers (Intel's classic compiler and compiler for E2K) can +// define __GNUC__ but the attribute must not be used with them. +// The new Clang-based ICX needs the attribute. +// +// NOTE: Build systems check for this too, keep them in sync with this. +#if (defined(__GNUC__) || defined(__clang__)) && !defined(__EDG__) +# define crc_attr_target \ + __attribute__((__target__("ssse3,sse4.1,pclmul"))) +#else +# define crc_attr_target +#endif + + +#define MASK_L(in, mask, r) r = _mm_shuffle_epi8(in, mask) + +#define MASK_H(in, mask, r) \ + r = _mm_shuffle_epi8(in, _mm_xor_si128(mask, vsign)) + +#define MASK_LH(in, mask, low, high) \ + MASK_L(in, mask, low); \ + MASK_H(in, mask, high) + + +crc_attr_target +crc_attr_no_sanitize_address +static lzma_always_inline void +crc_simd_body(const uint8_t *buf, const size_t size, __m128i *v0, __m128i *v1, + const __m128i vfold16, const __m128i initial_crc) +{ + // Create a vector with 8-bit values 0 to 15. This is used to + // construct control masks for _mm_blendv_epi8 and _mm_shuffle_epi8. + const __m128i vramp = _mm_setr_epi32( + 0x03020100, 0x07060504, 0x0b0a0908, 0x0f0e0d0c); + + // This is used to inverse the control mask of _mm_shuffle_epi8 + // so that bytes that wouldn't be picked with the original mask + // will be picked and vice versa. + const __m128i vsign = _mm_set1_epi8(-0x80); + + // Memory addresses A to D and the distances between them: + // + // A B C D + // [skip_start][size][skip_end] + // [ size2 ] + // + // A and D are 16-byte aligned. B and C are 1-byte aligned. + // skip_start and skip_end are 0-15 bytes. size is at least 1 byte. + // + // A = aligned_buf will initially point to this address. + // B = The address pointed by the caller-supplied buf. + // C = buf + size == aligned_buf + size2 + // D = buf + size + skip_end == aligned_buf + size2 + skip_end + const size_t skip_start = (size_t)((uintptr_t)buf & 15); + const size_t skip_end = (size_t)((0U - (uintptr_t)(buf + size)) & 15); + const __m128i *aligned_buf = (const __m128i *)( + (uintptr_t)buf & ~(uintptr_t)15); + + // If size2 <= 16 then the whole input fits into a single 16-byte + // vector. If size2 > 16 then at least two 16-byte vectors must + // be processed. If size2 > 16 && size <= 16 then there is only + // one 16-byte vector's worth of input but it is unaligned in memory. + // + // NOTE: There is no integer overflow here if the arguments + // are valid. If this overflowed, buf + size would too. + const size_t size2 = skip_start + size; + + // Masks to be used with _mm_blendv_epi8 and _mm_shuffle_epi8: + // The first skip_start or skip_end bytes in the vectors will have + // the high bit (0x80) set. _mm_blendv_epi8 and _mm_shuffle_epi8 + // will produce zeros for these positions. (Bitwise-xor of these + // masks with vsign will produce the opposite behavior.) + const __m128i mask_start + = _mm_sub_epi8(vramp, _mm_set1_epi8((char)skip_start)); + const __m128i mask_end + = _mm_sub_epi8(vramp, _mm_set1_epi8((char)skip_end)); + + // Get the first 1-16 bytes into data0. If loading less than 16 + // bytes, the bytes are loaded to the high bits of the vector and + // the least significant positions are filled with zeros. + const __m128i data0 = _mm_blendv_epi8(_mm_load_si128(aligned_buf), + _mm_setzero_si128(), mask_start); + aligned_buf++; + + __m128i v2, v3; + +#ifndef CRC_USE_GENERIC_FOR_SMALL_INPUTS + if (size <= 16) { + // Right-shift initial_crc by 1-16 bytes based on "size" + // and store the result in v1 (high bytes) and v0 (low bytes). + // + // NOTE: The highest 8 bytes of initial_crc are zeros so + // v1 will be filled with zeros if size >= 8. The highest + // 8 bytes of v1 will always become zeros. + // + // [ v1 ][ v0 ] + // [ initial_crc ] size == 1 + // [ initial_crc ] size == 2 + // [ initial_crc ] size == 15 + // [ initial_crc ] size == 16 (all in v0) + const __m128i mask_low = _mm_add_epi8( + vramp, _mm_set1_epi8((char)(size - 16))); + MASK_LH(initial_crc, mask_low, *v0, *v1); + + if (size2 <= 16) { + // There are 1-16 bytes of input and it is all + // in data0. Copy the input bytes to v3. If there + // are fewer than 16 bytes, the low bytes in v3 + // will be filled with zeros. That is, the input + // bytes are stored to the same position as + // (part of) initial_crc is in v0. + MASK_L(data0, mask_end, v3); + } else { + // There are 2-16 bytes of input but not all bytes + // are in data0. + const __m128i data1 = _mm_load_si128(aligned_buf); + + // Collect the 2-16 input bytes from data0 and data1 + // to v2 and v3, and bitwise-xor them with the + // low bits of initial_crc in v0. Note that the + // the second xor is below this else-block as it + // is shared with the other branch. + MASK_H(data0, mask_end, v2); + MASK_L(data1, mask_end, v3); + *v0 = _mm_xor_si128(*v0, v2); + } + + *v0 = _mm_xor_si128(*v0, v3); + *v1 = _mm_alignr_epi8(*v1, *v0, 8); + } else +#endif + { + // There is more than 16 bytes of input. + const __m128i data1 = _mm_load_si128(aligned_buf); + const __m128i *end = (const __m128i*)( + (const char *)aligned_buf - 16 + size2); + aligned_buf++; + + MASK_LH(initial_crc, mask_start, *v0, *v1); + *v0 = _mm_xor_si128(*v0, data0); + *v1 = _mm_xor_si128(*v1, data1); + + while (aligned_buf < end) { + *v1 = _mm_xor_si128(*v1, _mm_clmulepi64_si128( + *v0, vfold16, 0x00)); + *v0 = _mm_xor_si128(*v1, _mm_clmulepi64_si128( + *v0, vfold16, 0x11)); + *v1 = _mm_load_si128(aligned_buf++); + } + + if (aligned_buf != end) { + MASK_H(*v0, mask_end, v2); + MASK_L(*v0, mask_end, *v0); + MASK_L(*v1, mask_end, v3); + *v1 = _mm_or_si128(v2, v3); + } + + *v1 = _mm_xor_si128(*v1, _mm_clmulepi64_si128( + *v0, vfold16, 0x00)); + *v0 = _mm_xor_si128(*v1, _mm_clmulepi64_si128( + *v0, vfold16, 0x11)); + *v1 = _mm_srli_si128(*v0, 8); + } +} + + +///////////////////// +// x86 CLMUL CRC32 // +///////////////////// + +/* +// These functions were used to generate the constants +// at the top of crc32_arch_optimized(). +static uint64_t +calc_lo(uint64_t p, uint64_t a, int n) +{ + uint64_t b = 0; int i; + for (i = 0; i < n; i++) { + b = b >> 1 | (a & 1) << (n - 1); + a = (a >> 1) ^ ((0 - (a & 1)) & p); + } + return b; +} + +// same as ~crc(&a, sizeof(a), ~0) +static uint64_t +calc_hi(uint64_t p, uint64_t a, int n) +{ + int i; + for (i = 0; i < n; i++) + a = (a >> 1) ^ ((0 - (a & 1)) & p); + return a; +} +*/ + +#ifdef BUILDING_CRC32_CLMUL + +crc_attr_target +crc_attr_no_sanitize_address +static uint32_t +crc32_arch_optimized(const uint8_t *buf, size_t size, uint32_t crc) +{ +#ifndef CRC_USE_GENERIC_FOR_SMALL_INPUTS + // The code assumes that there is at least one byte of input. + if (size == 0) + return crc; +#endif + + // uint32_t poly = 0xedb88320; + const int64_t p = 0x1db710640; // p << 1 + const int64_t mu = 0x1f7011641; // calc_lo(p, p, 32) << 1 | 1 + const int64_t k5 = 0x163cd6124; // calc_hi(p, p, 32) << 1 + const int64_t k4 = 0x0ccaa009e; // calc_hi(p, p, 64) << 1 + const int64_t k3 = 0x1751997d0; // calc_hi(p, p, 128) << 1 + + const __m128i vfold4 = _mm_set_epi64x(mu, p); + const __m128i vfold8 = _mm_set_epi64x(0, k5); + const __m128i vfold16 = _mm_set_epi64x(k4, k3); + + __m128i v0, v1, v2; + + crc_simd_body(buf, size, &v0, &v1, vfold16, + _mm_cvtsi32_si128((int32_t)~crc)); + + v1 = _mm_xor_si128( + _mm_clmulepi64_si128(v0, vfold16, 0x10), v1); // xxx0 + v2 = _mm_shuffle_epi32(v1, 0xe7); // 0xx0 + v0 = _mm_slli_epi64(v1, 32); // [0] + v0 = _mm_clmulepi64_si128(v0, vfold8, 0x00); + v0 = _mm_xor_si128(v0, v2); // [1] [2] + v2 = _mm_clmulepi64_si128(v0, vfold4, 0x10); + v2 = _mm_clmulepi64_si128(v2, vfold4, 0x00); + v0 = _mm_xor_si128(v0, v2); // [2] + return ~(uint32_t)_mm_extract_epi32(v0, 2); +} +#endif // BUILDING_CRC32_CLMUL + + +///////////////////// +// x86 CLMUL CRC64 // +///////////////////// + +/* +// These functions were used to generate the constants +// at the top of crc64_arch_optimized(). +static uint64_t +calc_lo(uint64_t poly) +{ + uint64_t a = poly; + uint64_t b = 0; + + for (unsigned i = 0; i < 64; ++i) { + b = (b >> 1) | (a << 63); + a = (a >> 1) ^ (a & 1 ? poly : 0); + } + + return b; +} + +static uint64_t +calc_hi(uint64_t poly, uint64_t a) +{ + for (unsigned i = 0; i < 64; ++i) + a = (a >> 1) ^ (a & 1 ? poly : 0); + + return a; +} +*/ + +#ifdef BUILDING_CRC64_CLMUL + +// MSVC (VS2015 - VS2022) produces bad 32-bit x86 code from the CLMUL CRC +// code when optimizations are enabled (release build). According to the bug +// report, the ebx register is corrupted and the calculated result is wrong. +// Trying to workaround the problem with "__asm mov ebx, ebx" didn't help. +// The following pragma works and performance is still good. x86-64 builds +// and CRC32 CLMUL aren't affected by this problem. The problem does not +// happen in crc_simd_body() either (which is shared with CRC32 CLMUL anyway). +// +// NOTE: Another pragma after crc64_arch_optimized() restores +// the optimizations. If the #if condition here is updated, +// the other one must be updated too. +#if defined(_MSC_VER) && !defined(__INTEL_COMPILER) && !defined(__clang__) \ + && defined(_M_IX86) +# pragma optimize("g", off) +#endif + +crc_attr_target +crc_attr_no_sanitize_address +static uint64_t +crc64_arch_optimized(const uint8_t *buf, size_t size, uint64_t crc) +{ +#ifndef CRC_USE_GENERIC_FOR_SMALL_INPUTS + // The code assumes that there is at least one byte of input. + if (size == 0) + return crc; +#endif + + // const uint64_t poly = 0xc96c5795d7870f42; // CRC polynomial + const uint64_t p = 0x92d8af2baf0e1e85; // (poly << 1) | 1 + const uint64_t mu = 0x9c3e466c172963d5; // (calc_lo(poly) << 1) | 1 + const uint64_t k2 = 0xdabe95afc7875f40; // calc_hi(poly, 1) + const uint64_t k1 = 0xe05dd497ca393ae4; // calc_hi(poly, k2) + + const __m128i vfold8 = _mm_set_epi64x((int64_t)p, (int64_t)mu); + const __m128i vfold16 = _mm_set_epi64x((int64_t)k2, (int64_t)k1); + + __m128i v0, v1, v2; + +#if defined(__i386__) || defined(_M_IX86) + crc_simd_body(buf, size, &v0, &v1, vfold16, + _mm_set_epi64x(0, (int64_t)~crc)); +#else + // GCC and Clang would produce good code with _mm_set_epi64x + // but MSVC needs _mm_cvtsi64_si128 on x86-64. + crc_simd_body(buf, size, &v0, &v1, vfold16, + _mm_cvtsi64_si128((int64_t)~crc)); +#endif + + v1 = _mm_xor_si128(_mm_clmulepi64_si128(v0, vfold16, 0x10), v1); + v0 = _mm_clmulepi64_si128(v1, vfold8, 0x00); + v2 = _mm_clmulepi64_si128(v0, vfold8, 0x10); + v0 = _mm_xor_si128(_mm_xor_si128(v1, _mm_slli_si128(v0, 8)), v2); + +#if defined(__i386__) || defined(_M_IX86) + return ~(((uint64_t)(uint32_t)_mm_extract_epi32(v0, 3) << 32) | + (uint64_t)(uint32_t)_mm_extract_epi32(v0, 2)); +#else + return ~(uint64_t)_mm_extract_epi64(v0, 1); +#endif +} + +#if defined(_MSC_VER) && !defined(__INTEL_COMPILER) && !defined(__clang__) \ + && defined(_M_IX86) +# pragma optimize("", on) +#endif + +#endif // BUILDING_CRC64_CLMUL + + +// Even though this is an inline function, compile it only when needed. +// This way it won't appear in E2K builds at all. +#if defined(CRC32_GENERIC) || defined(CRC64_GENERIC) +// Inlining this function duplicates the function body in crc32_resolve() and +// crc64_resolve(), but this is acceptable because this is a tiny function. +static inline bool +is_arch_extension_supported(void) +{ + int success = 1; + uint32_t r[4]; // eax, ebx, ecx, edx + +#if defined(_MSC_VER) + // This needs <intrin.h> with MSVC. ICC has it as a built-in + // on all platforms. + __cpuid(r, 1); +#elif defined(HAVE_CPUID_H) + // Compared to just using __asm__ to run CPUID, this also checks + // that CPUID is supported and saves and restores ebx as that is + // needed with GCC < 5 with position-independent code (PIC). + success = __get_cpuid(1, &r[0], &r[1], &r[2], &r[3]); +#else + // Just a fallback that shouldn't be needed. + __asm__("cpuid\n\t" + : "=a"(r[0]), "=b"(r[1]), "=c"(r[2]), "=d"(r[3]) + : "a"(1), "c"(0)); +#endif + + // Returns true if these are supported: + // CLMUL (bit 1 in ecx) + // SSSE3 (bit 9 in ecx) + // SSE4.1 (bit 19 in ecx) + const uint32_t ecx_mask = (1 << 1) | (1 << 9) | (1 << 19); + return success && (r[2] & ecx_mask) == ecx_mask; + + // Alternative methods that weren't used: + // - ICC's _may_i_use_cpu_feature: the other methods should work too. + // - GCC >= 6 / Clang / ICX __builtin_cpu_supports("pclmul") + // + // CPUID decoding is needed with MSVC anyway and older GCC. This keeps + // the feature checks in the build system simpler too. The nice thing + // about __builtin_cpu_supports would be that it generates very short + // code as is it only reads a variable set at startup but a few bytes + // doesn't matter here. +} +#endif
diff --git a/Utilities/cmliblzma/liblzma/check/sha256.c b/Utilities/cmliblzma/liblzma/check/sha256.c index 5eede5c..c067a3a 100644 --- a/Utilities/cmliblzma/liblzma/check/sha256.c +++ b/Utilities/cmliblzma/liblzma/check/sha256.c
@@ -1,24 +1,17 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file sha256.c /// \brief SHA-256 -/// -/// \todo Crypto++ has x86 ASM optimizations. They use SSE so if they -/// are imported to liblzma, SSE instructions need to be used -/// conditionally to keep the code working on older boxes. // -// This code is based on the code found from 7-Zip, which has a modified -// version of the SHA-256 found from Crypto++ <http://www.cryptopp.com/>. -// The code was modified a little to fit into liblzma. +// The C code is based on the public domain SHA-256 code found from +// Crypto++ Library 5.5.1 released in 2007: https://www.cryptopp.com/ +// A few minor tweaks have been made in liblzma. // -// Authors: Kevin Springle -// Wei Dai -// Igor Pavlov +// Authors: Wei Dai // Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "check.h" @@ -28,7 +21,7 @@ static inline uint32_t rotr_32(uint32_t num, unsigned amount) { - return (num >> amount) | (num << (32 - amount)); + return (num >> amount) | (num << (32 - amount)); } #define blk0(i) (W[i] = conv32be(data[i]))
diff --git a/Utilities/cmliblzma/liblzma/common/alone_decoder.c b/Utilities/cmliblzma/liblzma/common/alone_decoder.c index 239b230..78af651 100644 --- a/Utilities/cmliblzma/liblzma/common/alone_decoder.c +++ b/Utilities/cmliblzma/liblzma/common/alone_decoder.c
@@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file alone_decoder.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "alone_decoder.h" @@ -110,12 +109,24 @@ // Another hack to ditch false positives: Assume that // if the uncompressed size is known, it must be less // than 256 GiB. + // + // FIXME? Without picky we allow > LZMA_VLI_MAX which doesn't + // really matter in this specific situation (> LZMA_VLI_MAX is + // safe in the LZMA decoder) but it's somewhat weird still. if (coder->picky && coder->uncompressed_size != LZMA_VLI_UNKNOWN && coder->uncompressed_size >= (LZMA_VLI_C(1) << 38)) return LZMA_FORMAT_ERROR; + // Use LZMA_FILTER_LZMA1EXT features to specify the + // uncompressed size and that the end marker is allowed + // even when the uncompressed size is known. Both .lzma + // header and LZMA1EXT use UINT64_MAX indicate that size + // is unknown. + coder->options.ext_flags = LZMA_LZMA1EXT_ALLOW_EOPM; + lzma_set_ext_size(coder->options, coder->uncompressed_size); + // Calculate the memory usage so that it is ready // for SEQ_CODER_INIT. coder->memusage = lzma_lzma_decoder_memusage(&coder->options) @@ -132,6 +143,7 @@ lzma_filter_info filters[2] = { { + .id = LZMA_FILTER_LZMA1EXT, .init = &lzma_lzma_decoder_init, .options = &coder->options, }, { @@ -139,14 +151,8 @@ } }; - const lzma_ret ret = lzma_next_filter_init(&coder->next, - allocator, filters); - if (ret != LZMA_OK) - return ret; - - // Use a hack to set the uncompressed size. - lzma_lz_decoder_uncompressed(coder->next.coder, - coder->uncompressed_size); + return_if_error(lzma_next_filter_init(&coder->next, + allocator, filters)); coder->sequence = SEQ_CODE; break;
diff --git a/Utilities/cmliblzma/liblzma/common/alone_decoder.h b/Utilities/cmliblzma/liblzma/common/alone_decoder.h index dfa031a..61ee24d 100644 --- a/Utilities/cmliblzma/liblzma/common/alone_decoder.h +++ b/Utilities/cmliblzma/liblzma/common/alone_decoder.h
@@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file alone_decoder.h @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef LZMA_ALONE_DECODER_H
diff --git a/Utilities/cmliblzma/liblzma/common/alone_encoder.c b/Utilities/cmliblzma/liblzma/common/alone_encoder.c index 96c1db7..21b0395 100644 --- a/Utilities/cmliblzma/liblzma/common/alone_encoder.c +++ b/Utilities/cmliblzma/liblzma/common/alone_encoder.c
@@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file alone_encoder.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "common.h" @@ -75,7 +74,6 @@ } -// At least for now, this is not used by any internal function. static lzma_ret alone_encoder_init(lzma_next_coder *next, const lzma_allocator *allocator, const lzma_options_lzma *options) @@ -129,6 +127,7 @@ // Initialize the LZMA encoder. const lzma_filter_info filters[2] = { { + .id = LZMA_FILTER_LZMA1, .init = &lzma_lzma_encoder_init, .options = (void *)(options), }, { @@ -140,16 +139,6 @@ } -/* -extern lzma_ret -lzma_alone_encoder_init(lzma_next_coder *next, const lzma_allocator *allocator, - const lzma_options_alone *options) -{ - lzma_next_coder_init(&alone_encoder_init, next, allocator, options); -} -*/ - - extern LZMA_API(lzma_ret) lzma_alone_encoder(lzma_stream *strm, const lzma_options_lzma *options) {
diff --git a/Utilities/cmliblzma/liblzma/common/auto_decoder.c b/Utilities/cmliblzma/liblzma/common/auto_decoder.c index 6895c7c..fdd520f 100644 --- a/Utilities/cmliblzma/liblzma/common/auto_decoder.c +++ b/Utilities/cmliblzma/liblzma/common/auto_decoder.c
@@ -1,21 +1,23 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file auto_decoder.c -/// \brief Autodetect between .xz Stream and .lzma (LZMA_Alone) formats +/// \brief Autodetect between .xz, .lzma (LZMA_Alone), and .lz (lzip) // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "stream_decoder.h" #include "alone_decoder.h" +#ifdef HAVE_LZIP_DECODER +# include "lzip_decoder.h" +#endif typedef struct { - /// Stream decoder or LZMA_Alone decoder + /// .xz Stream decoder, LZMA_Alone decoder, or lzip decoder lzma_next_coder next; uint64_t memlimit; @@ -46,14 +48,22 @@ // SEQ_CODE even if we return some LZMA_*_CHECK. coder->sequence = SEQ_CODE; - // Detect the file format. For now this is simple, since if - // it doesn't start with 0xFD (the first magic byte of the - // new format), it has to be LZMA_Alone, or something that - // we don't support at all. + // Detect the file format. .xz files start with 0xFD which + // cannot be the first byte of .lzma (LZMA_Alone) format. + // The .lz format starts with 0x4C which could be the + // first byte of a .lzma file but luckily it would mean + // lc/lp/pb being 4/3/1 which liblzma doesn't support because + // lc + lp > 4. So using just 0x4C to detect .lz is OK here. if (in[*in_pos] == 0xFD) { return_if_error(lzma_stream_decoder_init( &coder->next, allocator, coder->memlimit, coder->flags)); +#ifdef HAVE_LZIP_DECODER + } else if (in[*in_pos] == 0x4C) { + return_if_error(lzma_lzip_decoder_init( + &coder->next, allocator, + coder->memlimit, coder->flags)); +#endif } else { return_if_error(lzma_alone_decoder_init(&coder->next, allocator, coder->memlimit, true)); @@ -86,8 +96,8 @@ // Fall through case SEQ_FINISH: - // When LZMA_DECODE_CONCATENATED was used and we were decoding - // LZMA_Alone file, we need to check check that there is no + // When LZMA_CONCATENATED was used and we were decoding + // a LZMA_Alone file, we need to check that there is no // trailing garbage and wait for LZMA_FINISH. if (*in_pos < in_size) return LZMA_DATA_ERROR;
diff --git a/Utilities/cmliblzma/liblzma/common/block_buffer_decoder.c b/Utilities/cmliblzma/liblzma/common/block_buffer_decoder.c index b0ded90..55566cd 100644 --- a/Utilities/cmliblzma/liblzma/common/block_buffer_decoder.c +++ b/Utilities/cmliblzma/liblzma/common/block_buffer_decoder.c
@@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file block_buffer_decoder.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "block_decoder.h"
diff --git a/Utilities/cmliblzma/liblzma/common/block_buffer_encoder.c b/Utilities/cmliblzma/liblzma/common/block_buffer_encoder.c index 39e263a..df3b90e 100644 --- a/Utilities/cmliblzma/liblzma/common/block_buffer_encoder.c +++ b/Utilities/cmliblzma/liblzma/common/block_buffer_encoder.c
@@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file block_buffer_encoder.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "block_buffer_encoder.h" @@ -277,7 +276,7 @@ if (ret != LZMA_BUF_ERROR) return ret; - // The data was uncompressible (at least with the options + // The data was incompressible (at least with the options // given to us) or the output buffer was too small. Use the // uncompressed chunks of LZMA2 to wrap the data into a valid // Block. If we haven't been given enough output space, even @@ -325,6 +324,24 @@ } +#ifdef HAVE_SYMBOL_VERSIONS_LINUX +// This is for compatibility with binaries linked against liblzma that +// has been patched with xz-5.2.2-compat-libs.patch from RHEL/CentOS 7. +LZMA_SYMVER_API("lzma_block_uncomp_encode@XZ_5.2.2", + lzma_ret, lzma_block_uncomp_encode_522)(lzma_block *block, + const uint8_t *in, size_t in_size, + uint8_t *out, size_t *out_pos, size_t out_size) + lzma_nothrow lzma_attr_warn_unused_result + __attribute__((__alias__("lzma_block_uncomp_encode_52"))); + +LZMA_SYMVER_API("lzma_block_uncomp_encode@@XZ_5.2", + lzma_ret, lzma_block_uncomp_encode_52)(lzma_block *block, + const uint8_t *in, size_t in_size, + uint8_t *out, size_t *out_pos, size_t out_size) + lzma_nothrow lzma_attr_warn_unused_result; + +#define lzma_block_uncomp_encode lzma_block_uncomp_encode_52 +#endif extern LZMA_API(lzma_ret) lzma_block_uncomp_encode(lzma_block *block, const uint8_t *in, size_t in_size,
diff --git a/Utilities/cmliblzma/liblzma/common/block_buffer_encoder.h b/Utilities/cmliblzma/liblzma/common/block_buffer_encoder.h index 653207f..5274ac4 100644 --- a/Utilities/cmliblzma/liblzma/common/block_buffer_encoder.h +++ b/Utilities/cmliblzma/liblzma/common/block_buffer_encoder.h
@@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file block_buffer_encoder.h @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef LZMA_BLOCK_BUFFER_ENCODER_H
diff --git a/Utilities/cmliblzma/liblzma/common/block_decoder.c b/Utilities/cmliblzma/liblzma/common/block_decoder.c index 075bd27..2e369d3 100644 --- a/Utilities/cmliblzma/liblzma/common/block_decoder.c +++ b/Utilities/cmliblzma/liblzma/common/block_decoder.c
@@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file block_decoder.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "block_decoder.h" @@ -40,6 +39,9 @@ /// is unknown. lzma_vli compressed_limit; + /// Maximum allowed Uncompressed Size. + lzma_vli uncompressed_limit; + /// Position when reading the Check field size_t check_pos; @@ -52,21 +54,6 @@ static inline bool -update_size(lzma_vli *size, lzma_vli add, lzma_vli limit) -{ - if (limit > LZMA_VLI_MAX) - limit = LZMA_VLI_MAX; - - if (limit < *size || limit - *size < add) - return true; - - *size += add; - - return false; -} - - -static inline bool is_size_valid(lzma_vli size, lzma_vli reference) { return reference == LZMA_VLI_UNKNOWN || reference == size; @@ -86,23 +73,59 @@ const size_t in_start = *in_pos; const size_t out_start = *out_pos; + // Limit the amount of input and output space that we give + // to the raw decoder based on the information we have + // (or don't have) from Block Header. + const size_t in_stop = *in_pos + (size_t)my_min( + in_size - *in_pos, + coder->compressed_limit - coder->compressed_size); + const size_t out_stop = *out_pos + (size_t)my_min( + out_size - *out_pos, + coder->uncompressed_limit - coder->uncompressed_size); + const lzma_ret ret = coder->next.code(coder->next.coder, - allocator, in, in_pos, in_size, - out, out_pos, out_size, action); + allocator, in, in_pos, in_stop, + out, out_pos, out_stop, action); const size_t in_used = *in_pos - in_start; const size_t out_used = *out_pos - out_start; - // NOTE: We compare to compressed_limit here, which prevents - // the total size of the Block growing past LZMA_VLI_MAX. - if (update_size(&coder->compressed_size, in_used, - coder->compressed_limit) - || update_size(&coder->uncompressed_size, - out_used, - coder->block->uncompressed_size)) - return LZMA_DATA_ERROR; + // Because we have limited the input and output sizes, + // we know that these cannot grow too big or overflow. + coder->compressed_size += in_used; + coder->uncompressed_size += out_used; - if (!coder->ignore_check) + if (ret == LZMA_OK) { + const bool comp_done = coder->compressed_size + == coder->block->compressed_size; + const bool uncomp_done = coder->uncompressed_size + == coder->block->uncompressed_size; + + // If both input and output amounts match the sizes + // in Block Header but we still got LZMA_OK instead + // of LZMA_STREAM_END, the file is broken. + if (comp_done && uncomp_done) + return LZMA_DATA_ERROR; + + // If the decoder has consumed all the input that it + // needs but it still couldn't fill the output buffer + // or return LZMA_STREAM_END, the file is broken. + if (comp_done && *out_pos < out_size) + return LZMA_DATA_ERROR; + + // If the decoder has produced all the output but + // it still didn't return LZMA_STREAM_END or consume + // more input (for example, detecting an end of + // payload marker may need more input but produce + // no output) the file is broken. + if (uncomp_done && *in_pos < in_size) + return LZMA_DATA_ERROR; + } + + // Don't waste time updating the integrity check if it will be + // ignored. Also skip it if no new output was produced. This + // avoids null pointer + 0 (undefined behavior) when out == 0. + if (!coder->ignore_check && out_used > 0) lzma_check_update(&coder->check, coder->block->check, out + out_start, out_used); @@ -230,6 +253,14 @@ - lzma_check_size(block->check) : block->compressed_size; + // With Uncompressed Size this is simpler. If Block Header lacks + // the size info, then LZMA_VLI_MAX is the maximum possible + // Uncompressed Size. + coder->uncompressed_limit + = block->uncompressed_size == LZMA_VLI_UNKNOWN + ? LZMA_VLI_MAX + : block->uncompressed_size; + // Initialize the check. It's caller's problem if the Check ID is not // supported, and the Block decoder cannot verify the Check field. // Caller can test lzma_check_is_supported(block->check).
diff --git a/Utilities/cmliblzma/liblzma/common/block_decoder.h b/Utilities/cmliblzma/liblzma/common/block_decoder.h index 718c5ce..2cbf9ba 100644 --- a/Utilities/cmliblzma/liblzma/common/block_decoder.h +++ b/Utilities/cmliblzma/liblzma/common/block_decoder.h
@@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file block_decoder.h @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef LZMA_BLOCK_DECODER_H
diff --git a/Utilities/cmliblzma/liblzma/common/block_encoder.c b/Utilities/cmliblzma/liblzma/common/block_encoder.c index 168846a..ce8c1de 100644 --- a/Utilities/cmliblzma/liblzma/common/block_encoder.c +++ b/Utilities/cmliblzma/liblzma/common/block_encoder.c
@@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file block_encoder.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "block_encoder.h" @@ -77,8 +76,11 @@ // checked it at the beginning of this function. coder->uncompressed_size += in_used; - lzma_check_update(&coder->check, coder->block->check, - in + in_start, in_used); + // Call lzma_check_update() only if input was consumed. This + // avoids null pointer + 0 (undefined behavior) when in == 0. + if (in_used > 0) + lzma_check_update(&coder->check, coder->block->check, + in + in_start, in_used); if (ret != LZMA_STREAM_END || action == LZMA_SYNC_FLUSH) return ret; @@ -217,6 +219,7 @@ lzma_next_strm_init(lzma_block_encoder_init, strm, block); strm->internal->supported_actions[LZMA_RUN] = true; + strm->internal->supported_actions[LZMA_SYNC_FLUSH] = true; strm->internal->supported_actions[LZMA_FINISH] = true; return LZMA_OK;
diff --git a/Utilities/cmliblzma/liblzma/common/block_encoder.h b/Utilities/cmliblzma/liblzma/common/block_encoder.h index bd97c18..b7dfe9a 100644 --- a/Utilities/cmliblzma/liblzma/common/block_encoder.h +++ b/Utilities/cmliblzma/liblzma/common/block_encoder.h
@@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file block_encoder.h @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef LZMA_BLOCK_ENCODER_H
diff --git a/Utilities/cmliblzma/liblzma/common/block_header_decoder.c b/Utilities/cmliblzma/liblzma/common/block_header_decoder.c index 2e1135d..f0b2fbe 100644 --- a/Utilities/cmliblzma/liblzma/common/block_header_decoder.c +++ b/Utilities/cmliblzma/liblzma/common/block_header_decoder.c
@@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file block_header_decoder.c @@ -5,31 +7,12 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "common.h" #include "check.h" -static void -free_properties(lzma_block *block, const lzma_allocator *allocator) -{ - // Free allocated filter options. The last array member is not - // touched after the initialization in the beginning of - // lzma_block_header_decode(), so we don't need to touch that here. - for (size_t i = 0; i < LZMA_FILTERS_MAX; ++i) { - lzma_free(block->filters[i].options, allocator); - block->filters[i].id = LZMA_VLI_UNKNOWN; - block->filters[i].options = NULL; - } - - return; -} - - extern LZMA_API(lzma_ret) lzma_block_header_decode(lzma_block *block, const lzma_allocator *allocator, const uint8_t *in) @@ -39,6 +22,10 @@ // are invalid or over 63 bits, or if the header is too small // to contain the claimed information. + // Catch unexpected NULL pointers. + if (block == NULL || block->filters == NULL || in == NULL) + return LZMA_PROG_ERROR; + // Initialize the filter options array. This way the caller can // safely free() the options even if an error occurs in this function. for (size_t i = 0; i <= LZMA_FILTERS_MAX; ++i) { @@ -67,8 +54,11 @@ const size_t in_size = block->header_size - 4; // Verify CRC32 - if (lzma_crc32(in, in_size, 0) != read32le(in + in_size)) + if (lzma_crc32(in, in_size, 0) != read32le(in + in_size)) { +#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION return LZMA_DATA_ERROR; +#endif + } // Check for unsupported flags. if (in[1] & 0x3C) @@ -104,7 +94,7 @@ &block->filters[i], allocator, in, &in_pos, in_size); if (ret != LZMA_OK) { - free_properties(block, allocator); + lzma_filters_free(block->filters, allocator); return ret; } } @@ -112,7 +102,7 @@ // Padding while (in_pos < in_size) { if (in[in_pos++] != 0x00) { - free_properties(block, allocator); + lzma_filters_free(block->filters, allocator); // Possibly some new field present so use // LZMA_OPTIONS_ERROR instead of LZMA_DATA_ERROR.
diff --git a/Utilities/cmliblzma/liblzma/common/block_header_encoder.c b/Utilities/cmliblzma/liblzma/common/block_header_encoder.c index 160425d..45e57a2 100644 --- a/Utilities/cmliblzma/liblzma/common/block_header_encoder.c +++ b/Utilities/cmliblzma/liblzma/common/block_header_encoder.c
@@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file block_header_encoder.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "common.h"
diff --git a/Utilities/cmliblzma/liblzma/common/block_util.c b/Utilities/cmliblzma/liblzma/common/block_util.c index acb3111..191f6d4 100644 --- a/Utilities/cmliblzma/liblzma/common/block_util.c +++ b/Utilities/cmliblzma/liblzma/common/block_util.c
@@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file block_util.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "common.h"
diff --git a/Utilities/cmliblzma/liblzma/common/common.c b/Utilities/cmliblzma/liblzma/common/common.c index cf714e5..cc0e06a 100644 --- a/Utilities/cmliblzma/liblzma/common/common.c +++ b/Utilities/cmliblzma/liblzma/common/common.c
@@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file common.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "common.h" @@ -35,7 +34,8 @@ // Memory allocation // /////////////////////// -extern void * lzma_attribute((__malloc__)) lzma_attr_alloc_size(1) +lzma_attr_alloc_size(1) +extern void * lzma_alloc(size_t size, const lzma_allocator *allocator) { // Some malloc() variants return NULL if called with size == 0. @@ -53,7 +53,8 @@ } -extern void * lzma_attribute((__malloc__)) lzma_attr_alloc_size(1) +lzma_attr_alloc_size(1) +extern void * lzma_alloc_zero(size_t size, const lzma_allocator *allocator) { // Some calloc() variants return NULL if called with size == 0. @@ -211,7 +212,6 @@ || strm->reserved_ptr2 != NULL || strm->reserved_ptr3 != NULL || strm->reserved_ptr4 != NULL - || strm->reserved_int1 != 0 || strm->reserved_int2 != 0 || strm->reserved_int3 != 0 || strm->reserved_int4 != 0 @@ -289,19 +289,25 @@ strm->next_in, &in_pos, strm->avail_in, strm->next_out, &out_pos, strm->avail_out, action); - strm->next_in += in_pos; - strm->avail_in -= in_pos; - strm->total_in += in_pos; + // Updating next_in and next_out has to be skipped when they are NULL + // to avoid null pointer + 0 (undefined behavior). Do this by checking + // in_pos > 0 and out_pos > 0 because this way NULL + non-zero (a bug) + // will get caught one way or other. + if (in_pos > 0) { + strm->next_in += in_pos; + strm->avail_in -= in_pos; + strm->total_in += in_pos; + } - strm->next_out += out_pos; - strm->avail_out -= out_pos; - strm->total_out += out_pos; + if (out_pos > 0) { + strm->next_out += out_pos; + strm->avail_out -= out_pos; + strm->total_out += out_pos; + } strm->internal->avail_in = strm->avail_in; - // Cast is needed to silence a warning about LZMA_TIMED_OUT, which - // isn't part of lzma_ret enumeration. - switch ((unsigned int)(ret)) { + switch (ret) { case LZMA_OK: // Don't return LZMA_BUF_ERROR when it happens the first time. // This is to avoid returning LZMA_BUF_ERROR when avail_out @@ -322,6 +328,17 @@ ret = LZMA_OK; break; + case LZMA_SEEK_NEEDED: + strm->internal->allow_buf_error = false; + + // If LZMA_FINISH was used, reset it back to the + // LZMA_RUN-based state so that new input can be supplied + // by the application. + if (strm->internal->sequence == ISEQ_FINISH) + strm->internal->sequence = ISEQ_RUN; + + break; + case LZMA_STREAM_END: if (strm->internal->sequence == ISEQ_SYNC_FLUSH || strm->internal->sequence == ISEQ_FULL_FLUSH @@ -366,6 +383,20 @@ } +#ifdef HAVE_SYMBOL_VERSIONS_LINUX +// This is for compatibility with binaries linked against liblzma that +// has been patched with xz-5.2.2-compat-libs.patch from RHEL/CentOS 7. +LZMA_SYMVER_API("lzma_get_progress@XZ_5.2.2", + void, lzma_get_progress_522)(lzma_stream *strm, + uint64_t *progress_in, uint64_t *progress_out) lzma_nothrow + __attribute__((__alias__("lzma_get_progress_52"))); + +LZMA_SYMVER_API("lzma_get_progress@@XZ_5.2", + void, lzma_get_progress_52)(lzma_stream *strm, + uint64_t *progress_in, uint64_t *progress_out) lzma_nothrow; + +#define lzma_get_progress lzma_get_progress_52 +#endif extern LZMA_API(void) lzma_get_progress(lzma_stream *strm, uint64_t *progress_in, uint64_t *progress_out)
diff --git a/Utilities/cmliblzma/liblzma/common/common.h b/Utilities/cmliblzma/liblzma/common/common.h index b3d3b7a..20af32f 100644 --- a/Utilities/cmliblzma/liblzma/common/common.h +++ b/Utilities/cmliblzma/liblzma/common/common.h
@@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file common.h @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef LZMA_COMMON_H @@ -17,23 +16,104 @@ #include "mythread.h" #include "tuklib_integer.h" +// LZMA_API_EXPORT is used to mark the exported API functions. +// It's used to define the LZMA_API macro. +// +// lzma_attr_visibility_hidden is used for marking *declarations* of extern +// variables that are internal to liblzma (-fvisibility=hidden alone is +// enough to hide the *definitions*). Such markings allow slightly more +// efficient code to accesses those variables in ELF shared libraries. #if defined(_WIN32) || defined(__CYGWIN__) # ifdef DLL_EXPORT # define LZMA_API_EXPORT __declspec(dllexport) # else # define LZMA_API_EXPORT # endif +# define lzma_attr_visibility_hidden // Don't use ifdef or defined() below. #elif HAVE_VISIBILITY # define LZMA_API_EXPORT __attribute__((__visibility__("default"))) +# define lzma_attr_visibility_hidden \ + __attribute__((__visibility__("hidden"))) #else # define LZMA_API_EXPORT +# define lzma_attr_visibility_hidden #endif #define LZMA_API(type) LZMA_API_EXPORT type LZMA_API_CALL #include "lzma.h" +// This is for detecting modern GCC and Clang attributes +// like __symver__ in GCC >= 10. +#ifdef __has_attribute +# define lzma_has_attribute(attr) __has_attribute(attr) +#else +# define lzma_has_attribute(attr) 0 +#endif + +// The extra symbol versioning in the C files may only be used when +// building a shared library. If HAVE_SYMBOL_VERSIONS_LINUX is defined +// to 2 then symbol versioning is done only if also PIC is defined. +// By default Libtool defines PIC when building a shared library and +// doesn't define it when building a static library but it can be +// overridden with --with-pic and --without-pic. configure let's rely +// on PIC if neither --with-pic or --without-pic was used. +#if defined(HAVE_SYMBOL_VERSIONS_LINUX) \ + && (HAVE_SYMBOL_VERSIONS_LINUX == 2 && !defined(PIC)) +# undef HAVE_SYMBOL_VERSIONS_LINUX +#endif + +#ifdef HAVE_SYMBOL_VERSIONS_LINUX +// To keep link-time optimization (LTO, -flto) working with GCC, +// the __symver__ attribute must be used instead of __asm__(".symver ..."). +// Otherwise the symbol versions may be lost, resulting in broken liblzma +// that has wrong default versions in the exported symbol list! +// The attribute was added in GCC 10; LTO with older GCC is not supported. +// +// To keep -Wmissing-prototypes happy, use LZMA_SYMVER_API only with function +// declarations (including those with __alias__ attribute) and LZMA_API with +// the function definitions. This means a little bit of silly copy-and-paste +// between declarations and definitions though. +// +// As of GCC 12.2, the __symver__ attribute supports only @ and @@ but the +// very convenient @@@ isn't supported (it's supported by GNU assembler +// since 2000). When using @@ instead of @@@, the internal name must not be +// the same as the external name to avoid problems in some situations. This +// is why "#define foo_52 foo" is needed for the default symbol versions. +// +// __has_attribute is supported before GCC 10 and it is supported in Clang 14 +// too (which doesn't support __symver__) so use it to detect if __symver__ +// is available. This should be far more reliable than looking at compiler +// version macros as nowadays especially __GNUC__ is defined by many compilers. +# if lzma_has_attribute(__symver__) +# define LZMA_SYMVER_API(extnamever, type, intname) \ + extern __attribute__((__symver__(extnamever))) \ + LZMA_API(type) intname +# else +# define LZMA_SYMVER_API(extnamever, type, intname) \ + __asm__(".symver " #intname "," extnamever); \ + extern LZMA_API(type) intname +# endif +#endif + +// MSVC has __forceinline which shouldn't be combined with the inline keyword +// (results in a warning). +// +// GCC 3.1 added always_inline attribute so we don't need to check +// for __GNUC__ version. Similarly, all relevant Clang versions +// support it (at least Clang 3.0.0 does already). +// Other compilers might support too which also support __has_attribute +// (Solaris Studio) so do that check too. +#if defined(_MSC_VER) +# define lzma_always_inline __forceinline +#elif defined(__GNUC__) || defined(__clang__) || defined(__INTEL_COMPILER) \ + || lzma_has_attribute(__always_inline__) +# define lzma_always_inline inline __attribute__((__always_inline__)) +#else +# define lzma_always_inline inline +#endif + // These allow helping the compiler in some often-executed branches, whose // result is almost always the same. #ifdef __GNUC__ @@ -67,14 +147,15 @@ #define LZMA_FILTER_RESERVED_START (LZMA_VLI_C(1) << 62) -/// Supported flags that can be passed to lzma_stream_decoder() -/// or lzma_auto_decoder(). +/// Supported flags that can be passed to lzma_stream_decoder(), +/// lzma_auto_decoder(), or lzma_stream_decoder_mt(). #define LZMA_SUPPORTED_FLAGS \ ( LZMA_TELL_NO_CHECK \ | LZMA_TELL_UNSUPPORTED_CHECK \ | LZMA_TELL_ANY_CHECK \ | LZMA_IGNORE_CHECK \ - | LZMA_CONCATENATED ) + | LZMA_CONCATENATED \ + | LZMA_FAIL_FAST ) /// Largest valid lzma_action value as unsigned integer. @@ -83,9 +164,12 @@ /// Special return value (lzma_ret) to indicate that a timeout was reached /// and lzma_code() must not return LZMA_BUF_ERROR. This is converted to -/// LZMA_OK in lzma_code(). This is not in the lzma_ret enumeration because -/// there's no need to have it in the public API. -#define LZMA_TIMED_OUT 32 +/// LZMA_OK in lzma_code(). +#define LZMA_TIMED_OUT LZMA_RET_INTERNAL1 + +/// Special return value (lzma_ret) for use in stream_decoder_mt.c to +/// indicate Index was detected instead of a Block Header. +#define LZMA_INDEX_DETECTED LZMA_RET_INTERNAL2 typedef struct lzma_next_coder_s lzma_next_coder; @@ -118,8 +202,11 @@ /// an array of lzma_filter_info structures. This array is used with /// lzma_next_filter_init to initialize the filter chain. struct lzma_filter_info_s { - /// Filter ID. This is used only by the encoder - /// with lzma_filters_update(). + /// Filter ID. This can be used to share the same initiazation + /// function *and* data structures with different Filter IDs + /// (LZMA_FILTER_LZMA1EXT does it), and also by the encoder + /// with lzma_filters_update() if filter chain is updated + /// in the middle of a raw stream or Block (LZMA_SYNC_FLUSH). lzma_vli id; /// Pointer to function used to initialize the filter. @@ -173,6 +260,16 @@ lzma_ret (*update)(void *coder, const lzma_allocator *allocator, const lzma_filter *filters, const lzma_filter *reversed_filters); + + /// Set how many bytes of output this coder may produce at maximum. + /// On success LZMA_OK must be returned. + /// If the filter chain as a whole cannot support this feature, + /// this must return LZMA_OPTIONS_ERROR. + /// If no input has been given to the coder and the requested limit + /// is too small, this must return LZMA_BUF_ERROR. If input has been + /// seen, LZMA_OK is allowed too. + lzma_ret (*set_out_limit)(void *coder, uint64_t *uncomp_size, + uint64_t out_limit); }; @@ -188,6 +285,7 @@ .get_check = NULL, \ .memconfig = NULL, \ .update = NULL, \ + .set_out_limit = NULL, \ } @@ -226,14 +324,14 @@ /// Allocates memory -extern void *lzma_alloc(size_t size, const lzma_allocator *allocator) - lzma_attribute((__malloc__)) lzma_attr_alloc_size(1); +lzma_attr_alloc_size(1) +extern void *lzma_alloc(size_t size, const lzma_allocator *allocator); /// Allocates memory and zeroes it (like calloc()). This can be faster /// than lzma_alloc() + memzero() while being backward compatible with /// custom allocators. -extern void * lzma_attribute((__malloc__)) lzma_attr_alloc_size(1) - lzma_alloc_zero(size_t size, const lzma_allocator *allocator); +lzma_attr_alloc_size(1) +extern void *lzma_alloc_zero(size_t size, const lzma_allocator *allocator); /// Frees memory extern void lzma_free(void *ptr, const lzma_allocator *allocator);
diff --git a/Utilities/cmliblzma/liblzma/common/easy_buffer_encoder.c b/Utilities/cmliblzma/liblzma/common/easy_buffer_encoder.c index 48eb56f..da610ce 100644 --- a/Utilities/cmliblzma/liblzma/common/easy_buffer_encoder.c +++ b/Utilities/cmliblzma/liblzma/common/easy_buffer_encoder.c
@@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file easy_buffer_encoder.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "easy_preset.h"
diff --git a/Utilities/cmliblzma/liblzma/common/easy_decoder_memusage.c b/Utilities/cmliblzma/liblzma/common/easy_decoder_memusage.c index 20bcd5b..0c76f10 100644 --- a/Utilities/cmliblzma/liblzma/common/easy_decoder_memusage.c +++ b/Utilities/cmliblzma/liblzma/common/easy_decoder_memusage.c
@@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file easy_decoder_memusage.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "easy_preset.h"
diff --git a/Utilities/cmliblzma/liblzma/common/easy_encoder.c b/Utilities/cmliblzma/liblzma/common/easy_encoder.c index 5cb492d..8dfe296 100644 --- a/Utilities/cmliblzma/liblzma/common/easy_encoder.c +++ b/Utilities/cmliblzma/liblzma/common/easy_encoder.c
@@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file easy_encoder.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "easy_preset.h"
diff --git a/Utilities/cmliblzma/liblzma/common/easy_encoder_memusage.c b/Utilities/cmliblzma/liblzma/common/easy_encoder_memusage.c index e910575..1184ac6 100644 --- a/Utilities/cmliblzma/liblzma/common/easy_encoder_memusage.c +++ b/Utilities/cmliblzma/liblzma/common/easy_encoder_memusage.c
@@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file easy_encoder_memusage.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "easy_preset.h"
diff --git a/Utilities/cmliblzma/liblzma/common/easy_preset.c b/Utilities/cmliblzma/liblzma/common/easy_preset.c index 2f98598..7908a2b 100644 --- a/Utilities/cmliblzma/liblzma/common/easy_preset.c +++ b/Utilities/cmliblzma/liblzma/common/easy_preset.c
@@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file easy_preset.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "easy_preset.h"
diff --git a/Utilities/cmliblzma/liblzma/common/easy_preset.h b/Utilities/cmliblzma/liblzma/common/easy_preset.h index 382ade8..4ef6d04 100644 --- a/Utilities/cmliblzma/liblzma/common/easy_preset.h +++ b/Utilities/cmliblzma/liblzma/common/easy_preset.h
@@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file easy_preset.h @@ -5,11 +7,11 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// +#ifndef LZMA_EASY_PRESET_H +#define LZMA_EASY_PRESET_H + #include "common.h" @@ -30,3 +32,5 @@ /// Set *easy to the settings given by the preset. Returns true on error, /// false on success. extern bool lzma_easy_preset(lzma_options_easy *easy, uint32_t preset); + +#endif
diff --git a/Utilities/cmliblzma/liblzma/common/file_info.c b/Utilities/cmliblzma/liblzma/common/file_info.c new file mode 100644 index 0000000..7c85084 --- /dev/null +++ b/Utilities/cmliblzma/liblzma/common/file_info.c
@@ -0,0 +1,854 @@ +// SPDX-License-Identifier: 0BSD + +/////////////////////////////////////////////////////////////////////////////// +// +/// \file file_info.c +/// \brief Decode .xz file information into a lzma_index structure +// +// Author: Lasse Collin +// +/////////////////////////////////////////////////////////////////////////////// + +#include "index_decoder.h" + + +typedef struct { + enum { + SEQ_MAGIC_BYTES, + SEQ_PADDING_SEEK, + SEQ_PADDING_DECODE, + SEQ_FOOTER, + SEQ_INDEX_INIT, + SEQ_INDEX_DECODE, + SEQ_HEADER_DECODE, + SEQ_HEADER_COMPARE, + } sequence; + + /// Absolute position of in[*in_pos] in the file. All code that + /// modifies *in_pos also updates this. seek_to_pos() needs this + /// to determine if we need to request the application to seek for + /// us or if we can do the seeking internally by adjusting *in_pos. + uint64_t file_cur_pos; + + /// This refers to absolute positions of interesting parts of the + /// input file. Sometimes it points to the *beginning* of a specific + /// field and sometimes to the *end* of a field. The current target + /// position at each moment is explained in the comments. + uint64_t file_target_pos; + + /// Size of the .xz file (from the application). + uint64_t file_size; + + /// Index decoder + lzma_next_coder index_decoder; + + /// Number of bytes remaining in the Index field that is currently + /// being decoded. + lzma_vli index_remaining; + + /// The Index decoder will store the decoded Index in this pointer. + lzma_index *this_index; + + /// Amount of Stream Padding in the current Stream. + lzma_vli stream_padding; + + /// The final combined index is collected here. + lzma_index *combined_index; + + /// Pointer from the application where to store the index information + /// after successful decoding. + lzma_index **dest_index; + + /// Pointer to lzma_stream.seek_pos to be used when returning + /// LZMA_SEEK_NEEDED. This is set by seek_to_pos() when needed. + uint64_t *external_seek_pos; + + /// Memory usage limit + uint64_t memlimit; + + /// Stream Flags from the very beginning of the file. + lzma_stream_flags first_header_flags; + + /// Stream Flags from Stream Header of the current Stream. + lzma_stream_flags header_flags; + + /// Stream Flags from Stream Footer of the current Stream. + lzma_stream_flags footer_flags; + + size_t temp_pos; + size_t temp_size; + uint8_t temp[8192]; + +} lzma_file_info_coder; + + +/// Copies data from in[*in_pos] into coder->temp until +/// coder->temp_pos == coder->temp_size. This also keeps coder->file_cur_pos +/// in sync with *in_pos. Returns true if more input is needed. +static bool +fill_temp(lzma_file_info_coder *coder, const uint8_t *restrict in, + size_t *restrict in_pos, size_t in_size) +{ + coder->file_cur_pos += lzma_bufcpy(in, in_pos, in_size, + coder->temp, &coder->temp_pos, coder->temp_size); + return coder->temp_pos < coder->temp_size; +} + + +/// Seeks to the absolute file position specified by target_pos. +/// This tries to do the seeking by only modifying *in_pos, if possible. +/// The main benefit of this is that if one passes the whole file at once +/// to lzma_code(), the decoder will never need to return LZMA_SEEK_NEEDED +/// as all the seeking can be done by adjusting *in_pos in this function. +/// +/// Returns true if an external seek is needed and the caller must return +/// LZMA_SEEK_NEEDED. +static bool +seek_to_pos(lzma_file_info_coder *coder, uint64_t target_pos, + size_t in_start, size_t *in_pos, size_t in_size) +{ + // The input buffer doesn't extend beyond the end of the file. + // This has been checked by file_info_decode() already. + assert(coder->file_size - coder->file_cur_pos >= in_size - *in_pos); + + const uint64_t pos_min = coder->file_cur_pos - (*in_pos - in_start); + const uint64_t pos_max = coder->file_cur_pos + (in_size - *in_pos); + + bool external_seek_needed; + + if (target_pos >= pos_min && target_pos <= pos_max) { + // The requested position is available in the current input + // buffer or right after it. That is, in a corner case we + // end up setting *in_pos == in_size and thus will immediately + // need new input bytes from the application. + *in_pos += (size_t)(target_pos - coder->file_cur_pos); + external_seek_needed = false; + } else { + // Ask the application to seek the input file. + *coder->external_seek_pos = target_pos; + external_seek_needed = true; + + // Mark the whole input buffer as used. This way + // lzma_stream.total_in will have a better estimate + // of the amount of data read. It still won't be perfect + // as the value will depend on the input buffer size that + // the application uses, but it should be good enough for + // those few who want an estimate. + *in_pos = in_size; + } + + // After seeking (internal or external) the current position + // will match the requested target position. + coder->file_cur_pos = target_pos; + + return external_seek_needed; +} + + +/// The caller sets coder->file_target_pos so that it points to the *end* +/// of the desired file position. This function then determines how far +/// backwards from that position we can seek. After seeking fill_temp() +/// can be used to read data into coder->temp. When fill_temp() has finished, +/// coder->temp[coder->temp_size] will match coder->file_target_pos. +/// +/// This also validates that coder->target_file_pos is sane in sense that +/// we aren't trying to seek too far backwards (too close or beyond the +/// beginning of the file). +static lzma_ret +reverse_seek(lzma_file_info_coder *coder, + size_t in_start, size_t *in_pos, size_t in_size) +{ + // Check that there is enough data before the target position + // to contain at least Stream Header and Stream Footer. If there + // isn't, the file cannot be valid. + if (coder->file_target_pos < 2 * LZMA_STREAM_HEADER_SIZE) + return LZMA_DATA_ERROR; + + coder->temp_pos = 0; + + // The Stream Header at the very beginning of the file gets handled + // specially in SEQ_MAGIC_BYTES and thus we will never need to seek + // there. By not seeking to the first LZMA_STREAM_HEADER_SIZE bytes + // we avoid a useless external seek after SEQ_MAGIC_BYTES if the + // application uses an extremely small input buffer and the input + // file is very small. + if (coder->file_target_pos - LZMA_STREAM_HEADER_SIZE + < sizeof(coder->temp)) + coder->temp_size = (size_t)(coder->file_target_pos + - LZMA_STREAM_HEADER_SIZE); + else + coder->temp_size = sizeof(coder->temp); + + // The above if-statements guarantee this. This is important because + // the Stream Header/Footer decoders assume that there's at least + // LZMA_STREAM_HEADER_SIZE bytes in coder->temp. + assert(coder->temp_size >= LZMA_STREAM_HEADER_SIZE); + + if (seek_to_pos(coder, coder->file_target_pos - coder->temp_size, + in_start, in_pos, in_size)) + return LZMA_SEEK_NEEDED; + + return LZMA_OK; +} + + +/// Gets the number of zero-bytes at the end of the buffer. +static size_t +get_padding_size(const uint8_t *buf, size_t buf_size) +{ + size_t padding = 0; + while (buf_size > 0 && buf[--buf_size] == 0x00) + ++padding; + + return padding; +} + + +/// With the Stream Header at the very beginning of the file, LZMA_FORMAT_ERROR +/// is used to tell the application that Magic Bytes didn't match. In other +/// Stream Header/Footer fields (in the middle/end of the file) it could be +/// a bit confusing to return LZMA_FORMAT_ERROR as we already know that there +/// is a valid Stream Header at the beginning of the file. For those cases +/// this function is used to convert LZMA_FORMAT_ERROR to LZMA_DATA_ERROR. +static lzma_ret +hide_format_error(lzma_ret ret) +{ + if (ret == LZMA_FORMAT_ERROR) + ret = LZMA_DATA_ERROR; + + return ret; +} + + +/// Calls the Index decoder and updates coder->index_remaining. +/// This is a separate function because the input can be either directly +/// from the application or from coder->temp. +static lzma_ret +decode_index(lzma_file_info_coder *coder, const lzma_allocator *allocator, + const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, bool update_file_cur_pos) +{ + const size_t in_start = *in_pos; + + const lzma_ret ret = coder->index_decoder.code( + coder->index_decoder.coder, + allocator, in, in_pos, in_size, + NULL, NULL, 0, LZMA_RUN); + + coder->index_remaining -= *in_pos - in_start; + + if (update_file_cur_pos) + coder->file_cur_pos += *in_pos - in_start; + + return ret; +} + + +static lzma_ret +file_info_decode(void *coder_ptr, const lzma_allocator *allocator, + const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, + uint8_t *restrict out lzma_attribute((__unused__)), + size_t *restrict out_pos lzma_attribute((__unused__)), + size_t out_size lzma_attribute((__unused__)), + lzma_action action lzma_attribute((__unused__))) +{ + lzma_file_info_coder *coder = coder_ptr; + const size_t in_start = *in_pos; + + // If the caller provides input past the end of the file, trim + // the extra bytes from the buffer so that we won't read too far. + assert(coder->file_size >= coder->file_cur_pos); + if (coder->file_size - coder->file_cur_pos < in_size - in_start) + in_size = in_start + + (size_t)(coder->file_size - coder->file_cur_pos); + + while (true) + switch (coder->sequence) { + case SEQ_MAGIC_BYTES: + // Decode the Stream Header at the beginning of the file + // first to check if the Magic Bytes match. The flags + // are stored in coder->first_header_flags so that we + // don't need to seek to it again. + // + // Check that the file is big enough to contain at least + // Stream Header. + if (coder->file_size < LZMA_STREAM_HEADER_SIZE) + return LZMA_FORMAT_ERROR; + + // Read the Stream Header field into coder->temp. + if (fill_temp(coder, in, in_pos, in_size)) + return LZMA_OK; + + // This is the only Stream Header/Footer decoding where we + // want to return LZMA_FORMAT_ERROR if the Magic Bytes don't + // match. Elsewhere it will be converted to LZMA_DATA_ERROR. + return_if_error(lzma_stream_header_decode( + &coder->first_header_flags, coder->temp)); + + // Now that we know that the Magic Bytes match, check the + // file size. It's better to do this here after checking the + // Magic Bytes since this way we can give LZMA_FORMAT_ERROR + // instead of LZMA_DATA_ERROR when the Magic Bytes don't + // match in a file that is too big or isn't a multiple of + // four bytes. + if (coder->file_size > LZMA_VLI_MAX || (coder->file_size & 3)) + return LZMA_DATA_ERROR; + + // Start looking for Stream Padding and Stream Footer + // at the end of the file. + coder->file_target_pos = coder->file_size; + + // Fall through + + case SEQ_PADDING_SEEK: + coder->sequence = SEQ_PADDING_DECODE; + return_if_error(reverse_seek( + coder, in_start, in_pos, in_size)); + + // Fall through + + case SEQ_PADDING_DECODE: { + // Copy to coder->temp first. This keeps the code simpler if + // the application only provides input a few bytes at a time. + if (fill_temp(coder, in, in_pos, in_size)) + return LZMA_OK; + + // Scan the buffer backwards to get the size of the + // Stream Padding field (if any). + const size_t new_padding = get_padding_size( + coder->temp, coder->temp_size); + coder->stream_padding += new_padding; + + // Set the target position to the beginning of Stream Padding + // that has been observed so far. If all Stream Padding has + // been seen, then the target position will be at the end + // of the Stream Footer field. + coder->file_target_pos -= new_padding; + + if (new_padding == coder->temp_size) { + // The whole buffer was padding. Seek backwards in + // the file to get more input. + coder->sequence = SEQ_PADDING_SEEK; + break; + } + + // Size of Stream Padding must be a multiple of 4 bytes. + if (coder->stream_padding & 3) + return LZMA_DATA_ERROR; + + coder->sequence = SEQ_FOOTER; + + // Calculate the amount of non-padding data in coder->temp. + coder->temp_size -= new_padding; + coder->temp_pos = coder->temp_size; + + // We can avoid an external seek if the whole Stream Footer + // is already in coder->temp. In that case SEQ_FOOTER won't + // read more input and will find the Stream Footer from + // coder->temp[coder->temp_size - LZMA_STREAM_HEADER_SIZE]. + // + // Otherwise we will need to seek. The seeking is done so + // that Stream Footer will be at the end of coder->temp. + // This way it's likely that we also get a complete Index + // field into coder->temp without needing a separate seek + // for that (unless the Index field is big). + if (coder->temp_size < LZMA_STREAM_HEADER_SIZE) + return_if_error(reverse_seek( + coder, in_start, in_pos, in_size)); + } + + // Fall through + + case SEQ_FOOTER: + // Copy the Stream Footer field into coder->temp. + // If Stream Footer was already available in coder->temp + // in SEQ_PADDING_DECODE, then this does nothing. + if (fill_temp(coder, in, in_pos, in_size)) + return LZMA_OK; + + // Make coder->file_target_pos and coder->temp_size point + // to the beginning of Stream Footer and thus to the end + // of the Index field. coder->temp_pos will be updated + // a bit later. + coder->file_target_pos -= LZMA_STREAM_HEADER_SIZE; + coder->temp_size -= LZMA_STREAM_HEADER_SIZE; + + // Decode Stream Footer. + return_if_error(hide_format_error(lzma_stream_footer_decode( + &coder->footer_flags, + coder->temp + coder->temp_size))); + + // Check that we won't seek past the beginning of the file. + // + // LZMA_STREAM_HEADER_SIZE is added because there must be + // space for Stream Header too even though we won't seek + // there before decoding the Index field. + // + // There's no risk of integer overflow here because + // Backward Size cannot be greater than 2^34. + if (coder->file_target_pos < coder->footer_flags.backward_size + + LZMA_STREAM_HEADER_SIZE) + return LZMA_DATA_ERROR; + + // Set the target position to the beginning of the Index field. + coder->file_target_pos -= coder->footer_flags.backward_size; + coder->sequence = SEQ_INDEX_INIT; + + // We can avoid an external seek if the whole Index field is + // already available in coder->temp. + if (coder->temp_size >= coder->footer_flags.backward_size) { + // Set coder->temp_pos to point to the beginning + // of the Index. + coder->temp_pos = coder->temp_size + - coder->footer_flags.backward_size; + } else { + // These are set to zero to indicate that there's no + // useful data (Index or anything else) in coder->temp. + coder->temp_pos = 0; + coder->temp_size = 0; + + // Seek to the beginning of the Index field. + if (seek_to_pos(coder, coder->file_target_pos, + in_start, in_pos, in_size)) + return LZMA_SEEK_NEEDED; + } + + // Fall through + + case SEQ_INDEX_INIT: { + // Calculate the amount of memory already used by the earlier + // Indexes so that we know how big memory limit to pass to + // the Index decoder. + // + // NOTE: When there are multiple Streams, the separate + // lzma_index structures can use more RAM (as measured by + // lzma_index_memused()) than the final combined lzma_index. + // Thus memlimit may need to be slightly higher than the final + // calculated memory usage will be. This is perhaps a bit + // confusing to the application, but I think it shouldn't + // cause problems in practice. + uint64_t memused = 0; + if (coder->combined_index != NULL) { + memused = lzma_index_memused(coder->combined_index); + assert(memused <= coder->memlimit); + if (memused > coder->memlimit) // Extra sanity check + return LZMA_PROG_ERROR; + } + + // Initialize the Index decoder. + return_if_error(lzma_index_decoder_init( + &coder->index_decoder, allocator, + &coder->this_index, + coder->memlimit - memused)); + + coder->index_remaining = coder->footer_flags.backward_size; + coder->sequence = SEQ_INDEX_DECODE; + } + + // Fall through + + case SEQ_INDEX_DECODE: { + // Decode (a part of) the Index. If the whole Index is already + // in coder->temp, read it from there. Otherwise read from + // in[*in_pos] onwards. Note that index_decode() updates + // coder->index_remaining and optionally coder->file_cur_pos. + lzma_ret ret; + if (coder->temp_size != 0) { + assert(coder->temp_size - coder->temp_pos + == coder->index_remaining); + ret = decode_index(coder, allocator, coder->temp, + &coder->temp_pos, coder->temp_size, + false); + } else { + // Don't give the decoder more input than the known + // remaining size of the Index field. + size_t in_stop = in_size; + if (in_size - *in_pos > coder->index_remaining) + in_stop = *in_pos + + (size_t)(coder->index_remaining); + + ret = decode_index(coder, allocator, + in, in_pos, in_stop, true); + } + + switch (ret) { + case LZMA_OK: + // If the Index docoder asks for more input when we + // have already given it as much input as Backward Size + // indicated, the file is invalid. + if (coder->index_remaining == 0) + return LZMA_DATA_ERROR; + + // We cannot get here if we were reading Index from + // coder->temp because when reading from coder->temp + // we give the Index decoder exactly + // coder->index_remaining bytes of input. + assert(coder->temp_size == 0); + + return LZMA_OK; + + case LZMA_STREAM_END: + // If the decoding seems to be successful, check also + // that the Index decoder consumed as much input as + // indicated by the Backward Size field. + if (coder->index_remaining != 0) + return LZMA_DATA_ERROR; + + break; + + default: + return ret; + } + + // Calculate how much the Index tells us to seek backwards + // (relative to the beginning of the Index): Total size of + // all Blocks plus the size of the Stream Header field. + // No integer overflow here because lzma_index_total_size() + // cannot return a value greater than LZMA_VLI_MAX. + const uint64_t seek_amount + = lzma_index_total_size(coder->this_index) + + LZMA_STREAM_HEADER_SIZE; + + // Check that Index is sane in sense that seek_amount won't + // make us seek past the beginning of the file when locating + // the Stream Header. + // + // coder->file_target_pos still points to the beginning of + // the Index field. + if (coder->file_target_pos < seek_amount) + return LZMA_DATA_ERROR; + + // Set the target to the beginning of Stream Header. + coder->file_target_pos -= seek_amount; + + if (coder->file_target_pos == 0) { + // We would seek to the beginning of the file, but + // since we already decoded that Stream Header in + // SEQ_MAGIC_BYTES, we can use the cached value from + // coder->first_header_flags to avoid the seek. + coder->header_flags = coder->first_header_flags; + coder->sequence = SEQ_HEADER_COMPARE; + break; + } + + coder->sequence = SEQ_HEADER_DECODE; + + // Make coder->file_target_pos point to the end of + // the Stream Header field. + coder->file_target_pos += LZMA_STREAM_HEADER_SIZE; + + // If coder->temp_size is non-zero, it points to the end + // of the Index field. Then the beginning of the Index + // field is at coder->temp[coder->temp_size + // - coder->footer_flags.backward_size]. + assert(coder->temp_size == 0 || coder->temp_size + >= coder->footer_flags.backward_size); + + // If coder->temp contained the whole Index, see if it has + // enough data to contain also the Stream Header. If so, + // we avoid an external seek. + // + // NOTE: This can happen only with small .xz files and only + // for the non-first Stream as the Stream Flags of the first + // Stream are cached and already handled a few lines above. + // So this isn't as useful as the other seek-avoidance cases. + if (coder->temp_size != 0 && coder->temp_size + - coder->footer_flags.backward_size + >= seek_amount) { + // Make temp_pos and temp_size point to the *end* of + // Stream Header so that SEQ_HEADER_DECODE will find + // the start of Stream Header from coder->temp[ + // coder->temp_size - LZMA_STREAM_HEADER_SIZE]. + coder->temp_pos = coder->temp_size + - coder->footer_flags.backward_size + - seek_amount + + LZMA_STREAM_HEADER_SIZE; + coder->temp_size = coder->temp_pos; + } else { + // Seek so that Stream Header will be at the end of + // coder->temp. With typical multi-Stream files we + // will usually also get the Stream Footer and Index + // of the *previous* Stream in coder->temp and thus + // won't need a separate seek for them. + return_if_error(reverse_seek(coder, + in_start, in_pos, in_size)); + } + } + + // Fall through + + case SEQ_HEADER_DECODE: + // Copy the Stream Header field into coder->temp. + // If Stream Header was already available in coder->temp + // in SEQ_INDEX_DECODE, then this does nothing. + if (fill_temp(coder, in, in_pos, in_size)) + return LZMA_OK; + + // Make all these point to the beginning of Stream Header. + coder->file_target_pos -= LZMA_STREAM_HEADER_SIZE; + coder->temp_size -= LZMA_STREAM_HEADER_SIZE; + coder->temp_pos = coder->temp_size; + + // Decode the Stream Header. + return_if_error(hide_format_error(lzma_stream_header_decode( + &coder->header_flags, + coder->temp + coder->temp_size))); + + coder->sequence = SEQ_HEADER_COMPARE; + + // Fall through + + case SEQ_HEADER_COMPARE: + // Compare Stream Header against Stream Footer. They must + // match. + return_if_error(lzma_stream_flags_compare( + &coder->header_flags, &coder->footer_flags)); + + // Store the decoded Stream Flags into the Index. Use the + // Footer Flags because it contains Backward Size, although + // it shouldn't matter in practice. + if (lzma_index_stream_flags(coder->this_index, + &coder->footer_flags) != LZMA_OK) + return LZMA_PROG_ERROR; + + // Store also the size of the Stream Padding field. It is + // needed to calculate the offsets of the Streams correctly. + if (lzma_index_stream_padding(coder->this_index, + coder->stream_padding) != LZMA_OK) + return LZMA_PROG_ERROR; + + // Reset it so that it's ready for the next Stream. + coder->stream_padding = 0; + + // Append the earlier decoded Indexes after this_index. + if (coder->combined_index != NULL) + return_if_error(lzma_index_cat(coder->this_index, + coder->combined_index, allocator)); + + coder->combined_index = coder->this_index; + coder->this_index = NULL; + + // If the whole file was decoded, tell the caller that we + // are finished. + if (coder->file_target_pos == 0) { + // The combined index must indicate the same file + // size as was told to us at initialization. + assert(lzma_index_file_size(coder->combined_index) + == coder->file_size); + + // Make the combined index available to + // the application. + *coder->dest_index = coder->combined_index; + coder->combined_index = NULL; + + // Mark the input buffer as used since we may have + // done internal seeking and thus don't know how + // many input bytes were actually used. This way + // lzma_stream.total_in gets a slightly better + // estimate of the amount of input used. + *in_pos = in_size; + return LZMA_STREAM_END; + } + + // We didn't hit the beginning of the file yet, so continue + // reading backwards in the file. If we have unprocessed + // data in coder->temp, use it before requesting more data + // from the application. + // + // coder->file_target_pos, coder->temp_size, and + // coder->temp_pos all point to the beginning of Stream Header + // and thus the end of the previous Stream in the file. + coder->sequence = coder->temp_size > 0 + ? SEQ_PADDING_DECODE : SEQ_PADDING_SEEK; + break; + + default: + assert(0); + return LZMA_PROG_ERROR; + } +} + + +static lzma_ret +file_info_decoder_memconfig(void *coder_ptr, uint64_t *memusage, + uint64_t *old_memlimit, uint64_t new_memlimit) +{ + lzma_file_info_coder *coder = coder_ptr; + + // The memory usage calculation comes from three things: + // + // (1) The Indexes that have already been decoded and processed into + // coder->combined_index. + // + // (2) The latest Index in coder->this_index that has been decoded but + // not yet put into coder->combined_index. + // + // (3) The latest Index that we have started decoding but haven't + // finished and thus isn't available in coder->this_index yet. + // Memory usage and limit information needs to be communicated + // from/to coder->index_decoder. + // + // Care has to be taken to not do both (2) and (3) when calculating + // the memory usage. + uint64_t combined_index_memusage = 0; + uint64_t this_index_memusage = 0; + + // (1) If we have already successfully decoded one or more Indexes, + // get their memory usage. + if (coder->combined_index != NULL) + combined_index_memusage = lzma_index_memused( + coder->combined_index); + + // Choose between (2), (3), or neither. + if (coder->this_index != NULL) { + // (2) The latest Index is available. Use its memory usage. + this_index_memusage = lzma_index_memused(coder->this_index); + + } else if (coder->sequence == SEQ_INDEX_DECODE) { + // (3) The Index decoder is activate and hasn't yet stored + // the new index in coder->this_index. Get the memory usage + // information from the Index decoder. + // + // NOTE: If the Index decoder doesn't yet know how much memory + // it will eventually need, it will return a tiny value here. + uint64_t dummy; + if (coder->index_decoder.memconfig(coder->index_decoder.coder, + &this_index_memusage, &dummy, 0) + != LZMA_OK) { + assert(0); + return LZMA_PROG_ERROR; + } + } + + // Now we know the total memory usage/requirement. If we had neither + // old Indexes nor a new Index, this will be zero which isn't + // acceptable as lzma_memusage() has to return non-zero on success + // and even with an empty .xz file we will end up with a lzma_index + // that takes some memory. + *memusage = combined_index_memusage + this_index_memusage; + if (*memusage == 0) + *memusage = lzma_index_memusage(1, 0); + + *old_memlimit = coder->memlimit; + + // If requested, set a new memory usage limit. + if (new_memlimit != 0) { + if (new_memlimit < *memusage) + return LZMA_MEMLIMIT_ERROR; + + // In the condition (3) we need to tell the Index decoder + // its new memory usage limit. + if (coder->this_index == NULL + && coder->sequence == SEQ_INDEX_DECODE) { + const uint64_t idec_new_memlimit = new_memlimit + - combined_index_memusage; + + assert(this_index_memusage > 0); + assert(idec_new_memlimit > 0); + + uint64_t dummy1; + uint64_t dummy2; + + if (coder->index_decoder.memconfig( + coder->index_decoder.coder, + &dummy1, &dummy2, idec_new_memlimit) + != LZMA_OK) { + assert(0); + return LZMA_PROG_ERROR; + } + } + + coder->memlimit = new_memlimit; + } + + return LZMA_OK; +} + + +static void +file_info_decoder_end(void *coder_ptr, const lzma_allocator *allocator) +{ + lzma_file_info_coder *coder = coder_ptr; + + lzma_next_end(&coder->index_decoder, allocator); + lzma_index_end(coder->this_index, allocator); + lzma_index_end(coder->combined_index, allocator); + + lzma_free(coder, allocator); + return; +} + + +static lzma_ret +lzma_file_info_decoder_init(lzma_next_coder *next, + const lzma_allocator *allocator, uint64_t *seek_pos, + lzma_index **dest_index, + uint64_t memlimit, uint64_t file_size) +{ + lzma_next_coder_init(&lzma_file_info_decoder_init, next, allocator); + + if (dest_index == NULL) + return LZMA_PROG_ERROR; + + lzma_file_info_coder *coder = next->coder; + if (coder == NULL) { + coder = lzma_alloc(sizeof(lzma_file_info_coder), allocator); + if (coder == NULL) + return LZMA_MEM_ERROR; + + next->coder = coder; + next->code = &file_info_decode; + next->end = &file_info_decoder_end; + next->memconfig = &file_info_decoder_memconfig; + + coder->index_decoder = LZMA_NEXT_CODER_INIT; + coder->this_index = NULL; + coder->combined_index = NULL; + } + + coder->sequence = SEQ_MAGIC_BYTES; + coder->file_cur_pos = 0; + coder->file_target_pos = 0; + coder->file_size = file_size; + + lzma_index_end(coder->this_index, allocator); + coder->this_index = NULL; + + lzma_index_end(coder->combined_index, allocator); + coder->combined_index = NULL; + + coder->stream_padding = 0; + + coder->dest_index = dest_index; + coder->external_seek_pos = seek_pos; + + // If memlimit is 0, make it 1 to ensure that lzma_memlimit_get() + // won't return 0 (which would indicate an error). + coder->memlimit = my_max(1, memlimit); + + // Prepare these for reading the first Stream Header into coder->temp. + coder->temp_pos = 0; + coder->temp_size = LZMA_STREAM_HEADER_SIZE; + + return LZMA_OK; +} + + +extern LZMA_API(lzma_ret) +lzma_file_info_decoder(lzma_stream *strm, lzma_index **dest_index, + uint64_t memlimit, uint64_t file_size) +{ + lzma_next_strm_init(lzma_file_info_decoder_init, strm, &strm->seek_pos, + dest_index, memlimit, file_size); + + // We allow LZMA_FINISH in addition to LZMA_RUN for convenience. + // lzma_code() is able to handle the LZMA_FINISH + LZMA_SEEK_NEEDED + // combination in a sane way. Applications still need to be careful + // if they use LZMA_FINISH so that they remember to reset it back + // to LZMA_RUN after seeking if needed. + strm->internal->supported_actions[LZMA_RUN] = true; + strm->internal->supported_actions[LZMA_FINISH] = true; + + return LZMA_OK; +}
diff --git a/Utilities/cmliblzma/liblzma/common/filter_buffer_decoder.c b/Utilities/cmliblzma/liblzma/common/filter_buffer_decoder.c index 6620986..cc0d88c 100644 --- a/Utilities/cmliblzma/liblzma/common/filter_buffer_decoder.c +++ b/Utilities/cmliblzma/liblzma/common/filter_buffer_decoder.c
@@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file filter_buffer_decoder.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "filter_decoder.h" @@ -24,7 +23,7 @@ || out_pos == NULL || *out_pos > out_size) return LZMA_PROG_ERROR; - // Initialize the decoer. + // Initialize the decoder. lzma_next_coder next = LZMA_NEXT_CODER_INIT; return_if_error(lzma_raw_decoder_init(&next, allocator, filters));
diff --git a/Utilities/cmliblzma/liblzma/common/filter_buffer_encoder.c b/Utilities/cmliblzma/liblzma/common/filter_buffer_encoder.c index dda18e3..7fb8922 100644 --- a/Utilities/cmliblzma/liblzma/common/filter_buffer_encoder.c +++ b/Utilities/cmliblzma/liblzma/common/filter_buffer_encoder.c
@@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file filter_buffer_encoder.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "filter_encoder.h"
diff --git a/Utilities/cmliblzma/liblzma/common/filter_common.c b/Utilities/cmliblzma/liblzma/common/filter_common.c index 9ad5d5d..d15d9cc 100644 --- a/Utilities/cmliblzma/liblzma/common/filter_common.c +++ b/Utilities/cmliblzma/liblzma/common/filter_common.c
@@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file filter_common.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "filter_common.h" @@ -42,6 +41,13 @@ .last_ok = true, .changes_size = true, }, + { + .id = LZMA_FILTER_LZMA1EXT, + .options_size = sizeof(lzma_options_lzma), + .non_last_ok = false, + .last_ok = true, + .changes_size = true, + }, #endif #if defined(HAVE_ENCODER_LZMA2) || defined(HAVE_DECODER_LZMA2) { @@ -97,6 +103,15 @@ .changes_size = false, }, #endif +#if defined(HAVE_ENCODER_ARM64) || defined(HAVE_DECODER_ARM64) + { + .id = LZMA_FILTER_ARM64, + .options_size = sizeof(lzma_options_bcj), + .non_last_ok = true, + .last_ok = false, + .changes_size = false, + }, +#endif #if defined(HAVE_ENCODER_SPARC) || defined(HAVE_DECODER_SPARC) { .id = LZMA_FILTER_SPARC, @@ -106,6 +121,15 @@ .changes_size = false, }, #endif +#if defined(HAVE_ENCODER_RISCV) || defined(HAVE_DECODER_RISCV) + { + .id = LZMA_FILTER_RISCV, + .options_size = sizeof(lzma_options_bcj), + .non_last_ok = true, + .last_ok = false, + .changes_size = false, + }, +#endif #if defined(HAVE_ENCODER_DELTA) || defined(HAVE_DECODER_DELTA) { .id = LZMA_FILTER_DELTA, @@ -122,12 +146,16 @@ extern LZMA_API(lzma_ret) -lzma_filters_copy(const lzma_filter *src, lzma_filter *dest, +lzma_filters_copy(const lzma_filter *src, lzma_filter *real_dest, const lzma_allocator *allocator) { - if (src == NULL || dest == NULL) + if (src == NULL || real_dest == NULL) return LZMA_PROG_ERROR; + // Use a temporary destination so that the real destination + // will never be modified if an error occurs. + lzma_filter dest[LZMA_FILTERS_MAX + 1]; + lzma_ret ret; size_t i; for (i = 0; src[i].id != LZMA_VLI_UNKNOWN; ++i) { @@ -173,25 +201,53 @@ } // Terminate the filter array. - assert(i <= LZMA_FILTERS_MAX + 1); + assert(i < LZMA_FILTERS_MAX + 1); dest[i].id = LZMA_VLI_UNKNOWN; dest[i].options = NULL; + // Copy it to the caller-supplied array now that we know that + // no errors occurred. + memcpy(real_dest, dest, (i + 1) * sizeof(lzma_filter)); + return LZMA_OK; error: // Free the options which we have already allocated. - while (i-- > 0) { + while (i-- > 0) lzma_free(dest[i].options, allocator); - dest[i].options = NULL; - } return ret; } -static lzma_ret -validate_chain(const lzma_filter *filters, size_t *count) +extern LZMA_API(void) +lzma_filters_free(lzma_filter *filters, const lzma_allocator *allocator) +{ + if (filters == NULL) + return; + + for (size_t i = 0; filters[i].id != LZMA_VLI_UNKNOWN; ++i) { + if (i == LZMA_FILTERS_MAX) { + // The API says that LZMA_FILTERS_MAX + 1 is the + // maximum allowed size including the terminating + // element. Thus, we should never get here but in + // case there is a bug and we do anyway, don't go + // past the (probable) end of the array. + assert(0); + break; + } + + lzma_free(filters[i].options, allocator); + filters[i].options = NULL; + filters[i].id = LZMA_VLI_UNKNOWN; + } + + return; +} + + +extern lzma_ret +lzma_validate_chain(const lzma_filter *filters, size_t *count) { // There must be at least one filter. if (filters == NULL || filters[0].id == LZMA_VLI_UNKNOWN) @@ -245,7 +301,7 @@ { // Do some basic validation and get the number of filters. size_t count; - return_if_error(validate_chain(options, &count)); + return_if_error(lzma_validate_chain(options, &count)); // Set the filter functions and copy the options pointer. lzma_filter_info filters[LZMA_FILTERS_MAX + 1]; @@ -298,7 +354,7 @@ // The chain has to have at least one filter. { size_t tmp; - if (validate_chain(filters, &tmp) != LZMA_OK) + if (lzma_validate_chain(filters, &tmp) != LZMA_OK) return UINT64_MAX; }
diff --git a/Utilities/cmliblzma/liblzma/common/filter_common.h b/Utilities/cmliblzma/liblzma/common/filter_common.h index 9390305..95f9fe2 100644 --- a/Utilities/cmliblzma/liblzma/common/filter_common.h +++ b/Utilities/cmliblzma/liblzma/common/filter_common.h
@@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file filter_common.h @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef LZMA_FILTER_COMMON_H @@ -35,6 +34,9 @@ typedef const lzma_filter_coder *(*lzma_filter_find)(lzma_vli id); +extern lzma_ret lzma_validate_chain(const lzma_filter *filters, size_t *count); + + extern lzma_ret lzma_raw_coder_init( lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter *filters,
diff --git a/Utilities/cmliblzma/liblzma/common/filter_decoder.c b/Utilities/cmliblzma/liblzma/common/filter_decoder.c index c75b0a8..cbdeb58 100644 --- a/Utilities/cmliblzma/liblzma/common/filter_decoder.c +++ b/Utilities/cmliblzma/liblzma/common/filter_decoder.c
@@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file filter_decoder.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "filter_decoder.h" @@ -50,6 +49,12 @@ .memusage = &lzma_lzma_decoder_memusage, .props_decode = &lzma_lzma_props_decode, }, + { + .id = LZMA_FILTER_LZMA1EXT, + .init = &lzma_lzma_decoder_init, + .memusage = &lzma_lzma_decoder_memusage, + .props_decode = &lzma_lzma_props_decode, + }, #endif #ifdef HAVE_DECODER_LZMA2 { @@ -99,6 +104,14 @@ .props_decode = &lzma_simple_props_decode, }, #endif +#ifdef HAVE_DECODER_ARM64 + { + .id = LZMA_FILTER_ARM64, + .init = &lzma_simple_arm64_decoder_init, + .memusage = NULL, + .props_decode = &lzma_simple_props_decode, + }, +#endif #ifdef HAVE_DECODER_SPARC { .id = LZMA_FILTER_SPARC, @@ -107,6 +120,14 @@ .props_decode = &lzma_simple_props_decode, }, #endif +#ifdef HAVE_DECODER_RISCV + { + .id = LZMA_FILTER_RISCV, + .init = &lzma_simple_riscv_decoder_init, + .memusage = NULL, + .props_decode = &lzma_simple_props_decode, + }, +#endif #ifdef HAVE_DECODER_DELTA { .id = LZMA_FILTER_DELTA, @@ -129,6 +150,16 @@ } +// lzma_filter_coder begins with the same members as lzma_filter_decoder. +// This function is a wrapper with a type that is compatible with the +// typedef of lzma_filter_find in filter_common.h. +static const lzma_filter_coder * +coder_find(lzma_vli id) +{ + return (const lzma_filter_coder *)decoder_find(id); +} + + extern LZMA_API(lzma_bool) lzma_filter_decoder_is_supported(lzma_vli id) { @@ -141,7 +172,7 @@ const lzma_filter *options) { return lzma_raw_coder_init(next, allocator, - options, (lzma_filter_find)(&decoder_find), false); + options, &coder_find, false); } @@ -160,8 +191,7 @@ extern LZMA_API(uint64_t) lzma_raw_decoder_memusage(const lzma_filter *filters) { - return lzma_raw_coder_memusage( - (lzma_filter_find)(&decoder_find), filters); + return lzma_raw_coder_memusage(&coder_find, filters); }
diff --git a/Utilities/cmliblzma/liblzma/common/filter_decoder.h b/Utilities/cmliblzma/liblzma/common/filter_decoder.h index 2dac602..e610bc1 100644 --- a/Utilities/cmliblzma/liblzma/common/filter_decoder.h +++ b/Utilities/cmliblzma/liblzma/common/filter_decoder.h
@@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file filter_decoder.h @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef LZMA_FILTER_DECODER_H
diff --git a/Utilities/cmliblzma/liblzma/common/filter_encoder.c b/Utilities/cmliblzma/liblzma/common/filter_encoder.c index c5d8f39..bc39444 100644 --- a/Utilities/cmliblzma/liblzma/common/filter_encoder.c +++ b/Utilities/cmliblzma/liblzma/common/filter_encoder.c
@@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file filter_decoder.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "filter_encoder.h" @@ -33,13 +32,17 @@ /// Calculates the recommended Uncompressed Size for .xz Blocks to /// which the input data can be split to make multithreaded /// encoding possible. If this is NULL, it is assumed that - /// the encoder is fast enough with single thread. + /// the encoder is fast enough with single thread. If the options + /// are invalid, UINT64_MAX is returned. uint64_t (*block_size)(const void *options); /// Tells the size of the Filter Properties field. If options are - /// invalid, UINT32_MAX is returned. If this is NULL, props_size_fixed - /// is used. + /// invalid, LZMA_OPTIONS_ERROR is returned and size is set to + /// UINT32_MAX. lzma_ret (*props_size_get)(uint32_t *size, const void *options); + + /// Some filters will always have the same size Filter Properties + /// field. If props_size_get is NULL, this value is used. uint32_t props_size_fixed; /// Encodes Filter Properties. @@ -59,7 +62,16 @@ .id = LZMA_FILTER_LZMA1, .init = &lzma_lzma_encoder_init, .memusage = &lzma_lzma_encoder_memusage, - .block_size = NULL, // FIXME + .block_size = NULL, // Not needed for LZMA1 + .props_size_get = NULL, + .props_size_fixed = 5, + .props_encode = &lzma_lzma_props_encode, + }, + { + .id = LZMA_FILTER_LZMA1EXT, + .init = &lzma_lzma_encoder_init, + .memusage = &lzma_lzma_encoder_memusage, + .block_size = NULL, // Not needed for LZMA1 .props_size_get = NULL, .props_size_fixed = 5, .props_encode = &lzma_lzma_props_encode, @@ -70,7 +82,7 @@ .id = LZMA_FILTER_LZMA2, .init = &lzma_lzma2_encoder_init, .memusage = &lzma_lzma2_encoder_memusage, - .block_size = &lzma_lzma2_block_size, // FIXME + .block_size = &lzma_lzma2_block_size, .props_size_get = NULL, .props_size_fixed = 1, .props_encode = &lzma_lzma2_props_encode, @@ -126,6 +138,16 @@ .props_encode = &lzma_simple_props_encode, }, #endif +#ifdef HAVE_ENCODER_ARM64 + { + .id = LZMA_FILTER_ARM64, + .init = &lzma_simple_arm64_encoder_init, + .memusage = NULL, + .block_size = NULL, + .props_size_get = &lzma_simple_props_size, + .props_encode = &lzma_simple_props_encode, + }, +#endif #ifdef HAVE_ENCODER_SPARC { .id = LZMA_FILTER_SPARC, @@ -136,6 +158,16 @@ .props_encode = &lzma_simple_props_encode, }, #endif +#ifdef HAVE_ENCODER_RISCV + { + .id = LZMA_FILTER_RISCV, + .init = &lzma_simple_riscv_encoder_init, + .memusage = NULL, + .block_size = NULL, + .props_size_get = &lzma_simple_props_size, + .props_encode = &lzma_simple_props_encode, + }, +#endif #ifdef HAVE_ENCODER_DELTA { .id = LZMA_FILTER_DELTA, @@ -161,6 +193,16 @@ } +// lzma_filter_coder begins with the same members as lzma_filter_encoder. +// This function is a wrapper with a type that is compatible with the +// typedef of lzma_filter_find in filter_common.h. +static const lzma_filter_coder * +coder_find(lzma_vli id) +{ + return (const lzma_filter_coder *)encoder_find(id); +} + + extern LZMA_API(lzma_bool) lzma_filter_encoder_is_supported(lzma_vli id) { @@ -197,18 +239,18 @@ extern lzma_ret lzma_raw_encoder_init(lzma_next_coder *next, const lzma_allocator *allocator, - const lzma_filter *options) + const lzma_filter *filters) { return lzma_raw_coder_init(next, allocator, - options, (lzma_filter_find)(&encoder_find), true); + filters, &coder_find, true); } extern LZMA_API(lzma_ret) -lzma_raw_encoder(lzma_stream *strm, const lzma_filter *options) +lzma_raw_encoder(lzma_stream *strm, const lzma_filter *filters) { - lzma_next_strm_init(lzma_raw_coder_init, strm, options, - (lzma_filter_find)(&encoder_find), true); + lzma_next_strm_init(lzma_raw_coder_init, strm, filters, + &coder_find, true); strm->internal->supported_actions[LZMA_RUN] = true; strm->internal->supported_actions[LZMA_SYNC_FLUSH] = true; @@ -221,31 +263,33 @@ extern LZMA_API(uint64_t) lzma_raw_encoder_memusage(const lzma_filter *filters) { - return lzma_raw_coder_memusage( - (lzma_filter_find)(&encoder_find), filters); + return lzma_raw_coder_memusage(&coder_find, filters); } -extern uint64_t +extern LZMA_API(uint64_t) lzma_mt_block_size(const lzma_filter *filters) { + if (filters == NULL) + return UINT64_MAX; + uint64_t max = 0; for (size_t i = 0; filters[i].id != LZMA_VLI_UNKNOWN; ++i) { const lzma_filter_encoder *const fe = encoder_find(filters[i].id); + if (fe == NULL) + return UINT64_MAX; + if (fe->block_size != NULL) { const uint64_t size = fe->block_size(filters[i].options); - if (size == 0) - return 0; - if (size > max) max = size; } } - return max; + return max == 0 ? UINT64_MAX : max; }
diff --git a/Utilities/cmliblzma/liblzma/common/filter_encoder.h b/Utilities/cmliblzma/liblzma/common/filter_encoder.h index f1d5683..88f2daf 100644 --- a/Utilities/cmliblzma/liblzma/common/filter_encoder.h +++ b/Utilities/cmliblzma/liblzma/common/filter_encoder.h
@@ -1,13 +1,12 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // -/// \file filter_encoder.c +/// \file filter_encoder.h /// \brief Filter ID mapping to filter-specific functions // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef LZMA_FILTER_ENCODER_H @@ -16,10 +15,6 @@ #include "common.h" -// FIXME: Might become a part of the public API. -extern uint64_t lzma_mt_block_size(const lzma_filter *filters); - - extern lzma_ret lzma_raw_encoder_init( lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter *filters);
diff --git a/Utilities/cmliblzma/liblzma/common/filter_flags_decoder.c b/Utilities/cmliblzma/liblzma/common/filter_flags_decoder.c index ddfb085..0f5d204 100644 --- a/Utilities/cmliblzma/liblzma/common/filter_flags_decoder.c +++ b/Utilities/cmliblzma/liblzma/common/filter_flags_decoder.c
@@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file filter_flags_decoder.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "filter_decoder.h"
diff --git a/Utilities/cmliblzma/liblzma/common/filter_flags_encoder.c b/Utilities/cmliblzma/liblzma/common/filter_flags_encoder.c index b57b9fd..e1d6588 100644 --- a/Utilities/cmliblzma/liblzma/common/filter_flags_encoder.c +++ b/Utilities/cmliblzma/liblzma/common/filter_flags_encoder.c
@@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file filter_flags_encoder.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "filter_encoder.h"
diff --git a/Utilities/cmliblzma/liblzma/common/hardware_cputhreads.c b/Utilities/cmliblzma/liblzma/common/hardware_cputhreads.c index f468366..4ce852b 100644 --- a/Utilities/cmliblzma/liblzma/common/hardware_cputhreads.c +++ b/Utilities/cmliblzma/liblzma/common/hardware_cputhreads.c
@@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file hardware_cputhreads.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "common.h" @@ -15,6 +14,18 @@ #include "tuklib_cpucores.h" +#ifdef HAVE_SYMBOL_VERSIONS_LINUX +// This is for compatibility with binaries linked against liblzma that +// has been patched with xz-5.2.2-compat-libs.patch from RHEL/CentOS 7. +LZMA_SYMVER_API("lzma_cputhreads@XZ_5.2.2", + uint32_t, lzma_cputhreads_522)(void) lzma_nothrow + __attribute__((__alias__("lzma_cputhreads_52"))); + +LZMA_SYMVER_API("lzma_cputhreads@@XZ_5.2", + uint32_t, lzma_cputhreads_52)(void) lzma_nothrow; + +#define lzma_cputhreads lzma_cputhreads_52 +#endif extern LZMA_API(uint32_t) lzma_cputhreads(void) {
diff --git a/Utilities/cmliblzma/liblzma/common/hardware_physmem.c b/Utilities/cmliblzma/liblzma/common/hardware_physmem.c index a2bbbe2..1bc3486 100644 --- a/Utilities/cmliblzma/liblzma/common/hardware_physmem.c +++ b/Utilities/cmliblzma/liblzma/common/hardware_physmem.c
@@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file hardware_physmem.c @@ -5,9 +7,6 @@ // // Author: Jonathan Nieder // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "common.h"
diff --git a/Utilities/cmliblzma/liblzma/common/index.c b/Utilities/cmliblzma/liblzma/common/index.c index 4c463ec..f823c03 100644 --- a/Utilities/cmliblzma/liblzma/common/index.c +++ b/Utilities/cmliblzma/liblzma/common/index.c
@@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file index.c @@ -5,11 +7,9 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// +#include "common.h" #include "index.h" #include "stream_flags_common.h" @@ -659,6 +659,16 @@ const uint32_t index_list_size_add = lzma_vli_size(unpadded_size) + lzma_vli_size(uncompressed_size); + // Check that uncompressed size will not overflow. + if (uncompressed_base + uncompressed_size > LZMA_VLI_MAX) + return LZMA_DATA_ERROR; + + // Check that the new unpadded sum will not overflow. This is + // checked again in index_file_size(), but the unpadded sum is + // passed to vli_ceil4() which expects a valid lzma_vli value. + if (compressed_base + unpadded_size > UNPADDED_SIZE_MAX) + return LZMA_DATA_ERROR; + // Check that the file size will stay within limits. if (index_file_size(s->node.compressed_base, compressed_base + unpadded_size, s->record_count + 1, @@ -770,6 +780,9 @@ lzma_index_cat(lzma_index *restrict dest, lzma_index *restrict src, const lzma_allocator *allocator) { + if (dest == NULL || src == NULL) + return LZMA_PROG_ERROR; + const lzma_vli dest_file_size = lzma_index_file_size(dest); // Check that we don't exceed the file size limits. @@ -838,6 +851,11 @@ } } + // dest->checks includes the check types of all except the last Stream + // in dest. Set the bit for the check type of the last Stream now so + // that it won't get lost when Stream(s) from src are appended to dest. + dest->checks = lzma_index_checks(dest); + // Add all the Streams from src to dest. Update the base offsets // of each Stream from src. const index_cat_info info = { @@ -854,7 +872,7 @@ dest->total_size += src->total_size; dest->record_count += src->record_count; dest->index_list_size += src->index_list_size; - dest->checks = lzma_index_checks(dest) | src->checks; + dest->checks |= src->checks; // There's nothing else left in src than the base structure. lzma_free(src, allocator); @@ -1229,7 +1247,7 @@ // Use binary search to locate the exact Record. It is the first // Record whose uncompressed_sum is greater than target. - // This is because we want the rightmost Record that fullfills the + // This is because we want the rightmost Record that fulfills the // search criterion. It is possible that there are empty Blocks; // we don't want to return them. size_t left = 0;
diff --git a/Utilities/cmliblzma/liblzma/common/index.h b/Utilities/cmliblzma/liblzma/common/index.h index 64e9724..007e118 100644 --- a/Utilities/cmliblzma/liblzma/common/index.h +++ b/Utilities/cmliblzma/liblzma/common/index.h
@@ -1,20 +1,24 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file index.h /// \brief Handling of Index +/// \note This header file does not include common.h or lzma.h because +/// this file is needed by both liblzma internally and by the +/// tests. Including common.h will include and define many things +/// the tests do not need and prevents issues with header file +/// include order. This way, if lzma.h or common.h are not +/// included before this file it will break on every OS instead +/// of causing more subtle errors. // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef LZMA_INDEX_H #define LZMA_INDEX_H -#include "common.h" - /// Minimum Unpadded Size #define UNPADDED_SIZE_MIN LZMA_VLI_C(5) @@ -22,6 +26,9 @@ /// Maximum Unpadded Size #define UNPADDED_SIZE_MAX (LZMA_VLI_MAX & ~LZMA_VLI_C(3)) +/// Index Indicator based on xz specification +#define INDEX_INDICATOR 0 + /// Get the size of the Index Padding field. This is needed by Index encoder /// and decoder, but applications should have no use for this. @@ -38,7 +45,7 @@ static inline lzma_vli vli_ceil4(lzma_vli vli) { - assert(vli <= LZMA_VLI_MAX); + assert(vli <= UNPADDED_SIZE_MAX); return (vli + 3) & ~LZMA_VLI_C(3); }
diff --git a/Utilities/cmliblzma/liblzma/common/index_decoder.c b/Utilities/cmliblzma/liblzma/common/index_decoder.c index cc07a1b..4bcb306 100644 --- a/Utilities/cmliblzma/liblzma/common/index_decoder.c +++ b/Utilities/cmliblzma/liblzma/common/index_decoder.c
@@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file index_decoder.c @@ -5,12 +7,9 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// -#include "index.h" +#include "index_decoder.h" #include "check.h" @@ -80,7 +79,7 @@ // format". One could argue that the application should // verify the Index Indicator before trying to decode the // Index, but well, I suppose it is simpler this way. - if (in[(*in_pos)++] != 0x00) + if (in[(*in_pos)++] != INDEX_INDICATOR) return LZMA_DATA_ERROR; coder->sequence = SEQ_COUNT; @@ -180,8 +179,11 @@ return LZMA_OK; if (((coder->crc32 >> (coder->pos * 8)) & 0xFF) - != in[(*in_pos)++]) + != in[(*in_pos)++]) { +#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION return LZMA_DATA_ERROR; +#endif + } } while (++coder->pos < 4); @@ -200,9 +202,16 @@ } out: - // Update the CRC32, - coder->crc32 = lzma_crc32(in + in_start, - *in_pos - in_start, coder->crc32); + // Update the CRC32. + // + // Avoid null pointer + 0 (undefined behavior) in "in + in_start". + // In such a case we had no input and thus in_used == 0. + { + const size_t in_used = *in_pos - in_start; + if (in_used > 0) + coder->crc32 = lzma_crc32(in + in_start, + in_used, coder->crc32); + } return ret; } @@ -265,11 +274,11 @@ } -static lzma_ret -index_decoder_init(lzma_next_coder *next, const lzma_allocator *allocator, +extern lzma_ret +lzma_index_decoder_init(lzma_next_coder *next, const lzma_allocator *allocator, lzma_index **i, uint64_t memlimit) { - lzma_next_coder_init(&index_decoder_init, next, allocator); + lzma_next_coder_init(&lzma_index_decoder_init, next, allocator); if (i == NULL) return LZMA_PROG_ERROR; @@ -296,7 +305,13 @@ extern LZMA_API(lzma_ret) lzma_index_decoder(lzma_stream *strm, lzma_index **i, uint64_t memlimit) { - lzma_next_strm_init(index_decoder_init, strm, i, memlimit); + // If i isn't NULL, *i must always be initialized due to + // the wording in the API docs. This way it is initialized + // if we return LZMA_PROG_ERROR due to strm == NULL. + if (i != NULL) + *i = NULL; + + lzma_next_strm_init(lzma_index_decoder_init, strm, i, memlimit); strm->internal->supported_actions[LZMA_RUN] = true; strm->internal->supported_actions[LZMA_FINISH] = true; @@ -310,6 +325,11 @@ const lzma_allocator *allocator, const uint8_t *in, size_t *in_pos, size_t in_size) { + // If i isn't NULL, *i must always be initialized due to + // the wording in the API docs. + if (i != NULL) + *i = NULL; + // Sanity checks if (i == NULL || memlimit == NULL || in == NULL || in_pos == NULL || *in_pos > in_size)
diff --git a/Utilities/cmliblzma/liblzma/common/index_decoder.h b/Utilities/cmliblzma/liblzma/common/index_decoder.h new file mode 100644 index 0000000..5351d2f --- /dev/null +++ b/Utilities/cmliblzma/liblzma/common/index_decoder.h
@@ -0,0 +1,24 @@ +// SPDX-License-Identifier: 0BSD + +/////////////////////////////////////////////////////////////////////////////// +// +/// \file index_decoder.h +/// \brief Decodes the Index field +// +// Author: Lasse Collin +// +/////////////////////////////////////////////////////////////////////////////// + +#ifndef LZMA_INDEX_DECODER_H +#define LZMA_INDEX_DECODER_H + +#include "common.h" +#include "index.h" + + +extern lzma_ret lzma_index_decoder_init(lzma_next_coder *next, + const lzma_allocator *allocator, + lzma_index **i, uint64_t memlimit); + + +#endif
diff --git a/Utilities/cmliblzma/liblzma/common/index_encoder.c b/Utilities/cmliblzma/liblzma/common/index_encoder.c index 5e822cb..20dc6f4 100644 --- a/Utilities/cmliblzma/liblzma/common/index_encoder.c +++ b/Utilities/cmliblzma/liblzma/common/index_encoder.c
@@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file index_encoder.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "index_encoder.h" @@ -65,7 +64,7 @@ while (*out_pos < out_size) switch (coder->sequence) { case SEQ_INDICATOR: - out[*out_pos] = 0x00; + out[*out_pos] = INDEX_INDICATOR; ++*out_pos; coder->sequence = SEQ_COUNT; break; @@ -153,8 +152,15 @@ out: // Update the CRC32. - coder->crc32 = lzma_crc32(out + out_start, - *out_pos - out_start, coder->crc32); + // + // Avoid null pointer + 0 (undefined behavior) in "out + out_start". + // In such a case we had no input and thus out_used == 0. + { + const size_t out_used = *out_pos - out_start; + if (out_used > 0) + coder->crc32 = lzma_crc32(out + out_start, + out_used, coder->crc32); + } return ret; }
diff --git a/Utilities/cmliblzma/liblzma/common/index_encoder.h b/Utilities/cmliblzma/liblzma/common/index_encoder.h index 4d55cd1..29ba110 100644 --- a/Utilities/cmliblzma/liblzma/common/index_encoder.h +++ b/Utilities/cmliblzma/liblzma/common/index_encoder.h
@@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file index_encoder.h @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef LZMA_INDEX_ENCODER_H
diff --git a/Utilities/cmliblzma/liblzma/common/index_hash.c b/Utilities/cmliblzma/liblzma/common/index_hash.c index d7a0344..caa5967 100644 --- a/Utilities/cmliblzma/liblzma/common/index_hash.c +++ b/Utilities/cmliblzma/liblzma/common/index_hash.c
@@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file index_hash.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "common.h" @@ -122,7 +121,7 @@ /// Updates the sizes and the hash without any validation. -static lzma_ret +static void hash_append(lzma_index_hash_info *info, lzma_vli unpadded_size, lzma_vli uncompressed_size) { @@ -136,7 +135,7 @@ lzma_check_update(&info->check, LZMA_CHECK_BEST, (const uint8_t *)(sizes), sizeof(sizes)); - return LZMA_OK; + return; } @@ -145,15 +144,14 @@ lzma_vli uncompressed_size) { // Validate the arguments. - if (index_hash->sequence != SEQ_BLOCK + if (index_hash == NULL || index_hash->sequence != SEQ_BLOCK || unpadded_size < UNPADDED_SIZE_MIN || unpadded_size > UNPADDED_SIZE_MAX || uncompressed_size > LZMA_VLI_MAX) return LZMA_PROG_ERROR; // Update the hash. - return_if_error(hash_append(&index_hash->blocks, - unpadded_size, uncompressed_size)); + hash_append(&index_hash->blocks, unpadded_size, uncompressed_size); // Validate the properties of *info are still in allowed limits. if (index_hash->blocks.blocks_size > LZMA_VLI_MAX @@ -191,7 +189,7 @@ switch (index_hash->sequence) { case SEQ_BLOCK: // Check the Index Indicator is present. - if (in[(*in_pos)++] != 0x00) + if (in[(*in_pos)++] != INDEX_INDICATOR) return LZMA_DATA_ERROR; index_hash->sequence = SEQ_COUNT; @@ -239,9 +237,9 @@ index_hash->sequence = SEQ_UNCOMPRESSED; } else { // Update the hash. - return_if_error(hash_append(&index_hash->records, + hash_append(&index_hash->records, index_hash->unpadded_size, - index_hash->uncompressed_size)); + index_hash->uncompressed_size); // Verify that we don't go over the known sizes. Note // that this validation is simpler than the one used @@ -313,8 +311,11 @@ return LZMA_OK; if (((index_hash->crc32 >> (index_hash->pos * 8)) - & 0xFF) != in[(*in_pos)++]) + & 0xFF) != in[(*in_pos)++]) { +#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION return LZMA_DATA_ERROR; +#endif + } } while (++index_hash->pos < 4); @@ -326,9 +327,16 @@ } out: - // Update the CRC32, - index_hash->crc32 = lzma_crc32(in + in_start, - *in_pos - in_start, index_hash->crc32); + // Update the CRC32. + // + // Avoid null pointer + 0 (undefined behavior) in "in + in_start". + // In such a case we had no input and thus in_used == 0. + { + const size_t in_used = *in_pos - in_start; + if (in_used > 0) + index_hash->crc32 = lzma_crc32(in + in_start, + in_used, index_hash->crc32); + } return ret; }
diff --git a/Utilities/cmliblzma/liblzma/common/lzip_decoder.c b/Utilities/cmliblzma/liblzma/common/lzip_decoder.c new file mode 100644 index 0000000..651a0ae --- /dev/null +++ b/Utilities/cmliblzma/liblzma/common/lzip_decoder.c
@@ -0,0 +1,417 @@ +// SPDX-License-Identifier: 0BSD + +/////////////////////////////////////////////////////////////////////////////// +// +/// \file lzip_decoder.c +/// \brief Decodes .lz (lzip) files +// +// Author: Michał Górny +// Lasse Collin +// +/////////////////////////////////////////////////////////////////////////////// + +#include "lzip_decoder.h" +#include "lzma_decoder.h" +#include "check.h" + + +// .lz format version 0 lacks the 64-bit Member size field in the footer. +#define LZIP_V0_FOOTER_SIZE 12 +#define LZIP_V1_FOOTER_SIZE 20 +#define LZIP_FOOTER_SIZE_MAX LZIP_V1_FOOTER_SIZE + +// lc/lp/pb are hardcoded in the .lz format. +#define LZIP_LC 3 +#define LZIP_LP 0 +#define LZIP_PB 2 + + +typedef struct { + enum { + SEQ_ID_STRING, + SEQ_VERSION, + SEQ_DICT_SIZE, + SEQ_CODER_INIT, + SEQ_LZMA_STREAM, + SEQ_MEMBER_FOOTER, + } sequence; + + /// .lz member format version + uint32_t version; + + /// CRC32 of the uncompressed data in the .lz member + uint32_t crc32; + + /// Uncompressed size of the .lz member + uint64_t uncompressed_size; + + /// Compressed size of the .lz member + uint64_t member_size; + + /// Memory usage limit + uint64_t memlimit; + + /// Amount of memory actually needed + uint64_t memusage; + + /// If true, LZMA_GET_CHECK is returned after decoding the header + /// fields. As all files use CRC32 this is redundant but it's + /// implemented anyway since the initialization functions supports + /// all other flags in addition to LZMA_TELL_ANY_CHECK. + bool tell_any_check; + + /// If true, we won't calculate or verify the CRC32 of + /// the uncompressed data. + bool ignore_check; + + /// If true, we will decode concatenated .lz members and stop if + /// non-.lz data is seen after at least one member has been + /// successfully decoded. + bool concatenated; + + /// When decoding concatenated .lz members, this is true as long as + /// we are decoding the first .lz member. This is needed to avoid + /// incorrect LZMA_FORMAT_ERROR in case there is non-.lz data at + /// the end of the file. + bool first_member; + + /// Reading position in the header and footer fields + size_t pos; + + /// Buffer to hold the .lz footer fields + uint8_t buffer[LZIP_FOOTER_SIZE_MAX]; + + /// Options decoded from the .lz header that needed to initialize + /// the LZMA1 decoder. + lzma_options_lzma options; + + /// LZMA1 decoder + lzma_next_coder lzma_decoder; + +} lzma_lzip_coder; + + +static lzma_ret +lzip_decode(void *coder_ptr, const lzma_allocator *allocator, + const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size, lzma_action action) +{ + lzma_lzip_coder *coder = coder_ptr; + + while (true) + switch (coder->sequence) { + case SEQ_ID_STRING: { + // The "ID string" or magic bytes are "LZIP" in US-ASCII. + const uint8_t lzip_id_string[4] = { 0x4C, 0x5A, 0x49, 0x50 }; + + while (coder->pos < sizeof(lzip_id_string)) { + if (*in_pos >= in_size) { + // If we are on the 2nd+ concatenated member + // and the input ends before we can read + // the magic bytes, we discard the bytes that + // were already read (up to 3) and finish. + // See the reasoning below. + return !coder->first_member + && action == LZMA_FINISH + ? LZMA_STREAM_END : LZMA_OK; + } + + if (in[*in_pos] != lzip_id_string[coder->pos]) { + // The .lz format allows putting non-.lz data + // at the end of the file. If we have seen + // at least one valid .lz member already, + // then we won't consume the byte at *in_pos + // and will return LZMA_STREAM_END. This way + // apps can easily locate and read the non-.lz + // data after the .lz member(s). + // + // NOTE: If the first 1-3 bytes of the non-.lz + // data match the .lz ID string then the first + // 1-3 bytes of the junk will get ignored by + // us. If apps want to properly locate the + // trailing data they must ensure that the + // first byte of their custom data isn't the + // same as the first byte of .lz ID string. + // With the liblzma API we cannot rewind the + // input position across calls to lzma_code(). + return !coder->first_member + ? LZMA_STREAM_END : LZMA_FORMAT_ERROR; + } + + ++*in_pos; + ++coder->pos; + } + + coder->pos = 0; + + coder->crc32 = 0; + coder->uncompressed_size = 0; + coder->member_size = sizeof(lzip_id_string); + + coder->sequence = SEQ_VERSION; + } + + // Fall through + + case SEQ_VERSION: + if (*in_pos >= in_size) + return LZMA_OK; + + coder->version = in[(*in_pos)++]; + + // We support version 0 and unextended version 1. + if (coder->version > 1) + return LZMA_OPTIONS_ERROR; + + ++coder->member_size; + coder->sequence = SEQ_DICT_SIZE; + + // .lz versions 0 and 1 use CRC32 as the integrity check + // so if the application wanted to know that + // (LZMA_TELL_ANY_CHECK) we can tell it now. + if (coder->tell_any_check) + return LZMA_GET_CHECK; + + // Fall through + + case SEQ_DICT_SIZE: { + if (*in_pos >= in_size) + return LZMA_OK; + + const uint32_t ds = in[(*in_pos)++]; + ++coder->member_size; + + // The five lowest bits are for the base-2 logarithm of + // the dictionary size and the highest three bits are + // the fractional part (0/16 to 7/16) that will be + // subtracted to get the final value. + // + // For example, with 0xB5: + // b2log = 21 + // fracnum = 5 + // dict_size = 2^21 - 2^21 * 5 / 16 = 1408 KiB + const uint32_t b2log = ds & 0x1F; + const uint32_t fracnum = ds >> 5; + + // The format versions 0 and 1 allow dictionary size in the + // range [4 KiB, 512 MiB]. + if (b2log < 12 || b2log > 29 || (b2log == 12 && fracnum > 0)) + return LZMA_DATA_ERROR; + + // 2^[b2log] - 2^[b2log] * [fracnum] / 16 + // = 2^[b2log] - [fracnum] * 2^([b2log] - 4) + coder->options.dict_size = (UINT32_C(1) << b2log) + - (fracnum << (b2log - 4)); + + assert(coder->options.dict_size >= 4096); + assert(coder->options.dict_size <= (UINT32_C(512) << 20)); + + coder->options.preset_dict = NULL; + coder->options.lc = LZIP_LC; + coder->options.lp = LZIP_LP; + coder->options.pb = LZIP_PB; + + // Calculate the memory usage. + coder->memusage = lzma_lzma_decoder_memusage(&coder->options) + + LZMA_MEMUSAGE_BASE; + + // Initialization is a separate step because if we return + // LZMA_MEMLIMIT_ERROR we need to be able to restart after + // the memlimit has been increased. + coder->sequence = SEQ_CODER_INIT; + } + + // Fall through + + case SEQ_CODER_INIT: { + if (coder->memusage > coder->memlimit) + return LZMA_MEMLIMIT_ERROR; + + const lzma_filter_info filters[2] = { + { + .id = LZMA_FILTER_LZMA1, + .init = &lzma_lzma_decoder_init, + .options = &coder->options, + }, { + .init = NULL, + } + }; + + return_if_error(lzma_next_filter_init(&coder->lzma_decoder, + allocator, filters)); + + coder->crc32 = 0; + coder->sequence = SEQ_LZMA_STREAM; + } + + // Fall through + + case SEQ_LZMA_STREAM: { + const size_t in_start = *in_pos; + const size_t out_start = *out_pos; + + const lzma_ret ret = coder->lzma_decoder.code( + coder->lzma_decoder.coder, allocator, + in, in_pos, in_size, out, out_pos, out_size, + action); + + const size_t out_used = *out_pos - out_start; + + coder->member_size += *in_pos - in_start; + coder->uncompressed_size += out_used; + + // Don't update the CRC32 if the integrity check will be + // ignored or if there was no new output. The latter is + // important in case out == NULL to avoid null pointer + 0 + // which is undefined behavior. + if (!coder->ignore_check && out_used > 0) + coder->crc32 = lzma_crc32(out + out_start, out_used, + coder->crc32); + + if (ret != LZMA_STREAM_END) + return ret; + + coder->sequence = SEQ_MEMBER_FOOTER; + } + + // Fall through + + case SEQ_MEMBER_FOOTER: { + // The footer of .lz version 0 lacks the Member size field. + // This is the only difference between version 0 and + // unextended version 1 formats. + const size_t footer_size = coder->version == 0 + ? LZIP_V0_FOOTER_SIZE + : LZIP_V1_FOOTER_SIZE; + + // Copy the CRC32, Data size, and Member size fields to + // the internal buffer. + lzma_bufcpy(in, in_pos, in_size, coder->buffer, &coder->pos, + footer_size); + + // Return if we didn't get the whole footer yet. + if (coder->pos < footer_size) + return LZMA_OK; + + coder->pos = 0; + coder->member_size += footer_size; + + // Check that the footer fields match the observed data. + if (!coder->ignore_check + && coder->crc32 != read32le(&coder->buffer[0])) + return LZMA_DATA_ERROR; + + if (coder->uncompressed_size != read64le(&coder->buffer[4])) + return LZMA_DATA_ERROR; + + if (coder->version > 0) { + // .lz version 0 has no Member size field. + if (coder->member_size != read64le(&coder->buffer[12])) + return LZMA_DATA_ERROR; + } + + // Decoding is finished if we weren't requested to decode + // more than one .lz member. + if (!coder->concatenated) + return LZMA_STREAM_END; + + coder->first_member = false; + coder->sequence = SEQ_ID_STRING; + break; + } + + default: + assert(0); + return LZMA_PROG_ERROR; + } + + // Never reached +} + + +static void +lzip_decoder_end(void *coder_ptr, const lzma_allocator *allocator) +{ + lzma_lzip_coder *coder = coder_ptr; + lzma_next_end(&coder->lzma_decoder, allocator); + lzma_free(coder, allocator); + return; +} + + +static lzma_check +lzip_decoder_get_check(const void *coder_ptr lzma_attribute((__unused__))) +{ + return LZMA_CHECK_CRC32; +} + + +static lzma_ret +lzip_decoder_memconfig(void *coder_ptr, uint64_t *memusage, + uint64_t *old_memlimit, uint64_t new_memlimit) +{ + lzma_lzip_coder *coder = coder_ptr; + + *memusage = coder->memusage; + *old_memlimit = coder->memlimit; + + if (new_memlimit != 0) { + if (new_memlimit < coder->memusage) + return LZMA_MEMLIMIT_ERROR; + + coder->memlimit = new_memlimit; + } + + return LZMA_OK; +} + + +extern lzma_ret +lzma_lzip_decoder_init( + lzma_next_coder *next, const lzma_allocator *allocator, + uint64_t memlimit, uint32_t flags) +{ + lzma_next_coder_init(&lzma_lzip_decoder_init, next, allocator); + + if (flags & ~LZMA_SUPPORTED_FLAGS) + return LZMA_OPTIONS_ERROR; + + lzma_lzip_coder *coder = next->coder; + if (coder == NULL) { + coder = lzma_alloc(sizeof(lzma_lzip_coder), allocator); + if (coder == NULL) + return LZMA_MEM_ERROR; + + next->coder = coder; + next->code = &lzip_decode; + next->end = &lzip_decoder_end; + next->get_check = &lzip_decoder_get_check; + next->memconfig = &lzip_decoder_memconfig; + + coder->lzma_decoder = LZMA_NEXT_CODER_INIT; + } + + coder->sequence = SEQ_ID_STRING; + coder->memlimit = my_max(1, memlimit); + coder->memusage = LZMA_MEMUSAGE_BASE; + coder->tell_any_check = (flags & LZMA_TELL_ANY_CHECK) != 0; + coder->ignore_check = (flags & LZMA_IGNORE_CHECK) != 0; + coder->concatenated = (flags & LZMA_CONCATENATED) != 0; + coder->first_member = true; + coder->pos = 0; + + return LZMA_OK; +} + + +extern LZMA_API(lzma_ret) +lzma_lzip_decoder(lzma_stream *strm, uint64_t memlimit, uint32_t flags) +{ + lzma_next_strm_init(lzma_lzip_decoder_init, strm, memlimit, flags); + + strm->internal->supported_actions[LZMA_RUN] = true; + strm->internal->supported_actions[LZMA_FINISH] = true; + + return LZMA_OK; +}
diff --git a/Utilities/cmliblzma/liblzma/common/lzip_decoder.h b/Utilities/cmliblzma/liblzma/common/lzip_decoder.h new file mode 100644 index 0000000..0e1f7be --- /dev/null +++ b/Utilities/cmliblzma/liblzma/common/lzip_decoder.h
@@ -0,0 +1,21 @@ +// SPDX-License-Identifier: 0BSD + +/////////////////////////////////////////////////////////////////////////////// +// +/// \file lzip_decoder.h +/// \brief Decodes .lz (lzip) files +// +// Author: Michał Górny +// +/////////////////////////////////////////////////////////////////////////////// + +#ifndef LZMA_LZIP_DECODER_H +#define LZMA_LZIP_DECODER_H + +#include "common.h" + +extern lzma_ret lzma_lzip_decoder_init( + lzma_next_coder *next, const lzma_allocator *allocator, + uint64_t memlimit, uint32_t flags); + +#endif
diff --git a/Utilities/cmliblzma/liblzma/common/memcmplen.h b/Utilities/cmliblzma/liblzma/common/memcmplen.h index dcfd8d6..394a485 100644 --- a/Utilities/cmliblzma/liblzma/common/memcmplen.h +++ b/Utilities/cmliblzma/liblzma/common/memcmplen.h
@@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file memcmplen.h @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef LZMA_MEMCMPLEN_H @@ -19,6 +18,17 @@ # include <immintrin.h> #endif +// Only include <intrin.h> if it is needed. The header is only needed +// on Windows when using an MSVC compatible compiler. The Intel compiler +// can use the intrinsics without the header file. +#if defined(TUKLIB_FAST_UNALIGNED_ACCESS) \ + && defined(_MSC_VER) \ + && (defined(_M_X64) \ + || defined(_M_ARM64) || defined(_M_ARM64EC)) \ + && !defined(__INTEL_COMPILER) +# include <intrin.h> +#endif + /// Find out how many equal bytes the two buffers have. /// @@ -39,7 +49,7 @@ /// It's rounded up to 2^n. This extra amount needs to be /// allocated in the buffers being used. It needs to be /// initialized too to keep Valgrind quiet. -static inline uint32_t lzma_attribute((__always_inline__)) +static lzma_always_inline uint32_t lzma_memcmplen(const uint8_t *buf1, const uint8_t *buf2, uint32_t len, uint32_t limit) { @@ -47,27 +57,40 @@ assert(limit <= UINT32_MAX / 2); #if defined(TUKLIB_FAST_UNALIGNED_ACCESS) \ - && ((TUKLIB_GNUC_REQ(3, 4) && defined(__x86_64__)) \ + && (((TUKLIB_GNUC_REQ(3, 4) || defined(__clang__)) \ + && (defined(__x86_64__) \ + || defined(__aarch64__))) \ || (defined(__INTEL_COMPILER) && defined(__x86_64__)) \ || (defined(__INTEL_COMPILER) && defined(_M_X64)) \ - || (defined(_MSC_VER) && defined(_M_X64))) - // NOTE: This will use 64-bit unaligned access which - // TUKLIB_FAST_UNALIGNED_ACCESS wasn't meant to permit, but - // it's convenient here at least as long as it's x86-64 only. + || (defined(_MSC_VER) && (defined(_M_X64) \ + || defined(_M_ARM64) || defined(_M_ARM64EC)))) + // This is only for x86-64 and ARM64 for now. This might be fine on + // other 64-bit processors too. On big endian one should use xor + // instead of subtraction and switch to __builtin_clzll(). // - // I keep this x86-64 only for now since that's where I know this - // to be a good method. This may be fine on other 64-bit CPUs too. - // On big endian one should use xor instead of subtraction and switch - // to __builtin_clzll(). + // Reasons to use subtraction instead of xor: + // + // - On some x86-64 processors (Intel Sandy Bridge to Tiger Lake), + // sub+jz and sub+jnz can be fused but xor+jz or xor+jnz cannot. + // Thus using subtraction has potential to be a tiny amount faster + // since the code checks if the quotient is non-zero. + // + // - Some processors (Intel Pentium 4) used to have more ALU + // resources for add/sub instructions than and/or/xor. + // + // The processor info is based on Agner Fog's microarchitecture.pdf + // version 2023-05-26. https://www.agner.org/optimize/ #define LZMA_MEMCMPLEN_EXTRA 8 while (len < limit) { const uint64_t x = read64ne(buf1 + len) - read64ne(buf2 + len); if (x != 0) { -# if defined(_M_X64) // MSVC or Intel C compiler on Windows + // MSVC or Intel C compiler on Windows +# if defined(_MSC_VER) || defined(__INTEL_COMPILER) unsigned long tmp; _BitScanForward64(&tmp, x); len += (uint32_t)tmp >> 3; -# else // GCC, clang, or Intel C compiler + // GCC, Clang, or Intel C compiler +# else len += (uint32_t)__builtin_ctzll(x) >> 3; # endif return my_min(len, limit); @@ -80,12 +103,12 @@ #elif defined(TUKLIB_FAST_UNALIGNED_ACCESS) \ && defined(HAVE__MM_MOVEMASK_EPI8) \ - && ((defined(__GNUC__) && defined(__SSE2_MATH__)) \ - || (defined(__INTEL_COMPILER) && defined(__SSE2__)) \ + && (defined(__SSE2__) \ || (defined(_MSC_VER) && defined(_M_IX86_FP) \ && _M_IX86_FP >= 2)) - // NOTE: Like above, this will use 128-bit unaligned access which - // TUKLIB_FAST_UNALIGNED_ACCESS wasn't meant to permit. + // NOTE: This will use 128-bit unaligned access which + // TUKLIB_FAST_UNALIGNED_ACCESS wasn't meant to permit, + // but it's convenient here since this is x86-only. // // SSE2 version for 32-bit and 64-bit x86. On x86-64 the above // version is sometimes significantly faster and sometimes @@ -93,7 +116,8 @@ // version isn't used on x86-64. # define LZMA_MEMCMPLEN_EXTRA 16 while (len < limit) { - const uint32_t x = 0xFFFF ^ _mm_movemask_epi8(_mm_cmpeq_epi8( + const uint32_t x = 0xFFFF ^ (uint32_t)_mm_movemask_epi8( + _mm_cmpeq_epi8( _mm_loadu_si128((const __m128i *)(buf1 + len)), _mm_loadu_si128((const __m128i *)(buf2 + len))));
diff --git a/Utilities/cmliblzma/liblzma/common/microlzma_decoder.c b/Utilities/cmliblzma/liblzma/common/microlzma_decoder.c new file mode 100644 index 0000000..882cb2c --- /dev/null +++ b/Utilities/cmliblzma/liblzma/common/microlzma_decoder.c
@@ -0,0 +1,220 @@ +// SPDX-License-Identifier: 0BSD + +/////////////////////////////////////////////////////////////////////////////// +// +/// \file microlzma_decoder.c +/// \brief Decode MicroLZMA format +// +// Author: Lasse Collin +// +/////////////////////////////////////////////////////////////////////////////// + +#include "lzma_decoder.h" +#include "lz_decoder.h" + + +typedef struct { + /// LZMA1 decoder + lzma_next_coder lzma; + + /// Compressed size of the stream as given by the application. + /// This must be exactly correct. + /// + /// This will be decremented when input is read. + uint64_t comp_size; + + /// Uncompressed size of the stream as given by the application. + /// This may be less than the actual uncompressed size if + /// uncomp_size_is_exact is false. + /// + /// This will be decremented when output is produced. + lzma_vli uncomp_size; + + /// LZMA dictionary size as given by the application + uint32_t dict_size; + + /// If true, the exact uncompressed size is known. If false, + /// uncomp_size may be smaller than the real uncompressed size; + /// uncomp_size may never be bigger than the real uncompressed size. + bool uncomp_size_is_exact; + + /// True once the first byte of the MicroLZMA stream + /// has been processed. + bool props_decoded; +} lzma_microlzma_coder; + + +static lzma_ret +microlzma_decode(void *coder_ptr, const lzma_allocator *allocator, + const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size, lzma_action action) +{ + lzma_microlzma_coder *coder = coder_ptr; + + // Remember the in start position so that we can update comp_size. + const size_t in_start = *in_pos; + + // Remember the out start position so that we can update uncomp_size. + const size_t out_start = *out_pos; + + // Limit the amount of input so that the decoder won't read more than + // comp_size. This is required when uncomp_size isn't exact because + // in that case the LZMA decoder will try to decode more input even + // when it has no output space (it can be looking for EOPM). + if (in_size - *in_pos > coder->comp_size) + in_size = *in_pos + (size_t)(coder->comp_size); + + // When the exact uncompressed size isn't known, we must limit + // the available output space to prevent the LZMA decoder from + // trying to decode too much. + if (!coder->uncomp_size_is_exact + && out_size - *out_pos > coder->uncomp_size) + out_size = *out_pos + (size_t)(coder->uncomp_size); + + if (!coder->props_decoded) { + // There must be at least one byte of input to decode + // the properties byte. + if (*in_pos >= in_size) + return LZMA_OK; + + lzma_options_lzma options = { + .dict_size = coder->dict_size, + .preset_dict = NULL, + .preset_dict_size = 0, + .ext_flags = 0, // EOPM not allowed when size is known + .ext_size_low = UINT32_MAX, // Unknown size by default + .ext_size_high = UINT32_MAX, + }; + + if (coder->uncomp_size_is_exact) + lzma_set_ext_size(options, coder->uncomp_size); + + // The properties are stored as bitwise-negation + // of the typical encoding. + if (lzma_lzma_lclppb_decode(&options, ~in[*in_pos])) + return LZMA_OPTIONS_ERROR; + + ++*in_pos; + + // Initialize the decoder. + lzma_filter_info filters[2] = { + { + .id = LZMA_FILTER_LZMA1EXT, + .init = &lzma_lzma_decoder_init, + .options = &options, + }, { + .init = NULL, + } + }; + + return_if_error(lzma_next_filter_init(&coder->lzma, + allocator, filters)); + + // Pass one dummy 0x00 byte to the LZMA decoder since that + // is what it expects the first byte to be. + const uint8_t dummy_in = 0; + size_t dummy_in_pos = 0; + if (coder->lzma.code(coder->lzma.coder, allocator, + &dummy_in, &dummy_in_pos, 1, + out, out_pos, out_size, LZMA_RUN) != LZMA_OK) + return LZMA_PROG_ERROR; + + assert(dummy_in_pos == 1); + coder->props_decoded = true; + } + + // The rest is normal LZMA decoding. + lzma_ret ret = coder->lzma.code(coder->lzma.coder, allocator, + in, in_pos, in_size, + out, out_pos, out_size, action); + + // Update the remaining compressed size. + assert(coder->comp_size >= *in_pos - in_start); + coder->comp_size -= *in_pos - in_start; + + if (coder->uncomp_size_is_exact) { + // After successful decompression of the complete stream + // the compressed size must match. + if (ret == LZMA_STREAM_END && coder->comp_size != 0) + ret = LZMA_DATA_ERROR; + } else { + // Update the amount of output remaining. + assert(coder->uncomp_size >= *out_pos - out_start); + coder->uncomp_size -= *out_pos - out_start; + + // - We must not get LZMA_STREAM_END because the stream + // shouldn't have EOPM. + // - We must use uncomp_size to determine when to + // return LZMA_STREAM_END. + if (ret == LZMA_STREAM_END) + ret = LZMA_DATA_ERROR; + else if (coder->uncomp_size == 0) + ret = LZMA_STREAM_END; + } + + return ret; +} + + +static void +microlzma_decoder_end(void *coder_ptr, const lzma_allocator *allocator) +{ + lzma_microlzma_coder *coder = coder_ptr; + lzma_next_end(&coder->lzma, allocator); + lzma_free(coder, allocator); + return; +} + + +static lzma_ret +microlzma_decoder_init(lzma_next_coder *next, const lzma_allocator *allocator, + uint64_t comp_size, + uint64_t uncomp_size, bool uncomp_size_is_exact, + uint32_t dict_size) +{ + lzma_next_coder_init(µlzma_decoder_init, next, allocator); + + lzma_microlzma_coder *coder = next->coder; + + if (coder == NULL) { + coder = lzma_alloc(sizeof(lzma_microlzma_coder), allocator); + if (coder == NULL) + return LZMA_MEM_ERROR; + + next->coder = coder; + next->code = µlzma_decode; + next->end = µlzma_decoder_end; + + coder->lzma = LZMA_NEXT_CODER_INIT; + } + + // The public API is uint64_t but the internal LZ decoder API uses + // lzma_vli. + if (uncomp_size > LZMA_VLI_MAX) + return LZMA_OPTIONS_ERROR; + + coder->comp_size = comp_size; + coder->uncomp_size = uncomp_size; + coder->uncomp_size_is_exact = uncomp_size_is_exact; + coder->dict_size = dict_size; + + coder->props_decoded = false; + + return LZMA_OK; +} + + +extern LZMA_API(lzma_ret) +lzma_microlzma_decoder(lzma_stream *strm, uint64_t comp_size, + uint64_t uncomp_size, lzma_bool uncomp_size_is_exact, + uint32_t dict_size) +{ + lzma_next_strm_init(microlzma_decoder_init, strm, comp_size, + uncomp_size, uncomp_size_is_exact, dict_size); + + strm->internal->supported_actions[LZMA_RUN] = true; + strm->internal->supported_actions[LZMA_FINISH] = true; + + return LZMA_OK; +}
diff --git a/Utilities/cmliblzma/liblzma/common/microlzma_encoder.c b/Utilities/cmliblzma/liblzma/common/microlzma_encoder.c new file mode 100644 index 0000000..45ec0b1 --- /dev/null +++ b/Utilities/cmliblzma/liblzma/common/microlzma_encoder.c
@@ -0,0 +1,140 @@ +// SPDX-License-Identifier: 0BSD + +/////////////////////////////////////////////////////////////////////////////// +// +/// \file microlzma_encoder.c +/// \brief Encode into MicroLZMA format +// +// Author: Lasse Collin +// +/////////////////////////////////////////////////////////////////////////////// + +#include "lzma_encoder.h" + + +typedef struct { + /// LZMA1 encoder + lzma_next_coder lzma; + + /// LZMA properties byte (lc/lp/pb) + uint8_t props; +} lzma_microlzma_coder; + + +static lzma_ret +microlzma_encode(void *coder_ptr, const lzma_allocator *allocator, + const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size, lzma_action action) +{ + lzma_microlzma_coder *coder = coder_ptr; + + // Remember *out_pos so that we can overwrite the first byte with + // the LZMA properties byte. + const size_t out_start = *out_pos; + + // Remember *in_pos so that we can set it based on how many + // uncompressed bytes were actually encoded. + const size_t in_start = *in_pos; + + // Set the output size limit based on the available output space. + // We know that the encoder supports set_out_limit() so + // LZMA_OPTIONS_ERROR isn't possible. LZMA_BUF_ERROR is possible + // but lzma_code() has an assertion to not allow it to be returned + // from here and I don't want to change that for now, so + // LZMA_BUF_ERROR becomes LZMA_PROG_ERROR. + uint64_t uncomp_size; + if (coder->lzma.set_out_limit(coder->lzma.coder, + &uncomp_size, out_size - *out_pos) != LZMA_OK) + return LZMA_PROG_ERROR; + + // set_out_limit fails if this isn't true. + assert(out_size - *out_pos >= 6); + + // Encode as much as possible. + const lzma_ret ret = coder->lzma.code(coder->lzma.coder, allocator, + in, in_pos, in_size, out, out_pos, out_size, action); + + if (ret != LZMA_STREAM_END) { + if (ret == LZMA_OK) { + assert(0); + return LZMA_PROG_ERROR; + } + + return ret; + } + + // The first output byte is bitwise-negation of the properties byte. + // We know that there is space for this byte because set_out_limit + // and the actual encoding succeeded. + out[out_start] = (uint8_t)(~coder->props); + + // The LZMA encoder likely read more input than it was able to encode. + // Set *in_pos based on uncomp_size. + assert(uncomp_size <= in_size - in_start); + *in_pos = in_start + (size_t)(uncomp_size); + + return ret; +} + + +static void +microlzma_encoder_end(void *coder_ptr, const lzma_allocator *allocator) +{ + lzma_microlzma_coder *coder = coder_ptr; + lzma_next_end(&coder->lzma, allocator); + lzma_free(coder, allocator); + return; +} + + +static lzma_ret +microlzma_encoder_init(lzma_next_coder *next, const lzma_allocator *allocator, + const lzma_options_lzma *options) +{ + lzma_next_coder_init(µlzma_encoder_init, next, allocator); + + lzma_microlzma_coder *coder = next->coder; + + if (coder == NULL) { + coder = lzma_alloc(sizeof(lzma_microlzma_coder), allocator); + if (coder == NULL) + return LZMA_MEM_ERROR; + + next->coder = coder; + next->code = µlzma_encode; + next->end = µlzma_encoder_end; + + coder->lzma = LZMA_NEXT_CODER_INIT; + } + + // Encode the properties byte. Bitwise-negation of it will be the + // first output byte. + if (lzma_lzma_lclppb_encode(options, &coder->props)) + return LZMA_OPTIONS_ERROR; + + // Initialize the LZMA encoder. + const lzma_filter_info filters[2] = { + { + .id = LZMA_FILTER_LZMA1, + .init = &lzma_lzma_encoder_init, + .options = (void *)(options), + }, { + .init = NULL, + } + }; + + return lzma_next_filter_init(&coder->lzma, allocator, filters); +} + + +extern LZMA_API(lzma_ret) +lzma_microlzma_encoder(lzma_stream *strm, const lzma_options_lzma *options) +{ + lzma_next_strm_init(microlzma_encoder_init, strm, options); + + strm->internal->supported_actions[LZMA_FINISH] = true; + + return LZMA_OK; + +}
diff --git a/Utilities/cmliblzma/liblzma/common/outqueue.c b/Utilities/cmliblzma/liblzma/common/outqueue.c index 2dc8a38..eb018eb 100644 --- a/Utilities/cmliblzma/liblzma/common/outqueue.c +++ b/Utilities/cmliblzma/liblzma/common/outqueue.c
@@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file outqueue.c @@ -5,92 +7,126 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "outqueue.h" -/// This is to ease integer overflow checking: We may allocate up to -/// 2 * LZMA_THREADS_MAX buffers and we need some extra memory for other -/// data structures (that's the second /2). -#define BUF_SIZE_MAX (UINT64_MAX / LZMA_THREADS_MAX / 2 / 2) - - -static lzma_ret -get_options(uint64_t *bufs_alloc_size, uint32_t *bufs_count, - uint64_t buf_size_max, uint32_t threads) -{ - if (threads > LZMA_THREADS_MAX || buf_size_max > BUF_SIZE_MAX) - return LZMA_OPTIONS_ERROR; - - // The number of buffers is twice the number of threads. - // This wastes RAM but keeps the threads busy when buffers - // finish out of order. - // - // NOTE: If this is changed, update BUF_SIZE_MAX too. - *bufs_count = threads * 2; - *bufs_alloc_size = *bufs_count * buf_size_max; - - return LZMA_OK; -} +/// Get the maximum number of buffers that may be allocated based +/// on the number of threads. For now this is twice the number of threads. +/// It's a compromise between RAM usage and keeping the worker threads busy +/// when buffers finish out of order. +#define GET_BUFS_LIMIT(threads) (2 * (threads)) extern uint64_t lzma_outq_memusage(uint64_t buf_size_max, uint32_t threads) { - uint64_t bufs_alloc_size; - uint32_t bufs_count; + // This is to ease integer overflow checking: We may allocate up to + // GET_BUFS_LIMIT(LZMA_THREADS_MAX) buffers and we need some extra + // memory for other data structures too (that's the /2). + // + // lzma_outq_prealloc_buf() will still accept bigger buffers than this. + const uint64_t limit + = UINT64_MAX / GET_BUFS_LIMIT(LZMA_THREADS_MAX) / 2; - if (get_options(&bufs_alloc_size, &bufs_count, buf_size_max, threads) - != LZMA_OK) + if (threads > LZMA_THREADS_MAX || buf_size_max > limit) return UINT64_MAX; - return sizeof(lzma_outq) + bufs_count * sizeof(lzma_outbuf) - + bufs_alloc_size; + return GET_BUFS_LIMIT(threads) + * lzma_outq_outbuf_memusage(buf_size_max); +} + + +static void +move_head_to_cache(lzma_outq *outq, const lzma_allocator *allocator) +{ + assert(outq->head != NULL); + assert(outq->tail != NULL); + assert(outq->bufs_in_use > 0); + + lzma_outbuf *buf = outq->head; + outq->head = buf->next; + if (outq->head == NULL) + outq->tail = NULL; + + if (outq->cache != NULL && outq->cache->allocated != buf->allocated) + lzma_outq_clear_cache(outq, allocator); + + buf->next = outq->cache; + outq->cache = buf; + + --outq->bufs_in_use; + outq->mem_in_use -= lzma_outq_outbuf_memusage(buf->allocated); + + return; +} + + +static void +free_one_cached_buffer(lzma_outq *outq, const lzma_allocator *allocator) +{ + assert(outq->cache != NULL); + + lzma_outbuf *buf = outq->cache; + outq->cache = buf->next; + + --outq->bufs_allocated; + outq->mem_allocated -= lzma_outq_outbuf_memusage(buf->allocated); + + lzma_free(buf, allocator); + return; +} + + +extern void +lzma_outq_clear_cache(lzma_outq *outq, const lzma_allocator *allocator) +{ + while (outq->cache != NULL) + free_one_cached_buffer(outq, allocator); + + return; +} + + +extern void +lzma_outq_clear_cache2(lzma_outq *outq, const lzma_allocator *allocator, + size_t keep_size) +{ + if (outq->cache == NULL) + return; + + // Free all but one. + while (outq->cache->next != NULL) + free_one_cached_buffer(outq, allocator); + + // Free the last one only if its size doesn't equal to keep_size. + if (outq->cache->allocated != keep_size) + free_one_cached_buffer(outq, allocator); + + return; } extern lzma_ret lzma_outq_init(lzma_outq *outq, const lzma_allocator *allocator, - uint64_t buf_size_max, uint32_t threads) + uint32_t threads) { - uint64_t bufs_alloc_size; - uint32_t bufs_count; + if (threads > LZMA_THREADS_MAX) + return LZMA_OPTIONS_ERROR; - // Set bufs_count and bufs_alloc_size. - return_if_error(get_options(&bufs_alloc_size, &bufs_count, - buf_size_max, threads)); + const uint32_t bufs_limit = GET_BUFS_LIMIT(threads); - // Allocate memory if needed. - if (outq->buf_size_max != buf_size_max - || outq->bufs_allocated != bufs_count) { - lzma_outq_end(outq, allocator); + // Clear head/tail. + while (outq->head != NULL) + move_head_to_cache(outq, allocator); -#if SIZE_MAX < UINT64_MAX - if (bufs_alloc_size > SIZE_MAX) - return LZMA_MEM_ERROR; -#endif + // If new buf_limit is lower than the old one, we may need to free + // a few cached buffers. + while (bufs_limit < outq->bufs_allocated) + free_one_cached_buffer(outq, allocator); - outq->bufs = lzma_alloc(bufs_count * sizeof(lzma_outbuf), - allocator); - outq->bufs_mem = lzma_alloc((size_t)(bufs_alloc_size), - allocator); - - if (outq->bufs == NULL || outq->bufs_mem == NULL) { - lzma_outq_end(outq, allocator); - return LZMA_MEM_ERROR; - } - } - - // Initialize the rest of the main structure. Initialization of - // outq->bufs[] is done when they are actually needed. - outq->buf_size_max = (size_t)(buf_size_max); - outq->bufs_allocated = bufs_count; - outq->bufs_pos = 0; - outq->bufs_used = 0; + outq->bufs_limit = bufs_limit; outq->read_pos = 0; return LZMA_OK; @@ -100,33 +136,81 @@ extern void lzma_outq_end(lzma_outq *outq, const lzma_allocator *allocator) { - lzma_free(outq->bufs, allocator); - outq->bufs = NULL; + while (outq->head != NULL) + move_head_to_cache(outq, allocator); - lzma_free(outq->bufs_mem, allocator); - outq->bufs_mem = NULL; - + lzma_outq_clear_cache(outq, allocator); return; } -extern lzma_outbuf * -lzma_outq_get_buf(lzma_outq *outq) +extern lzma_ret +lzma_outq_prealloc_buf(lzma_outq *outq, const lzma_allocator *allocator, + size_t size) { // Caller must have checked it with lzma_outq_has_buf(). - assert(outq->bufs_used < outq->bufs_allocated); + assert(outq->bufs_in_use < outq->bufs_limit); - // Initialize the new buffer. - lzma_outbuf *buf = &outq->bufs[outq->bufs_pos]; - buf->buf = outq->bufs_mem + outq->bufs_pos * outq->buf_size_max; - buf->size = 0; + // If there already is appropriately-sized buffer in the cache, + // we need to do nothing. + if (outq->cache != NULL && outq->cache->allocated == size) + return LZMA_OK; + + if (size > SIZE_MAX - sizeof(lzma_outbuf)) + return LZMA_MEM_ERROR; + + const size_t alloc_size = lzma_outq_outbuf_memusage(size); + + // The cache may have buffers but their size is wrong. + lzma_outq_clear_cache(outq, allocator); + + outq->cache = lzma_alloc(alloc_size, allocator); + if (outq->cache == NULL) + return LZMA_MEM_ERROR; + + outq->cache->next = NULL; + outq->cache->allocated = size; + + ++outq->bufs_allocated; + outq->mem_allocated += alloc_size; + + return LZMA_OK; +} + + +extern lzma_outbuf * +lzma_outq_get_buf(lzma_outq *outq, void *worker) +{ + // Caller must have used lzma_outq_prealloc_buf() to ensure these. + assert(outq->bufs_in_use < outq->bufs_limit); + assert(outq->bufs_in_use < outq->bufs_allocated); + assert(outq->cache != NULL); + + lzma_outbuf *buf = outq->cache; + outq->cache = buf->next; + buf->next = NULL; + + if (outq->tail != NULL) { + assert(outq->head != NULL); + outq->tail->next = buf; + } else { + assert(outq->head == NULL); + outq->head = buf; + } + + outq->tail = buf; + + buf->worker = worker; buf->finished = false; + buf->finish_ret = LZMA_STREAM_END; + buf->pos = 0; + buf->decoder_in_pos = 0; - // Update the queue state. - if (++outq->bufs_pos == outq->bufs_allocated) - outq->bufs_pos = 0; + buf->unpadded_size = 0; + buf->uncompressed_size = 0; - ++outq->bufs_used; + ++outq->bufs_in_use; + outq->mem_in_use += lzma_outq_outbuf_memusage(buf->allocated); return buf; } @@ -135,50 +219,68 @@ extern bool lzma_outq_is_readable(const lzma_outq *outq) { - uint32_t i = outq->bufs_pos - outq->bufs_used; - if (outq->bufs_pos < outq->bufs_used) - i += outq->bufs_allocated; + if (outq->head == NULL) + return false; - return outq->bufs[i].finished; + return outq->read_pos < outq->head->pos || outq->head->finished; } extern lzma_ret -lzma_outq_read(lzma_outq *restrict outq, uint8_t *restrict out, - size_t *restrict out_pos, size_t out_size, +lzma_outq_read(lzma_outq *restrict outq, + const lzma_allocator *restrict allocator, + uint8_t *restrict out, size_t *restrict out_pos, + size_t out_size, lzma_vli *restrict unpadded_size, lzma_vli *restrict uncompressed_size) { // There must be at least one buffer from which to read. - if (outq->bufs_used == 0) + if (outq->bufs_in_use == 0) return LZMA_OK; // Get the buffer. - uint32_t i = outq->bufs_pos - outq->bufs_used; - if (outq->bufs_pos < outq->bufs_used) - i += outq->bufs_allocated; - - lzma_outbuf *buf = &outq->bufs[i]; - - // If it isn't finished yet, we cannot read from it. - if (!buf->finished) - return LZMA_OK; + lzma_outbuf *buf = outq->head; // Copy from the buffer to output. - lzma_bufcpy(buf->buf, &outq->read_pos, buf->size, + // + // FIXME? In threaded decoder it may be bad to do this copy while + // the mutex is being held. + lzma_bufcpy(buf->buf, &outq->read_pos, buf->pos, out, out_pos, out_size); // Return if we didn't get all the data from the buffer. - if (outq->read_pos < buf->size) + if (!buf->finished || outq->read_pos < buf->pos) return LZMA_OK; // The buffer was finished. Tell the caller its size information. - *unpadded_size = buf->unpadded_size; - *uncompressed_size = buf->uncompressed_size; + if (unpadded_size != NULL) + *unpadded_size = buf->unpadded_size; + + if (uncompressed_size != NULL) + *uncompressed_size = buf->uncompressed_size; + + // Remember the return value. + const lzma_ret finish_ret = buf->finish_ret; // Free this buffer for further use. - --outq->bufs_used; + move_head_to_cache(outq, allocator); outq->read_pos = 0; - return LZMA_STREAM_END; + return finish_ret; +} + + +extern void +lzma_outq_enable_partial_output(lzma_outq *outq, + void (*enable_partial_output)(void *worker)) +{ + if (outq->head != NULL && !outq->head->finished + && outq->head->worker != NULL) { + enable_partial_output(outq->head->worker); + + // Set it to NULL since calling it twice is pointless. + outq->head->worker = NULL; + } + + return; }
diff --git a/Utilities/cmliblzma/liblzma/common/outqueue.h b/Utilities/cmliblzma/liblzma/common/outqueue.h index 079634d..25f0719 100644 --- a/Utilities/cmliblzma/liblzma/common/outqueue.h +++ b/Utilities/cmliblzma/liblzma/common/outqueue.h
@@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file outqueue.h @@ -5,25 +7,45 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// +#ifndef LZMA_OUTQUEUE_H +#define LZMA_OUTQUEUE_H + #include "common.h" /// Output buffer for a single thread -typedef struct { - /// Pointer to the output buffer of lzma_outq.buf_size_max bytes - uint8_t *buf; +typedef struct lzma_outbuf_s lzma_outbuf; +struct lzma_outbuf_s { + /// Pointer to the next buffer. This is used for the cached buffers. + /// The worker thread must not modify this. + lzma_outbuf *next; - /// Amount of data written to buf - size_t size; + /// This initialized by lzma_outq_get_buf() and + /// is used by lzma_outq_enable_partial_output(). + /// The worker thread must not modify this. + void *worker; - /// Additional size information - lzma_vli unpadded_size; - lzma_vli uncompressed_size; + /// Amount of memory allocated for buf[]. + /// The worker thread must not modify this. + size_t allocated; + + /// Writing position in the worker thread or, in other words, the + /// amount of finished data written to buf[] which can be copied out + /// + /// \note This is read by another thread and thus access + /// to this variable needs a mutex. + size_t pos; + + /// Decompression: Position in the input buffer in the worker thread + /// that matches the output "pos" above. This is used to detect if + /// more output might be possible from the worker thread: if it has + /// consumed all its input, then more output isn't possible. + /// + /// \note This is read by another thread and thus access + /// to this variable needs a mutex. + size_t decoder_in_pos; /// True when no more data will be written into this buffer. /// @@ -31,32 +53,55 @@ /// to this variable needs a mutex. bool finished; -} lzma_outbuf; + /// Return value for lzma_outq_read() when the last byte from + /// a finished buffer has been read. Defaults to LZMA_STREAM_END. + /// This must *not* be LZMA_OK. The idea is to allow a decoder to + /// pass an error code to the main thread, setting the code here + /// together with finished = true. + lzma_ret finish_ret; + + /// Additional size information. lzma_outq_read() may read these + /// when "finished" is true. + lzma_vli unpadded_size; + lzma_vli uncompressed_size; + + /// Buffer of "allocated" bytes + uint8_t buf[]; +}; typedef struct { - /// Array of buffers that are used cyclically. - lzma_outbuf *bufs; + /// Linked list of buffers in use. The next output byte will be + /// read from the head and buffers for the next thread will be + /// appended to the tail. tail->next is always NULL. + lzma_outbuf *head; + lzma_outbuf *tail; - /// Memory allocated for all the buffers - uint8_t *bufs_mem; - - /// Amount of buffer space available in each buffer - size_t buf_size_max; - - /// Number of buffers allocated - uint32_t bufs_allocated; - - /// Position in the bufs array. The next buffer to be taken - /// into use is bufs[bufs_pos]. - uint32_t bufs_pos; - - /// Number of buffers in use - uint32_t bufs_used; - - /// Position in the buffer in lzma_outq_read() + /// Number of bytes read from head->buf[] in lzma_outq_read() size_t read_pos; + /// Linked list of allocated buffers that aren't currently used. + /// This way buffers of similar size can be reused and don't + /// need to be reallocated every time. For simplicity, all + /// cached buffers in the list have the same allocated size. + lzma_outbuf *cache; + + /// Total amount of memory allocated for buffers + uint64_t mem_allocated; + + /// Amount of memory used by the buffers that are in use in + /// the head...tail linked list. + uint64_t mem_in_use; + + /// Number of buffers in use in the head...tail list. If and only if + /// this is zero, the pointers head and tail above are NULL. + uint32_t bufs_in_use; + + /// Number of buffers allocated (in use + cached) + uint32_t bufs_allocated; + + /// Maximum allowed number of allocated buffers + uint32_t bufs_limit; } lzma_outq; @@ -76,32 +121,60 @@ /// function knows that there are no previous /// allocations to free. /// \param allocator Pointer to allocator or NULL -/// \param buf_size_max Maximum amount of data that a single buffer -/// in the queue may need to store. /// \param threads Number of buffers that may be in use /// concurrently. Note that more than this number -/// of buffers will actually get allocated to +/// of buffers may actually get allocated to /// improve performance when buffers finish -/// out of order. +/// out of order. The actual maximum number of +/// allocated buffers is derived from the number +/// of threads. /// /// \return - LZMA_OK /// - LZMA_MEM_ERROR /// -extern lzma_ret lzma_outq_init( - lzma_outq *outq, const lzma_allocator *allocator, - uint64_t buf_size_max, uint32_t threads); +extern lzma_ret lzma_outq_init(lzma_outq *outq, + const lzma_allocator *allocator, uint32_t threads); /// \brief Free the memory associated with the output queue extern void lzma_outq_end(lzma_outq *outq, const lzma_allocator *allocator); +/// \brief Free all cached buffers that consume memory but aren't in use +extern void lzma_outq_clear_cache( + lzma_outq *outq, const lzma_allocator *allocator); + + +/// \brief Like lzma_outq_clear_cache() but might keep one buffer +/// +/// One buffer is not freed if its size is equal to keep_size. +/// This is useful if the caller knows that it will soon need a buffer of +/// keep_size bytes. This way it won't be freed and immediately reallocated. +extern void lzma_outq_clear_cache2( + lzma_outq *outq, const lzma_allocator *allocator, + size_t keep_size); + + +/// \brief Preallocate a new buffer into cache +/// +/// Splitting the buffer allocation into a separate function makes it +/// possible to ensure that way lzma_outq_get_buf() cannot fail. +/// If the preallocated buffer isn't actually used (for example, some +/// other error occurs), the caller has to do nothing as the buffer will +/// be used later or cleared from the cache when not needed. +/// +/// \return LZMA_OK on success, LZMA_MEM_ERROR if allocation fails +/// +extern lzma_ret lzma_outq_prealloc_buf( + lzma_outq *outq, const lzma_allocator *allocator, size_t size); + + /// \brief Get a new buffer /// -/// lzma_outq_has_buf() must be used to check that there is a buffer +/// lzma_outq_prealloc_buf() must be used to ensure that there is a buffer /// available before calling lzma_outq_get_buf(). /// -extern lzma_outbuf *lzma_outq_get_buf(lzma_outq *outq); +extern lzma_outbuf *lzma_outq_get_buf(lzma_outq *outq, void *worker); /// \brief Test if there is data ready to be read @@ -126,17 +199,32 @@ /// \return - LZMA: All OK. Either no data was available or the buffer /// being read didn't become empty yet. /// - LZMA_STREAM_END: The buffer being read was finished. -/// *unpadded_size and *uncompressed_size were set. +/// *unpadded_size and *uncompressed_size were set if they +/// were not NULL. /// -/// \note This reads lzma_outbuf.finished variables and thus call -/// to this function needs to be protected with a mutex. +/// \note This reads lzma_outbuf.finished and .pos variables and thus +/// calls to this function need to be protected with a mutex. /// extern lzma_ret lzma_outq_read(lzma_outq *restrict outq, + const lzma_allocator *restrict allocator, uint8_t *restrict out, size_t *restrict out_pos, size_t out_size, lzma_vli *restrict unpadded_size, lzma_vli *restrict uncompressed_size); +/// \brief Enable partial output from a worker thread +/// +/// If the buffer at the head of the output queue isn't finished, +/// this will call enable_partial_output on the worker associated with +/// that output buffer. +/// +/// \note This reads a lzma_outbuf.finished variable and thus +/// calls to this function need to be protected with a mutex. +/// +extern void lzma_outq_enable_partial_output(lzma_outq *outq, + void (*enable_partial_output)(void *worker)); + + /// \brief Test if there is at least one buffer free /// /// This must be used before getting a new buffer with lzma_outq_get_buf(). @@ -144,7 +232,7 @@ static inline bool lzma_outq_has_buf(const lzma_outq *outq) { - return outq->bufs_used < outq->bufs_allocated; + return outq->bufs_in_use < outq->bufs_limit; } @@ -152,5 +240,19 @@ static inline bool lzma_outq_is_empty(const lzma_outq *outq) { - return outq->bufs_used == 0; + return outq->bufs_in_use == 0; } + + +/// \brief Get the amount of memory needed for a single lzma_outbuf +/// +/// \note Caller must check that the argument is significantly less +/// than SIZE_MAX to avoid an integer overflow! +static inline uint64_t +lzma_outq_outbuf_memusage(size_t buf_size) +{ + assert(buf_size <= SIZE_MAX - sizeof(lzma_outbuf)); + return sizeof(lzma_outbuf) + buf_size; +} + +#endif
diff --git a/Utilities/cmliblzma/liblzma/common/stream_buffer_decoder.c b/Utilities/cmliblzma/liblzma/common/stream_buffer_decoder.c index b9745b5..c4f91fb 100644 --- a/Utilities/cmliblzma/liblzma/common/stream_buffer_decoder.c +++ b/Utilities/cmliblzma/liblzma/common/stream_buffer_decoder.c
@@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file stream_buffer_decoder.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "stream_decoder.h"
diff --git a/Utilities/cmliblzma/liblzma/common/stream_buffer_encoder.c b/Utilities/cmliblzma/liblzma/common/stream_buffer_encoder.c index af49554..04d5869 100644 --- a/Utilities/cmliblzma/liblzma/common/stream_buffer_encoder.c +++ b/Utilities/cmliblzma/liblzma/common/stream_buffer_encoder.c
@@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file stream_buffer_encoder.c @@ -5,11 +7,9 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// +#include "common.h" #include "index.h"
diff --git a/Utilities/cmliblzma/liblzma/common/stream_decoder.c b/Utilities/cmliblzma/liblzma/common/stream_decoder.c index fdd8ff2..7f42684 100644 --- a/Utilities/cmliblzma/liblzma/common/stream_decoder.c +++ b/Utilities/cmliblzma/liblzma/common/stream_decoder.c
@@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file stream_decoder.c @@ -5,28 +7,25 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "stream_decoder.h" #include "block_decoder.h" +#include "index.h" typedef struct { enum { SEQ_STREAM_HEADER, SEQ_BLOCK_HEADER, - SEQ_BLOCK, + SEQ_BLOCK_INIT, + SEQ_BLOCK_RUN, SEQ_INDEX, SEQ_STREAM_FOOTER, SEQ_STREAM_PADDING, } sequence; - /// Block or Metadata decoder. This takes little memory and the same - /// data structure can be used to decode every Block Header, so it's - /// a good idea to have a separate lzma_next_coder structure for it. + /// Block decoder lzma_next_coder block_decoder; /// Block options decoded by the Block Header decoder and used by @@ -63,9 +62,9 @@ /// If true, we will decode concatenated Streams that possibly have /// Stream Padding between or after them. LZMA_STREAM_END is returned - /// once the application isn't giving us any new input, and we aren't - /// in the middle of a Stream, and possible Stream Padding is a - /// multiple of four bytes. + /// once the application isn't giving us any new input (LZMA_FINISH), + /// and we aren't in the middle of a Stream, and possible + /// Stream Padding is a multiple of four bytes. bool concatenated; /// When decoding concatenated Streams, this is true as long as we @@ -165,7 +164,7 @@ if (coder->pos == 0) { // Detect if it's Index. - if (in[*in_pos] == 0x00) { + if (in[*in_pos] == INDEX_INDICATOR) { coder->sequence = SEQ_INDEX; break; } @@ -187,6 +186,15 @@ return LZMA_OK; coder->pos = 0; + coder->sequence = SEQ_BLOCK_INIT; + } + + // Fall through + + case SEQ_BLOCK_INIT: { + // Checking memusage and doing the initialization needs + // its own sequence point because we need to be able to + // retry if we return LZMA_MEMLIMIT_ERROR. // Version 1 is needed to support the .ignore_check option. coder->block_options.version = 1; @@ -235,22 +243,20 @@ // Free the allocated filter options since they are needed // only to initialize the Block decoder. - for (size_t i = 0; i < LZMA_FILTERS_MAX; ++i) - lzma_free(filters[i].options, allocator); - + lzma_filters_free(filters, allocator); coder->block_options.filters = NULL; - // Check if memory usage calculation and Block enocoder + // Check if memory usage calculation and Block decoder // initialization succeeded. if (ret != LZMA_OK) return ret; - coder->sequence = SEQ_BLOCK; + coder->sequence = SEQ_BLOCK_RUN; } // Fall through - case SEQ_BLOCK: { + case SEQ_BLOCK_RUN: { const lzma_ret ret = coder->block_decoder.code( coder->block_decoder.coder, allocator, in, in_pos, in_size, out, out_pos, out_size,
diff --git a/Utilities/cmliblzma/liblzma/common/stream_decoder.h b/Utilities/cmliblzma/liblzma/common/stream_decoder.h index c13c6ba..5803715 100644 --- a/Utilities/cmliblzma/liblzma/common/stream_decoder.h +++ b/Utilities/cmliblzma/liblzma/common/stream_decoder.h
@@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file stream_decoder.h @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef LZMA_STREAM_DECODER_H
diff --git a/Utilities/cmliblzma/liblzma/common/stream_decoder_mt.c b/Utilities/cmliblzma/liblzma/common/stream_decoder_mt.c new file mode 100644 index 0000000..244624a --- /dev/null +++ b/Utilities/cmliblzma/liblzma/common/stream_decoder_mt.c
@@ -0,0 +1,2017 @@ +// SPDX-License-Identifier: 0BSD + +/////////////////////////////////////////////////////////////////////////////// +// +/// \file stream_decoder_mt.c +/// \brief Multithreaded .xz Stream decoder +// +// Authors: Sebastian Andrzej Siewior +// Lasse Collin +// +/////////////////////////////////////////////////////////////////////////////// + +#include "common.h" +#include "block_decoder.h" +#include "stream_decoder.h" +#include "index.h" +#include "outqueue.h" + + +typedef enum { + /// Waiting for work. + /// Main thread may change this to THR_RUN or THR_EXIT. + THR_IDLE, + + /// Decoding is in progress. + /// Main thread may change this to THR_STOP or THR_EXIT. + /// The worker thread may change this to THR_IDLE. + THR_RUN, + + /// The main thread wants the thread to stop whatever it was doing + /// but not exit. Main thread may change this to THR_EXIT. + /// The worker thread may change this to THR_IDLE. + THR_STOP, + + /// The main thread wants the thread to exit. + THR_EXIT, + +} worker_state; + + +typedef enum { + /// Partial updates (storing of worker thread progress + /// to lzma_outbuf) are disabled. + PARTIAL_DISABLED, + + /// Main thread requests partial updates to be enabled but + /// no partial update has been done by the worker thread yet. + /// + /// Changing from PARTIAL_DISABLED to PARTIAL_START requires + /// use of the worker-thread mutex. Other transitions don't + /// need a mutex. + PARTIAL_START, + + /// Partial updates are enabled and the worker thread has done + /// at least one partial update. + PARTIAL_ENABLED, + +} partial_update_mode; + + +struct worker_thread { + /// Worker state is protected with our mutex. + worker_state state; + + /// Input buffer that will contain the whole Block except Block Header. + uint8_t *in; + + /// Amount of memory allocated for "in" + size_t in_size; + + /// Number of bytes written to "in" by the main thread + size_t in_filled; + + /// Number of bytes consumed from "in" by the worker thread. + size_t in_pos; + + /// Amount of uncompressed data that has been decoded. This local + /// copy is needed because updating outbuf->pos requires locking + /// the main mutex (coder->mutex). + size_t out_pos; + + /// Pointer to the main structure is needed to (1) lock the main + /// mutex (coder->mutex) when updating outbuf->pos and (2) when + /// putting this thread back to the stack of free threads. + struct lzma_stream_coder *coder; + + /// The allocator is set by the main thread. Since a copy of the + /// pointer is kept here, the application must not change the + /// allocator before calling lzma_end(). + const lzma_allocator *allocator; + + /// Output queue buffer to which the uncompressed data is written. + lzma_outbuf *outbuf; + + /// Amount of compressed data that has already been decompressed. + /// This is updated from in_pos when our mutex is locked. + /// This is size_t, not uint64_t, because per-thread progress + /// is limited to sizes of allocated buffers. + size_t progress_in; + + /// Like progress_in but for uncompressed data. + size_t progress_out; + + /// Updating outbuf->pos requires locking the main mutex + /// (coder->mutex). Since the main thread will only read output + /// from the oldest outbuf in the queue, only the worker thread + /// that is associated with the oldest outbuf needs to update its + /// outbuf->pos. This avoids useless mutex contention that would + /// happen if all worker threads were frequently locking the main + /// mutex to update their outbuf->pos. + /// + /// Only when partial_update is something else than PARTIAL_DISABLED, + /// this worker thread will update outbuf->pos after each call to + /// the Block decoder. + partial_update_mode partial_update; + + /// Block decoder + lzma_next_coder block_decoder; + + /// Thread-specific Block options are needed because the Block + /// decoder modifies the struct given to it at initialization. + lzma_block block_options; + + /// Filter chain memory usage + uint64_t mem_filters; + + /// Next structure in the stack of free worker threads. + struct worker_thread *next; + + mythread_mutex mutex; + mythread_cond cond; + + /// The ID of this thread is used to join the thread + /// when it's not needed anymore. + mythread thread_id; +}; + + +struct lzma_stream_coder { + enum { + SEQ_STREAM_HEADER, + SEQ_BLOCK_HEADER, + SEQ_BLOCK_INIT, + SEQ_BLOCK_THR_INIT, + SEQ_BLOCK_THR_RUN, + SEQ_BLOCK_DIRECT_INIT, + SEQ_BLOCK_DIRECT_RUN, + SEQ_INDEX_WAIT_OUTPUT, + SEQ_INDEX_DECODE, + SEQ_STREAM_FOOTER, + SEQ_STREAM_PADDING, + SEQ_ERROR, + } sequence; + + /// Block decoder + lzma_next_coder block_decoder; + + /// Every Block Header will be decoded into this structure. + /// This is also used to initialize a Block decoder when in + /// direct mode. In threaded mode, a thread-specific copy will + /// be made for decoder initialization because the Block decoder + /// will modify the structure given to it. + lzma_block block_options; + + /// Buffer to hold a filter chain for Block Header decoding and + /// initialization. These are freed after successful Block decoder + /// initialization or at stream_decoder_mt_end(). The thread-specific + /// copy of block_options won't hold a pointer to filters[] after + /// initialization. + lzma_filter filters[LZMA_FILTERS_MAX + 1]; + + /// Stream Flags from Stream Header + lzma_stream_flags stream_flags; + + /// Index is hashed so that it can be compared to the sizes of Blocks + /// with O(1) memory usage. + lzma_index_hash *index_hash; + + + /// Maximum wait time if cannot use all the input and cannot + /// fill the output buffer. This is in milliseconds. + uint32_t timeout; + + + /// Error code from a worker thread. + /// + /// \note Use mutex. + lzma_ret thread_error; + + /// Error code to return after pending output has been copied out. If + /// set in read_output_and_wait(), this is a mirror of thread_error. + /// If set in stream_decode_mt() then it's, for example, error that + /// occurred when decoding Block Header. + lzma_ret pending_error; + + /// Number of threads that will be created at maximum. + uint32_t threads_max; + + /// Number of thread structures that have been initialized from + /// "threads", and thus the number of worker threads actually + /// created so far. + uint32_t threads_initialized; + + /// Array of allocated thread-specific structures. When no threads + /// are in use (direct mode) this is NULL. In threaded mode this + /// points to an array of threads_max number of worker_thread structs. + struct worker_thread *threads; + + /// Stack of free threads. When a thread finishes, it puts itself + /// back into this stack. This starts as empty because threads + /// are created only when actually needed. + /// + /// \note Use mutex. + struct worker_thread *threads_free; + + /// The most recent worker thread to which the main thread writes + /// the new input from the application. + struct worker_thread *thr; + + /// Output buffer queue for decompressed data from the worker threads + /// + /// \note Use mutex with operations that need it. + lzma_outq outq; + + mythread_mutex mutex; + mythread_cond cond; + + + /// Memory usage that will not be exceeded in multi-threaded mode. + /// Single-threaded mode can exceed this even by a large amount. + uint64_t memlimit_threading; + + /// Memory usage limit that should never be exceeded. + /// LZMA_MEMLIMIT_ERROR will be returned if decoding isn't possible + /// even in single-threaded mode without exceeding this limit. + uint64_t memlimit_stop; + + /// Amount of memory in use by the direct mode decoder + /// (coder->block_decoder). In threaded mode this is 0. + uint64_t mem_direct_mode; + + /// Amount of memory needed by the running worker threads. + /// This doesn't include the memory needed by the output buffer. + /// + /// \note Use mutex. + uint64_t mem_in_use; + + /// Amount of memory used by the idle (cached) threads. + /// + /// \note Use mutex. + uint64_t mem_cached; + + + /// Amount of memory needed for the filter chain of the next Block. + uint64_t mem_next_filters; + + /// Amount of memory needed for the thread-specific input buffer + /// for the next Block. + uint64_t mem_next_in; + + /// Amount of memory actually needed to decode the next Block + /// in threaded mode. This is + /// mem_next_filters + mem_next_in + memory needed for lzma_outbuf. + uint64_t mem_next_block; + + + /// Amount of compressed data in Stream Header + Blocks that have + /// already been finished. + /// + /// \note Use mutex. + uint64_t progress_in; + + /// Amount of uncompressed data in Blocks that have already + /// been finished. + /// + /// \note Use mutex. + uint64_t progress_out; + + + /// If true, LZMA_NO_CHECK is returned if the Stream has + /// no integrity check. + bool tell_no_check; + + /// If true, LZMA_UNSUPPORTED_CHECK is returned if the Stream has + /// an integrity check that isn't supported by this liblzma build. + bool tell_unsupported_check; + + /// If true, LZMA_GET_CHECK is returned after decoding Stream Header. + bool tell_any_check; + + /// If true, we will tell the Block decoder to skip calculating + /// and verifying the integrity check. + bool ignore_check; + + /// If true, we will decode concatenated Streams that possibly have + /// Stream Padding between or after them. LZMA_STREAM_END is returned + /// once the application isn't giving us any new input (LZMA_FINISH), + /// and we aren't in the middle of a Stream, and possible + /// Stream Padding is a multiple of four bytes. + bool concatenated; + + /// If true, we will return any errors immediately instead of first + /// producing all output before the location of the error. + bool fail_fast; + + + /// When decoding concatenated Streams, this is true as long as we + /// are decoding the first Stream. This is needed to avoid misleading + /// LZMA_FORMAT_ERROR in case the later Streams don't have valid magic + /// bytes. + bool first_stream; + + /// This is used to track if the previous call to stream_decode_mt() + /// had output space (*out_pos < out_size) and managed to fill the + /// output buffer (*out_pos == out_size). This may be set to true + /// in read_output_and_wait(). This is read and then reset to false + /// at the beginning of stream_decode_mt(). + /// + /// This is needed to support applications that call lzma_code() in + /// such a way that more input is provided only when lzma_code() + /// didn't fill the output buffer completely. Basically, this makes + /// it easier to convert such applications from single-threaded + /// decoder to multi-threaded decoder. + bool out_was_filled; + + /// Write position in buffer[] and position in Stream Padding + size_t pos; + + /// Buffer to hold Stream Header, Block Header, and Stream Footer. + /// Block Header has biggest maximum size. + uint8_t buffer[LZMA_BLOCK_HEADER_SIZE_MAX]; +}; + + +/// Enables updating of outbuf->pos. This is a callback function that is +/// used with lzma_outq_enable_partial_output(). +static void +worker_enable_partial_update(void *thr_ptr) +{ + struct worker_thread *thr = thr_ptr; + + mythread_sync(thr->mutex) { + thr->partial_update = PARTIAL_START; + mythread_cond_signal(&thr->cond); + } +} + + +/// Things do to at THR_STOP or when finishing a Block. +/// This is called with thr->mutex locked. +static void +worker_stop(struct worker_thread *thr) +{ + // Update memory usage counters. + thr->coder->mem_in_use -= thr->in_size; + thr->in_size = 0; // thr->in was freed above. + + thr->coder->mem_in_use -= thr->mem_filters; + thr->coder->mem_cached += thr->mem_filters; + + // Put this thread to the stack of free threads. + thr->next = thr->coder->threads_free; + thr->coder->threads_free = thr; + + mythread_cond_signal(&thr->coder->cond); + return; +} + + +static MYTHREAD_RET_TYPE +worker_decoder(void *thr_ptr) +{ + struct worker_thread *thr = thr_ptr; + size_t in_filled; + partial_update_mode partial_update; + lzma_ret ret; + +next_loop_lock: + + mythread_mutex_lock(&thr->mutex); +next_loop_unlocked: + + if (thr->state == THR_IDLE) { + mythread_cond_wait(&thr->cond, &thr->mutex); + goto next_loop_unlocked; + } + + if (thr->state == THR_EXIT) { + mythread_mutex_unlock(&thr->mutex); + + lzma_free(thr->in, thr->allocator); + lzma_next_end(&thr->block_decoder, thr->allocator); + + mythread_mutex_destroy(&thr->mutex); + mythread_cond_destroy(&thr->cond); + + return MYTHREAD_RET_VALUE; + } + + if (thr->state == THR_STOP) { + thr->state = THR_IDLE; + mythread_mutex_unlock(&thr->mutex); + + mythread_sync(thr->coder->mutex) { + worker_stop(thr); + } + + goto next_loop_lock; + } + + assert(thr->state == THR_RUN); + + // Update progress info for get_progress(). + thr->progress_in = thr->in_pos; + thr->progress_out = thr->out_pos; + + // If we don't have any new input, wait for a signal from the main + // thread except if partial output has just been enabled. In that + // case we will do one normal run so that the partial output info + // gets passed to the main thread. The call to block_decoder.code() + // is useless but harmless as it can occur only once per Block. + in_filled = thr->in_filled; + partial_update = thr->partial_update; + + if (in_filled == thr->in_pos && partial_update != PARTIAL_START) { + mythread_cond_wait(&thr->cond, &thr->mutex); + goto next_loop_unlocked; + } + + mythread_mutex_unlock(&thr->mutex); + + // Pass the input in small chunks to the Block decoder. + // This way we react reasonably fast if we are told to stop/exit, + // and (when partial update is enabled) we tell about our progress + // to the main thread frequently enough. + const size_t chunk_size = 16384; + if ((in_filled - thr->in_pos) > chunk_size) + in_filled = thr->in_pos + chunk_size; + + ret = thr->block_decoder.code( + thr->block_decoder.coder, thr->allocator, + thr->in, &thr->in_pos, in_filled, + thr->outbuf->buf, &thr->out_pos, + thr->outbuf->allocated, LZMA_RUN); + + if (ret == LZMA_OK) { + if (partial_update != PARTIAL_DISABLED) { + // The main thread uses thr->mutex to change from + // PARTIAL_DISABLED to PARTIAL_START. The main thread + // doesn't care about this variable after that so we + // can safely change it here to PARTIAL_ENABLED + // without a mutex. + thr->partial_update = PARTIAL_ENABLED; + + // The main thread is reading decompressed data + // from thr->outbuf. Tell the main thread about + // our progress. + // + // NOTE: It's possible that we consumed input without + // producing any new output so it's possible that + // only in_pos has changed. In case of PARTIAL_START + // it is possible that neither in_pos nor out_pos has + // changed. + mythread_sync(thr->coder->mutex) { + thr->outbuf->pos = thr->out_pos; + thr->outbuf->decoder_in_pos = thr->in_pos; + mythread_cond_signal(&thr->coder->cond); + } + } + + goto next_loop_lock; + } + + // Either we finished successfully (LZMA_STREAM_END) or an error + // occurred. Both cases are handled almost identically. The error + // case requires updating thr->coder->thread_error. + // + // The sizes are in the Block Header and the Block decoder + // checks that they match, thus we know these: + assert(ret != LZMA_STREAM_END || thr->in_pos == thr->in_size); + assert(ret != LZMA_STREAM_END + || thr->out_pos == thr->block_options.uncompressed_size); + + // Free the input buffer. Don't update in_size as we need + // it later to update thr->coder->mem_in_use. + lzma_free(thr->in, thr->allocator); + thr->in = NULL; + + mythread_sync(thr->mutex) { + if (thr->state != THR_EXIT) + thr->state = THR_IDLE; + } + + mythread_sync(thr->coder->mutex) { + // Move our progress info to the main thread. + thr->coder->progress_in += thr->in_pos; + thr->coder->progress_out += thr->out_pos; + thr->progress_in = 0; + thr->progress_out = 0; + + // Mark the outbuf as finished. + thr->outbuf->pos = thr->out_pos; + thr->outbuf->decoder_in_pos = thr->in_pos; + thr->outbuf->finished = true; + thr->outbuf->finish_ret = ret; + thr->outbuf = NULL; + + // If an error occurred, tell it to the main thread. + if (ret != LZMA_STREAM_END + && thr->coder->thread_error == LZMA_OK) + thr->coder->thread_error = ret; + + worker_stop(thr); + } + + goto next_loop_lock; +} + + +/// Tells the worker threads to exit and waits for them to terminate. +static void +threads_end(struct lzma_stream_coder *coder, const lzma_allocator *allocator) +{ + for (uint32_t i = 0; i < coder->threads_initialized; ++i) { + mythread_sync(coder->threads[i].mutex) { + coder->threads[i].state = THR_EXIT; + mythread_cond_signal(&coder->threads[i].cond); + } + } + + for (uint32_t i = 0; i < coder->threads_initialized; ++i) + mythread_join(coder->threads[i].thread_id); + + lzma_free(coder->threads, allocator); + coder->threads_initialized = 0; + coder->threads = NULL; + coder->threads_free = NULL; + + // The threads don't update these when they exit. Do it here. + coder->mem_in_use = 0; + coder->mem_cached = 0; + + return; +} + + +static void +threads_stop(struct lzma_stream_coder *coder) +{ + for (uint32_t i = 0; i < coder->threads_initialized; ++i) { + mythread_sync(coder->threads[i].mutex) { + // The state must be changed conditionally because + // THR_IDLE -> THR_STOP is not a valid state change. + if (coder->threads[i].state != THR_IDLE) { + coder->threads[i].state = THR_STOP; + mythread_cond_signal(&coder->threads[i].cond); + } + } + } + + return; +} + + +/// Initialize a new worker_thread structure and create a new thread. +static lzma_ret +initialize_new_thread(struct lzma_stream_coder *coder, + const lzma_allocator *allocator) +{ + // Allocate the coder->threads array if needed. It's done here instead + // of when initializing the decoder because we don't need this if we + // use the direct mode (we may even free coder->threads in the middle + // of the file if we switch from threaded to direct mode). + if (coder->threads == NULL) { + coder->threads = lzma_alloc( + coder->threads_max * sizeof(struct worker_thread), + allocator); + + if (coder->threads == NULL) + return LZMA_MEM_ERROR; + } + + // Pick a free structure. + assert(coder->threads_initialized < coder->threads_max); + struct worker_thread *thr + = &coder->threads[coder->threads_initialized]; + + if (mythread_mutex_init(&thr->mutex)) + goto error_mutex; + + if (mythread_cond_init(&thr->cond)) + goto error_cond; + + thr->state = THR_IDLE; + thr->in = NULL; + thr->in_size = 0; + thr->allocator = allocator; + thr->coder = coder; + thr->outbuf = NULL; + thr->block_decoder = LZMA_NEXT_CODER_INIT; + thr->mem_filters = 0; + + if (mythread_create(&thr->thread_id, worker_decoder, thr)) + goto error_thread; + + ++coder->threads_initialized; + coder->thr = thr; + + return LZMA_OK; + +error_thread: + mythread_cond_destroy(&thr->cond); + +error_cond: + mythread_mutex_destroy(&thr->mutex); + +error_mutex: + return LZMA_MEM_ERROR; +} + + +static lzma_ret +get_thread(struct lzma_stream_coder *coder, const lzma_allocator *allocator) +{ + // If there is a free structure on the stack, use it. + mythread_sync(coder->mutex) { + if (coder->threads_free != NULL) { + coder->thr = coder->threads_free; + coder->threads_free = coder->threads_free->next; + + // The thread is no longer in the cache so subtract + // it from the cached memory usage. Don't add it + // to mem_in_use though; the caller will handle it + // since it knows how much memory it will actually + // use (the filter chain might change). + coder->mem_cached -= coder->thr->mem_filters; + } + } + + if (coder->thr == NULL) { + assert(coder->threads_initialized < coder->threads_max); + + // Initialize a new thread. + return_if_error(initialize_new_thread(coder, allocator)); + } + + coder->thr->in_filled = 0; + coder->thr->in_pos = 0; + coder->thr->out_pos = 0; + + coder->thr->progress_in = 0; + coder->thr->progress_out = 0; + + coder->thr->partial_update = PARTIAL_DISABLED; + + return LZMA_OK; +} + + +static lzma_ret +read_output_and_wait(struct lzma_stream_coder *coder, + const lzma_allocator *allocator, + uint8_t *restrict out, size_t *restrict out_pos, + size_t out_size, + bool *input_is_possible, + bool waiting_allowed, + mythread_condtime *wait_abs, bool *has_blocked) +{ + lzma_ret ret = LZMA_OK; + + mythread_sync(coder->mutex) { + do { + // Get as much output from the queue as is possible + // without blocking. + const size_t out_start = *out_pos; + do { + ret = lzma_outq_read(&coder->outq, allocator, + out, out_pos, out_size, + NULL, NULL); + + // If a Block was finished, tell the worker + // thread of the next Block (if it is still + // running) to start telling the main thread + // when new output is available. + if (ret == LZMA_STREAM_END) + lzma_outq_enable_partial_output( + &coder->outq, + &worker_enable_partial_update); + + // Loop until a Block wasn't finished. + // It's important to loop around even if + // *out_pos == out_size because there could + // be an empty Block that will return + // LZMA_STREAM_END without needing any + // output space. + } while (ret == LZMA_STREAM_END); + + // Check if lzma_outq_read reported an error from + // the Block decoder. + if (ret != LZMA_OK) + break; + + // If the output buffer is now full but it wasn't full + // when this function was called, set out_was_filled. + // This way the next call to stream_decode_mt() knows + // that some output was produced and no output space + // remained in the previous call to stream_decode_mt(). + if (*out_pos == out_size && *out_pos != out_start) + coder->out_was_filled = true; + + // Check if any thread has indicated an error. + if (coder->thread_error != LZMA_OK) { + // If LZMA_FAIL_FAST was used, report errors + // from worker threads immediately. + if (coder->fail_fast) { + ret = coder->thread_error; + break; + } + + // Otherwise set pending_error. The value we + // set here will not actually get used other + // than working as a flag that an error has + // occurred. This is because in SEQ_ERROR + // all output before the error will be read + // first by calling this function, and once we + // reach the location of the (first) error the + // error code from the above lzma_outq_read() + // will be returned to the application. + // + // Use LZMA_PROG_ERROR since the value should + // never leak to the application. It's + // possible that pending_error has already + // been set but that doesn't matter: if we get + // here, pending_error only works as a flag. + coder->pending_error = LZMA_PROG_ERROR; + } + + // Check if decoding of the next Block can be started. + // The memusage of the active threads must be low + // enough, there must be a free buffer slot in the + // output queue, and there must be a free thread + // (that can be either created or an existing one + // reused). + // + // NOTE: This is checked after reading the output + // above because reading the output can free a slot in + // the output queue and also reduce active memusage. + // + // NOTE: If output queue is empty, then input will + // always be possible. + if (input_is_possible != NULL + && coder->memlimit_threading + - coder->mem_in_use + - coder->outq.mem_in_use + >= coder->mem_next_block + && lzma_outq_has_buf(&coder->outq) + && (coder->threads_initialized + < coder->threads_max + || coder->threads_free + != NULL)) { + *input_is_possible = true; + break; + } + + // If the caller doesn't want us to block, return now. + if (!waiting_allowed) + break; + + // This check is needed only when input_is_possible + // is NULL. We must return if we aren't waiting for + // input to become possible and there is no more + // output coming from the queue. + if (lzma_outq_is_empty(&coder->outq)) { + assert(input_is_possible == NULL); + break; + } + + // If there is more data available from the queue, + // our out buffer must be full and we need to return + // so that the application can provide more output + // space. + // + // NOTE: In general lzma_outq_is_readable() can return + // true also when there are no more bytes available. + // This can happen when a Block has finished without + // providing any new output. We know that this is not + // the case because in the beginning of this loop we + // tried to read as much as possible even when we had + // no output space left and the mutex has been locked + // all the time (so worker threads cannot have changed + // anything). Thus there must be actual pending output + // in the queue. + if (lzma_outq_is_readable(&coder->outq)) { + assert(*out_pos == out_size); + break; + } + + // If the application stops providing more input + // in the middle of a Block, there will eventually + // be one worker thread left that is stuck waiting for + // more input (that might never arrive) and a matching + // outbuf which the worker thread cannot finish due + // to lack of input. We must detect this situation, + // otherwise we would end up waiting indefinitely + // (if no timeout is in use) or keep returning + // LZMA_TIMED_OUT while making no progress. Thus, the + // application would never get LZMA_BUF_ERROR from + // lzma_code() which would tell the application that + // no more progress is possible. No LZMA_BUF_ERROR + // means that, for example, truncated .xz files could + // cause an infinite loop. + // + // A worker thread doing partial updates will + // store not only the output position in outbuf->pos + // but also the matching input position in + // outbuf->decoder_in_pos. Here we check if that + // input position matches the amount of input that + // the worker thread has been given (in_filled). + // If so, we must return and not wait as no more + // output will be coming without first getting more + // input to the worker thread. If the application + // keeps calling lzma_code() without providing more + // input, it will eventually get LZMA_BUF_ERROR. + // + // NOTE: We can read partial_update and in_filled + // without thr->mutex as only the main thread + // modifies these variables. decoder_in_pos requires + // coder->mutex which we are already holding. + if (coder->thr != NULL && coder->thr->partial_update + != PARTIAL_DISABLED) { + // There is exactly one outbuf in the queue. + assert(coder->thr->outbuf == coder->outq.head); + assert(coder->thr->outbuf == coder->outq.tail); + + if (coder->thr->outbuf->decoder_in_pos + == coder->thr->in_filled) + break; + } + + // Wait for input or output to become possible. + if (coder->timeout != 0) { + // See the comment in stream_encoder_mt.c + // about why mythread_condtime_set() is used + // like this. + // + // FIXME? + // In contrast to the encoder, this calls + // _condtime_set while the mutex is locked. + if (!*has_blocked) { + *has_blocked = true; + mythread_condtime_set(wait_abs, + &coder->cond, + coder->timeout); + } + + if (mythread_cond_timedwait(&coder->cond, + &coder->mutex, + wait_abs) != 0) { + ret = LZMA_TIMED_OUT; + break; + } + } else { + mythread_cond_wait(&coder->cond, + &coder->mutex); + } + } while (ret == LZMA_OK); + } + + // If we are returning an error, then the application cannot get + // more output from us and thus keeping the threads running is + // useless and waste of CPU time. + if (ret != LZMA_OK && ret != LZMA_TIMED_OUT) + threads_stop(coder); + + return ret; +} + + +static lzma_ret +decode_block_header(struct lzma_stream_coder *coder, + const lzma_allocator *allocator, const uint8_t *restrict in, + size_t *restrict in_pos, size_t in_size) +{ + if (*in_pos >= in_size) + return LZMA_OK; + + if (coder->pos == 0) { + // Detect if it's Index. + if (in[*in_pos] == INDEX_INDICATOR) + return LZMA_INDEX_DETECTED; + + // Calculate the size of the Block Header. Note that + // Block Header decoder wants to see this byte too + // so don't advance *in_pos. + coder->block_options.header_size + = lzma_block_header_size_decode( + in[*in_pos]); + } + + // Copy the Block Header to the internal buffer. + lzma_bufcpy(in, in_pos, in_size, coder->buffer, &coder->pos, + coder->block_options.header_size); + + // Return if we didn't get the whole Block Header yet. + if (coder->pos < coder->block_options.header_size) + return LZMA_OK; + + coder->pos = 0; + + // Version 1 is needed to support the .ignore_check option. + coder->block_options.version = 1; + + // Block Header decoder will initialize all members of this array + // so we don't need to do it here. + coder->block_options.filters = coder->filters; + + // Decode the Block Header. + return_if_error(lzma_block_header_decode(&coder->block_options, + allocator, coder->buffer)); + + // If LZMA_IGNORE_CHECK was used, this flag needs to be set. + // It has to be set after lzma_block_header_decode() because + // it always resets this to false. + coder->block_options.ignore_check = coder->ignore_check; + + // coder->block_options is ready now. + return LZMA_STREAM_END; +} + + +/// Get the size of the Compressed Data + Block Padding + Check. +static size_t +comp_blk_size(const struct lzma_stream_coder *coder) +{ + return vli_ceil4(coder->block_options.compressed_size) + + lzma_check_size(coder->stream_flags.check); +} + + +/// Returns true if the size (compressed or uncompressed) is such that +/// threaded decompression cannot be used. Sizes that are too big compared +/// to SIZE_MAX must be rejected to avoid integer overflows and truncations +/// when lzma_vli is assigned to a size_t. +static bool +is_direct_mode_needed(lzma_vli size) +{ + return size == LZMA_VLI_UNKNOWN || size > SIZE_MAX / 3; +} + + +static lzma_ret +stream_decoder_reset(struct lzma_stream_coder *coder, + const lzma_allocator *allocator) +{ + // Initialize the Index hash used to verify the Index. + coder->index_hash = lzma_index_hash_init(coder->index_hash, allocator); + if (coder->index_hash == NULL) + return LZMA_MEM_ERROR; + + // Reset the rest of the variables. + coder->sequence = SEQ_STREAM_HEADER; + coder->pos = 0; + + return LZMA_OK; +} + + +static lzma_ret +stream_decode_mt(void *coder_ptr, const lzma_allocator *allocator, + const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, + uint8_t *restrict out, size_t *restrict out_pos, + size_t out_size, lzma_action action) +{ + struct lzma_stream_coder *coder = coder_ptr; + + mythread_condtime wait_abs; + bool has_blocked = false; + + // Determine if in SEQ_BLOCK_HEADER and SEQ_BLOCK_THR_RUN we should + // tell read_output_and_wait() to wait until it can fill the output + // buffer (or a timeout occurs). Two conditions must be met: + // + // (1) If the caller provided no new input. The reason for this + // can be, for example, the end of the file or that there is + // a pause in the input stream and more input is available + // a little later. In this situation we should wait for output + // because otherwise we would end up in a busy-waiting loop where + // we make no progress and the application just calls us again + // without providing any new input. This would then result in + // LZMA_BUF_ERROR even though more output would be available + // once the worker threads decode more data. + // + // (2) Even if (1) is true, we will not wait if the previous call to + // this function managed to produce some output and the output + // buffer became full. This is for compatibility with applications + // that call lzma_code() in such a way that new input is provided + // only when the output buffer didn't become full. Without this + // trick such applications would have bad performance (bad + // parallelization due to decoder not getting input fast enough). + // + // NOTE: Such loops might require that timeout is disabled (0) + // if they assume that output-not-full implies that all input has + // been consumed. If and only if timeout is enabled, we may return + // when output isn't full *and* not all input has been consumed. + // + // However, if LZMA_FINISH is used, the above is ignored and we always + // wait (timeout can still cause us to return) because we know that + // we won't get any more input. This matters if the input file is + // truncated and we are doing single-shot decoding, that is, + // timeout = 0 and LZMA_FINISH is used on the first call to + // lzma_code() and the output buffer is known to be big enough + // to hold all uncompressed data: + // + // - If LZMA_FINISH wasn't handled specially, we could return + // LZMA_OK before providing all output that is possible with the + // truncated input. The rest would be available if lzma_code() was + // called again but then it's not single-shot decoding anymore. + // + // - By handling LZMA_FINISH specially here, the first call will + // produce all the output, matching the behavior of the + // single-threaded decoder. + // + // So it's a very specific corner case but also easy to avoid. Note + // that this special handling of LZMA_FINISH has no effect for + // single-shot decoding when the input file is valid (not truncated); + // premature LZMA_OK wouldn't be possible as long as timeout = 0. + const bool waiting_allowed = action == LZMA_FINISH + || (*in_pos == in_size && !coder->out_was_filled); + coder->out_was_filled = false; + + while (true) + switch (coder->sequence) { + case SEQ_STREAM_HEADER: { + // Copy the Stream Header to the internal buffer. + const size_t in_old = *in_pos; + lzma_bufcpy(in, in_pos, in_size, coder->buffer, &coder->pos, + LZMA_STREAM_HEADER_SIZE); + coder->progress_in += *in_pos - in_old; + + // Return if we didn't get the whole Stream Header yet. + if (coder->pos < LZMA_STREAM_HEADER_SIZE) + return LZMA_OK; + + coder->pos = 0; + + // Decode the Stream Header. + const lzma_ret ret = lzma_stream_header_decode( + &coder->stream_flags, coder->buffer); + if (ret != LZMA_OK) + return ret == LZMA_FORMAT_ERROR && !coder->first_stream + ? LZMA_DATA_ERROR : ret; + + // If we are decoding concatenated Streams, and the later + // Streams have invalid Header Magic Bytes, we give + // LZMA_DATA_ERROR instead of LZMA_FORMAT_ERROR. + coder->first_stream = false; + + // Copy the type of the Check so that Block Header and Block + // decoders see it. + coder->block_options.check = coder->stream_flags.check; + + // Even if we return LZMA_*_CHECK below, we want + // to continue from Block Header decoding. + coder->sequence = SEQ_BLOCK_HEADER; + + // Detect if there's no integrity check or if it is + // unsupported if those were requested by the application. + if (coder->tell_no_check && coder->stream_flags.check + == LZMA_CHECK_NONE) + return LZMA_NO_CHECK; + + if (coder->tell_unsupported_check + && !lzma_check_is_supported( + coder->stream_flags.check)) + return LZMA_UNSUPPORTED_CHECK; + + if (coder->tell_any_check) + return LZMA_GET_CHECK; + } + + // Fall through + + case SEQ_BLOCK_HEADER: { + const size_t in_old = *in_pos; + const lzma_ret ret = decode_block_header(coder, allocator, + in, in_pos, in_size); + coder->progress_in += *in_pos - in_old; + + if (ret == LZMA_OK) { + // We didn't decode the whole Block Header yet. + // + // Read output from the queue before returning. This + // is important because it is possible that the + // application doesn't have any new input available + // immediately. If we didn't try to copy output from + // the output queue here, lzma_code() could end up + // returning LZMA_BUF_ERROR even though queued output + // is available. + // + // If the lzma_code() call provided at least one input + // byte, only copy as much data from the output queue + // as is available immediately. This way the + // application will be able to provide more input + // without a delay. + // + // On the other hand, if lzma_code() was called with + // an empty input buffer(*), treat it specially: try + // to fill the output buffer even if it requires + // waiting for the worker threads to provide output + // (timeout, if specified, can still cause us to + // return). + // + // - This way the application will be able to get all + // data that can be decoded from the input provided + // so far. + // + // - We avoid both premature LZMA_BUF_ERROR and + // busy-waiting where the application repeatedly + // calls lzma_code() which immediately returns + // LZMA_OK without providing new data. + // + // - If the queue becomes empty, we won't wait + // anything and will return LZMA_OK immediately + // (coder->timeout is completely ignored). + // + // (*) See the comment at the beginning of this + // function how waiting_allowed is determined + // and why there is an exception to the rule + // of "called with an empty input buffer". + assert(*in_pos == in_size); + + // If LZMA_FINISH was used we know that we won't get + // more input, so the file must be truncated if we + // get here. If worker threads don't detect any + // errors, eventually there will be no more output + // while we keep returning LZMA_OK which gets + // converted to LZMA_BUF_ERROR in lzma_code(). + // + // If fail-fast is enabled then we will return + // immediately using LZMA_DATA_ERROR instead of + // LZMA_OK or LZMA_BUF_ERROR. Rationale for the + // error code: + // + // - Worker threads may have a large amount of + // not-yet-decoded input data and we don't + // know for sure if all data is valid. Bad + // data there would result in LZMA_DATA_ERROR + // when fail-fast isn't used. + // + // - Immediate LZMA_BUF_ERROR would be a bit weird + // considering the older liblzma code. lzma_code() + // even has an assertion to prevent coders from + // returning LZMA_BUF_ERROR directly. + // + // The downside of this is that with fail-fast apps + // cannot always distinguish between corrupt and + // truncated files. + if (action == LZMA_FINISH && coder->fail_fast) { + // We won't produce any more output. Stop + // the unfinished worker threads so they + // won't waste CPU time. + threads_stop(coder); + return LZMA_DATA_ERROR; + } + + // read_output_and_wait() will call threads_stop() + // if needed so with that we can use return_if_error. + return_if_error(read_output_and_wait(coder, allocator, + out, out_pos, out_size, + NULL, waiting_allowed, + &wait_abs, &has_blocked)); + + if (coder->pending_error != LZMA_OK) { + coder->sequence = SEQ_ERROR; + break; + } + + return LZMA_OK; + } + + if (ret == LZMA_INDEX_DETECTED) { + coder->sequence = SEQ_INDEX_WAIT_OUTPUT; + break; + } + + // See if an error occurred. + if (ret != LZMA_STREAM_END) { + // NOTE: Here and in all other places where + // pending_error is set, it may overwrite the value + // (LZMA_PROG_ERROR) set by read_output_and_wait(). + // That function might overwrite value set here too. + // These are fine because when read_output_and_wait() + // sets pending_error, it actually works as a flag + // variable only ("some error has occurred") and the + // actual value of pending_error is not used in + // SEQ_ERROR. In such cases SEQ_ERROR will eventually + // get the correct error code from the return value of + // a later read_output_and_wait() call. + coder->pending_error = ret; + coder->sequence = SEQ_ERROR; + break; + } + + // Calculate the memory usage of the filters / Block decoder. + coder->mem_next_filters = lzma_raw_decoder_memusage( + coder->filters); + + if (coder->mem_next_filters == UINT64_MAX) { + // One or more unknown Filter IDs. + coder->pending_error = LZMA_OPTIONS_ERROR; + coder->sequence = SEQ_ERROR; + break; + } + + coder->sequence = SEQ_BLOCK_INIT; + } + + // Fall through + + case SEQ_BLOCK_INIT: { + // Check if decoding is possible at all with the current + // memlimit_stop which we must never exceed. + // + // This needs to be the first thing in SEQ_BLOCK_INIT + // to make it possible to restart decoding after increasing + // memlimit_stop with lzma_memlimit_set(). + if (coder->mem_next_filters > coder->memlimit_stop) { + // Flush pending output before returning + // LZMA_MEMLIMIT_ERROR. If the application doesn't + // want to increase the limit, at least it will get + // all the output possible so far. + return_if_error(read_output_and_wait(coder, allocator, + out, out_pos, out_size, + NULL, true, &wait_abs, &has_blocked)); + + if (!lzma_outq_is_empty(&coder->outq)) + return LZMA_OK; + + return LZMA_MEMLIMIT_ERROR; + } + + // Check if the size information is available in Block Header. + // If it is, check if the sizes are small enough that we don't + // need to worry *too* much about integer overflows later in + // the code. If these conditions are not met, we must use the + // single-threaded direct mode. + if (is_direct_mode_needed(coder->block_options.compressed_size) + || is_direct_mode_needed( + coder->block_options.uncompressed_size)) { + coder->sequence = SEQ_BLOCK_DIRECT_INIT; + break; + } + + // Calculate the amount of memory needed for the input and + // output buffers in threaded mode. + // + // These cannot overflow because we already checked that + // the sizes are small enough using is_direct_mode_needed(). + coder->mem_next_in = comp_blk_size(coder); + const uint64_t mem_buffers = coder->mem_next_in + + lzma_outq_outbuf_memusage( + coder->block_options.uncompressed_size); + + // Add the amount needed by the filters. + // Avoid integer overflows. + if (UINT64_MAX - mem_buffers < coder->mem_next_filters) { + // Use direct mode if the memusage would overflow. + // This is a theoretical case that shouldn't happen + // in practice unless the input file is weird (broken + // or malicious). + coder->sequence = SEQ_BLOCK_DIRECT_INIT; + break; + } + + // Amount of memory needed to decode this Block in + // threaded mode: + coder->mem_next_block = coder->mem_next_filters + mem_buffers; + + // If this alone would exceed memlimit_threading, then we must + // use the single-threaded direct mode. + if (coder->mem_next_block > coder->memlimit_threading) { + coder->sequence = SEQ_BLOCK_DIRECT_INIT; + break; + } + + // Use the threaded mode. Free the direct mode decoder in + // case it has been initialized. + lzma_next_end(&coder->block_decoder, allocator); + coder->mem_direct_mode = 0; + + // Since we already know what the sizes are supposed to be, + // we can already add them to the Index hash. The Block + // decoder will verify the values while decoding. + const lzma_ret ret = lzma_index_hash_append(coder->index_hash, + lzma_block_unpadded_size( + &coder->block_options), + coder->block_options.uncompressed_size); + if (ret != LZMA_OK) { + coder->pending_error = ret; + coder->sequence = SEQ_ERROR; + break; + } + + coder->sequence = SEQ_BLOCK_THR_INIT; + } + + // Fall through + + case SEQ_BLOCK_THR_INIT: { + // We need to wait for a multiple conditions to become true + // until we can initialize the Block decoder and let a worker + // thread decode it: + // + // - Wait for the memory usage of the active threads to drop + // so that starting the decoding of this Block won't make + // us go over memlimit_threading. + // + // - Wait for at least one free output queue slot. + // + // - Wait for a free worker thread. + // + // While we wait, we must copy decompressed data to the out + // buffer and catch possible decoder errors. + // + // read_output_and_wait() does all the above. + bool block_can_start = false; + + return_if_error(read_output_and_wait(coder, allocator, + out, out_pos, out_size, + &block_can_start, true, + &wait_abs, &has_blocked)); + + if (coder->pending_error != LZMA_OK) { + coder->sequence = SEQ_ERROR; + break; + } + + if (!block_can_start) { + // It's not a timeout because return_if_error handles + // it already. Output queue cannot be empty either + // because in that case block_can_start would have + // been true. Thus the output buffer must be full and + // the queue isn't empty. + assert(*out_pos == out_size); + assert(!lzma_outq_is_empty(&coder->outq)); + return LZMA_OK; + } + + // We know that we can start decoding this Block without + // exceeding memlimit_threading. However, to stay below + // memlimit_threading may require freeing some of the + // cached memory. + // + // Get a local copy of variables that require locking the + // mutex. It is fine if the worker threads modify the real + // values after we read these as those changes can only be + // towards more favorable conditions (less memory in use, + // more in cache). + // + // These are initialized to silence warnings. + uint64_t mem_in_use = 0; + uint64_t mem_cached = 0; + struct worker_thread *thr = NULL; + + mythread_sync(coder->mutex) { + mem_in_use = coder->mem_in_use; + mem_cached = coder->mem_cached; + thr = coder->threads_free; + } + + // The maximum amount of memory that can be held by other + // threads and cached buffers while allowing us to start + // decoding the next Block. + const uint64_t mem_max = coder->memlimit_threading + - coder->mem_next_block; + + // If the existing allocations are so large that starting + // to decode this Block might exceed memlimit_threads, + // try to free memory from the output queue cache first. + // + // NOTE: This math assumes the worst case. It's possible + // that the limit wouldn't be exceeded if the existing cached + // allocations are reused. + if (mem_in_use + mem_cached + coder->outq.mem_allocated + > mem_max) { + // Clear the outq cache except leave one buffer in + // the cache if its size is correct. That way we + // don't free and almost immediately reallocate + // an identical buffer. + lzma_outq_clear_cache2(&coder->outq, allocator, + coder->block_options.uncompressed_size); + } + + // If there is at least one worker_thread in the cache and + // the existing allocations are so large that starting to + // decode this Block might exceed memlimit_threads, free + // memory by freeing cached Block decoders. + // + // NOTE: The comparison is different here than above. + // Here we don't care about cached buffers in outq anymore + // and only look at memory actually in use. This is because + // if there is something in outq cache, it's a single buffer + // that can be used as is. We ensured this in the above + // if-block. + uint64_t mem_freed = 0; + if (thr != NULL && mem_in_use + mem_cached + + coder->outq.mem_in_use > mem_max) { + // Don't free the first Block decoder if its memory + // usage isn't greater than what this Block will need. + // Typically the same filter chain is used for all + // Blocks so this way the allocations can be reused + // when get_thread() picks the first worker_thread + // from the cache. + if (thr->mem_filters <= coder->mem_next_filters) + thr = thr->next; + + while (thr != NULL) { + lzma_next_end(&thr->block_decoder, allocator); + mem_freed += thr->mem_filters; + thr->mem_filters = 0; + thr = thr->next; + } + } + + // Update the memory usage counters. Note that coder->mem_* + // may have changed since we read them so we must subtract + // or add the changes. + mythread_sync(coder->mutex) { + coder->mem_cached -= mem_freed; + + // Memory needed for the filters and the input buffer. + // The output queue takes care of its own counter so + // we don't touch it here. + // + // NOTE: After this, coder->mem_in_use + + // coder->mem_cached might count the same thing twice. + // If so, this will get corrected in get_thread() when + // a worker_thread is picked from coder->free_threads + // and its memory usage is subtracted from mem_cached. + coder->mem_in_use += coder->mem_next_in + + coder->mem_next_filters; + } + + // Allocate memory for the output buffer in the output queue. + lzma_ret ret = lzma_outq_prealloc_buf( + &coder->outq, allocator, + coder->block_options.uncompressed_size); + if (ret != LZMA_OK) { + threads_stop(coder); + return ret; + } + + // Set up coder->thr. + ret = get_thread(coder, allocator); + if (ret != LZMA_OK) { + threads_stop(coder); + return ret; + } + + // The new Block decoder memory usage is already counted in + // coder->mem_in_use. Store it in the thread too. + coder->thr->mem_filters = coder->mem_next_filters; + + // Initialize the Block decoder. + coder->thr->block_options = coder->block_options; + ret = lzma_block_decoder_init( + &coder->thr->block_decoder, allocator, + &coder->thr->block_options); + + // Free the allocated filter options since they are needed + // only to initialize the Block decoder. + lzma_filters_free(coder->filters, allocator); + coder->thr->block_options.filters = NULL; + + // Check if memory usage calculation and Block encoder + // initialization succeeded. + if (ret != LZMA_OK) { + coder->pending_error = ret; + coder->sequence = SEQ_ERROR; + break; + } + + // Allocate the input buffer. + coder->thr->in_size = coder->mem_next_in; + coder->thr->in = lzma_alloc(coder->thr->in_size, allocator); + if (coder->thr->in == NULL) { + threads_stop(coder); + return LZMA_MEM_ERROR; + } + + // Get the preallocated output buffer. + coder->thr->outbuf = lzma_outq_get_buf( + &coder->outq, coder->thr); + + // Start the decoder. + mythread_sync(coder->thr->mutex) { + assert(coder->thr->state == THR_IDLE); + coder->thr->state = THR_RUN; + mythread_cond_signal(&coder->thr->cond); + } + + // Enable output from the thread that holds the oldest output + // buffer in the output queue (if such a thread exists). + mythread_sync(coder->mutex) { + lzma_outq_enable_partial_output(&coder->outq, + &worker_enable_partial_update); + } + + coder->sequence = SEQ_BLOCK_THR_RUN; + } + + // Fall through + + case SEQ_BLOCK_THR_RUN: { + if (action == LZMA_FINISH && coder->fail_fast) { + // We know that we won't get more input and that + // the caller wants fail-fast behavior. If we see + // that we don't have enough input to finish this + // Block, return LZMA_DATA_ERROR immediately. + // See SEQ_BLOCK_HEADER for the error code rationale. + const size_t in_avail = in_size - *in_pos; + const size_t in_needed = coder->thr->in_size + - coder->thr->in_filled; + if (in_avail < in_needed) { + threads_stop(coder); + return LZMA_DATA_ERROR; + } + } + + // Copy input to the worker thread. + size_t cur_in_filled = coder->thr->in_filled; + lzma_bufcpy(in, in_pos, in_size, coder->thr->in, + &cur_in_filled, coder->thr->in_size); + + // Tell the thread how much we copied. + mythread_sync(coder->thr->mutex) { + coder->thr->in_filled = cur_in_filled; + + // NOTE: Most of the time we are copying input faster + // than the thread can decode so most of the time + // calling mythread_cond_signal() is useless but + // we cannot make it conditional because thr->in_pos + // is updated without a mutex. And the overhead should + // be very much negligible anyway. + mythread_cond_signal(&coder->thr->cond); + } + + // Read output from the output queue. Just like in + // SEQ_BLOCK_HEADER, we wait to fill the output buffer + // only if waiting_allowed was set to true in the beginning + // of this function (see the comment there). + return_if_error(read_output_and_wait(coder, allocator, + out, out_pos, out_size, + NULL, waiting_allowed, + &wait_abs, &has_blocked)); + + if (coder->pending_error != LZMA_OK) { + coder->sequence = SEQ_ERROR; + break; + } + + // Return if the input didn't contain the whole Block. + if (coder->thr->in_filled < coder->thr->in_size) { + assert(*in_pos == in_size); + return LZMA_OK; + } + + // The whole Block has been copied to the thread-specific + // buffer. Continue from the next Block Header or Index. + coder->thr = NULL; + coder->sequence = SEQ_BLOCK_HEADER; + break; + } + + case SEQ_BLOCK_DIRECT_INIT: { + // Wait for the threads to finish and that all decoded data + // has been copied to the output. That is, wait until the + // output queue becomes empty. + // + // NOTE: No need to check for coder->pending_error as + // we aren't consuming any input until the queue is empty + // and if there is a pending error, read_output_and_wait() + // will eventually return it before the queue is empty. + return_if_error(read_output_and_wait(coder, allocator, + out, out_pos, out_size, + NULL, true, &wait_abs, &has_blocked)); + if (!lzma_outq_is_empty(&coder->outq)) + return LZMA_OK; + + // Free the cached output buffers. + lzma_outq_clear_cache(&coder->outq, allocator); + + // Get rid of the worker threads, including the coder->threads + // array. + threads_end(coder, allocator); + + // Initialize the Block decoder. + const lzma_ret ret = lzma_block_decoder_init( + &coder->block_decoder, allocator, + &coder->block_options); + + // Free the allocated filter options since they are needed + // only to initialize the Block decoder. + lzma_filters_free(coder->filters, allocator); + coder->block_options.filters = NULL; + + // Check if Block decoder initialization succeeded. + if (ret != LZMA_OK) + return ret; + + // Make the memory usage visible to _memconfig(). + coder->mem_direct_mode = coder->mem_next_filters; + + coder->sequence = SEQ_BLOCK_DIRECT_RUN; + } + + // Fall through + + case SEQ_BLOCK_DIRECT_RUN: { + const size_t in_old = *in_pos; + const size_t out_old = *out_pos; + const lzma_ret ret = coder->block_decoder.code( + coder->block_decoder.coder, allocator, + in, in_pos, in_size, out, out_pos, out_size, + action); + coder->progress_in += *in_pos - in_old; + coder->progress_out += *out_pos - out_old; + + if (ret != LZMA_STREAM_END) + return ret; + + // Block decoded successfully. Add the new size pair to + // the Index hash. + return_if_error(lzma_index_hash_append(coder->index_hash, + lzma_block_unpadded_size( + &coder->block_options), + coder->block_options.uncompressed_size)); + + coder->sequence = SEQ_BLOCK_HEADER; + break; + } + + case SEQ_INDEX_WAIT_OUTPUT: + // Flush the output from all worker threads so that we can + // decode the Index without thinking about threading. + return_if_error(read_output_and_wait(coder, allocator, + out, out_pos, out_size, + NULL, true, &wait_abs, &has_blocked)); + + if (!lzma_outq_is_empty(&coder->outq)) + return LZMA_OK; + + coder->sequence = SEQ_INDEX_DECODE; + + // Fall through + + case SEQ_INDEX_DECODE: { + // If we don't have any input, don't call + // lzma_index_hash_decode() since it would return + // LZMA_BUF_ERROR, which we must not do here. + if (*in_pos >= in_size) + return LZMA_OK; + + // Decode the Index and compare it to the hash calculated + // from the sizes of the Blocks (if any). + const size_t in_old = *in_pos; + const lzma_ret ret = lzma_index_hash_decode(coder->index_hash, + in, in_pos, in_size); + coder->progress_in += *in_pos - in_old; + if (ret != LZMA_STREAM_END) + return ret; + + coder->sequence = SEQ_STREAM_FOOTER; + } + + // Fall through + + case SEQ_STREAM_FOOTER: { + // Copy the Stream Footer to the internal buffer. + const size_t in_old = *in_pos; + lzma_bufcpy(in, in_pos, in_size, coder->buffer, &coder->pos, + LZMA_STREAM_HEADER_SIZE); + coder->progress_in += *in_pos - in_old; + + // Return if we didn't get the whole Stream Footer yet. + if (coder->pos < LZMA_STREAM_HEADER_SIZE) + return LZMA_OK; + + coder->pos = 0; + + // Decode the Stream Footer. The decoder gives + // LZMA_FORMAT_ERROR if the magic bytes don't match, + // so convert that return code to LZMA_DATA_ERROR. + lzma_stream_flags footer_flags; + const lzma_ret ret = lzma_stream_footer_decode( + &footer_flags, coder->buffer); + if (ret != LZMA_OK) + return ret == LZMA_FORMAT_ERROR + ? LZMA_DATA_ERROR : ret; + + // Check that Index Size stored in the Stream Footer matches + // the real size of the Index field. + if (lzma_index_hash_size(coder->index_hash) + != footer_flags.backward_size) + return LZMA_DATA_ERROR; + + // Compare that the Stream Flags fields are identical in + // both Stream Header and Stream Footer. + return_if_error(lzma_stream_flags_compare( + &coder->stream_flags, &footer_flags)); + + if (!coder->concatenated) + return LZMA_STREAM_END; + + coder->sequence = SEQ_STREAM_PADDING; + } + + // Fall through + + case SEQ_STREAM_PADDING: + assert(coder->concatenated); + + // Skip over possible Stream Padding. + while (true) { + if (*in_pos >= in_size) { + // Unless LZMA_FINISH was used, we cannot + // know if there's more input coming later. + if (action != LZMA_FINISH) + return LZMA_OK; + + // Stream Padding must be a multiple of + // four bytes. + return coder->pos == 0 + ? LZMA_STREAM_END + : LZMA_DATA_ERROR; + } + + // If the byte is not zero, it probably indicates + // beginning of a new Stream (or the file is corrupt). + if (in[*in_pos] != 0x00) + break; + + ++*in_pos; + ++coder->progress_in; + coder->pos = (coder->pos + 1) & 3; + } + + // Stream Padding must be a multiple of four bytes (empty + // Stream Padding is OK). + if (coder->pos != 0) { + ++*in_pos; + ++coder->progress_in; + return LZMA_DATA_ERROR; + } + + // Prepare to decode the next Stream. + return_if_error(stream_decoder_reset(coder, allocator)); + break; + + case SEQ_ERROR: + if (!coder->fail_fast) { + // Let the application get all data before the point + // where the error was detected. This matches the + // behavior of single-threaded use. + // + // FIXME? Some errors (LZMA_MEM_ERROR) don't get here, + // they are returned immediately. Thus in rare cases + // the output will be less than in the single-threaded + // mode. Maybe this doesn't matter much in practice. + return_if_error(read_output_and_wait(coder, allocator, + out, out_pos, out_size, + NULL, true, &wait_abs, &has_blocked)); + + // We get here only if the error happened in the main + // thread, for example, unsupported Block Header. + if (!lzma_outq_is_empty(&coder->outq)) + return LZMA_OK; + } + + // We only get here if no errors were detected by the worker + // threads. Errors from worker threads would have already been + // returned by the call to read_output_and_wait() above. + return coder->pending_error; + + default: + assert(0); + return LZMA_PROG_ERROR; + } + + // Never reached +} + + +static void +stream_decoder_mt_end(void *coder_ptr, const lzma_allocator *allocator) +{ + struct lzma_stream_coder *coder = coder_ptr; + + threads_end(coder, allocator); + lzma_outq_end(&coder->outq, allocator); + + lzma_next_end(&coder->block_decoder, allocator); + lzma_filters_free(coder->filters, allocator); + lzma_index_hash_end(coder->index_hash, allocator); + + lzma_free(coder, allocator); + return; +} + + +static lzma_check +stream_decoder_mt_get_check(const void *coder_ptr) +{ + const struct lzma_stream_coder *coder = coder_ptr; + return coder->stream_flags.check; +} + + +static lzma_ret +stream_decoder_mt_memconfig(void *coder_ptr, uint64_t *memusage, + uint64_t *old_memlimit, uint64_t new_memlimit) +{ + // NOTE: This function gets/sets memlimit_stop. For now, + // memlimit_threading cannot be modified after initialization. + // + // *memusage will include cached memory too. Excluding cached memory + // would be misleading and it wouldn't help the applications to + // know how much memory is actually needed to decompress the file + // because the higher the number of threads and the memlimits are + // the more memory the decoder may use. + // + // Setting a new limit includes the cached memory too and too low + // limits will be rejected. Alternative could be to free the cached + // memory immediately if that helps to bring the limit down but + // the current way is the simplest. It's unlikely that limit needs + // to be lowered in the middle of a file anyway; the typical reason + // to want a new limit is to increase after LZMA_MEMLIMIT_ERROR + // and even such use isn't common. + struct lzma_stream_coder *coder = coder_ptr; + + mythread_sync(coder->mutex) { + *memusage = coder->mem_direct_mode + + coder->mem_in_use + + coder->mem_cached + + coder->outq.mem_allocated; + } + + // If no filter chains are allocated, *memusage may be zero. + // Always return at least LZMA_MEMUSAGE_BASE. + if (*memusage < LZMA_MEMUSAGE_BASE) + *memusage = LZMA_MEMUSAGE_BASE; + + *old_memlimit = coder->memlimit_stop; + + if (new_memlimit != 0) { + if (new_memlimit < *memusage) + return LZMA_MEMLIMIT_ERROR; + + coder->memlimit_stop = new_memlimit; + } + + return LZMA_OK; +} + + +static void +stream_decoder_mt_get_progress(void *coder_ptr, + uint64_t *progress_in, uint64_t *progress_out) +{ + struct lzma_stream_coder *coder = coder_ptr; + + // Lock coder->mutex to prevent finishing threads from moving their + // progress info from the worker_thread structure to lzma_stream_coder. + mythread_sync(coder->mutex) { + *progress_in = coder->progress_in; + *progress_out = coder->progress_out; + + for (size_t i = 0; i < coder->threads_initialized; ++i) { + mythread_sync(coder->threads[i].mutex) { + *progress_in += coder->threads[i].progress_in; + *progress_out += coder->threads[i] + .progress_out; + } + } + } + + return; +} + + +static lzma_ret +stream_decoder_mt_init(lzma_next_coder *next, const lzma_allocator *allocator, + const lzma_mt *options) +{ + struct lzma_stream_coder *coder; + + if (options->threads == 0 || options->threads > LZMA_THREADS_MAX) + return LZMA_OPTIONS_ERROR; + + if (options->flags & ~LZMA_SUPPORTED_FLAGS) + return LZMA_OPTIONS_ERROR; + + lzma_next_coder_init(&stream_decoder_mt_init, next, allocator); + + coder = next->coder; + if (!coder) { + coder = lzma_alloc(sizeof(struct lzma_stream_coder), allocator); + if (coder == NULL) + return LZMA_MEM_ERROR; + + next->coder = coder; + + if (mythread_mutex_init(&coder->mutex)) { + lzma_free(coder, allocator); + return LZMA_MEM_ERROR; + } + + if (mythread_cond_init(&coder->cond)) { + mythread_mutex_destroy(&coder->mutex); + lzma_free(coder, allocator); + return LZMA_MEM_ERROR; + } + + next->code = &stream_decode_mt; + next->end = &stream_decoder_mt_end; + next->get_check = &stream_decoder_mt_get_check; + next->memconfig = &stream_decoder_mt_memconfig; + next->get_progress = &stream_decoder_mt_get_progress; + + coder->filters[0].id = LZMA_VLI_UNKNOWN; + memzero(&coder->outq, sizeof(coder->outq)); + + coder->block_decoder = LZMA_NEXT_CODER_INIT; + coder->mem_direct_mode = 0; + + coder->index_hash = NULL; + coder->threads = NULL; + coder->threads_free = NULL; + coder->threads_initialized = 0; + } + + // Cleanup old filter chain if one remains after unfinished decoding + // of a previous Stream. + lzma_filters_free(coder->filters, allocator); + + // By allocating threads from scratch we can start memory-usage + // accounting from scratch, too. Changes in filter and block sizes may + // affect number of threads. + // + // FIXME? Reusing should be easy but unlike the single-threaded + // decoder, with some types of input file combinations reusing + // could leave quite a lot of memory allocated but unused (first + // file could allocate a lot, the next files could use fewer + // threads and some of the allocations from the first file would not + // get freed unless memlimit_threading forces us to clear caches). + // + // NOTE: The direct mode decoder isn't freed here if one exists. + // It will be reused or freed as needed in the main loop. + threads_end(coder, allocator); + + // All memusage counters start at 0 (including mem_direct_mode). + // The little extra that is needed for the structs in this file + // get accounted well enough by the filter chain memory usage + // which adds LZMA_MEMUSAGE_BASE for each chain. However, + // stream_decoder_mt_memconfig() has to handle this specially so that + // it will never return less than LZMA_MEMUSAGE_BASE as memory usage. + coder->mem_in_use = 0; + coder->mem_cached = 0; + coder->mem_next_block = 0; + + coder->progress_in = 0; + coder->progress_out = 0; + + coder->sequence = SEQ_STREAM_HEADER; + coder->thread_error = LZMA_OK; + coder->pending_error = LZMA_OK; + coder->thr = NULL; + + coder->timeout = options->timeout; + + coder->memlimit_threading = my_max(1, options->memlimit_threading); + coder->memlimit_stop = my_max(1, options->memlimit_stop); + if (coder->memlimit_threading > coder->memlimit_stop) + coder->memlimit_threading = coder->memlimit_stop; + + coder->tell_no_check = (options->flags & LZMA_TELL_NO_CHECK) != 0; + coder->tell_unsupported_check + = (options->flags & LZMA_TELL_UNSUPPORTED_CHECK) != 0; + coder->tell_any_check = (options->flags & LZMA_TELL_ANY_CHECK) != 0; + coder->ignore_check = (options->flags & LZMA_IGNORE_CHECK) != 0; + coder->concatenated = (options->flags & LZMA_CONCATENATED) != 0; + coder->fail_fast = (options->flags & LZMA_FAIL_FAST) != 0; + + coder->first_stream = true; + coder->out_was_filled = false; + coder->pos = 0; + + coder->threads_max = options->threads; + + return_if_error(lzma_outq_init(&coder->outq, allocator, + coder->threads_max)); + + return stream_decoder_reset(coder, allocator); +} + + +extern LZMA_API(lzma_ret) +lzma_stream_decoder_mt(lzma_stream *strm, const lzma_mt *options) +{ + lzma_next_strm_init(stream_decoder_mt_init, strm, options); + + strm->internal->supported_actions[LZMA_RUN] = true; + strm->internal->supported_actions[LZMA_FINISH] = true; + + return LZMA_OK; +}
diff --git a/Utilities/cmliblzma/liblzma/common/stream_encoder.c b/Utilities/cmliblzma/liblzma/common/stream_encoder.c index 858cba4..e7e5b3f 100644 --- a/Utilities/cmliblzma/liblzma/common/stream_encoder.c +++ b/Utilities/cmliblzma/liblzma/common/stream_encoder.c
@@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file stream_encoder.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "block_encoder.h" @@ -219,8 +218,7 @@ lzma_next_end(&coder->index_encoder, allocator); lzma_index_end(coder->index, allocator); - for (size_t i = 0; coder->filters[i].id != LZMA_VLI_UNKNOWN; ++i) - lzma_free(coder->filters[i].options, allocator); + lzma_filters_free(coder->filters, allocator); lzma_free(coder, allocator); return; @@ -233,6 +231,13 @@ const lzma_filter *reversed_filters) { lzma_stream_coder *coder = coder_ptr; + lzma_ret ret; + + // Make a copy to a temporary buffer first. This way it is easier + // to keep the encoder state unchanged if an error occurs with + // lzma_filters_copy(). + lzma_filter temp[LZMA_FILTERS_MAX + 1]; + return_if_error(lzma_filters_copy(filters, temp, allocator)); if (coder->sequence <= SEQ_BLOCK_INIT) { // There is no incomplete Block waiting to be finished, @@ -240,31 +245,40 @@ // trying to initialize the Block encoder with the new // chain. This way we detect if the chain is valid. coder->block_encoder_is_initialized = false; - coder->block_options.filters = (lzma_filter *)(filters); - const lzma_ret ret = block_encoder_init(coder, allocator); + coder->block_options.filters = temp; + ret = block_encoder_init(coder, allocator); coder->block_options.filters = coder->filters; if (ret != LZMA_OK) - return ret; + goto error; coder->block_encoder_is_initialized = true; } else if (coder->sequence <= SEQ_BLOCK_ENCODE) { // We are in the middle of a Block. Try to update only // the filter-specific options. - return_if_error(coder->block_encoder.update( + ret = coder->block_encoder.update( coder->block_encoder.coder, allocator, - filters, reversed_filters)); + filters, reversed_filters); + if (ret != LZMA_OK) + goto error; } else { // Trying to update the filter chain when we are already // encoding Index or Stream Footer. - return LZMA_PROG_ERROR; + ret = LZMA_PROG_ERROR; + goto error; } - // Free the copy of the old chain and make a copy of the new chain. - for (size_t i = 0; coder->filters[i].id != LZMA_VLI_UNKNOWN; ++i) - lzma_free(coder->filters[i].options, allocator); + // Free the options of the old chain. + lzma_filters_free(coder->filters, allocator); - return lzma_filters_copy(filters, coder->filters, allocator); + // Copy the new filter chain in place. + memcpy(coder->filters, temp, sizeof(temp)); + + return LZMA_OK; + +error: + lzma_filters_free(temp, allocator); + return ret; } @@ -319,7 +333,7 @@ // Initialize the Block encoder. This way we detect unsupported // filter chains when initializing the Stream encoder instead of - // giving an error after Stream Header has already written out. + // giving an error after Stream Header has already been written out. return stream_encoder_update(coder, allocator, filters, NULL); }
diff --git a/Utilities/cmliblzma/liblzma/common/stream_encoder_mt.c b/Utilities/cmliblzma/liblzma/common/stream_encoder_mt.c index 01e4033..f0fef15 100644 --- a/Utilities/cmliblzma/liblzma/common/stream_encoder_mt.c +++ b/Utilities/cmliblzma/liblzma/common/stream_encoder_mt.c
@@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file stream_encoder_mt.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "filter_encoder.h" @@ -85,6 +84,11 @@ /// Compression options for this Block lzma_block block_options; + /// Filter chain for this thread. By copying the filters array + /// to each thread it is possible to change the filter chain + /// between Blocks using lzma_filters_update(). + lzma_filter filters[LZMA_FILTERS_MAX + 1]; + /// Next structure in the stack of free worker threads. worker_thread *next; @@ -109,9 +113,22 @@ /// LZMA_FULL_FLUSH or LZMA_FULL_BARRIER is used earlier. size_t block_size; - /// The filter chain currently in use + /// The filter chain to use for the next Block. + /// This can be updated using lzma_filters_update() + /// after LZMA_FULL_BARRIER or LZMA_FULL_FLUSH. lzma_filter filters[LZMA_FILTERS_MAX + 1]; + /// A copy of filters[] will be put here when attempting to get + /// a new worker thread. This will be copied to a worker thread + /// when a thread becomes free and then this cache is marked as + /// empty by setting [0].id = LZMA_VLI_UNKNOWN. Without this cache + /// the filter options from filters[] would get uselessly copied + /// multiple times (allocated and freed) when waiting for a new free + /// worker thread. + /// + /// This is freed if filters[] is updated via lzma_filters_update(). + lzma_filter filters_cache[LZMA_FILTERS_MAX + 1]; + /// Index to hold sizes of the Blocks lzma_index *index; @@ -133,6 +150,9 @@ /// Output buffer queue for compressed data lzma_outq outq; + /// How much memory to allocate for each lzma_outbuf.buf + size_t outbuf_alloc_size; + /// Maximum wait time if cannot use all the input and cannot /// fill the output buffer. This is in milliseconds. @@ -196,7 +216,7 @@ static worker_state -worker_encode(worker_thread *thr, worker_state state) +worker_encode(worker_thread *thr, size_t *out_pos, worker_state state) { assert(thr->progress_in == 0); assert(thr->progress_out == 0); @@ -205,12 +225,9 @@ thr->block_options = (lzma_block){ .version = 0, .check = thr->coder->stream_flags.check, - .compressed_size = thr->coder->outq.buf_size_max, + .compressed_size = thr->outbuf->allocated, .uncompressed_size = thr->coder->block_size, - - // TODO: To allow changing the filter chain, the filters - // array must be copied to each worker_thread. - .filters = thr->coder->filters, + .filters = thr->filters, }; // Calculate maximum size of the Block Header. This amount is @@ -234,12 +251,12 @@ size_t in_pos = 0; size_t in_size = 0; - thr->outbuf->size = thr->block_options.header_size; - const size_t out_size = thr->coder->outq.buf_size_max; + *out_pos = thr->block_options.header_size; + const size_t out_size = thr->outbuf->allocated; do { mythread_sync(thr->mutex) { - // Store in_pos and out_pos into *thr so that + // Store in_pos and *out_pos into *thr so that // an application may read them via // lzma_get_progress() to get progress information. // @@ -247,7 +264,7 @@ // finishes. Instead, the final values are taken // later from thr->outbuf. thr->progress_in = in_pos; - thr->progress_out = thr->outbuf->size; + thr->progress_out = *out_pos; while (in_size == thr->in_size && thr->state == THR_RUN) @@ -277,8 +294,8 @@ ret = thr->block_encoder.code( thr->block_encoder.coder, thr->allocator, thr->in, &in_pos, in_limit, thr->outbuf->buf, - &thr->outbuf->size, out_size, action); - } while (ret == LZMA_OK && thr->outbuf->size < out_size); + out_pos, out_size, action); + } while (ret == LZMA_OK && *out_pos < out_size); switch (ret) { case LZMA_STREAM_END: @@ -313,10 +330,10 @@ return state; // Do the encoding. This takes care of the Block Header too. - thr->outbuf->size = 0; + *out_pos = 0; ret = lzma_block_uncomp_encode(&thr->block_options, thr->in, in_size, thr->outbuf->buf, - &thr->outbuf->size, out_size); + out_pos, out_size); // It shouldn't fail. if (ret != LZMA_OK) { @@ -367,11 +384,13 @@ } } + size_t out_pos = 0; + assert(state != THR_IDLE); assert(state != THR_STOP); if (state <= THR_FINISH) - state = worker_encode(thr, state); + state = worker_encode(thr, &out_pos, state); if (state == THR_EXIT) break; @@ -387,14 +406,17 @@ } mythread_sync(thr->coder->mutex) { - // Mark the output buffer as finished if - // no errors occurred. - thr->outbuf->finished = state == THR_FINISH; + // If no errors occurred, make the encoded data + // available to be copied out. + if (state == THR_FINISH) { + thr->outbuf->pos = out_pos; + thr->outbuf->finished = true; + } // Update the main progress info. thr->coder->progress_in += thr->outbuf->uncompressed_size; - thr->coder->progress_out += thr->outbuf->size; + thr->coder->progress_out += out_pos; thr->progress_in = 0; thr->progress_out = 0; @@ -407,6 +429,8 @@ } // Exiting, free the resources. + lzma_filters_free(thr->filters, thr->allocator); + mythread_mutex_destroy(&thr->mutex); mythread_cond_destroy(&thr->cond); @@ -490,6 +514,7 @@ thr->progress_in = 0; thr->progress_out = 0; thr->block_encoder = LZMA_NEXT_CODER_INIT; + thr->filters[0].id = LZMA_VLI_UNKNOWN; if (mythread_create(&thr->thread_id, &worker_start, thr)) goto error_thread; @@ -519,6 +544,18 @@ if (!lzma_outq_has_buf(&coder->outq)) return LZMA_OK; + // That's also true if we cannot allocate memory for the output + // buffer in the output queue. + return_if_error(lzma_outq_prealloc_buf(&coder->outq, allocator, + coder->outbuf_alloc_size)); + + // Make a thread-specific copy of the filter chain. Put it in + // the cache array first so that if we cannot get a new thread yet, + // the allocation is ready when we try again. + if (coder->filters_cache[0].id == LZMA_VLI_UNKNOWN) + return_if_error(lzma_filters_copy( + coder->filters, coder->filters_cache, allocator)); + // If there is a free structure on the stack, use it. mythread_sync(coder->mutex) { if (coder->threads_free != NULL) { @@ -541,7 +578,16 @@ mythread_sync(coder->thr->mutex) { coder->thr->state = THR_RUN; coder->thr->in_size = 0; - coder->thr->outbuf = lzma_outq_get_buf(&coder->outq); + coder->thr->outbuf = lzma_outq_get_buf(&coder->outq, NULL); + + // Free the old thread-specific filter options and replace + // them with the already-allocated new options from + // coder->filters_cache[]. Then mark the cache as empty. + lzma_filters_free(coder->thr->filters, allocator); + memcpy(coder->thr->filters, coder->filters_cache, + sizeof(coder->filters_cache)); + coder->filters_cache[0].id = LZMA_VLI_UNKNOWN; + mythread_cond_signal(&coder->thr->cond); } @@ -598,7 +644,7 @@ } if (block_error) { - lzma_ret ret; + lzma_ret ret = LZMA_OK; // Init to silence a warning. mythread_sync(coder->mutex) { ret = coder->thread_error; @@ -627,9 +673,13 @@ // to true here and calculate the absolute time when // we must return if there's nothing to do. // - // The idea of *has_blocked is to avoid unneeded calls - // to mythread_condtime_set(), which may do a syscall - // depending on the operating system. + // This way if we block multiple times for short moments + // less than "timeout" milliseconds, we will return once + // "timeout" amount of time has passed since the *first* + // blocking occurred. If the absolute time was calculated + // again every time we block, "timeout" would effectively + // be meaningless if we never consecutively block longer + // than "timeout" ms. *has_blocked = true; mythread_condtime_set(wait_abs, &coder->cond, coder->timeout); } @@ -692,7 +742,7 @@ // These are for wait_for_work(). bool has_blocked = false; - mythread_condtime wait_abs; + mythread_condtime wait_abs = { 0 }; while (true) { mythread_sync(coder->mutex) { @@ -704,7 +754,7 @@ } // Try to read compressed data to out[]. - ret = lzma_outq_read(&coder->outq, + ret = lzma_outq_read(&coder->outq, allocator, out, out_pos, out_size, &unpadded_size, &uncompressed_size); @@ -715,6 +765,10 @@ ret = lzma_index_append(coder->index, allocator, unpadded_size, uncompressed_size); + if (ret != LZMA_OK) { + threads_stop(coder, false); + return ret; + } // If we didn't fill the output buffer yet, // try to read more data. Maybe the next @@ -724,8 +778,7 @@ } if (ret != LZMA_OK) { - // coder->thread_error was set or - // lzma_index_append() failed. + // coder->thread_error was set. threads_stop(coder, false); return ret; } @@ -846,8 +899,8 @@ threads_end(coder, allocator); lzma_outq_end(&coder->outq, allocator); - for (size_t i = 0; coder->filters[i].id != LZMA_VLI_UNKNOWN; ++i) - lzma_free(coder->filters[i].options, allocator); + lzma_filters_free(coder->filters, allocator); + lzma_filters_free(coder->filters_cache, allocator); lzma_next_end(&coder->index_encoder, allocator); lzma_index_end(coder->index, allocator); @@ -860,6 +913,45 @@ } +static lzma_ret +stream_encoder_mt_update(void *coder_ptr, const lzma_allocator *allocator, + const lzma_filter *filters, + const lzma_filter *reversed_filters + lzma_attribute((__unused__))) +{ + lzma_stream_coder *coder = coder_ptr; + + // Applications shouldn't attempt to change the options when + // we are already encoding the Index or Stream Footer. + if (coder->sequence > SEQ_BLOCK) + return LZMA_PROG_ERROR; + + // For now the threaded encoder doesn't support changing + // the options in the middle of a Block. + if (coder->thr != NULL) + return LZMA_PROG_ERROR; + + // Check if the filter chain seems mostly valid. See the comment + // in stream_encoder_mt_init(). + if (lzma_raw_encoder_memusage(filters) == UINT64_MAX) + return LZMA_OPTIONS_ERROR; + + // Make a copy to a temporary buffer first. This way the encoder + // state stays unchanged if an error occurs in lzma_filters_copy(). + lzma_filter temp[LZMA_FILTERS_MAX + 1]; + return_if_error(lzma_filters_copy(filters, temp, allocator)); + + // Free the options of the old chain as well as the cache. + lzma_filters_free(coder->filters, allocator); + lzma_filters_free(coder->filters_cache, allocator); + + // Copy the new filter chain in place. + memcpy(coder->filters, temp, sizeof(temp)); + + return LZMA_OK; +} + + /// Options handling for lzma_stream_encoder_mt_init() and /// lzma_stream_encoder_mt_memusage() static lzma_ret @@ -886,20 +978,18 @@ *filters = opt_easy->filters; } - // Block size - if (options->block_size > 0) { - if (options->block_size > BLOCK_SIZE_MAX) - return LZMA_OPTIONS_ERROR; - + // If the Block size is not set, determine it from the filter chain. + if (options->block_size > 0) *block_size = options->block_size; - } else { - // Determine the Block size from the filter chain. + else *block_size = lzma_mt_block_size(*filters); - if (*block_size == 0) - return LZMA_OPTIONS_ERROR; - assert(*block_size <= BLOCK_SIZE_MAX); - } + // UINT64_MAX > BLOCK_SIZE_MAX, so the second condition + // should be optimized out by any reasonable compiler. + // The second condition should be there in the unlikely event that + // the macros change and UINT64_MAX < BLOCK_SIZE_MAX. + if (*block_size > BLOCK_SIZE_MAX || *block_size == UINT64_MAX) + return LZMA_OPTIONS_ERROR; // Calculate the maximum amount output that a single output buffer // may need to hold. This is the same as the maximum total size of @@ -951,14 +1041,16 @@ &block_size, &outbuf_size_max)); #if SIZE_MAX < UINT64_MAX - if (block_size > SIZE_MAX) + if (block_size > SIZE_MAX || outbuf_size_max > SIZE_MAX) return LZMA_MEM_ERROR; #endif // Validate the filter chain so that we can give an error in this // function instead of delaying it to the first call to lzma_code(). // The memory usage calculation verifies the filter chain as - // a side effect so we take advantage of that. + // a side effect so we take advantage of that. It's not a perfect + // check though as raw encoder allows LZMA1 too but such problems + // will be caught eventually with Block Header encoder. if (lzma_raw_encoder_memusage(filters) == UINT64_MAX) return LZMA_OPTIONS_ERROR; @@ -998,9 +1090,10 @@ next->code = &stream_encode_mt; next->end = &stream_encoder_mt_end; next->get_progress = &get_progress; -// next->update = &stream_encoder_mt_update; + next->update = &stream_encoder_mt_update; coder->filters[0].id = LZMA_VLI_UNKNOWN; + coder->filters_cache[0].id = LZMA_VLI_UNKNOWN; coder->index_encoder = LZMA_NEXT_CODER_INIT; coder->index = NULL; memzero(&coder->outq, sizeof(coder->outq)); @@ -1012,6 +1105,7 @@ // Basic initializations coder->sequence = SEQ_STREAM_HEADER; coder->block_size = (size_t)(block_size); + coder->outbuf_alloc_size = (size_t)(outbuf_size_max); coder->thread_error = LZMA_OK; coder->thr = NULL; @@ -1041,15 +1135,16 @@ // Output queue return_if_error(lzma_outq_init(&coder->outq, allocator, - outbuf_size_max, options->threads)); + options->threads)); // Timeout coder->timeout = options->timeout; - // Free the old filter chain and copy the new one. - for (size_t i = 0; coder->filters[i].id != LZMA_VLI_UNKNOWN; ++i) - lzma_free(coder->filters[i].options, allocator); + // Free the old filter chain and the cache. + lzma_filters_free(coder->filters, allocator); + lzma_filters_free(coder->filters_cache, allocator); + // Copy the new filter chain. return_if_error(lzma_filters_copy( filters, coder->filters, allocator)); @@ -1075,6 +1170,31 @@ } +#ifdef HAVE_SYMBOL_VERSIONS_LINUX +// These are for compatibility with binaries linked against liblzma that +// has been patched with xz-5.2.2-compat-libs.patch from RHEL/CentOS 7. +// Actually that patch didn't create lzma_stream_encoder_mt@XZ_5.2.2 +// but it has been added here anyway since someone might misread the +// RHEL patch and think both @XZ_5.1.2alpha and @XZ_5.2.2 exist. +LZMA_SYMVER_API("lzma_stream_encoder_mt@XZ_5.1.2alpha", + lzma_ret, lzma_stream_encoder_mt_512a)( + lzma_stream *strm, const lzma_mt *options) + lzma_nothrow lzma_attr_warn_unused_result + __attribute__((__alias__("lzma_stream_encoder_mt_52"))); + +LZMA_SYMVER_API("lzma_stream_encoder_mt@XZ_5.2.2", + lzma_ret, lzma_stream_encoder_mt_522)( + lzma_stream *strm, const lzma_mt *options) + lzma_nothrow lzma_attr_warn_unused_result + __attribute__((__alias__("lzma_stream_encoder_mt_52"))); + +LZMA_SYMVER_API("lzma_stream_encoder_mt@@XZ_5.2", + lzma_ret, lzma_stream_encoder_mt_52)( + lzma_stream *strm, const lzma_mt *options) + lzma_nothrow lzma_attr_warn_unused_result; + +#define lzma_stream_encoder_mt lzma_stream_encoder_mt_52 +#endif extern LZMA_API(lzma_ret) lzma_stream_encoder_mt(lzma_stream *strm, const lzma_mt *options) { @@ -1090,6 +1210,23 @@ } +#ifdef HAVE_SYMBOL_VERSIONS_LINUX +LZMA_SYMVER_API("lzma_stream_encoder_mt_memusage@XZ_5.1.2alpha", + uint64_t, lzma_stream_encoder_mt_memusage_512a)( + const lzma_mt *options) lzma_nothrow lzma_attr_pure + __attribute__((__alias__("lzma_stream_encoder_mt_memusage_52"))); + +LZMA_SYMVER_API("lzma_stream_encoder_mt_memusage@XZ_5.2.2", + uint64_t, lzma_stream_encoder_mt_memusage_522)( + const lzma_mt *options) lzma_nothrow lzma_attr_pure + __attribute__((__alias__("lzma_stream_encoder_mt_memusage_52"))); + +LZMA_SYMVER_API("lzma_stream_encoder_mt_memusage@@XZ_5.2", + uint64_t, lzma_stream_encoder_mt_memusage_52)( + const lzma_mt *options) lzma_nothrow lzma_attr_pure; + +#define lzma_stream_encoder_mt_memusage lzma_stream_encoder_mt_memusage_52 +#endif // This function name is a monster but it's consistent with the older // monster names. :-( 31 chars is the max that C99 requires so in that // sense it's not too long. ;-)
diff --git a/Utilities/cmliblzma/liblzma/common/stream_flags_common.c b/Utilities/cmliblzma/liblzma/common/stream_flags_common.c index fbe8eb8..41b8dcb 100644 --- a/Utilities/cmliblzma/liblzma/common/stream_flags_common.c +++ b/Utilities/cmliblzma/liblzma/common/stream_flags_common.c
@@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file stream_flags_common.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "stream_flags_common.h"
diff --git a/Utilities/cmliblzma/liblzma/common/stream_flags_common.h b/Utilities/cmliblzma/liblzma/common/stream_flags_common.h index 9f3122a..28729db 100644 --- a/Utilities/cmliblzma/liblzma/common/stream_flags_common.h +++ b/Utilities/cmliblzma/liblzma/common/stream_flags_common.h
@@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file stream_flags_common.h @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef LZMA_STREAM_FLAGS_COMMON_H @@ -18,7 +17,10 @@ /// Size of the Stream Flags field #define LZMA_STREAM_FLAGS_SIZE 2 +lzma_attr_visibility_hidden extern const uint8_t lzma_header_magic[6]; + +lzma_attr_visibility_hidden extern const uint8_t lzma_footer_magic[2];
diff --git a/Utilities/cmliblzma/liblzma/common/stream_flags_decoder.c b/Utilities/cmliblzma/liblzma/common/stream_flags_decoder.c index 4e43e35..522c98b 100644 --- a/Utilities/cmliblzma/liblzma/common/stream_flags_decoder.c +++ b/Utilities/cmliblzma/liblzma/common/stream_flags_decoder.c
@@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file stream_flags_decoder.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "stream_flags_common.h" @@ -39,8 +38,11 @@ const uint32_t crc = lzma_crc32(in + sizeof(lzma_header_magic), LZMA_STREAM_FLAGS_SIZE, 0); if (crc != read32le(in + sizeof(lzma_header_magic) - + LZMA_STREAM_FLAGS_SIZE)) + + LZMA_STREAM_FLAGS_SIZE)) { +#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION return LZMA_DATA_ERROR; +#endif + } // Stream Flags if (stream_flags_decode(options, in + sizeof(lzma_header_magic))) @@ -67,8 +69,11 @@ // CRC32 const uint32_t crc = lzma_crc32(in + sizeof(uint32_t), sizeof(uint32_t) + LZMA_STREAM_FLAGS_SIZE, 0); - if (crc != read32le(in)) + if (crc != read32le(in)) { +#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION return LZMA_DATA_ERROR; +#endif + } // Stream Flags if (stream_flags_decode(options, in + sizeof(uint32_t) * 2))
diff --git a/Utilities/cmliblzma/liblzma/common/stream_flags_encoder.c b/Utilities/cmliblzma/liblzma/common/stream_flags_encoder.c index b98ab17..f94b5cd 100644 --- a/Utilities/cmliblzma/liblzma/common/stream_flags_encoder.c +++ b/Utilities/cmliblzma/liblzma/common/stream_flags_encoder.c
@@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file stream_flags_encoder.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "stream_flags_common.h"
diff --git a/Utilities/cmliblzma/liblzma/common/string_conversion.c b/Utilities/cmliblzma/liblzma/common/string_conversion.c new file mode 100644 index 0000000..c899783 --- /dev/null +++ b/Utilities/cmliblzma/liblzma/common/string_conversion.c
@@ -0,0 +1,1338 @@ +// SPDX-License-Identifier: 0BSD + +/////////////////////////////////////////////////////////////////////////////// +// +/// \file string_conversion.c +/// \brief Conversion of strings to filter chain and vice versa +// +// Author: Lasse Collin +// +/////////////////////////////////////////////////////////////////////////////// + +#include "filter_common.h" + + +///////////////////// +// String building // +///////////////////// + +/// How much memory to allocate for strings. For now, no realloc is used +/// so this needs to be big enough even though there of course is +/// an overflow check still. +/// +/// FIXME? Using a fixed size is wasteful if the application doesn't free +/// the string fairly quickly but this can be improved later if needed. +#define STR_ALLOC_SIZE 800 + + +typedef struct { + char *buf; + size_t pos; +} lzma_str; + + +static lzma_ret +str_init(lzma_str *str, const lzma_allocator *allocator) +{ + str->buf = lzma_alloc(STR_ALLOC_SIZE, allocator); + if (str->buf == NULL) + return LZMA_MEM_ERROR; + + str->pos = 0; + return LZMA_OK; +} + + +static void +str_free(lzma_str *str, const lzma_allocator *allocator) +{ + lzma_free(str->buf, allocator); + return; +} + + +static bool +str_is_full(const lzma_str *str) +{ + return str->pos == STR_ALLOC_SIZE - 1; +} + + +static lzma_ret +str_finish(char **dest, lzma_str *str, const lzma_allocator *allocator) +{ + if (str_is_full(str)) { + // The preallocated buffer was too small. + // This shouldn't happen as STR_ALLOC_SIZE should + // be adjusted if new filters are added. + lzma_free(str->buf, allocator); + *dest = NULL; + assert(0); + return LZMA_PROG_ERROR; + } + + str->buf[str->pos] = '\0'; + *dest = str->buf; + return LZMA_OK; +} + + +static void +str_append_str(lzma_str *str, const char *s) +{ + const size_t len = strlen(s); + const size_t limit = STR_ALLOC_SIZE - 1 - str->pos; + const size_t copy_size = my_min(len, limit); + + memcpy(str->buf + str->pos, s, copy_size); + str->pos += copy_size; + return; +} + + +static void +str_append_u32(lzma_str *str, uint32_t v, bool use_byte_suffix) +{ + if (v == 0) { + str_append_str(str, "0"); + } else { + // NOTE: Don't use plain "B" because xz and the parser in this + // file don't support it and at glance it may look like 8 + // (there cannot be a space before the suffix). + static const char suffixes[4][4] = { "", "KiB", "MiB", "GiB" }; + + size_t suf = 0; + if (use_byte_suffix) { + while ((v & 1023) == 0 + && suf < ARRAY_SIZE(suffixes) - 1) { + v >>= 10; + ++suf; + } + } + + // UINT32_MAX in base 10 would need 10 + 1 bytes. Remember + // that initializing to "" initializes all elements to + // zero so '\0'-termination gets handled by this. + char buf[16] = ""; + size_t pos = sizeof(buf) - 1; + + do { + buf[--pos] = '0' + (v % 10); + v /= 10; + } while (v != 0); + + str_append_str(str, buf + pos); + str_append_str(str, suffixes[suf]); + } + + return; +} + + +////////////////////////////////////////////// +// Parsing and stringification declarations // +////////////////////////////////////////////// + +/// Maximum length for filter and option names. +/// 11 chars + terminating '\0' + sizeof(uint32_t) = 16 bytes +#define NAME_LEN_MAX 11 + + +/// For option_map.flags: Use .u.map to do convert the input value +/// to an integer. Without this flag, .u.range.{min,max} are used +/// as the allowed range for the integer. +#define OPTMAP_USE_NAME_VALUE_MAP 0x01 + +/// For option_map.flags: Allow KiB/MiB/GiB in input string and use them in +/// the stringified output if the value is an exact multiple of these. +/// This is used e.g. for LZMA1/2 dictionary size. +#define OPTMAP_USE_BYTE_SUFFIX 0x02 + +/// For option_map.flags: If the integer value is zero then this option +/// won't be included in the stringified output. It's used e.g. for +/// BCJ filter start offset which usually is zero. +#define OPTMAP_NO_STRFY_ZERO 0x04 + +/// Possible values for option_map.type. Since OPTMAP_TYPE_UINT32 is 0, +/// it doesn't need to be specified in the initializers as it is +/// the implicit value. +enum { + OPTMAP_TYPE_UINT32, + OPTMAP_TYPE_LZMA_MODE, + OPTMAP_TYPE_LZMA_MATCH_FINDER, + OPTMAP_TYPE_LZMA_PRESET, +}; + + +/// This is for mapping string values in options to integers. +/// The last element of an array must have "" as the name. +/// It's used e.g. for match finder names in LZMA1/2. +typedef struct { + const char name[NAME_LEN_MAX + 1]; + const uint32_t value; +} name_value_map; + + +/// Each filter that has options needs an array of option_map structures. +/// The array doesn't need to be terminated as the functions take the +/// length of the array as an argument. +/// +/// When converting a string to filter options structure, option values +/// will be handled in a few different ways: +/// +/// (1) If .type equals OPTMAP_TYPE_LZMA_PRESET then LZMA1/2 preset string +/// is handled specially. +/// +/// (2) If .flags has OPTMAP_USE_NAME_VALUE_MAP set then the string is +/// converted to an integer using the name_value_map pointed by .u.map. +/// The last element in .u.map must have .name = "" as the terminator. +/// +/// (3) Otherwise the string is treated as a non-negative unsigned decimal +/// integer which must be in the range set in .u.range. If .flags has +/// OPTMAP_USE_BYTE_SUFFIX then KiB, MiB, and GiB suffixes are allowed. +/// +/// The integer value from (2) or (3) is then stored to filter_options +/// at the offset specified in .offset using the type specified in .type +/// (default is uint32_t). +/// +/// Stringifying a filter is done by processing a given number of options +/// in order from the beginning of an option_map array. The integer is +/// read from filter_options at .offset using the type from .type. +/// +/// If the integer is zero and .flags has OPTMAP_NO_STRFY_ZERO then the +/// option is skipped. +/// +/// If .flags has OPTMAP_USE_NAME_VALUE_MAP set then .u.map will be used +/// to convert the option to a string. If the map doesn't contain a string +/// for the integer value then "UNKNOWN" is used. +/// +/// If .flags doesn't have OPTMAP_USE_NAME_VALUE_MAP set then the integer is +/// converted to a decimal value. If OPTMAP_USE_BYTE_SUFFIX is used then KiB, +/// MiB, or GiB suffix is used if the value is an exact multiple of these. +/// Plain "B" suffix is never used. +typedef struct { + char name[NAME_LEN_MAX + 1]; + uint8_t type; + uint8_t flags; + uint16_t offset; + + union { + // NVHPC has problems with unions that contain pointers that + // are not the first members, so keep "map" at the top. + const name_value_map *map; + + struct { + uint32_t min; + uint32_t max; + } range; + } u; +} option_map; + + +static const char *parse_options(const char **const str, const char *str_end, + void *filter_options, + const option_map *const optmap, const size_t optmap_size); + + +///////// +// BCJ // +///////// + +#if defined(HAVE_ENCODER_X86) \ + || defined(HAVE_DECODER_X86) \ + || defined(HAVE_ENCODER_ARM) \ + || defined(HAVE_DECODER_ARM) \ + || defined(HAVE_ENCODER_ARMTHUMB) \ + || defined(HAVE_DECODER_ARMTHUMB) \ + || defined(HAVE_ENCODER_ARM64) \ + || defined(HAVE_DECODER_ARM64) \ + || defined(HAVE_ENCODER_POWERPC) \ + || defined(HAVE_DECODER_POWERPC) \ + || defined(HAVE_ENCODER_IA64) \ + || defined(HAVE_DECODER_IA64) \ + || defined(HAVE_ENCODER_SPARC) \ + || defined(HAVE_DECODER_SPARC) \ + || defined(HAVE_ENCODER_RISCV) \ + || defined(HAVE_DECODER_RISCV) +static const option_map bcj_optmap[] = { + { + .name = "start", + .flags = OPTMAP_NO_STRFY_ZERO | OPTMAP_USE_BYTE_SUFFIX, + .offset = offsetof(lzma_options_bcj, start_offset), + .u.range.min = 0, + .u.range.max = UINT32_MAX, + } +}; + + +static const char * +parse_bcj(const char **const str, const char *str_end, void *filter_options) +{ + // filter_options was zeroed on allocation and that is enough + // for the default value. + return parse_options(str, str_end, filter_options, + bcj_optmap, ARRAY_SIZE(bcj_optmap)); +} +#endif + + +/////////// +// Delta // +/////////// + +#if defined(HAVE_ENCODER_DELTA) || defined(HAVE_DECODER_DELTA) +static const option_map delta_optmap[] = { + { + .name = "dist", + .offset = offsetof(lzma_options_delta, dist), + .u.range.min = LZMA_DELTA_DIST_MIN, + .u.range.max = LZMA_DELTA_DIST_MAX, + } +}; + + +static const char * +parse_delta(const char **const str, const char *str_end, void *filter_options) +{ + lzma_options_delta *opts = filter_options; + opts->type = LZMA_DELTA_TYPE_BYTE; + opts->dist = LZMA_DELTA_DIST_MIN; + + return parse_options(str, str_end, filter_options, + delta_optmap, ARRAY_SIZE(delta_optmap)); +} +#endif + + +/////////////////// +// LZMA1 & LZMA2 // +/////////////////// + +/// Help string for presets +#define LZMA12_PRESET_STR "0-9[e]" + + +static const char * +parse_lzma12_preset(const char **const str, const char *str_end, + uint32_t *preset) +{ + assert(*str < str_end); + *preset = (uint32_t)(**str - '0'); + + // NOTE: Remember to update LZMA12_PRESET_STR if this is modified! + while (++*str < str_end) { + switch (**str) { + case 'e': + *preset |= LZMA_PRESET_EXTREME; + break; + + default: + return "Unsupported preset flag"; + } + } + + return NULL; +} + + +static const char * +set_lzma12_preset(const char **const str, const char *str_end, + void *filter_options) +{ + uint32_t preset; + const char *errmsg = parse_lzma12_preset(str, str_end, &preset); + if (errmsg != NULL) + return errmsg; + + lzma_options_lzma *opts = filter_options; + if (lzma_lzma_preset(opts, preset)) + return "Unsupported preset"; + + return NULL; +} + + +static const name_value_map lzma12_mode_map[] = { + { "fast", LZMA_MODE_FAST }, + { "normal", LZMA_MODE_NORMAL }, + { "", 0 } +}; + + +static const name_value_map lzma12_mf_map[] = { + { "hc3", LZMA_MF_HC3 }, + { "hc4", LZMA_MF_HC4 }, + { "bt2", LZMA_MF_BT2 }, + { "bt3", LZMA_MF_BT3 }, + { "bt4", LZMA_MF_BT4 }, + { "", 0 } +}; + + +static const option_map lzma12_optmap[] = { + { + .name = "preset", + .type = OPTMAP_TYPE_LZMA_PRESET, + }, { + .name = "dict", + .flags = OPTMAP_USE_BYTE_SUFFIX, + .offset = offsetof(lzma_options_lzma, dict_size), + .u.range.min = LZMA_DICT_SIZE_MIN, + // FIXME? The max is really max for encoding but decoding + // would allow 4 GiB - 1 B. + .u.range.max = (UINT32_C(1) << 30) + (UINT32_C(1) << 29), + }, { + .name = "lc", + .offset = offsetof(lzma_options_lzma, lc), + .u.range.min = LZMA_LCLP_MIN, + .u.range.max = LZMA_LCLP_MAX, + }, { + .name = "lp", + .offset = offsetof(lzma_options_lzma, lp), + .u.range.min = LZMA_LCLP_MIN, + .u.range.max = LZMA_LCLP_MAX, + }, { + .name = "pb", + .offset = offsetof(lzma_options_lzma, pb), + .u.range.min = LZMA_PB_MIN, + .u.range.max = LZMA_PB_MAX, + }, { + .name = "mode", + .type = OPTMAP_TYPE_LZMA_MODE, + .flags = OPTMAP_USE_NAME_VALUE_MAP, + .offset = offsetof(lzma_options_lzma, mode), + .u.map = lzma12_mode_map, + }, { + .name = "nice", + .offset = offsetof(lzma_options_lzma, nice_len), + .u.range.min = 2, + .u.range.max = 273, + }, { + .name = "mf", + .type = OPTMAP_TYPE_LZMA_MATCH_FINDER, + .flags = OPTMAP_USE_NAME_VALUE_MAP, + .offset = offsetof(lzma_options_lzma, mf), + .u.map = lzma12_mf_map, + }, { + .name = "depth", + .offset = offsetof(lzma_options_lzma, depth), + .u.range.min = 0, + .u.range.max = UINT32_MAX, + } +}; + + +static const char * +parse_lzma12(const char **const str, const char *str_end, void *filter_options) +{ + lzma_options_lzma *opts = filter_options; + + // It cannot fail. + const bool preset_ret = lzma_lzma_preset(opts, LZMA_PRESET_DEFAULT); + assert(!preset_ret); + (void)preset_ret; + + const char *errmsg = parse_options(str, str_end, filter_options, + lzma12_optmap, ARRAY_SIZE(lzma12_optmap)); + if (errmsg != NULL) + return errmsg; + + if (opts->lc + opts->lp > LZMA_LCLP_MAX) + return "The sum of lc and lp must not exceed 4"; + + return NULL; +} + + +///////////////////////////////////////// +// Generic parsing and stringification // +///////////////////////////////////////// + +static const struct { + /// Name of the filter + char name[NAME_LEN_MAX + 1]; + + /// For lzma_str_to_filters: + /// Size of the filter-specific options structure. + uint32_t opts_size; + + /// Filter ID + lzma_vli id; + + /// For lzma_str_to_filters: + /// Function to parse the filter-specific options. The filter_options + /// will already have been allocated using lzma_alloc_zero(). + const char *(*parse)(const char **str, const char *str_end, + void *filter_options); + + /// For lzma_str_from_filters: + /// If the flag LZMA_STR_ENCODER is used then the first + /// strfy_encoder elements of optmap are stringified. + /// With LZMA_STR_DECODER strfy_decoder is used. + /// Currently encoders use all options that decoders do but if + /// that changes then this needs to be changed too, for example, + /// add a new OPTMAP flag to skip printing some decoder-only options. + const option_map *optmap; + uint8_t strfy_encoder; + uint8_t strfy_decoder; + + /// For lzma_str_from_filters: + /// If true, lzma_filter.options is allowed to be NULL. In that case, + /// only the filter name is printed without any options. + bool allow_null; + +} filter_name_map[] = { +#if defined (HAVE_ENCODER_LZMA1) || defined(HAVE_DECODER_LZMA1) + { "lzma1", sizeof(lzma_options_lzma), LZMA_FILTER_LZMA1, + &parse_lzma12, lzma12_optmap, 9, 5, false }, +#endif + +#if defined(HAVE_ENCODER_LZMA2) || defined(HAVE_DECODER_LZMA2) + { "lzma2", sizeof(lzma_options_lzma), LZMA_FILTER_LZMA2, + &parse_lzma12, lzma12_optmap, 9, 2, false }, +#endif + +#if defined(HAVE_ENCODER_X86) || defined(HAVE_DECODER_X86) + { "x86", sizeof(lzma_options_bcj), LZMA_FILTER_X86, + &parse_bcj, bcj_optmap, 1, 1, true }, +#endif + +#if defined(HAVE_ENCODER_ARM) || defined(HAVE_DECODER_ARM) + { "arm", sizeof(lzma_options_bcj), LZMA_FILTER_ARM, + &parse_bcj, bcj_optmap, 1, 1, true }, +#endif + +#if defined(HAVE_ENCODER_ARMTHUMB) || defined(HAVE_DECODER_ARMTHUMB) + { "armthumb", sizeof(lzma_options_bcj), LZMA_FILTER_ARMTHUMB, + &parse_bcj, bcj_optmap, 1, 1, true }, +#endif + +#if defined(HAVE_ENCODER_ARM64) || defined(HAVE_DECODER_ARM64) + { "arm64", sizeof(lzma_options_bcj), LZMA_FILTER_ARM64, + &parse_bcj, bcj_optmap, 1, 1, true }, +#endif + +#if defined(HAVE_ENCODER_RISCV) || defined(HAVE_DECODER_RISCV) + { "riscv", sizeof(lzma_options_bcj), LZMA_FILTER_RISCV, + &parse_bcj, bcj_optmap, 1, 1, true }, +#endif + +#if defined(HAVE_ENCODER_POWERPC) || defined(HAVE_DECODER_POWERPC) + { "powerpc", sizeof(lzma_options_bcj), LZMA_FILTER_POWERPC, + &parse_bcj, bcj_optmap, 1, 1, true }, +#endif + +#if defined(HAVE_ENCODER_IA64) || defined(HAVE_DECODER_IA64) + { "ia64", sizeof(lzma_options_bcj), LZMA_FILTER_IA64, + &parse_bcj, bcj_optmap, 1, 1, true }, +#endif + +#if defined(HAVE_ENCODER_SPARC) || defined(HAVE_DECODER_SPARC) + { "sparc", sizeof(lzma_options_bcj), LZMA_FILTER_SPARC, + &parse_bcj, bcj_optmap, 1, 1, true }, +#endif + +#if defined(HAVE_ENCODER_DELTA) || defined(HAVE_DECODER_DELTA) + { "delta", sizeof(lzma_options_delta), LZMA_FILTER_DELTA, + &parse_delta, delta_optmap, 1, 1, false }, +#endif +}; + + +/// Decodes options from a string for one filter (name1=value1,name2=value2). +/// Caller must have allocated memory for filter_options already and set +/// the initial default values. This is called from the filter-specific +/// parse_* functions. +/// +/// The input string starts at *str and the address in str_end is the first +/// char that is not part of the string anymore. So no '\0' terminator is +/// used. *str is advanced every time something has been decoded successfully. +static const char * +parse_options(const char **const str, const char *str_end, + void *filter_options, + const option_map *const optmap, const size_t optmap_size) +{ + while (*str < str_end && **str != '\0') { + // Each option is of the form name=value. + // Commas (',') separate options. Extra commas are ignored. + // Ignoring extra commas makes it simpler if an optional + // option stored in a shell variable which can be empty. + if (**str == ',') { + ++*str; + continue; + } + + // Find where the next name=value ends. + const size_t str_len = (size_t)(str_end - *str); + const char *name_eq_value_end = memchr(*str, ',', str_len); + if (name_eq_value_end == NULL) + name_eq_value_end = str_end; + + const char *equals_sign = memchr(*str, '=', + (size_t)(name_eq_value_end - *str)); + + // Fail if the '=' wasn't found or the option name is missing + // (the first char is '='). + if (equals_sign == NULL || **str == '=') + return "Options must be 'name=value' pairs separated " + "with commas"; + + // Reject a too long option name so that the memcmp() + // in the loop below won't read past the end of the + // string in optmap[i].name. + const size_t name_len = (size_t)(equals_sign - *str); + if (name_len > NAME_LEN_MAX) + return "Unknown option name"; + + // Find the option name from optmap[]. + size_t i = 0; + while (true) { + if (i == optmap_size) + return "Unknown option name"; + + if (memcmp(*str, optmap[i].name, name_len) == 0 + && optmap[i].name[name_len] == '\0') + break; + + ++i; + } + + // The input string is good at least until the start of + // the option value. + *str = equals_sign + 1; + + // The code assumes that the option value isn't an empty + // string so check it here. + const size_t value_len = (size_t)(name_eq_value_end - *str); + if (value_len == 0) + return "Option value cannot be empty"; + + // LZMA1/2 preset has its own parsing function. + if (optmap[i].type == OPTMAP_TYPE_LZMA_PRESET) { + const char *errmsg = set_lzma12_preset(str, + name_eq_value_end, filter_options); + if (errmsg != NULL) + return errmsg; + + continue; + } + + // It's an integer value. + uint32_t v; + if (optmap[i].flags & OPTMAP_USE_NAME_VALUE_MAP) { + // The integer is picked from a string-to-integer map. + // + // Reject a too long value string so that the memcmp() + // in the loop below won't read past the end of the + // string in optmap[i].u.map[j].name. + if (value_len > NAME_LEN_MAX) + return "Invalid option value"; + + const name_value_map *map = optmap[i].u.map; + size_t j = 0; + while (true) { + // The array is terminated with an empty name. + if (map[j].name[0] == '\0') + return "Invalid option value"; + + if (memcmp(*str, map[j].name, value_len) == 0 + && map[j].name[value_len] + == '\0') { + v = map[j].value; + break; + } + + ++j; + } + } else if (**str < '0' || **str > '9') { + // Note that "max" isn't supported while it is + // supported in xz. It's not useful here. + return "Value is not a non-negative decimal integer"; + } else { + // strtoul() has locale-specific behavior so it cannot + // be relied on to get reproducible results since we + // cannot change the locate in a thread-safe library. + // It also needs '\0'-termination. + // + // Use a temporary pointer so that *str will point + // to the beginning of the value string in case + // an error occurs. + const char *p = *str; + v = 0; + do { + if (v > UINT32_MAX / 10) + return "Value out of range"; + + v *= 10; + + const uint32_t add = (uint32_t)(*p - '0'); + if (UINT32_MAX - add < v) + return "Value out of range"; + + v += add; + ++p; + } while (p < name_eq_value_end + && *p >= '0' && *p <= '9'); + + if (p < name_eq_value_end) { + // Remember this position so that it can be + // used for error messages that are + // specifically about the suffix. (Out of + // range values are about the whole value + // and those error messages point to the + // beginning of the number part, + // not to the suffix.) + const char *multiplier_start = p; + + // If multiplier suffix shouldn't be used + // then don't allow them even if the value + // would stay within limits. This is a somewhat + // unnecessary check but it rejects silly + // things like lzma2:pb=0MiB which xz allows. + if ((optmap[i].flags & OPTMAP_USE_BYTE_SUFFIX) + == 0) { + *str = multiplier_start; + return "This option does not support " + "any integer suffixes"; + } + + uint32_t shift; + + switch (*p) { + case 'k': + case 'K': + shift = 10; + break; + + case 'm': + case 'M': + shift = 20; + break; + + case 'g': + case 'G': + shift = 30; + break; + + default: + *str = multiplier_start; + return "Invalid multiplier suffix " + "(KiB, MiB, or GiB)"; + } + + ++p; + + // Allow "M", "Mi", "MB", "MiB" and the same + // for the other five characters from the + // switch-statement above. All are handled + // as base-2 (perhaps a mistake, perhaps not). + // Note that 'i' and 'B' are case sensitive. + if (p < name_eq_value_end && *p == 'i') + ++p; + + if (p < name_eq_value_end && *p == 'B') + ++p; + + // Now we must have no chars remaining. + if (p < name_eq_value_end) { + *str = multiplier_start; + return "Invalid multiplier suffix " + "(KiB, MiB, or GiB)"; + } + + if (v > (UINT32_MAX >> shift)) + return "Value out of range"; + + v <<= shift; + } + + if (v < optmap[i].u.range.min + || v > optmap[i].u.range.max) + return "Value out of range"; + } + + // Set the value in filter_options. Enums are handled + // specially since the underlying type isn't the same + // as uint32_t on all systems. + void *ptr = (char *)filter_options + optmap[i].offset; + switch (optmap[i].type) { + case OPTMAP_TYPE_LZMA_MODE: + *(lzma_mode *)ptr = (lzma_mode)v; + break; + + case OPTMAP_TYPE_LZMA_MATCH_FINDER: + *(lzma_match_finder *)ptr = (lzma_match_finder)v; + break; + + default: + *(uint32_t *)ptr = v; + break; + } + + // This option has been successfully handled. + *str = name_eq_value_end; + } + + // No errors. + return NULL; +} + + +/// Finds the name of the filter at the beginning of the string and +/// calls filter_name_map[i].parse() to decode the filter-specific options. +/// The caller must have set str_end so that exactly one filter and its +/// options are present without any trailing characters. +static const char * +parse_filter(const char **const str, const char *str_end, lzma_filter *filter, + const lzma_allocator *allocator, bool only_xz) +{ + // Search for a colon or equals sign that would separate the filter + // name from filter options. If neither is found, then the input + // string only contains a filter name and there are no options. + // + // First assume that a colon or equals sign won't be found: + const char *name_end = str_end; + const char *opts_start = str_end; + + for (const char *p = *str; p < str_end; ++p) { + if (*p == ':' || *p == '=') { + name_end = p; + + // Filter options (name1=value1,name2=value2,...) + // begin after the colon or equals sign. + opts_start = p + 1; + break; + } + } + + // Reject a too long filter name so that the memcmp() + // in the loop below won't read past the end of the + // string in filter_name_map[i].name. + const size_t name_len = (size_t)(name_end - *str); + if (name_len > NAME_LEN_MAX) + return "Unknown filter name"; + + for (size_t i = 0; i < ARRAY_SIZE(filter_name_map); ++i) { + if (memcmp(*str, filter_name_map[i].name, name_len) == 0 + && filter_name_map[i].name[name_len] == '\0') { + if (only_xz && filter_name_map[i].id + >= LZMA_FILTER_RESERVED_START) + return "This filter cannot be used in " + "the .xz format"; + + // Allocate the filter-specific options and + // initialize the memory with zeros. + void *options = lzma_alloc_zero( + filter_name_map[i].opts_size, + allocator); + if (options == NULL) + return "Memory allocation failed"; + + // Filter name was found so the input string is good + // at least this far. + *str = opts_start; + + const char *errmsg = filter_name_map[i].parse( + str, str_end, options); + if (errmsg != NULL) { + lzma_free(options, allocator); + return errmsg; + } + + // *filter is modified only when parsing is successful. + filter->id = filter_name_map[i].id; + filter->options = options; + return NULL; + } + } + + return "Unknown filter name"; +} + + +/// Converts the string to a filter chain (array of lzma_filter structures). +/// +/// *str is advanced every time something has been decoded successfully. +/// This way the caller knows where in the string a possible error occurred. +static const char * +str_to_filters(const char **const str, lzma_filter *filters, uint32_t flags, + const lzma_allocator *allocator) +{ + const char *errmsg; + + // Skip leading spaces. + while (**str == ' ') + ++*str; + + if (**str == '\0') + return "Empty string is not allowed, " + "try \"6\" if a default value is needed"; + + // Detect the type of the string. + // + // A string beginning with a digit or a string beginning with + // one dash and a digit are treated as presets. Trailing spaces + // will be ignored too (leading spaces were already ignored above). + // + // For example, "6", "7 ", "-9e", or " -3 " are treated as presets. + // Strings like "-" or "- " aren't preset. +#define MY_IS_DIGIT(c) ((c) >= '0' && (c) <= '9') + if (MY_IS_DIGIT(**str) || (**str == '-' && MY_IS_DIGIT((*str)[1]))) { + if (**str == '-') + ++*str; + + // Ignore trailing spaces. + const size_t str_len = strlen(*str); + const char *str_end = memchr(*str, ' ', str_len); + if (str_end != NULL) { + // There is at least one trailing space. Check that + // there are no chars other than spaces. + for (size_t i = 1; str_end[i] != '\0'; ++i) + if (str_end[i] != ' ') + return "Unsupported preset"; + } else { + // There are no trailing spaces. Use the whole string. + str_end = *str + str_len; + } + + uint32_t preset; + errmsg = parse_lzma12_preset(str, str_end, &preset); + if (errmsg != NULL) + return errmsg; + + lzma_options_lzma *opts = lzma_alloc(sizeof(*opts), allocator); + if (opts == NULL) + return "Memory allocation failed"; + + if (lzma_lzma_preset(opts, preset)) { + lzma_free(opts, allocator); + return "Unsupported preset"; + } + + filters[0].id = LZMA_FILTER_LZMA2; + filters[0].options = opts; + filters[1].id = LZMA_VLI_UNKNOWN; + filters[1].options = NULL; + + return NULL; + } + + // Not a preset so it must be a filter chain. + // + // If LZMA_STR_ALL_FILTERS isn't used we allow only filters that + // can be used in .xz. + const bool only_xz = (flags & LZMA_STR_ALL_FILTERS) == 0; + + // Use a temporary array so that we don't modify the caller-supplied + // one until we know that no errors occurred. + lzma_filter temp_filters[LZMA_FILTERS_MAX + 1]; + + size_t i = 0; + do { + if (i == LZMA_FILTERS_MAX) { + errmsg = "The maximum number of filters is four"; + goto error; + } + + // Skip "--" if present. + if ((*str)[0] == '-' && (*str)[1] == '-') + *str += 2; + + // Locate the end of "filter:name1=value1,name2=value2", + // stopping at the first "--" or a single space. + const char *filter_end = *str; + while (filter_end[0] != '\0') { + if ((filter_end[0] == '-' && filter_end[1] == '-') + || filter_end[0] == ' ') + break; + + ++filter_end; + } + + // Inputs that have "--" at the end or "-- " in the middle + // will result in an empty filter name. + if (filter_end == *str) { + errmsg = "Filter name is missing"; + goto error; + } + + errmsg = parse_filter(str, filter_end, &temp_filters[i], + allocator, only_xz); + if (errmsg != NULL) + goto error; + + // Skip trailing spaces. + while (**str == ' ') + ++*str; + + ++i; + } while (**str != '\0'); + + // Seems to be good, terminate the array so that + // basic validation can be done. + temp_filters[i].id = LZMA_VLI_UNKNOWN; + temp_filters[i].options = NULL; + + // Do basic validation if the application didn't prohibit it. + if ((flags & LZMA_STR_NO_VALIDATION) == 0) { + size_t dummy; + const lzma_ret ret = lzma_validate_chain(temp_filters, &dummy); + assert(ret == LZMA_OK || ret == LZMA_OPTIONS_ERROR); + if (ret != LZMA_OK) { + errmsg = "Invalid filter chain " + "('lzma2' missing at the end?)"; + goto error; + } + } + + // All good. Copy the filters to the application supplied array. + memcpy(filters, temp_filters, (i + 1) * sizeof(lzma_filter)); + return NULL; + +error: + // Free the filter options that were successfully decoded. + while (i-- > 0) + lzma_free(temp_filters[i].options, allocator); + + return errmsg; +} + + +extern LZMA_API(const char *) +lzma_str_to_filters(const char *str, int *error_pos, lzma_filter *filters, + uint32_t flags, const lzma_allocator *allocator) +{ + // If error_pos isn't NULL, *error_pos must always be set. + // liblzma <= 5.4.6 and <= 5.6.1 have a bug and don't do this + // when str == NULL or filters == NULL or flags are unsupported. + if (error_pos != NULL) + *error_pos = 0; + + if (str == NULL || filters == NULL) + return "Unexpected NULL pointer argument(s) " + "to lzma_str_to_filters()"; + + // Validate the flags. + const uint32_t supported_flags + = LZMA_STR_ALL_FILTERS + | LZMA_STR_NO_VALIDATION; + + if (flags & ~supported_flags) + return "Unsupported flags to lzma_str_to_filters()"; + + const char *used = str; + const char *errmsg = str_to_filters(&used, filters, flags, allocator); + + if (error_pos != NULL) { + const size_t n = (size_t)(used - str); + *error_pos = n > INT_MAX ? INT_MAX : (int)n; + } + + return errmsg; +} + + +/// Converts options of one filter to a string. +/// +/// The caller must have already put the filter name in the destination +/// string. Since it is possible that no options will be needed, the caller +/// won't have put a delimiter character (':' or '=') in the string yet. +/// We will add it if at least one option will be added to the string. +static void +strfy_filter(lzma_str *dest, const char *delimiter, + const option_map *optmap, size_t optmap_count, + const void *filter_options) +{ + for (size_t i = 0; i < optmap_count; ++i) { + // No attempt is made to reverse LZMA1/2 preset. + if (optmap[i].type == OPTMAP_TYPE_LZMA_PRESET) + continue; + + // All options have integer values, some just are mapped + // to a string with a name_value_map. LZMA1/2 preset + // isn't reversed back to preset=PRESET form. + uint32_t v; + const void *ptr + = (const char *)filter_options + optmap[i].offset; + switch (optmap[i].type) { + case OPTMAP_TYPE_LZMA_MODE: + v = *(const lzma_mode *)ptr; + break; + + case OPTMAP_TYPE_LZMA_MATCH_FINDER: + v = *(const lzma_match_finder *)ptr; + break; + + default: + v = *(const uint32_t *)ptr; + break; + } + + // Skip this if this option should be omitted from + // the string when the value is zero. + if (v == 0 && (optmap[i].flags & OPTMAP_NO_STRFY_ZERO)) + continue; + + // Before the first option we add whatever delimiter + // the caller gave us. For later options a comma is used. + str_append_str(dest, delimiter); + delimiter = ","; + + // Add the option name and equals sign. + str_append_str(dest, optmap[i].name); + str_append_str(dest, "="); + + if (optmap[i].flags & OPTMAP_USE_NAME_VALUE_MAP) { + const name_value_map *map = optmap[i].u.map; + size_t j = 0; + while (true) { + if (map[j].name[0] == '\0') { + str_append_str(dest, "UNKNOWN"); + break; + } + + if (map[j].value == v) { + str_append_str(dest, map[j].name); + break; + } + + ++j; + } + } else { + str_append_u32(dest, v, + optmap[i].flags & OPTMAP_USE_BYTE_SUFFIX); + } + } + + return; +} + + +extern LZMA_API(lzma_ret) +lzma_str_from_filters(char **output_str, const lzma_filter *filters, + uint32_t flags, const lzma_allocator *allocator) +{ + // On error *output_str is always set to NULL. + // Do it as the very first step. + if (output_str == NULL) + return LZMA_PROG_ERROR; + + *output_str = NULL; + + if (filters == NULL) + return LZMA_PROG_ERROR; + + // Validate the flags. + const uint32_t supported_flags + = LZMA_STR_ENCODER + | LZMA_STR_DECODER + | LZMA_STR_GETOPT_LONG + | LZMA_STR_NO_SPACES; + + if (flags & ~supported_flags) + return LZMA_OPTIONS_ERROR; + + // There must be at least one filter. + if (filters[0].id == LZMA_VLI_UNKNOWN) + return LZMA_OPTIONS_ERROR; + + // Allocate memory for the output string. + lzma_str dest; + return_if_error(str_init(&dest, allocator)); + + const bool show_opts = (flags & (LZMA_STR_ENCODER | LZMA_STR_DECODER)); + + const char *opt_delim = (flags & LZMA_STR_GETOPT_LONG) ? "=" : ":"; + + for (size_t i = 0; filters[i].id != LZMA_VLI_UNKNOWN; ++i) { + // If we reach LZMA_FILTERS_MAX, then the filters array + // is too large since the ID cannot be LZMA_VLI_UNKNOWN here. + if (i == LZMA_FILTERS_MAX) { + str_free(&dest, allocator); + return LZMA_OPTIONS_ERROR; + } + + // Don't add a space between filters if the caller + // doesn't want them. + if (i > 0 && !(flags & LZMA_STR_NO_SPACES)) + str_append_str(&dest, " "); + + // Use dashes for xz getopt_long() compatible syntax but also + // use dashes to separate filters when spaces weren't wanted. + if ((flags & LZMA_STR_GETOPT_LONG) + || (i > 0 && (flags & LZMA_STR_NO_SPACES))) + str_append_str(&dest, "--"); + + size_t j = 0; + while (true) { + if (j == ARRAY_SIZE(filter_name_map)) { + // Filter ID in filters[i].id isn't supported. + str_free(&dest, allocator); + return LZMA_OPTIONS_ERROR; + } + + if (filter_name_map[j].id == filters[i].id) { + // Add the filter name. + str_append_str(&dest, filter_name_map[j].name); + + // If only the filter names were wanted then + // skip to the next filter. In this case + // .options is ignored and may be NULL even + // when the filter doesn't allow NULL options. + if (!show_opts) + break; + + if (filters[i].options == NULL) { + if (!filter_name_map[j].allow_null) { + // Filter-specific options + // are missing but with + // this filter the options + // structure is mandatory. + str_free(&dest, allocator); + return LZMA_OPTIONS_ERROR; + } + + // .options is allowed to be NULL. + // There is no need to add any + // options to the string. + break; + } + + // Options structure is available. Add + // the filter options to the string. + const size_t optmap_count + = (flags & LZMA_STR_ENCODER) + ? filter_name_map[j].strfy_encoder + : filter_name_map[j].strfy_decoder; + strfy_filter(&dest, opt_delim, + filter_name_map[j].optmap, + optmap_count, + filters[i].options); + break; + } + + ++j; + } + } + + return str_finish(output_str, &dest, allocator); +} + + +extern LZMA_API(lzma_ret) +lzma_str_list_filters(char **output_str, lzma_vli filter_id, uint32_t flags, + const lzma_allocator *allocator) +{ + // On error *output_str is always set to NULL. + // Do it as the very first step. + if (output_str == NULL) + return LZMA_PROG_ERROR; + + *output_str = NULL; + + // Validate the flags. + const uint32_t supported_flags + = LZMA_STR_ALL_FILTERS + | LZMA_STR_ENCODER + | LZMA_STR_DECODER + | LZMA_STR_GETOPT_LONG; + + if (flags & ~supported_flags) + return LZMA_OPTIONS_ERROR; + + // Allocate memory for the output string. + lzma_str dest; + return_if_error(str_init(&dest, allocator)); + + // If only listing the filter names then separate them with spaces. + // Otherwise use newlines. + const bool show_opts = (flags & (LZMA_STR_ENCODER | LZMA_STR_DECODER)); + const char *filter_delim = show_opts ? "\n" : " "; + + const char *opt_delim = (flags & LZMA_STR_GETOPT_LONG) ? "=" : ":"; + bool first_filter_printed = false; + + for (size_t i = 0; i < ARRAY_SIZE(filter_name_map); ++i) { + // If we are printing only one filter then skip others. + if (filter_id != LZMA_VLI_UNKNOWN + && filter_id != filter_name_map[i].id) + continue; + + // If we are printing only .xz filters then skip the others. + if (filter_name_map[i].id >= LZMA_FILTER_RESERVED_START + && (flags & LZMA_STR_ALL_FILTERS) == 0 + && filter_id == LZMA_VLI_UNKNOWN) + continue; + + // Add a new line if this isn't the first filter being + // written to the string. + if (first_filter_printed) + str_append_str(&dest, filter_delim); + + first_filter_printed = true; + + if (flags & LZMA_STR_GETOPT_LONG) + str_append_str(&dest, "--"); + + str_append_str(&dest, filter_name_map[i].name); + + // If only the filter names were wanted then continue + // to the next filter. + if (!show_opts) + continue; + + const option_map *optmap = filter_name_map[i].optmap; + const char *d = opt_delim; + + const size_t end = (flags & LZMA_STR_ENCODER) + ? filter_name_map[i].strfy_encoder + : filter_name_map[i].strfy_decoder; + + for (size_t j = 0; j < end; ++j) { + // The first option is delimited from the filter + // name using "=" or ":" and the rest of the options + // are separated with ",". + str_append_str(&dest, d); + d = ","; + + // optname=<possible_values> + str_append_str(&dest, optmap[j].name); + str_append_str(&dest, "=<"); + + if (optmap[j].type == OPTMAP_TYPE_LZMA_PRESET) { + // LZMA1/2 preset has its custom help string. + str_append_str(&dest, LZMA12_PRESET_STR); + } else if (optmap[j].flags + & OPTMAP_USE_NAME_VALUE_MAP) { + // Separate the possible option values by "|". + const name_value_map *m = optmap[j].u.map; + for (size_t k = 0; m[k].name[0] != '\0'; ++k) { + if (k > 0) + str_append_str(&dest, "|"); + + str_append_str(&dest, m[k].name); + } + } else { + // Integer range is shown as min-max. + const bool use_byte_suffix = optmap[j].flags + & OPTMAP_USE_BYTE_SUFFIX; + str_append_u32(&dest, optmap[j].u.range.min, + use_byte_suffix); + str_append_str(&dest, "-"); + str_append_u32(&dest, optmap[j].u.range.max, + use_byte_suffix); + } + + str_append_str(&dest, ">"); + } + } + + // If no filters were added to the string then it must be because + // the caller provided an unsupported Filter ID. + if (!first_filter_printed) { + str_free(&dest, allocator); + return LZMA_OPTIONS_ERROR; + } + + return str_finish(output_str, &dest, allocator); +}
diff --git a/Utilities/cmliblzma/liblzma/common/vli_decoder.c b/Utilities/cmliblzma/liblzma/common/vli_decoder.c index af2799d..3254ccc 100644 --- a/Utilities/cmliblzma/liblzma/common/vli_decoder.c +++ b/Utilities/cmliblzma/liblzma/common/vli_decoder.c
@@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file vli_decoder.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "common.h"
diff --git a/Utilities/cmliblzma/liblzma/common/vli_encoder.c b/Utilities/cmliblzma/liblzma/common/vli_encoder.c index f864269..3859006 100644 --- a/Utilities/cmliblzma/liblzma/common/vli_encoder.c +++ b/Utilities/cmliblzma/liblzma/common/vli_encoder.c
@@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file vli_encoder.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "common.h"
diff --git a/Utilities/cmliblzma/liblzma/common/vli_size.c b/Utilities/cmliblzma/liblzma/common/vli_size.c index ec1b4fa..c8cb2ec 100644 --- a/Utilities/cmliblzma/liblzma/common/vli_size.c +++ b/Utilities/cmliblzma/liblzma/common/vli_size.c
@@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file vli_size.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "common.h"
diff --git a/Utilities/cmliblzma/liblzma/delta/delta_common.c b/Utilities/cmliblzma/liblzma/delta/delta_common.c index 4768201..5dbe253 100644 --- a/Utilities/cmliblzma/liblzma/delta/delta_common.c +++ b/Utilities/cmliblzma/liblzma/delta/delta_common.c
@@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file delta_common.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "delta_common.h"
diff --git a/Utilities/cmliblzma/liblzma/delta/delta_common.h b/Utilities/cmliblzma/liblzma/delta/delta_common.h index 7e7e1ba..bd09127 100644 --- a/Utilities/cmliblzma/liblzma/delta/delta_common.h +++ b/Utilities/cmliblzma/liblzma/delta/delta_common.h
@@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file delta_common.h @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef LZMA_DELTA_COMMON_H
diff --git a/Utilities/cmliblzma/liblzma/delta/delta_decoder.c b/Utilities/cmliblzma/liblzma/delta/delta_decoder.c index 13d8a28..9f0d49c 100644 --- a/Utilities/cmliblzma/liblzma/delta/delta_decoder.c +++ b/Utilities/cmliblzma/liblzma/delta/delta_decoder.c
@@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file delta_decoder.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "delta_decoder.h" @@ -26,6 +25,11 @@ } +// For an unknown reason NVIDIA HPC Compiler needs this pragma +// to produce working code. +#ifdef __NVCOMPILER +# pragma routine novector +#endif static lzma_ret delta_decode(void *coder_ptr, const lzma_allocator *allocator, const uint8_t *restrict in, size_t *restrict in_pos, @@ -42,7 +46,12 @@ in, in_pos, in_size, out, out_pos, out_size, action); - decode_buffer(coder, out + out_start, *out_pos - out_start); + // out might be NULL. In that case size == 0. Null pointer + 0 is + // undefined behavior so skip the call in that case as it would + // do nothing anyway. + const size_t size = *out_pos - out_start; + if (size > 0) + decode_buffer(coder, out + out_start, size); return ret; }
diff --git a/Utilities/cmliblzma/liblzma/delta/delta_decoder.h b/Utilities/cmliblzma/liblzma/delta/delta_decoder.h index ad89cc6..e2268ed 100644 --- a/Utilities/cmliblzma/liblzma/delta/delta_decoder.h +++ b/Utilities/cmliblzma/liblzma/delta/delta_decoder.h
@@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file delta_decoder.h @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef LZMA_DELTA_DECODER_H
diff --git a/Utilities/cmliblzma/liblzma/delta/delta_encoder.c b/Utilities/cmliblzma/liblzma/delta/delta_encoder.c index 3841651..ba4a50b 100644 --- a/Utilities/cmliblzma/liblzma/delta/delta_encoder.c +++ b/Utilities/cmliblzma/liblzma/delta/delta_encoder.c
@@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file delta_encoder.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "delta_encoder.h" @@ -63,7 +62,12 @@ const size_t out_avail = out_size - *out_pos; const size_t size = my_min(in_avail, out_avail); - copy_and_encode(coder, in + *in_pos, out + *out_pos, size); + // in and out might be NULL. In such cases size == 0. + // Null pointer + 0 is undefined behavior so skip + // the call in that case as it would do nothing anyway. + if (size > 0) + copy_and_encode(coder, in + *in_pos, out + *out_pos, + size); *in_pos += size; *out_pos += size; @@ -78,7 +82,10 @@ in, in_pos, in_size, out, out_pos, out_size, action); - encode_in_place(coder, out + out_start, *out_pos - out_start); + // Like above, avoid null pointer + 0. + const size_t size = *out_pos - out_start; + if (size > 0) + encode_in_place(coder, out + out_start, size); } return ret;
diff --git a/Utilities/cmliblzma/liblzma/delta/delta_encoder.h b/Utilities/cmliblzma/liblzma/delta/delta_encoder.h index 4ab9847..735f0ed 100644 --- a/Utilities/cmliblzma/liblzma/delta/delta_encoder.h +++ b/Utilities/cmliblzma/liblzma/delta/delta_encoder.h
@@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file delta_encoder.h @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef LZMA_DELTA_ENCODER_H
diff --git a/Utilities/cmliblzma/liblzma/delta/delta_private.h b/Utilities/cmliblzma/liblzma/delta/delta_private.h index 0d6cb38..e54721a 100644 --- a/Utilities/cmliblzma/liblzma/delta/delta_private.h +++ b/Utilities/cmliblzma/liblzma/delta/delta_private.h
@@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file delta_private.h @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef LZMA_DELTA_PRIVATE_H
diff --git a/Utilities/cmliblzma/liblzma/liblzma.pc.in b/Utilities/cmliblzma/liblzma/liblzma.pc.in index 9fa4891..a432992 100644 --- a/Utilities/cmliblzma/liblzma/liblzma.pc.in +++ b/Utilities/cmliblzma/liblzma/liblzma.pc.in
@@ -1,9 +1,5 @@ -# +# SPDX-License-Identifier: 0BSD # Author: Lasse Collin -# -# This file has been put into the public domain. -# You can do whatever you want with this file. -# prefix=@prefix@ exec_prefix=@exec_prefix@ @@ -15,5 +11,6 @@ URL: @PACKAGE_URL@ Version: @PACKAGE_VERSION@ Cflags: -I${includedir} +Cflags.private: -DLZMA_API_STATIC Libs: -L${libdir} -llzma Libs.private: @PTHREAD_CFLAGS@ @LIBS@
diff --git a/Utilities/cmliblzma/liblzma/liblzma_generic.map b/Utilities/cmliblzma/liblzma/liblzma_generic.map new file mode 100644 index 0000000..f74c154 --- /dev/null +++ b/Utilities/cmliblzma/liblzma/liblzma_generic.map
@@ -0,0 +1,128 @@ +/* SPDX-License-Identifier: 0BSD */ + +XZ_5.0 { +global: + lzma_alone_decoder; + lzma_alone_encoder; + lzma_auto_decoder; + lzma_block_buffer_bound; + lzma_block_buffer_decode; + lzma_block_buffer_encode; + lzma_block_compressed_size; + lzma_block_decoder; + lzma_block_encoder; + lzma_block_header_decode; + lzma_block_header_encode; + lzma_block_header_size; + lzma_block_total_size; + lzma_block_unpadded_size; + lzma_check_is_supported; + lzma_check_size; + lzma_code; + lzma_crc32; + lzma_crc64; + lzma_easy_buffer_encode; + lzma_easy_decoder_memusage; + lzma_easy_encoder; + lzma_easy_encoder_memusage; + lzma_end; + lzma_filter_decoder_is_supported; + lzma_filter_encoder_is_supported; + lzma_filter_flags_decode; + lzma_filter_flags_encode; + lzma_filter_flags_size; + lzma_filters_copy; + lzma_filters_update; + lzma_get_check; + lzma_index_append; + lzma_index_block_count; + lzma_index_buffer_decode; + lzma_index_buffer_encode; + lzma_index_cat; + lzma_index_checks; + lzma_index_decoder; + lzma_index_dup; + lzma_index_encoder; + lzma_index_end; + lzma_index_file_size; + lzma_index_hash_append; + lzma_index_hash_decode; + lzma_index_hash_end; + lzma_index_hash_init; + lzma_index_hash_size; + lzma_index_init; + lzma_index_iter_init; + lzma_index_iter_locate; + lzma_index_iter_next; + lzma_index_iter_rewind; + lzma_index_memusage; + lzma_index_memused; + lzma_index_size; + lzma_index_stream_count; + lzma_index_stream_flags; + lzma_index_stream_padding; + lzma_index_stream_size; + lzma_index_total_size; + lzma_index_uncompressed_size; + lzma_lzma_preset; + lzma_memlimit_get; + lzma_memlimit_set; + lzma_memusage; + lzma_mf_is_supported; + lzma_mode_is_supported; + lzma_physmem; + lzma_properties_decode; + lzma_properties_encode; + lzma_properties_size; + lzma_raw_buffer_decode; + lzma_raw_buffer_encode; + lzma_raw_decoder; + lzma_raw_decoder_memusage; + lzma_raw_encoder; + lzma_raw_encoder_memusage; + lzma_stream_buffer_bound; + lzma_stream_buffer_decode; + lzma_stream_buffer_encode; + lzma_stream_decoder; + lzma_stream_encoder; + lzma_stream_flags_compare; + lzma_stream_footer_decode; + lzma_stream_footer_encode; + lzma_stream_header_decode; + lzma_stream_header_encode; + lzma_version_number; + lzma_version_string; + lzma_vli_decode; + lzma_vli_encode; + lzma_vli_size; + +local: + *; +}; + +XZ_5.2 { +global: + lzma_block_uncomp_encode; + lzma_cputhreads; + lzma_get_progress; + lzma_stream_encoder_mt; + lzma_stream_encoder_mt_memusage; +} XZ_5.0; + +XZ_5.4 { +global: + lzma_file_info_decoder; + lzma_filters_free; + lzma_lzip_decoder; + lzma_microlzma_decoder; + lzma_microlzma_encoder; + lzma_stream_decoder_mt; + lzma_str_from_filters; + lzma_str_list_filters; + lzma_str_to_filters; +} XZ_5.2; + +XZ_5.6.0 { +global: + lzma_mt_block_size; +} XZ_5.4;
diff --git a/Utilities/cmliblzma/liblzma/liblzma_linux.map b/Utilities/cmliblzma/liblzma/liblzma_linux.map new file mode 100644 index 0000000..7e4b25e --- /dev/null +++ b/Utilities/cmliblzma/liblzma/liblzma_linux.map
@@ -0,0 +1,143 @@ +/* SPDX-License-Identifier: 0BSD */ + +XZ_5.0 { +global: + lzma_alone_decoder; + lzma_alone_encoder; + lzma_auto_decoder; + lzma_block_buffer_bound; + lzma_block_buffer_decode; + lzma_block_buffer_encode; + lzma_block_compressed_size; + lzma_block_decoder; + lzma_block_encoder; + lzma_block_header_decode; + lzma_block_header_encode; + lzma_block_header_size; + lzma_block_total_size; + lzma_block_unpadded_size; + lzma_check_is_supported; + lzma_check_size; + lzma_code; + lzma_crc32; + lzma_crc64; + lzma_easy_buffer_encode; + lzma_easy_decoder_memusage; + lzma_easy_encoder; + lzma_easy_encoder_memusage; + lzma_end; + lzma_filter_decoder_is_supported; + lzma_filter_encoder_is_supported; + lzma_filter_flags_decode; + lzma_filter_flags_encode; + lzma_filter_flags_size; + lzma_filters_copy; + lzma_filters_update; + lzma_get_check; + lzma_index_append; + lzma_index_block_count; + lzma_index_buffer_decode; + lzma_index_buffer_encode; + lzma_index_cat; + lzma_index_checks; + lzma_index_decoder; + lzma_index_dup; + lzma_index_encoder; + lzma_index_end; + lzma_index_file_size; + lzma_index_hash_append; + lzma_index_hash_decode; + lzma_index_hash_end; + lzma_index_hash_init; + lzma_index_hash_size; + lzma_index_init; + lzma_index_iter_init; + lzma_index_iter_locate; + lzma_index_iter_next; + lzma_index_iter_rewind; + lzma_index_memusage; + lzma_index_memused; + lzma_index_size; + lzma_index_stream_count; + lzma_index_stream_flags; + lzma_index_stream_padding; + lzma_index_stream_size; + lzma_index_total_size; + lzma_index_uncompressed_size; + lzma_lzma_preset; + lzma_memlimit_get; + lzma_memlimit_set; + lzma_memusage; + lzma_mf_is_supported; + lzma_mode_is_supported; + lzma_physmem; + lzma_properties_decode; + lzma_properties_encode; + lzma_properties_size; + lzma_raw_buffer_decode; + lzma_raw_buffer_encode; + lzma_raw_decoder; + lzma_raw_decoder_memusage; + lzma_raw_encoder; + lzma_raw_encoder_memusage; + lzma_stream_buffer_bound; + lzma_stream_buffer_decode; + lzma_stream_buffer_encode; + lzma_stream_decoder; + lzma_stream_encoder; + lzma_stream_flags_compare; + lzma_stream_footer_decode; + lzma_stream_footer_encode; + lzma_stream_header_decode; + lzma_stream_header_encode; + lzma_version_number; + lzma_version_string; + lzma_vli_decode; + lzma_vli_encode; + lzma_vli_size; + +local: + *; +}; + +XZ_5.2 { +global: + lzma_block_uncomp_encode; + lzma_cputhreads; + lzma_get_progress; + lzma_stream_encoder_mt; + lzma_stream_encoder_mt_memusage; +} XZ_5.0; + +XZ_5.1.2alpha { +global: + lzma_stream_encoder_mt; + lzma_stream_encoder_mt_memusage; +} XZ_5.0; + +XZ_5.2.2 { +global: + lzma_block_uncomp_encode; + lzma_cputhreads; + lzma_get_progress; + lzma_stream_encoder_mt; + lzma_stream_encoder_mt_memusage; +} XZ_5.1.2alpha; + +XZ_5.4 { +global: + lzma_file_info_decoder; + lzma_filters_free; + lzma_lzip_decoder; + lzma_microlzma_decoder; + lzma_microlzma_encoder; + lzma_stream_decoder_mt; + lzma_str_from_filters; + lzma_str_list_filters; + lzma_str_to_filters; +} XZ_5.2; + +XZ_5.6.0 { +global: + lzma_mt_block_size; +} XZ_5.4;
diff --git a/Utilities/cmliblzma/liblzma/liblzma_w32res.rc b/Utilities/cmliblzma/liblzma/liblzma_w32res.rc index 773caf8..76c1cdf 100644 --- a/Utilities/cmliblzma/liblzma/liblzma_w32res.rc +++ b/Utilities/cmliblzma/liblzma/liblzma_w32res.rc
@@ -1,12 +1,19 @@ +/* SPDX-License-Identifier: 0BSD */ + /* * Author: Lasse Collin - * - * This file has been put into the public domain. - * You can do whatever you want with this file. */ #define MY_TYPE VFT_DLL -#define MY_NAME "liblzma" + +#if defined(__MSYS__) +# define MY_NAME "msys-lzma-5" +#elif defined(__CYGWIN__) +# define MY_NAME "cyglzma-5" +#else +# define MY_NAME "liblzma" +#endif + #define MY_SUFFIX ".dll" #define MY_DESC "liblzma data compression library" #define PACKAGE_NAME "XZ Utils"
diff --git a/Utilities/cmliblzma/liblzma/lz/lz_decoder.c b/Utilities/cmliblzma/liblzma/lz/lz_decoder.c index 09b5743..92913f2 100644 --- a/Utilities/cmliblzma/liblzma/lz/lz_decoder.c +++ b/Utilities/cmliblzma/liblzma/lz/lz_decoder.c
@@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file lz_decoder.c @@ -6,9 +8,6 @@ // Authors: Igor Pavlov // Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// // liblzma supports multiple LZ77-based filters. The LZ part is shared @@ -54,9 +53,10 @@ static void lz_decoder_reset(lzma_coder *coder) { - coder->dict.pos = 0; + coder->dict.pos = 2 * LZ_DICT_REPEAT_MAX; coder->dict.full = 0; - coder->dict.buf[coder->dict.size - 1] = '\0'; + coder->dict.buf[2 * LZ_DICT_REPEAT_MAX - 1] = '\0'; + coder->dict.has_wrapped = false; coder->dict.need_reset = false; return; } @@ -70,8 +70,15 @@ { while (true) { // Wrap the dictionary if needed. - if (coder->dict.pos == coder->dict.size) - coder->dict.pos = 0; + if (coder->dict.pos == coder->dict.size) { + // See the comment of #define LZ_DICT_REPEAT_MAX. + coder->dict.pos = LZ_DICT_REPEAT_MAX; + coder->dict.has_wrapped = true; + memcpy(coder->dict.buf, coder->dict.buf + + coder->dict.size + - LZ_DICT_REPEAT_MAX, + LZ_DICT_REPEAT_MAX); + } // Store the current dictionary position. It is needed to know // where to start copying to the out[] buffer. @@ -212,7 +219,8 @@ lzma_lz_decoder_init(lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter_info *filters, lzma_ret (*lz_init)(lzma_lz_decoder *lz, - const lzma_allocator *allocator, const void *options, + const lzma_allocator *allocator, + lzma_vli id, const void *options, lzma_lz_options *lz_options)) { // Allocate the base structure if it isn't already allocated. @@ -236,7 +244,7 @@ // us the dictionary size. lzma_lz_options lz_options; return_if_error(lz_init(&coder->lz, allocator, - filters[0].options, &lz_options)); + filters[0].id, filters[0].options, &lz_options)); // If the dictionary size is very small, increase it to 4096 bytes. // This is to prevent constant wrapping of the dictionary, which @@ -252,21 +260,31 @@ // dictionary to the output buffer, since applications are // recommended to give aligned buffers to liblzma. // + // Reserve 2 * LZ_DICT_REPEAT_MAX bytes of extra space which is + // needed for alloc_size. + // // Avoid integer overflow. - if (lz_options.dict_size > SIZE_MAX - 15) + if (lz_options.dict_size > SIZE_MAX - 15 - 2 * LZ_DICT_REPEAT_MAX) return LZMA_MEM_ERROR; lz_options.dict_size = (lz_options.dict_size + 15) & ~((size_t)(15)); + // Reserve extra space as explained in the comment + // of #define LZ_DICT_REPEAT_MAX. + const size_t alloc_size + = lz_options.dict_size + 2 * LZ_DICT_REPEAT_MAX; + // Allocate and initialize the dictionary. - if (coder->dict.size != lz_options.dict_size) { + if (coder->dict.size != alloc_size) { lzma_free(coder->dict.buf, allocator); - coder->dict.buf - = lzma_alloc(lz_options.dict_size, allocator); + coder->dict.buf = lzma_alloc(alloc_size, allocator); if (coder->dict.buf == NULL) return LZMA_MEM_ERROR; - coder->dict.size = lz_options.dict_size; + // NOTE: Yes, alloc_size, not lz_options.dict_size. The way + // coder->dict.full is updated will take care that we will + // still reject distances larger than lz_options.dict_size. + coder->dict.size = alloc_size; } lz_decoder_reset(next->coder); @@ -279,9 +297,12 @@ const size_t copy_size = my_min(lz_options.preset_dict_size, lz_options.dict_size); const size_t offset = lz_options.preset_dict_size - copy_size; - memcpy(coder->dict.buf, lz_options.preset_dict + offset, + memcpy(coder->dict.buf + coder->dict.pos, + lz_options.preset_dict + offset, copy_size); - coder->dict.pos = copy_size; + + // dict.pos isn't zero after lz_decoder_reset(). + coder->dict.pos += copy_size; coder->dict.full = copy_size; } @@ -301,11 +322,3 @@ { return sizeof(lzma_coder) + (uint64_t)(dictionary_size); } - - -extern void -lzma_lz_decoder_uncompressed(void *coder_ptr, lzma_vli uncompressed_size) -{ - lzma_coder *coder = coder_ptr; - coder->lz.set_uncompressed(coder->lz.coder, uncompressed_size); -}
diff --git a/Utilities/cmliblzma/liblzma/lz/lz_decoder.h b/Utilities/cmliblzma/liblzma/lz/lz_decoder.h index 754ccf3..cb61b6e 100644 --- a/Utilities/cmliblzma/liblzma/lz/lz_decoder.h +++ b/Utilities/cmliblzma/liblzma/lz/lz_decoder.h
@@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file lz_decoder.h @@ -6,9 +8,6 @@ // Authors: Igor Pavlov // Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef LZMA_LZ_DECODER_H @@ -17,10 +16,28 @@ #include "common.h" +/// Maximum length of a match rounded up to a nice power of 2 which is +/// a good size for aligned memcpy(). The allocated dictionary buffer will +/// be 2 * LZ_DICT_REPEAT_MAX bytes larger than the actual dictionary size: +/// +/// (1) Every time the decoder reaches the end of the dictionary buffer, +/// the last LZ_DICT_REPEAT_MAX bytes will be copied to the beginning. +/// This way dict_repeat() will only need to copy from one place, +/// never from both the end and beginning of the buffer. +/// +/// (2) The other LZ_DICT_REPEAT_MAX bytes is kept as a buffer between +/// the oldest byte still in the dictionary and the current write +/// position. This way dict_repeat(dict, dict->size - 1, &len) +/// won't need memmove() as the copying cannot overlap. +/// +/// Note that memcpy() still cannot be used if distance < len. +/// +/// LZMA's longest match length is 273 so pick a multiple of 16 above that. +#define LZ_DICT_REPEAT_MAX 288 + + typedef struct { - /// Pointer to the dictionary buffer. It can be an allocated buffer - /// internal to liblzma, or it can a be a buffer given by the - /// application when in single-call mode (not implemented yet). + /// Pointer to the dictionary buffer. uint8_t *buf; /// Write position in dictionary. The next byte will be written to @@ -35,9 +52,16 @@ /// Write limit size_t limit; - /// Size of the dictionary + /// Allocated size of buf. This is 2 * LZ_DICT_REPEAT_MAX bytes + /// larger than the actual dictionary size. This is enforced by + /// how the value for "full" is set; it can be at most + /// "size - 2 * LZ_DICT_REPEAT_MAX". size_t size; + /// True once the dictionary has become full and the writing position + /// has been wrapped in decode_buffer() in lz_decoder.c. + bool has_wrapped; + /// True when dictionary should be reset before decoding more data. bool need_reset; @@ -62,8 +86,10 @@ void (*reset)(void *coder, const void *options); - /// Set the uncompressed size - void (*set_uncompressed)(void *coder, lzma_vli uncompressed_size); + /// Set the uncompressed size. If uncompressed_size == LZMA_VLI_UNKNOWN + /// then allow_eopm will always be true. + void (*set_uncompressed)(void *coder, lzma_vli uncompressed_size, + bool allow_eopm); /// Free allocated resources void (*end)(void *coder, const lzma_allocator *allocator); @@ -85,14 +111,12 @@ const lzma_allocator *allocator, const lzma_filter_info *filters, lzma_ret (*lz_init)(lzma_lz_decoder *lz, - const lzma_allocator *allocator, const void *options, + const lzma_allocator *allocator, + lzma_vli id, const void *options, lzma_lz_options *lz_options)); extern uint64_t lzma_lz_decoder_memusage(size_t dictionary_size); -extern void lzma_lz_decoder_uncompressed( - void *coder, lzma_vli uncompressed_size); - ////////////////////// // Inline functions // @@ -103,7 +127,16 @@ dict_get(const lzma_dict *const dict, const uint32_t distance) { return dict->buf[dict->pos - distance - 1 - + (distance < dict->pos ? 0 : dict->size)]; + + (distance < dict->pos + ? 0 : dict->size - LZ_DICT_REPEAT_MAX)]; +} + + +/// Optimized version of dict_get(dict, 0) +static inline uint8_t +dict_get0(const lzma_dict *const dict) +{ + return dict->buf[dict->pos - 1]; } @@ -132,68 +165,51 @@ uint32_t left = my_min(dict_avail, *len); *len -= left; + size_t back = dict->pos - distance - 1; + if (distance >= dict->pos) + back += dict->size - LZ_DICT_REPEAT_MAX; + // Repeat a block of data from the history. Because memcpy() is faster // than copying byte by byte in a loop, the copying process gets split - // into three cases. + // into two cases. if (distance < left) { // Source and target areas overlap, thus we can't use // memcpy() nor even memmove() safely. do { - dict->buf[dict->pos] = dict_get(dict, distance); - ++dict->pos; + dict->buf[dict->pos++] = dict->buf[back++]; } while (--left > 0); - - } else if (distance < dict->pos) { - // The easiest and fastest case - memcpy(dict->buf + dict->pos, - dict->buf + dict->pos - distance - 1, - left); - dict->pos += left; - } else { - // The bigger the dictionary, the more rare this - // case occurs. We need to "wrap" the dict, thus - // we might need two memcpy() to copy all the data. - assert(dict->full == dict->size); - const uint32_t copy_pos - = dict->pos - distance - 1 + dict->size; - uint32_t copy_size = dict->size - copy_pos; - - if (copy_size < left) { - memmove(dict->buf + dict->pos, dict->buf + copy_pos, - copy_size); - dict->pos += copy_size; - copy_size = left - copy_size; - memcpy(dict->buf + dict->pos, dict->buf, copy_size); - dict->pos += copy_size; - } else { - memmove(dict->buf + dict->pos, dict->buf + copy_pos, - left); - dict->pos += left; - } + memcpy(dict->buf + dict->pos, dict->buf + back, left); + dict->pos += left; } // Update how full the dictionary is. - if (dict->full < dict->pos) - dict->full = dict->pos; + if (!dict->has_wrapped) + dict->full = dict->pos - 2 * LZ_DICT_REPEAT_MAX; - return unlikely(*len != 0); + return *len != 0; +} + + +static inline void +dict_put(lzma_dict *dict, uint8_t byte) +{ + dict->buf[dict->pos++] = byte; + + if (!dict->has_wrapped) + dict->full = dict->pos - 2 * LZ_DICT_REPEAT_MAX; } /// Puts one byte into the dictionary. Returns true if the dictionary was /// already full and the byte couldn't be added. static inline bool -dict_put(lzma_dict *dict, uint8_t byte) +dict_put_safe(lzma_dict *dict, uint8_t byte) { if (unlikely(dict->pos == dict->limit)) return true; - dict->buf[dict->pos++] = byte; - - if (dict->pos > dict->full) - dict->full = dict->pos; - + dict_put(dict, byte); return false; } @@ -217,8 +233,8 @@ *left -= lzma_bufcpy(in, in_pos, in_size, dict->buf, &dict->pos, dict->limit); - if (dict->pos > dict->full) - dict->full = dict->pos; + if (!dict->has_wrapped) + dict->full = dict->pos - 2 * LZ_DICT_REPEAT_MAX; return; }
diff --git a/Utilities/cmliblzma/liblzma/lz/lz_encoder.c b/Utilities/cmliblzma/liblzma/lz/lz_encoder.c index 9a74b7c..4af23e1 100644 --- a/Utilities/cmliblzma/liblzma/lz/lz_encoder.c +++ b/Utilities/cmliblzma/liblzma/lz/lz_encoder.c
@@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file lz_encoder.c @@ -6,9 +8,6 @@ // Authors: Igor Pavlov // Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "lz_encoder.h" @@ -196,9 +195,7 @@ // For now, the dictionary size is limited to 1.5 GiB. This may grow // in the future if needed, but it needs a little more work than just // changing this check. - if (lz_options->dict_size < LZMA_DICT_SIZE_MIN - || lz_options->dict_size - > (UINT32_C(1) << 30) + (UINT32_C(1) << 29) + if (!IS_ENC_DICT_SIZE_VALID(lz_options->dict_size) || lz_options->nice_len > lz_options->match_len_max) return true; @@ -293,11 +290,15 @@ return true; } - // Calculate the sizes of mf->hash and mf->son and check that - // nice_len is big enough for the selected match finder. - const uint32_t hash_bytes = lz_options->match_finder & 0x0F; - if (hash_bytes > mf->nice_len) - return true; + // Calculate the sizes of mf->hash and mf->son. + // + // NOTE: Since 5.3.5beta the LZMA encoder ensures that nice_len + // is big enough for the selected match finder. This makes it + // easier for applications as nice_len = 2 will always be accepted + // even though the effective value can be slightly bigger. + const uint32_t hash_bytes + = mf_get_hash_bytes(lz_options->match_finder); + assert(hash_bytes <= mf->nice_len); const bool is_bt = (lz_options->match_finder & 0x10) != 0; uint32_t hs; @@ -521,15 +522,31 @@ } +static lzma_ret +lz_encoder_set_out_limit(void *coder_ptr, uint64_t *uncomp_size, + uint64_t out_limit) +{ + lzma_coder *coder = coder_ptr; + + // This is supported only if there are no other filters chained. + if (coder->next.code == NULL && coder->lz.set_out_limit != NULL) + return coder->lz.set_out_limit( + coder->lz.coder, uncomp_size, out_limit); + + return LZMA_OPTIONS_ERROR; +} + + extern lzma_ret lzma_lz_encoder_init(lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter_info *filters, lzma_ret (*lz_init)(lzma_lz_encoder *lz, - const lzma_allocator *allocator, const void *options, + const lzma_allocator *allocator, + lzma_vli id, const void *options, lzma_lz_options *lz_options)) { -#ifdef HAVE_SMALL - // We need that the CRC32 table has been initialized. +#if defined(HAVE_SMALL) && !defined(HAVE_FUNC_ATTRIBUTE_CONSTRUCTOR) + // The CRC32 table must be initialized. lzma_crc32_init(); #endif @@ -544,10 +561,13 @@ next->code = &lz_encode; next->end = &lz_encoder_end; next->update = &lz_encoder_update; + next->set_out_limit = &lz_encoder_set_out_limit; coder->lz.coder = NULL; coder->lz.code = NULL; coder->lz.end = NULL; + coder->lz.options_update = NULL; + coder->lz.set_out_limit = NULL; // mf.size is initialized to silence Valgrind // when used on optimized binaries (GCC may reorder @@ -565,7 +585,7 @@ // Initialize the LZ-based encoder. lzma_lz_options lz_options; return_if_error(lz_init(&coder->lz, allocator, - filters[0].options, &lz_options)); + filters[0].id, filters[0].options, &lz_options)); // Setup the size information into coder->mf and deallocate // old buffers if they have wrong size. @@ -585,32 +605,28 @@ extern LZMA_API(lzma_bool) lzma_mf_is_supported(lzma_match_finder mf) { - bool ret = false; - + switch (mf) { #ifdef HAVE_MF_HC3 - if (mf == LZMA_MF_HC3) - ret = true; + case LZMA_MF_HC3: + return true; #endif - #ifdef HAVE_MF_HC4 - if (mf == LZMA_MF_HC4) - ret = true; + case LZMA_MF_HC4: + return true; #endif - #ifdef HAVE_MF_BT2 - if (mf == LZMA_MF_BT2) - ret = true; + case LZMA_MF_BT2: + return true; #endif - #ifdef HAVE_MF_BT3 - if (mf == LZMA_MF_BT3) - ret = true; + case LZMA_MF_BT3: + return true; #endif - #ifdef HAVE_MF_BT4 - if (mf == LZMA_MF_BT4) - ret = true; + case LZMA_MF_BT4: + return true; #endif - - return ret; + default: + return false; + } }
diff --git a/Utilities/cmliblzma/liblzma/lz/lz_encoder.h b/Utilities/cmliblzma/liblzma/lz/lz_encoder.h index 426dcd8..eb197c6 100644 --- a/Utilities/cmliblzma/liblzma/lz/lz_encoder.h +++ b/Utilities/cmliblzma/liblzma/lz/lz_encoder.h
@@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file lz_encoder.h @@ -6,9 +8,6 @@ // Authors: Igor Pavlov // Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef LZMA_LZ_ENCODER_H @@ -17,6 +16,14 @@ #include "common.h" +// For now, the dictionary size is limited to 1.5 GiB. This may grow +// in the future if needed, but it needs a little more work than just +// changing this check. +#define IS_ENC_DICT_SIZE_VALID(size) \ + ((size) >= LZMA_DICT_SIZE_MIN \ + && (size) <= (UINT32_C(1) << 30) + (UINT32_C(1) << 29)) + + /// A table of these is used by the LZ-based encoder to hold /// the length-distance pairs found by the match finder. typedef struct { @@ -153,9 +160,13 @@ /// Maximum search depth uint32_t depth; - /// TODO: Comment + /// Initial dictionary for the match finder to search. const uint8_t *preset_dict; + /// If the preset dictionary is NULL, this value is ignored. + /// Otherwise this member must indicate the preset dictionary's + /// buffer size. If this size is larger than dict_size, then only + /// the dict_size sized tail of the preset_dict will be used. uint32_t preset_dict_size; } lzma_lz_options; @@ -184,7 +195,7 @@ // // Algorithms such as LZMA2 first try to compress a chunk, and then check // if the encoded result is smaller than the uncompressed one. If the chunk -// was uncompressible, it is better to store it in uncompressed form in +// was incompressible, it is better to store it in uncompressed form in // the output stream. To do this, the whole uncompressed chunk has to be // still available in the history buffer. before_size achieves that. @@ -204,6 +215,10 @@ /// Update the options in the middle of the encoding. lzma_ret (*options_update)(void *coder, const lzma_filter *filter); + /// Set maximum allowed output size + lzma_ret (*set_out_limit)(void *coder, uint64_t *uncomp_size, + uint64_t out_limit); + } lzma_lz_encoder; @@ -213,7 +228,16 @@ // 3. The literals and matches are encoded using e.g. LZMA. // // The bytes that have been ran through the match finder, but not encoded yet, -// are called `read ahead'. +// are called 'read ahead'. + + +/// Get how many bytes the match finder hashes in its initial step. +/// This is also the minimum nice_len value with the match finder. +static inline uint32_t +mf_get_hash_bytes(lzma_match_finder match_finder) +{ + return (uint32_t)match_finder & 0x0F; +} /// Get pointer to the first byte not ran through the match finder @@ -298,7 +322,8 @@ lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter_info *filters, lzma_ret (*lz_init)(lzma_lz_encoder *lz, - const lzma_allocator *allocator, const void *options, + const lzma_allocator *allocator, + lzma_vli id, const void *options, lzma_lz_options *lz_options));
diff --git a/Utilities/cmliblzma/liblzma/lz/lz_encoder_hash.h b/Utilities/cmliblzma/liblzma/lz/lz_encoder_hash.h index fb15c58..8ace82b 100644 --- a/Utilities/cmliblzma/liblzma/lz/lz_encoder_hash.h +++ b/Utilities/cmliblzma/liblzma/lz/lz_encoder_hash.h
@@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file lz_encoder_hash.h @@ -5,9 +7,6 @@ // // Author: Igor Pavlov // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef LZMA_LZ_ENCODER_HASH_H @@ -17,6 +16,7 @@ // This is to make liblzma produce the same output on big endian // systems that it does on little endian systems. lz_encoder.c // takes care of including the actual table. + lzma_attr_visibility_hidden extern const uint32_t lzma_lz_hash_table[256]; # define hash_table lzma_lz_hash_table #else
diff --git a/Utilities/cmliblzma/liblzma/lz/lz_encoder_hash_table.h b/Utilities/cmliblzma/liblzma/lz/lz_encoder_hash_table.h index 8c51717..2b3a60e 100644 --- a/Utilities/cmliblzma/liblzma/lz/lz_encoder_hash_table.h +++ b/Utilities/cmliblzma/liblzma/lz/lz_encoder_hash_table.h
@@ -1,4 +1,6 @@ -/* This file has been automatically generated by crc32_tablegen.c. */ +// SPDX-License-Identifier: 0BSD + +// This file has been generated by crc32_tablegen.c. const uint32_t lzma_lz_hash_table[256] = { 0x00000000, 0x77073096, 0xEE0E612C, 0x990951BA,
diff --git a/Utilities/cmliblzma/liblzma/lz/lz_encoder_mf.c b/Utilities/cmliblzma/liblzma/lz/lz_encoder_mf.c index d03657a..557c261 100644 --- a/Utilities/cmliblzma/liblzma/lz/lz_encoder_mf.c +++ b/Utilities/cmliblzma/liblzma/lz/lz_encoder_mf.c
@@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file lz_encoder_mf.c @@ -6,9 +8,6 @@ // Authors: Igor Pavlov // Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "lz_encoder.h" @@ -220,10 +219,11 @@ /// of matches found. #define call_find(func, len_best) \ do { \ - matches_count = func(len_limit, pos, cur, cur_match, mf->depth, \ - mf->son, mf->cyclic_pos, mf->cyclic_size, \ + matches_count = (uint32_t)(func(len_limit, pos, cur, cur_match, \ + mf->depth, mf->son, \ + mf->cyclic_pos, mf->cyclic_size, \ matches + matches_count, len_best) \ - - matches; \ + - matches); \ move_pos(mf); \ return matches_count; \ } while (0) @@ -242,8 +242,8 @@ /// \param cur_match Start position of the current match candidate /// \param depth Maximum length of the hash chain /// \param son lzma_mf.son (contains the hash chain) -/// \param cyclic_pos -/// \param cyclic_size +/// \param cyclic_pos lzma_mf.cyclic_pos +/// \param cyclic_size lzma_mf_cyclic_size /// \param matches Array to hold the matches. /// \param len_best The length of the longest match found so far. static lzma_match *
diff --git a/Utilities/cmliblzma/liblzma/lzma/fastpos.h b/Utilities/cmliblzma/liblzma/lzma/fastpos.h index cba442c..d3969a7 100644 --- a/Utilities/cmliblzma/liblzma/lzma/fastpos.h +++ b/Utilities/cmliblzma/liblzma/lzma/fastpos.h
@@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file fastpos.h @@ -6,9 +8,6 @@ // Authors: Igor Pavlov // Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef LZMA_FASTPOS_H @@ -91,6 +90,7 @@ #define FASTPOS_BITS 13 +lzma_attr_visibility_hidden extern const uint8_t lzma_fastpos[1 << FASTPOS_BITS];
diff --git a/Utilities/cmliblzma/liblzma/lzma/fastpos_table.c b/Utilities/cmliblzma/liblzma/lzma/fastpos_table.c index 6a3ceac..4e10e37 100644 --- a/Utilities/cmliblzma/liblzma/lzma/fastpos_table.c +++ b/Utilities/cmliblzma/liblzma/lzma/fastpos_table.c
@@ -1,4 +1,6 @@ -/* This file has been automatically generated by fastpos_tablegen.c. */ +// SPDX-License-Identifier: 0BSD + +// This file has been generated by fastpos_tablegen.c. #include "common.h" #include "fastpos.h"
diff --git a/Utilities/cmliblzma/liblzma/lzma/fastpos_tablegen.c b/Utilities/cmliblzma/liblzma/lzma/fastpos_tablegen.c index d4484c8..957ccb7 100644 --- a/Utilities/cmliblzma/liblzma/lzma/fastpos_tablegen.c +++ b/Utilities/cmliblzma/liblzma/lzma/fastpos_tablegen.c
@@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file fastpos_tablegen.c @@ -6,13 +8,12 @@ // Authors: Igor Pavlov // Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include <inttypes.h> #include <stdio.h> + +#define lzma_attr_visibility_hidden #include "fastpos.h" @@ -33,11 +34,13 @@ fastpos[c] = slot_fast; } - printf("/* This file has been automatically generated " - "by fastpos_tablegen.c. */\n\n" - "#include \"common.h\"\n" - "#include \"fastpos.h\"\n\n" - "const uint8_t lzma_fastpos[1 << FASTPOS_BITS] = {"); + // Split the SPDX string so that it won't accidentally match + // when tools search for the string. + printf("// SPDX" "-License-Identifier" ": 0BSD\n\n" + "// This file has been generated by fastpos_tablegen.c.\n\n" + "#include \"common.h\"\n" + "#include \"fastpos.h\"\n\n" + "const uint8_t lzma_fastpos[1 << FASTPOS_BITS] = {"); for (size_t i = 0; i < (1 << FASTPOS_BITS); ++i) { if (i % 16 == 0)
diff --git a/Utilities/cmliblzma/liblzma/lzma/lzma2_decoder.c b/Utilities/cmliblzma/liblzma/lzma/lzma2_decoder.c index cf1b511..37ab253 100644 --- a/Utilities/cmliblzma/liblzma/lzma/lzma2_decoder.c +++ b/Utilities/cmliblzma/liblzma/lzma/lzma2_decoder.c
@@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file lzma2_decoder.c @@ -6,9 +8,6 @@ // Authors: Igor Pavlov // Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "lzma2_decoder.h" @@ -139,7 +138,7 @@ coder->uncompressed_size += in[(*in_pos)++] + 1U; coder->sequence = SEQ_COMPRESSED_0; coder->lzma.set_uncompressed(coder->lzma.coder, - coder->uncompressed_size); + coder->uncompressed_size, false); break; case SEQ_COMPRESSED_0: @@ -226,7 +225,8 @@ static lzma_ret lzma2_decoder_init(lzma_lz_decoder *lz, const lzma_allocator *allocator, - const void *opt, lzma_lz_options *lz_options) + lzma_vli id lzma_attribute((__unused__)), const void *opt, + lzma_lz_options *lz_options) { lzma_lzma2_coder *coder = lz->coder; if (coder == NULL) {
diff --git a/Utilities/cmliblzma/liblzma/lzma/lzma2_decoder.h b/Utilities/cmliblzma/liblzma/lzma/lzma2_decoder.h index ef2dcbf..cdd8b46 100644 --- a/Utilities/cmliblzma/liblzma/lzma/lzma2_decoder.h +++ b/Utilities/cmliblzma/liblzma/lzma/lzma2_decoder.h
@@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file lzma2_decoder.h @@ -6,9 +8,6 @@ // Authors: Igor Pavlov // Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef LZMA_LZMA2_DECODER_H
diff --git a/Utilities/cmliblzma/liblzma/lzma/lzma2_encoder.c b/Utilities/cmliblzma/liblzma/lzma/lzma2_encoder.c index 63588ee..e20b75b 100644 --- a/Utilities/cmliblzma/liblzma/lzma/lzma2_encoder.c +++ b/Utilities/cmliblzma/liblzma/lzma/lzma2_encoder.c
@@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file lzma2_encoder.c @@ -6,9 +8,6 @@ // Authors: Igor Pavlov // Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "lz_encoder.h" @@ -310,7 +309,8 @@ static lzma_ret lzma2_encoder_init(lzma_lz_encoder *lz, const lzma_allocator *allocator, - const void *options, lzma_lz_options *lz_options) + lzma_vli id lzma_attribute((__unused__)), const void *options, + lzma_lz_options *lz_options) { if (options == NULL) return LZMA_PROG_ERROR; @@ -340,7 +340,7 @@ // Initialize LZMA encoder return_if_error(lzma_lzma_encoder_create(&coder->lzma, allocator, - &coder->opt_cur, lz_options)); + LZMA_FILTER_LZMA2, &coder->opt_cur, lz_options)); // Make sure that we will always have enough history available in // case we need to use uncompressed chunks. They are used when the @@ -378,6 +378,9 @@ extern lzma_ret lzma_lzma2_props_encode(const void *options, uint8_t *out) { + if (options == NULL) + return LZMA_PROG_ERROR; + const lzma_options_lzma *const opt = options; uint32_t d = my_max(opt->dict_size, LZMA_DICT_SIZE_MIN); @@ -405,6 +408,9 @@ { const lzma_options_lzma *const opt = options; + if (!IS_ENC_DICT_SIZE_VALID(opt->dict_size)) + return UINT64_MAX; + // Use at least 1 MiB to keep compression ratio better. return my_max((uint64_t)(opt->dict_size) * 3, UINT64_C(1) << 20); }
diff --git a/Utilities/cmliblzma/liblzma/lzma/lzma2_encoder.h b/Utilities/cmliblzma/liblzma/lzma/lzma2_encoder.h index 515f183..29966a6 100644 --- a/Utilities/cmliblzma/liblzma/lzma/lzma2_encoder.h +++ b/Utilities/cmliblzma/liblzma/lzma/lzma2_encoder.h
@@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file lzma2_encoder.h @@ -6,9 +8,6 @@ // Authors: Igor Pavlov // Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef LZMA_LZMA2_ENCODER_H
diff --git a/Utilities/cmliblzma/liblzma/lzma/lzma_common.h b/Utilities/cmliblzma/liblzma/lzma/lzma_common.h index 9d040d9..c3c587f 100644 --- a/Utilities/cmliblzma/liblzma/lzma/lzma_common.h +++ b/Utilities/cmliblzma/liblzma/lzma/lzma_common.h
@@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file lzma_common.h @@ -6,9 +8,6 @@ // Authors: Igor Pavlov // Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef LZMA_LZMA_COMMON_H @@ -84,6 +83,20 @@ ? (state) - 3 \ : (state) - 6)) +/// Like update_literal(state) but when it is already known that +/// is_literal_state(state) is true. +#define update_literal_normal(state) \ + state = ((state) <= STATE_SHORTREP_LIT_LIT \ + ? STATE_LIT_LIT \ + : (state) - 3); + +/// Like update_literal(state) but when it is already known that +/// is_literal_state(state) is false. +#define update_literal_matched(state) \ + state = ((state) <= STATE_LIT_SHORTREP \ + ? (state) - 3 \ + : (state) - 6); + /// Indicate that the latest state was a match. #define update_match(state) \ state = ((state) < LIT_STATES ? STATE_LIT_MATCH : STATE_NONLIT_MATCH) @@ -112,31 +125,33 @@ /// /// Match byte is used when the previous LZMA symbol was something else than /// a literal (that is, it was some kind of match). -#define LITERAL_CODER_SIZE 0x300 +#define LITERAL_CODER_SIZE UINT32_C(0x300) /// Maximum number of literal coders #define LITERAL_CODERS_MAX (1 << LZMA_LCLP_MAX) +/// Calculates the literal_mask that literal_subcoder() needs. +#define literal_mask_calc(lc, lp) \ + ((UINT32_C(0x100) << (lp)) - (UINT32_C(0x100) >> (lc))) + /// Locate the literal coder for the next literal byte. The choice depends on /// - the lowest literal_pos_bits bits of the position of the current /// byte; and /// - the highest literal_context_bits bits of the previous byte. -#define literal_subcoder(probs, lc, lp_mask, pos, prev_byte) \ - ((probs)[(((pos) & (lp_mask)) << (lc)) \ - + ((uint32_t)(prev_byte) >> (8U - (lc)))]) +#define literal_subcoder(probs, lc, literal_mask, pos, prev_byte) \ + ((probs) + UINT32_C(3) * \ + (((((pos) << 8) + (prev_byte)) & (literal_mask)) << (lc))) static inline void -literal_init(probability (*probs)[LITERAL_CODER_SIZE], - uint32_t lc, uint32_t lp) +literal_init(probability *probs, uint32_t lc, uint32_t lp) { assert(lc + lp <= LZMA_LCLP_MAX); - const uint32_t coders = 1U << (lc + lp); + const size_t coders = LITERAL_CODER_SIZE << (lc + lp); - for (uint32_t i = 0; i < coders; ++i) - for (uint32_t j = 0; j < LITERAL_CODER_SIZE; ++j) - bit_reset(probs[i][j]); + for (size_t i = 0; i < coders; ++i) + bit_reset(probs[i]); return; }
diff --git a/Utilities/cmliblzma/liblzma/lzma/lzma_decoder.c b/Utilities/cmliblzma/liblzma/lzma/lzma_decoder.c index e605a0a..0abed02 100644 --- a/Utilities/cmliblzma/liblzma/lzma/lzma_decoder.c +++ b/Utilities/cmliblzma/liblzma/lzma/lzma_decoder.c
@@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file lzma_decoder.c @@ -5,9 +7,7 @@ /// // Authors: Igor Pavlov // Lasse Collin -// -// This file has been put into the public domain. -// You can do whatever you want with this file. +// Jia Tan // /////////////////////////////////////////////////////////////////////////////// @@ -22,25 +22,20 @@ # pragma GCC diagnostic ignored "-Wimplicit-fallthrough" #endif +// Minimum number of input bytes to safely decode one LZMA symbol. +// The worst case is that we decode 22 bits using probabilities and 26 +// direct bits. This may decode at maximum 20 bytes of input. +#define LZMA_IN_REQUIRED 20 -#ifdef HAVE_SMALL // Macros for (somewhat) size-optimized code. -#define seq_4(seq) seq - -#define seq_6(seq) seq - -#define seq_8(seq) seq - -#define seq_len(seq) \ - seq ## _CHOICE, \ - seq ## _CHOICE2, \ - seq ## _BITTREE - +// This is used to decode the match length (how many bytes must be repeated +// from the dictionary). This version is used in the Resumable mode and +// does not unroll any loops. #define len_decode(target, ld, pos_state, seq) \ do { \ case seq ## _CHOICE: \ - rc_if_0(ld.choice, seq ## _CHOICE) { \ + rc_if_0_safe(ld.choice, seq ## _CHOICE) { \ rc_update_0(ld.choice); \ probs = ld.low[pos_state];\ limit = LEN_LOW_SYMBOLS; \ @@ -48,7 +43,7 @@ } else { \ rc_update_1(ld.choice); \ case seq ## _CHOICE2: \ - rc_if_0(ld.choice2, seq ## _CHOICE2) { \ + rc_if_0_safe(ld.choice2, seq ## _CHOICE2) { \ rc_update_0(ld.choice2); \ probs = ld.mid[pos_state]; \ limit = LEN_MID_SYMBOLS; \ @@ -64,98 +59,39 @@ symbol = 1; \ case seq ## _BITTREE: \ do { \ - rc_bit(probs[symbol], , , seq ## _BITTREE); \ + rc_bit_safe(probs[symbol], , , seq ## _BITTREE); \ } while (symbol < limit); \ target += symbol - limit; \ } while (0) -#else // HAVE_SMALL -// Unrolled versions -#define seq_4(seq) \ - seq ## 0, \ - seq ## 1, \ - seq ## 2, \ - seq ## 3 - -#define seq_6(seq) \ - seq ## 0, \ - seq ## 1, \ - seq ## 2, \ - seq ## 3, \ - seq ## 4, \ - seq ## 5 - -#define seq_8(seq) \ - seq ## 0, \ - seq ## 1, \ - seq ## 2, \ - seq ## 3, \ - seq ## 4, \ - seq ## 5, \ - seq ## 6, \ - seq ## 7 - -#define seq_len(seq) \ - seq ## _CHOICE, \ - seq ## _LOW0, \ - seq ## _LOW1, \ - seq ## _LOW2, \ - seq ## _CHOICE2, \ - seq ## _MID0, \ - seq ## _MID1, \ - seq ## _MID2, \ - seq ## _HIGH0, \ - seq ## _HIGH1, \ - seq ## _HIGH2, \ - seq ## _HIGH3, \ - seq ## _HIGH4, \ - seq ## _HIGH5, \ - seq ## _HIGH6, \ - seq ## _HIGH7 - -#define len_decode(target, ld, pos_state, seq) \ +// This is the faster version of the match length decoder that does not +// worry about being resumable. It unrolls the bittree decoding loop. +#define len_decode_fast(target, ld, pos_state) \ do { \ symbol = 1; \ -case seq ## _CHOICE: \ - rc_if_0(ld.choice, seq ## _CHOICE) { \ + rc_if_0(ld.choice) { \ rc_update_0(ld.choice); \ - rc_bit_case(ld.low[pos_state][symbol], , , seq ## _LOW0); \ - rc_bit_case(ld.low[pos_state][symbol], , , seq ## _LOW1); \ - rc_bit_case(ld.low[pos_state][symbol], , , seq ## _LOW2); \ - target = symbol - LEN_LOW_SYMBOLS + MATCH_LEN_MIN; \ + rc_bittree3(ld.low[pos_state], \ + -LEN_LOW_SYMBOLS + MATCH_LEN_MIN); \ + target = symbol; \ } else { \ rc_update_1(ld.choice); \ -case seq ## _CHOICE2: \ - rc_if_0(ld.choice2, seq ## _CHOICE2) { \ + rc_if_0(ld.choice2) { \ rc_update_0(ld.choice2); \ - rc_bit_case(ld.mid[pos_state][symbol], , , \ - seq ## _MID0); \ - rc_bit_case(ld.mid[pos_state][symbol], , , \ - seq ## _MID1); \ - rc_bit_case(ld.mid[pos_state][symbol], , , \ - seq ## _MID2); \ - target = symbol - LEN_MID_SYMBOLS \ - + MATCH_LEN_MIN + LEN_LOW_SYMBOLS; \ + rc_bittree3(ld.mid[pos_state], -LEN_MID_SYMBOLS \ + + MATCH_LEN_MIN + LEN_LOW_SYMBOLS); \ + target = symbol; \ } else { \ rc_update_1(ld.choice2); \ - rc_bit_case(ld.high[symbol], , , seq ## _HIGH0); \ - rc_bit_case(ld.high[symbol], , , seq ## _HIGH1); \ - rc_bit_case(ld.high[symbol], , , seq ## _HIGH2); \ - rc_bit_case(ld.high[symbol], , , seq ## _HIGH3); \ - rc_bit_case(ld.high[symbol], , , seq ## _HIGH4); \ - rc_bit_case(ld.high[symbol], , , seq ## _HIGH5); \ - rc_bit_case(ld.high[symbol], , , seq ## _HIGH6); \ - rc_bit_case(ld.high[symbol], , , seq ## _HIGH7); \ - target = symbol - LEN_HIGH_SYMBOLS \ + rc_bittree8(ld.high, -LEN_HIGH_SYMBOLS \ + MATCH_LEN_MIN \ - + LEN_LOW_SYMBOLS + LEN_MID_SYMBOLS; \ + + LEN_LOW_SYMBOLS + LEN_MID_SYMBOLS); \ + target = symbol; \ } \ } \ } while (0) -#endif // HAVE_SMALL - /// Length decoder probabilities; see comments in lzma_common.h. typedef struct { @@ -173,7 +109,7 @@ /////////////////// /// Literals; see comments in lzma_common.h. - probability literal[LITERAL_CODERS_MAX][LITERAL_CODER_SIZE]; + probability literal[LITERAL_CODERS_MAX * LITERAL_CODER_SIZE]; /// If 1, it's a match. Otherwise it's a single 8-bit literal. probability is_match[STATES][POS_STATES_MAX]; @@ -232,12 +168,17 @@ uint32_t pos_mask; // (1U << pb) - 1 uint32_t literal_context_bits; - uint32_t literal_pos_mask; + uint32_t literal_mask; /// Uncompressed size as bytes, or LZMA_VLI_UNKNOWN if end of /// payload marker is expected. lzma_vli uncompressed_size; + /// True if end of payload marker (EOPM) is allowed even when + /// uncompressed_size is known; false if EOPM must not be present. + /// This is ignored if uncompressed_size == LZMA_VLI_UNKNOWN. + bool allow_eopm; + //////////////////////////////// // State of incomplete symbol // //////////////////////////////// @@ -246,22 +187,26 @@ enum { SEQ_NORMALIZE, SEQ_IS_MATCH, - seq_8(SEQ_LITERAL), - seq_8(SEQ_LITERAL_MATCHED), + SEQ_LITERAL, + SEQ_LITERAL_MATCHED, SEQ_LITERAL_WRITE, SEQ_IS_REP, - seq_len(SEQ_MATCH_LEN), - seq_6(SEQ_DIST_SLOT), + SEQ_MATCH_LEN_CHOICE, + SEQ_MATCH_LEN_CHOICE2, + SEQ_MATCH_LEN_BITTREE, + SEQ_DIST_SLOT, SEQ_DIST_MODEL, SEQ_DIRECT, - seq_4(SEQ_ALIGN), + SEQ_ALIGN, SEQ_EOPM, SEQ_IS_REP0, SEQ_SHORTREP, SEQ_IS_REP0_LONG, SEQ_IS_REP1, SEQ_IS_REP2, - seq_len(SEQ_REP_LEN), + SEQ_REP_LEN_CHOICE, + SEQ_REP_LEN_CHOICE2, + SEQ_REP_LEN_BITTREE, SEQ_COPY, } sequence; @@ -316,7 +261,7 @@ const size_t dict_start = dict.pos; // Range decoder - rc_to_local(coder->rc, *in_pos); + rc_to_local(coder->rc, *in_pos, LZMA_IN_REQUIRED); // State uint32_t state = coder->state; @@ -335,7 +280,7 @@ uint32_t offset = coder->offset; uint32_t len = coder->len; - const uint32_t literal_pos_mask = coder->literal_pos_mask; + const uint32_t literal_mask = coder->literal_mask; const uint32_t literal_context_bits = coder->literal_context_bits; // Temporary variables @@ -343,15 +288,43 @@ lzma_ret ret = LZMA_OK; - // If uncompressed size is known, there must be no end of payload - // marker. - const bool no_eopm = coder->uncompressed_size - != LZMA_VLI_UNKNOWN; - if (no_eopm && coder->uncompressed_size < dict.limit - dict.pos) - dict.limit = dict.pos + (size_t)(coder->uncompressed_size); + // This is true when the next LZMA symbol is allowed to be EOPM. + // That is, if this is false, then EOPM is considered + // an invalid symbol and we will return LZMA_DATA_ERROR. + // + // EOPM is always required (not just allowed) when + // the uncompressed size isn't known. When uncompressed size + // is known, eopm_is_valid may be set to true later. + bool eopm_is_valid = coder->uncompressed_size == LZMA_VLI_UNKNOWN; - // The main decoder loop. The "switch" is used to restart the decoder at - // correct location. Once restarted, the "switch" is no longer used. + // If uncompressed size is known and there is enough output space + // to decode all the data, limit the available buffer space so that + // the main loop won't try to decode past the end of the stream. + bool might_finish_without_eopm = false; + if (coder->uncompressed_size != LZMA_VLI_UNKNOWN + && coder->uncompressed_size <= dict.limit - dict.pos) { + dict.limit = dict.pos + (size_t)(coder->uncompressed_size); + might_finish_without_eopm = true; + } + + // The main decoder loop. The "switch" is used to resume the decoder at + // correct location. Once resumed, the "switch" is no longer used. + // The decoder loops is split into two modes: + // + // 1 - Non-resumable mode (fast). This is used when it is guaranteed + // there is enough input to decode the next symbol. If the output + // limit is reached, then the decoder loop will save the place + // for the resumable mode to continue. This mode is not used if + // HAVE_SMALL is defined. This is faster than Resumable mode + // because it reduces the number of branches needed and allows + // for more compiler optimizations. + // + // 2 - Resumable mode (slow). This is used when a previous decoder + // loop did not have enough space in the input or output buffers + // to complete. It uses sequence enum values to set remind + // coder->sequence where to resume in the decoder loop. This + // is the only mode used when HAVE_SMALL is defined. + switch (coder->sequence) while (true) { // Calculate new pos_state. This is skipped on the first loop @@ -359,54 +332,392 @@ // variables. pos_state = dict.pos & pos_mask; - case SEQ_NORMALIZE: - case SEQ_IS_MATCH: - if (unlikely(no_eopm && dict.pos == dict.limit)) - break; +#ifndef HAVE_SMALL - rc_if_0(coder->is_match[state][pos_state], SEQ_IS_MATCH) { + /////////////////////////////// + // Non-resumable Mode (fast) // + /////////////////////////////// + + // Go to Resumable mode (1) if there is not enough input to + // safely decode any possible LZMA symbol or (2) if the + // dictionary is full, which may need special checks that + // are only done in the Resumable mode. + if (unlikely(!rc_is_fast_allowed() + || dict.pos == dict.limit)) + goto slow; + + // Decode the first bit from the next LZMA symbol. + // If the bit is a 0, then we handle it as a literal. + // If the bit is a 1, then it is a match of previously + // decoded data. + rc_if_0(coder->is_match[state][pos_state]) { + ///////////////////// + // Decode literal. // + ///////////////////// + + // Update the RC that we have decoded a 0. rc_update_0(coder->is_match[state][pos_state]); - // It's a literal i.e. a single 8-bit byte. + // Get the correct probability array from lp and + // lc params. + probs = literal_subcoder(coder->literal, + literal_context_bits, literal_mask, + dict.pos, dict_get0(&dict)); + + if (is_literal_state(state)) { + update_literal_normal(state); + + // Decode literal without match byte. + rc_bittree8(probs, 0); + } else { + update_literal_matched(state); + + // Decode literal with match byte. + rc_matched_literal(probs, + dict_get(&dict, rep0)); + } + + // Write decoded literal to dictionary + dict_put(&dict, symbol); + continue; + } + + /////////////////// + // Decode match. // + /////////////////// + + // Instead of a new byte we are going to decode a + // distance-length pair. The distance represents how far + // back in the dictionary to begin copying. The length + // represents how many bytes to copy. + + rc_update_1(coder->is_match[state][pos_state]); + + rc_if_0(coder->is_rep[state]) { + /////////////////// + // Simple match. // + /////////////////// + + // Not a repeated match. In this case, + // the length (how many bytes to copy) must be + // decoded first. Then, the distance (where to + // start copying) is decoded. + // + // This is also how we know when we are done + // decoding. If the distance decodes to UINT32_MAX, + // then we know to stop decoding (end of payload + // marker). + + rc_update_0(coder->is_rep[state]); + update_match(state); + + // The latest three match distances are kept in + // memory in case there are repeated matches. + rep3 = rep2; + rep2 = rep1; + rep1 = rep0; + + // Decode the length of the match. + len_decode_fast(len, coder->match_len_decoder, + pos_state); + + // Next, decode the distance into rep0. + + // The next 6 bits determine how to decode the + // rest of the distance. + probs = coder->dist_slot[get_dist_state(len)]; + + rc_bittree6(probs, -DIST_SLOTS); + assert(symbol <= 63); + + if (symbol < DIST_MODEL_START) { + // If the decoded symbol is < DIST_MODEL_START + // then we use its value directly as the + // match distance. No other bits are needed. + // The only possible distance values + // are [0, 3]. + rep0 = symbol; + } else { + // Use the first two bits of symbol as the + // highest bits of the match distance. + + // "limit" represents the number of low bits + // to decode. + limit = (symbol >> 1) - 1; + assert(limit >= 1 && limit <= 30); + rep0 = 2 + (symbol & 1); + + if (symbol < DIST_MODEL_END) { + // When symbol is > DIST_MODEL_START, + // but symbol < DIST_MODEL_END, then + // it can decode distances between + // [4, 127]. + assert(limit <= 5); + rep0 <<= limit; + assert(rep0 <= 96); + + // -1 is fine, because we start + // decoding at probs[1], not probs[0]. + // NOTE: This violates the C standard, + // since we are doing pointer + // arithmetic past the beginning of + // the array. + assert((int32_t)(rep0 - symbol - 1) + >= -1); + assert((int32_t)(rep0 - symbol - 1) + <= 82); + probs = coder->pos_special + rep0 + - symbol - 1; + symbol = 1; + offset = 1; + + // Variable number (1-5) of bits + // from a reverse bittree. This + // isn't worth manual unrolling. + // + // NOTE: Making one or many of the + // variables (probs, symbol, offset, + // or limit) local here (instead of + // using those declared outside the + // main loop) can affect code size + // and performance which isn't a + // surprise but it's not so clear + // what is the best. + do { + rc_bit_add_if_1(probs, + rep0, offset); + offset <<= 1; + } while (--limit > 0); + } else { + // The distance is >= 128. Decode the + // lower bits without probabilities + // except the lowest four bits. + assert(symbol >= 14); + assert(limit >= 6); + + limit -= ALIGN_BITS; + assert(limit >= 2); + + rc_direct(rep0, limit); + + // Decode the lowest four bits using + // probabilities. + rep0 <<= ALIGN_BITS; + rc_bittree_rev4(coder->pos_align); + rep0 += symbol; + + // If the end of payload marker (EOPM) + // is detected, jump to the safe code. + // The EOPM handling isn't speed + // critical at all. + // + // A final normalization is needed + // after the EOPM (there can be a + // dummy byte to read in some cases). + // If the normalization was done here + // in the fast code, it would need to + // be taken into account in the value + // of LZMA_IN_REQUIRED. Using the + // safe code allows keeping + // LZMA_IN_REQUIRED as 20 instead of + // 21. + if (rep0 == UINT32_MAX) + goto eopm; + } + } + + // Validate the distance we just decoded. + if (unlikely(!dict_is_distance_valid(&dict, rep0))) { + ret = LZMA_DATA_ERROR; + goto out; + } + + } else { + rc_update_1(coder->is_rep[state]); + + ///////////////////// + // Repeated match. // + ///////////////////// + + // The match distance is a value that we have decoded + // recently. The latest four match distances are + // available as rep0, rep1, rep2 and rep3. We will + // now decode which of them is the new distance. + // + // There cannot be a match if we haven't produced + // any output, so check that first. + if (unlikely(!dict_is_distance_valid(&dict, 0))) { + ret = LZMA_DATA_ERROR; + goto out; + } + + rc_if_0(coder->is_rep0[state]) { + rc_update_0(coder->is_rep0[state]); + // The distance is rep0. + + // Decode the next bit to determine if 1 byte + // should be copied from rep0 distance or + // if the number of bytes needs to be decoded. + + // If the next bit is 0, then it is a + // "Short Rep Match" and only 1 bit is copied. + // Otherwise, the length of the match is + // decoded after the "else" statement. + rc_if_0(coder->is_rep0_long[state][pos_state]) { + rc_update_0(coder->is_rep0_long[ + state][pos_state]); + + update_short_rep(state); + dict_put(&dict, dict_get(&dict, rep0)); + continue; + } + + // Repeating more than one byte at + // distance of rep0. + rc_update_1(coder->is_rep0_long[ + state][pos_state]); + + } else { + rc_update_1(coder->is_rep0[state]); + + // The distance is rep1, rep2 or rep3. Once + // we find out which one of these three, it + // is stored to rep0 and rep1, rep2 and rep3 + // are updated accordingly. There is no + // "Short Rep Match" option, so the length + // of the match must always be decoded next. + rc_if_0(coder->is_rep1[state]) { + // The distance is rep1. + rc_update_0(coder->is_rep1[state]); + + const uint32_t distance = rep1; + rep1 = rep0; + rep0 = distance; + + } else { + rc_update_1(coder->is_rep1[state]); + + rc_if_0(coder->is_rep2[state]) { + // The distance is rep2. + rc_update_0(coder->is_rep2[ + state]); + + const uint32_t distance = rep2; + rep2 = rep1; + rep1 = rep0; + rep0 = distance; + + } else { + // The distance is rep3. + rc_update_1(coder->is_rep2[ + state]); + + const uint32_t distance = rep3; + rep3 = rep2; + rep2 = rep1; + rep1 = rep0; + rep0 = distance; + } + } + } + + update_long_rep(state); + + // Decode the length of the repeated match. + len_decode_fast(len, coder->rep_len_decoder, + pos_state); + } + + ///////////////////////////////// + // Repeat from history buffer. // + ///////////////////////////////// + + // The length is always between these limits. There is no way + // to trigger the algorithm to set len outside this range. + assert(len >= MATCH_LEN_MIN); + assert(len <= MATCH_LEN_MAX); + + // Repeat len bytes from distance of rep0. + if (unlikely(dict_repeat(&dict, rep0, &len))) { + coder->sequence = SEQ_COPY; + goto out; + } + + continue; + +slow: +#endif + /////////////////////////// + // Resumable Mode (slow) // + /////////////////////////// + + // This is very similar to Non-resumable Mode, so most of the + // comments are not repeated. The main differences are: + // - case labels are used to resume at the correct location. + // - Loops are not unrolled. + // - Range coder macros take an extra sequence argument + // so they can save to coder->sequence the location to + // resume in case there is not enough input. + case SEQ_NORMALIZE: + case SEQ_IS_MATCH: + if (unlikely(might_finish_without_eopm + && dict.pos == dict.limit)) { + // In rare cases there is a useless byte that needs + // to be read anyway. + rc_normalize_safe(SEQ_NORMALIZE); + + // If the range decoder state is such that we can + // be at the end of the LZMA stream, then the + // decoding is finished. + if (rc_is_finished(rc)) { + ret = LZMA_STREAM_END; + goto out; + } + + // If the caller hasn't allowed EOPM to be present + // together with known uncompressed size, then the + // LZMA stream is corrupt. + if (!coder->allow_eopm) { + ret = LZMA_DATA_ERROR; + goto out; + } + + // Otherwise continue decoding with the expectation + // that the next LZMA symbol is EOPM. + eopm_is_valid = true; + } + + rc_if_0_safe(coder->is_match[state][pos_state], SEQ_IS_MATCH) { + ///////////////////// + // Decode literal. // + ///////////////////// + + rc_update_0(coder->is_match[state][pos_state]); probs = literal_subcoder(coder->literal, - literal_context_bits, literal_pos_mask, - dict.pos, dict_get(&dict, 0)); + literal_context_bits, literal_mask, + dict.pos, dict_get0(&dict)); symbol = 1; if (is_literal_state(state)) { + update_literal_normal(state); + // Decode literal without match byte. -#ifdef HAVE_SMALL + // The "slow" version does not unroll + // the loop. case SEQ_LITERAL: do { - rc_bit(probs[symbol], , , SEQ_LITERAL); + rc_bit_safe(probs[symbol], , , + SEQ_LITERAL); } while (symbol < (1 << 8)); -#else - rc_bit_case(probs[symbol], , , SEQ_LITERAL0); - rc_bit_case(probs[symbol], , , SEQ_LITERAL1); - rc_bit_case(probs[symbol], , , SEQ_LITERAL2); - rc_bit_case(probs[symbol], , , SEQ_LITERAL3); - rc_bit_case(probs[symbol], , , SEQ_LITERAL4); - rc_bit_case(probs[symbol], , , SEQ_LITERAL5); - rc_bit_case(probs[symbol], , , SEQ_LITERAL6); - rc_bit_case(probs[symbol], , , SEQ_LITERAL7); -#endif } else { + update_literal_matched(state); + // Decode literal with match byte. - // - // We store the byte we compare against - // ("match byte") to "len" to minimize the - // number of variables we need to store - // between decoder calls. len = (uint32_t)(dict_get(&dict, rep0)) << 1; - // The usage of "offset" allows omitting some - // branches, which should give tiny speed - // improvement on some CPUs. "offset" gets - // set to zero if match_bit didn't match. offset = 0x100; -#ifdef HAVE_SMALL case SEQ_LITERAL_MATCHED: do { const uint32_t match_bit @@ -415,7 +726,7 @@ = offset + match_bit + symbol; - rc_bit(probs[subcoder_index], + rc_bit_safe(probs[subcoder_index], offset &= ~match_bit, offset &= match_bit, SEQ_LITERAL_MATCHED); @@ -428,61 +739,10 @@ len <<= 1; } while (symbol < (1 << 8)); -#else - // Unroll the loop. - uint32_t match_bit; - uint32_t subcoder_index; - -# define d(seq) \ - case seq: \ - match_bit = len & offset; \ - subcoder_index = offset + match_bit + symbol; \ - rc_bit(probs[subcoder_index], \ - offset &= ~match_bit, \ - offset &= match_bit, \ - seq) - - d(SEQ_LITERAL_MATCHED0); - len <<= 1; - d(SEQ_LITERAL_MATCHED1); - len <<= 1; - d(SEQ_LITERAL_MATCHED2); - len <<= 1; - d(SEQ_LITERAL_MATCHED3); - len <<= 1; - d(SEQ_LITERAL_MATCHED4); - len <<= 1; - d(SEQ_LITERAL_MATCHED5); - len <<= 1; - d(SEQ_LITERAL_MATCHED6); - len <<= 1; - d(SEQ_LITERAL_MATCHED7); -# undef d -#endif } - //update_literal(state); - // Use a lookup table to update to literal state, - // since compared to other state updates, this would - // need two branches. - static const lzma_lzma_state next_state[] = { - STATE_LIT_LIT, - STATE_LIT_LIT, - STATE_LIT_LIT, - STATE_LIT_LIT, - STATE_MATCH_LIT_LIT, - STATE_REP_LIT_LIT, - STATE_SHORTREP_LIT_LIT, - STATE_MATCH_LIT, - STATE_REP_LIT, - STATE_SHORTREP_LIT, - STATE_MATCH_LIT, - STATE_REP_LIT - }; - state = next_state[state]; - case SEQ_LITERAL_WRITE: - if (unlikely(dict_put(&dict, symbol))) { + if (dict_put_safe(&dict, symbol)) { coder->sequence = SEQ_LITERAL_WRITE; goto out; } @@ -490,64 +750,47 @@ continue; } - // Instead of a new byte we are going to get a byte range - // (distance and length) which will be repeated from our - // output history. + /////////////////// + // Decode match. // + /////////////////// rc_update_1(coder->is_match[state][pos_state]); case SEQ_IS_REP: - rc_if_0(coder->is_rep[state], SEQ_IS_REP) { - // Not a repeated match + rc_if_0_safe(coder->is_rep[state], SEQ_IS_REP) { + /////////////////// + // Simple match. // + /////////////////// + rc_update_0(coder->is_rep[state]); update_match(state); - // The latest three match distances are kept in - // memory in case there are repeated matches. rep3 = rep2; rep2 = rep1; rep1 = rep0; - // Decode the length of the match. len_decode(len, coder->match_len_decoder, pos_state, SEQ_MATCH_LEN); - // Prepare to decode the highest two bits of the - // match distance. probs = coder->dist_slot[get_dist_state(len)]; symbol = 1; -#ifdef HAVE_SMALL case SEQ_DIST_SLOT: do { - rc_bit(probs[symbol], , , SEQ_DIST_SLOT); + rc_bit_safe(probs[symbol], , , SEQ_DIST_SLOT); } while (symbol < DIST_SLOTS); -#else - rc_bit_case(probs[symbol], , , SEQ_DIST_SLOT0); - rc_bit_case(probs[symbol], , , SEQ_DIST_SLOT1); - rc_bit_case(probs[symbol], , , SEQ_DIST_SLOT2); - rc_bit_case(probs[symbol], , , SEQ_DIST_SLOT3); - rc_bit_case(probs[symbol], , , SEQ_DIST_SLOT4); - rc_bit_case(probs[symbol], , , SEQ_DIST_SLOT5); -#endif - // Get rid of the highest bit that was needed for - // indexing of the probability array. + symbol -= DIST_SLOTS; assert(symbol <= 63); if (symbol < DIST_MODEL_START) { - // Match distances [0, 3] have only two bits. rep0 = symbol; } else { - // Decode the lowest [1, 29] bits of - // the match distance. limit = (symbol >> 1) - 1; assert(limit >= 1 && limit <= 30); rep0 = 2 + (symbol & 1); if (symbol < DIST_MODEL_END) { - // Prepare to decode the low bits for - // a distance of [4, 127]. assert(limit <= 5); rep0 <<= limit; assert(rep0 <= 96); @@ -566,103 +809,54 @@ symbol = 1; offset = 0; case SEQ_DIST_MODEL: -#ifdef HAVE_SMALL do { - rc_bit(probs[symbol], , + rc_bit_safe(probs[symbol], , rep0 += 1U << offset, SEQ_DIST_MODEL); } while (++offset < limit); -#else - switch (limit) { - case 5: - assert(offset == 0); - rc_bit(probs[symbol], , - rep0 += 1U, - SEQ_DIST_MODEL); - ++offset; - --limit; - case 4: - rc_bit(probs[symbol], , - rep0 += 1U << offset, - SEQ_DIST_MODEL); - ++offset; - --limit; - case 3: - rc_bit(probs[symbol], , - rep0 += 1U << offset, - SEQ_DIST_MODEL); - ++offset; - --limit; - case 2: - rc_bit(probs[symbol], , - rep0 += 1U << offset, - SEQ_DIST_MODEL); - ++offset; - --limit; - case 1: - // We need "symbol" only for - // indexing the probability - // array, thus we can use - // rc_bit_last() here to omit - // the unneeded updating of - // "symbol". - rc_bit_last(probs[symbol], , - rep0 += 1U << offset, - SEQ_DIST_MODEL); - } -#endif } else { - // The distance is >= 128. Decode the - // lower bits without probabilities - // except the lowest four bits. assert(symbol >= 14); assert(limit >= 6); limit -= ALIGN_BITS; assert(limit >= 2); case SEQ_DIRECT: - // Not worth manual unrolling - do { - rc_direct(rep0, SEQ_DIRECT); - } while (--limit > 0); + rc_direct_safe(rep0, limit, + SEQ_DIRECT); - // Decode the lowest four bits using - // probabilities. rep0 <<= ALIGN_BITS; - symbol = 1; -#ifdef HAVE_SMALL - offset = 0; + symbol = 0; + offset = 1; case SEQ_ALIGN: do { - rc_bit(coder->pos_align[ - symbol], , - rep0 += 1U << offset, + rc_bit_last_safe( + coder->pos_align[ + offset + + symbol], + , + symbol += offset, SEQ_ALIGN); - } while (++offset < ALIGN_BITS); -#else - case SEQ_ALIGN0: - rc_bit(coder->pos_align[symbol], , - rep0 += 1, SEQ_ALIGN0); - case SEQ_ALIGN1: - rc_bit(coder->pos_align[symbol], , - rep0 += 2, SEQ_ALIGN1); - case SEQ_ALIGN2: - rc_bit(coder->pos_align[symbol], , - rep0 += 4, SEQ_ALIGN2); - case SEQ_ALIGN3: - // Like in SEQ_DIST_MODEL, we don't - // need "symbol" for anything else - // than indexing the probability array. - rc_bit_last(coder->pos_align[symbol], , - rep0 += 8, SEQ_ALIGN3); -#endif + offset <<= 1; + } while (offset < ALIGN_SIZE); + + rep0 += symbol; if (rep0 == UINT32_MAX) { // End of payload marker was - // found. It must not be - // present if uncompressed - // size is known. - if (coder->uncompressed_size - != LZMA_VLI_UNKNOWN) { + // found. It may only be + // present if + // - uncompressed size is + // unknown or + // - after known uncompressed + // size amount of bytes has + // been decompressed and + // caller has indicated + // that EOPM might be used + // (it's not allowed in + // LZMA2). +#ifndef HAVE_SMALL +eopm: +#endif + if (!eopm_is_valid) { ret = LZMA_DATA_ERROR; goto out; } @@ -670,43 +864,39 @@ case SEQ_EOPM: // LZMA1 stream with // end-of-payload marker. - rc_normalize(SEQ_EOPM); - ret = LZMA_STREAM_END; + rc_normalize_safe(SEQ_EOPM); + ret = rc_is_finished(rc) + ? LZMA_STREAM_END + : LZMA_DATA_ERROR; goto out; } } } - // Validate the distance we just decoded. if (unlikely(!dict_is_distance_valid(&dict, rep0))) { ret = LZMA_DATA_ERROR; goto out; } } else { + ///////////////////// + // Repeated match. // + ///////////////////// + rc_update_1(coder->is_rep[state]); - // Repeated match - // - // The match distance is a value that we have had - // earlier. The latest four match distances are - // available as rep0, rep1, rep2 and rep3. We will - // now decode which of them is the new distance. - // - // There cannot be a match if we haven't produced - // any output, so check that first. if (unlikely(!dict_is_distance_valid(&dict, 0))) { ret = LZMA_DATA_ERROR; goto out; } case SEQ_IS_REP0: - rc_if_0(coder->is_rep0[state], SEQ_IS_REP0) { + rc_if_0_safe(coder->is_rep0[state], SEQ_IS_REP0) { rc_update_0(coder->is_rep0[state]); - // The distance is rep0. case SEQ_IS_REP0_LONG: - rc_if_0(coder->is_rep0_long[state][pos_state], + rc_if_0_safe(coder->is_rep0_long + [state][pos_state], SEQ_IS_REP0_LONG) { rc_update_0(coder->is_rep0_long[ state][pos_state]); @@ -714,8 +904,9 @@ update_short_rep(state); case SEQ_SHORTREP: - if (unlikely(dict_put(&dict, dict_get( - &dict, rep0)))) { + if (dict_put_safe(&dict, + dict_get(&dict, + rep0))) { coder->sequence = SEQ_SHORTREP; goto out; } @@ -723,8 +914,6 @@ continue; } - // Repeating more than one byte at - // distance of rep0. rc_update_1(coder->is_rep0_long[ state][pos_state]); @@ -732,11 +921,7 @@ rc_update_1(coder->is_rep0[state]); case SEQ_IS_REP1: - // The distance is rep1, rep2 or rep3. Once - // we find out which one of these three, it - // is stored to rep0 and rep1, rep2 and rep3 - // are updated accordingly. - rc_if_0(coder->is_rep1[state], SEQ_IS_REP1) { + rc_if_0_safe(coder->is_rep1[state], SEQ_IS_REP1) { rc_update_0(coder->is_rep1[state]); const uint32_t distance = rep1; @@ -746,7 +931,7 @@ } else { rc_update_1(coder->is_rep1[state]); case SEQ_IS_REP2: - rc_if_0(coder->is_rep2[state], + rc_if_0_safe(coder->is_rep2[state], SEQ_IS_REP2) { rc_update_0(coder->is_rep2[ state]); @@ -771,7 +956,6 @@ update_long_rep(state); - // Decode the length of the repeated match. len_decode(len, coder->rep_len_decoder, pos_state, SEQ_REP_LEN); } @@ -780,22 +964,16 @@ // Repeat from history buffer. // ///////////////////////////////// - // The length is always between these limits. There is no way - // to trigger the algorithm to set len outside this range. assert(len >= MATCH_LEN_MIN); assert(len <= MATCH_LEN_MAX); case SEQ_COPY: - // Repeat len bytes from distance of rep0. if (unlikely(dict_repeat(&dict, rep0, &len))) { coder->sequence = SEQ_COPY; goto out; } } - rc_normalize(SEQ_NORMALIZE); - coder->sequence = SEQ_IS_MATCH; - out: // Save state @@ -822,36 +1000,34 @@ if (coder->uncompressed_size != LZMA_VLI_UNKNOWN) { coder->uncompressed_size -= dict.pos - dict_start; - // Since there cannot be end of payload marker if the - // uncompressed size was known, we check here if we - // finished decoding. + // If we have gotten all the output but the decoder wants + // to write more output, the file is corrupt. There are + // three SEQ values where output is produced. if (coder->uncompressed_size == 0 && ret == LZMA_OK - && coder->sequence != SEQ_NORMALIZE) - ret = coder->sequence == SEQ_IS_MATCH - ? LZMA_STREAM_END : LZMA_DATA_ERROR; + && (coder->sequence == SEQ_LITERAL_WRITE + || coder->sequence == SEQ_SHORTREP + || coder->sequence == SEQ_COPY)) + ret = LZMA_DATA_ERROR; } - // We can do an additional check in the range decoder to catch some - // corrupted files. if (ret == LZMA_STREAM_END) { - if (!rc_is_finished(coder->rc)) - ret = LZMA_DATA_ERROR; - // Reset the range decoder so that it is ready to reinitialize // for a new LZMA2 chunk. rc_reset(coder->rc); + coder->sequence = SEQ_IS_MATCH; } return ret; } - static void -lzma_decoder_uncompressed(void *coder_ptr, lzma_vli uncompressed_size) +lzma_decoder_uncompressed(void *coder_ptr, lzma_vli uncompressed_size, + bool allow_eopm) { lzma_lzma1_decoder *coder = coder_ptr; coder->uncompressed_size = uncompressed_size; + coder->allow_eopm = allow_eopm; } @@ -871,7 +1047,7 @@ literal_init(coder->literal, options->lc, options->lp); coder->literal_context_bits = options->lc; - coder->literal_pos_mask = (1U << options->lp) - 1; + coder->literal_mask = literal_mask_calc(options->lc, options->lp); // State coder->state = STATE_LIT_LIT; @@ -940,7 +1116,7 @@ extern lzma_ret lzma_lzma_decoder_create(lzma_lz_decoder *lz, const lzma_allocator *allocator, - const void *opt, lzma_lz_options *lz_options) + const lzma_options_lzma *options, lzma_lz_options *lz_options) { if (lz->coder == NULL) { lz->coder = lzma_alloc(sizeof(lzma_lzma1_decoder), allocator); @@ -954,7 +1130,6 @@ // All dictionary sizes are OK here. LZ decoder will take care of // the special cases. - const lzma_options_lzma *options = opt; lz_options->dict_size = options->dict_size; lz_options->preset_dict = options->preset_dict; lz_options->preset_dict_size = options->preset_dict_size; @@ -968,16 +1143,40 @@ /// the LZ initialization). static lzma_ret lzma_decoder_init(lzma_lz_decoder *lz, const lzma_allocator *allocator, - const void *options, lzma_lz_options *lz_options) + lzma_vli id, const void *options, lzma_lz_options *lz_options) { if (!is_lclppb_valid(options)) return LZMA_PROG_ERROR; + lzma_vli uncomp_size = LZMA_VLI_UNKNOWN; + bool allow_eopm = true; + + if (id == LZMA_FILTER_LZMA1EXT) { + const lzma_options_lzma *opt = options; + + // Only one flag is supported. + if (opt->ext_flags & ~LZMA_LZMA1EXT_ALLOW_EOPM) + return LZMA_OPTIONS_ERROR; + + // FIXME? Using lzma_vli instead of uint64_t is weird because + // this has nothing to do with .xz headers and variable-length + // integer encoding. On the other hand, using LZMA_VLI_UNKNOWN + // instead of UINT64_MAX is clearer when unknown size is + // meant. A problem with using lzma_vli is that now we + // allow > LZMA_VLI_MAX which is fine in this file but + // it's still confusing. Note that alone_decoder.c also + // allows > LZMA_VLI_MAX when setting uncompressed size. + uncomp_size = opt->ext_size_low + + ((uint64_t)(opt->ext_size_high) << 32); + allow_eopm = (opt->ext_flags & LZMA_LZMA1EXT_ALLOW_EOPM) != 0 + || uncomp_size == LZMA_VLI_UNKNOWN; + } + return_if_error(lzma_lzma_decoder_create( lz, allocator, options, lz_options)); lzma_decoder_reset(lz->coder, options); - lzma_decoder_uncompressed(lz->coder, LZMA_VLI_UNKNOWN); + lzma_decoder_uncompressed(lz->coder, uncomp_size, allow_eopm); return LZMA_OK; }
diff --git a/Utilities/cmliblzma/liblzma/lzma/lzma_decoder.h b/Utilities/cmliblzma/liblzma/lzma/lzma_decoder.h index fa8ecb2..9730f56 100644 --- a/Utilities/cmliblzma/liblzma/lzma/lzma_decoder.h +++ b/Utilities/cmliblzma/liblzma/lzma/lzma_decoder.h
@@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file lzma_decoder.h @@ -6,9 +8,6 @@ // Authors: Igor Pavlov // Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef LZMA_LZMA_DECODER_H @@ -42,7 +41,7 @@ /// LZMA2 decoders. extern lzma_ret lzma_lzma_decoder_create( lzma_lz_decoder *lz, const lzma_allocator *allocator, - const void *opt, lzma_lz_options *lz_options); + const lzma_options_lzma *opt, lzma_lz_options *lz_options); /// Gets memory usage without validating lc/lp/pb. This is used by LZMA2 /// decoder, because raw LZMA2 decoding doesn't need lc/lp/pb.
diff --git a/Utilities/cmliblzma/liblzma/lzma/lzma_encoder.c b/Utilities/cmliblzma/liblzma/lzma/lzma_encoder.c index 07d2b87..543ca32 100644 --- a/Utilities/cmliblzma/liblzma/lzma/lzma_encoder.c +++ b/Utilities/cmliblzma/liblzma/lzma/lzma_encoder.c
@@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file lzma_encoder.c @@ -6,9 +8,6 @@ // Authors: Igor Pavlov // Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "lzma2_encoder.h" @@ -49,24 +48,24 @@ const uint8_t cur_byte = mf->buffer[ mf->read_pos - mf->read_ahead]; probability *subcoder = literal_subcoder(coder->literal, - coder->literal_context_bits, coder->literal_pos_mask, + coder->literal_context_bits, coder->literal_mask, position, mf->buffer[mf->read_pos - mf->read_ahead - 1]); if (is_literal_state(coder->state)) { // Previous LZMA-symbol was a literal. Encode a normal // literal without a match byte. + update_literal_normal(coder->state); rc_bittree(&coder->rc, subcoder, 8, cur_byte); } else { // Previous LZMA-symbol was a match. Use the last byte of // the match as a "match byte". That is, compare the bits // of the current literal and the match byte. + update_literal_matched(coder->state); const uint8_t match_byte = mf->buffer[ mf->read_pos - coder->reps[0] - 1 - mf->read_ahead]; literal_matched(&coder->rc, subcoder, match_byte, cur_byte); } - - update_literal(coder->state); } @@ -268,6 +267,7 @@ encode_init(lzma_lzma1_encoder *coder, lzma_mf *mf) { assert(mf_position(mf) == 0); + assert(coder->uncomp_size == 0); if (mf->read_pos == mf->read_limit) { if (mf->action == LZMA_RUN) @@ -282,7 +282,8 @@ mf_skip(mf, 1); mf->read_ahead = 0; rc_bit(&coder->rc, &coder->is_match[0][0], 0); - rc_bittree(&coder->rc, coder->literal[0], 8, mf->buffer[0]); + rc_bittree(&coder->rc, coder->literal + 0, 8, mf->buffer[0]); + ++coder->uncomp_size; } // Initialization is done (except if empty file). @@ -317,21 +318,28 @@ if (!coder->is_initialized && !encode_init(coder, mf)) return LZMA_OK; - // Get the lowest bits of the uncompressed offset from the LZ layer. - uint32_t position = mf_position(mf); + // Encode pending output bytes from the range encoder. + // At the start of the stream, encode_init() encodes one literal. + // Later there can be pending output only with LZMA1 because LZMA2 + // ensures that there is always enough output space. Thus when using + // LZMA2, rc_encode() calls in this function will always return false. + if (rc_encode(&coder->rc, out, out_pos, out_size)) { + // We don't get here with LZMA2. + assert(limit == UINT32_MAX); + return LZMA_OK; + } + + // If the range encoder was flushed in an earlier call to this + // function but there wasn't enough output buffer space, those + // bytes would have now been encoded by the above rc_encode() call + // and the stream has now been finished. This can only happen with + // LZMA1 as LZMA2 always provides enough output buffer space. + if (coder->is_flushed) { + assert(limit == UINT32_MAX); + return LZMA_STREAM_END; + } while (true) { - // Encode pending bits, if any. Calling this before encoding - // the next symbol is needed only with plain LZMA, since - // LZMA2 always provides big enough buffer to flush - // everything out from the range encoder. For the same reason, - // rc_encode() never returns true when this function is used - // as part of LZMA2 encoder. - if (rc_encode(&coder->rc, out, out_pos, out_size)) { - assert(limit == UINT32_MAX); - return LZMA_OK; - } - // With LZMA2 we need to take care that compressed size of // a chunk doesn't get too big. // FIXME? Check if this could be improved. @@ -365,37 +373,64 @@ if (coder->fast_mode) lzma_lzma_optimum_fast(coder, mf, &back, &len); else - lzma_lzma_optimum_normal( - coder, mf, &back, &len, position); + lzma_lzma_optimum_normal(coder, mf, &back, &len, + (uint32_t)(coder->uncomp_size)); - encode_symbol(coder, mf, back, len, position); + encode_symbol(coder, mf, back, len, + (uint32_t)(coder->uncomp_size)); - position += len; - } + // If output size limiting is active (out_limit != 0), check + // if encoding this LZMA symbol would make the output size + // exceed the specified limit. + if (coder->out_limit != 0 && rc_encode_dummy( + &coder->rc, coder->out_limit)) { + // The most recent LZMA symbol would make the output + // too big. Throw it away. + rc_forget(&coder->rc); - if (!coder->is_flushed) { - coder->is_flushed = true; + // FIXME: Tell the LZ layer to not read more input as + // it would be waste of time. This doesn't matter if + // output-size-limited encoding is done with a single + // call though. - // We don't support encoding plain LZMA streams without EOPM, - // and LZMA2 doesn't use EOPM at LZMA level. - if (limit == UINT32_MAX) - encode_eopm(coder, position); + break; + } - // Flush the remaining bytes from the range encoder. - rc_flush(&coder->rc); + // This symbol will be encoded so update the uncompressed size. + coder->uncomp_size += len; - // Copy the remaining bytes to the output buffer. If there - // isn't enough output space, we will copy out the remaining - // bytes on the next call to this function by using - // the rc_encode() call in the encoding loop above. + // Encode the LZMA symbol. if (rc_encode(&coder->rc, out, out_pos, out_size)) { + // Once again, this can only happen with LZMA1. assert(limit == UINT32_MAX); return LZMA_OK; } } - // Make it ready for the next LZMA2 chunk. - coder->is_flushed = false; + // Make the uncompressed size available to the application. + if (coder->uncomp_size_ptr != NULL) + *coder->uncomp_size_ptr = coder->uncomp_size; + + // LZMA2 doesn't use EOPM at LZMA level. + // + // Plain LZMA streams without EOPM aren't supported except when + // output size limiting is enabled. + if (coder->use_eopm) + encode_eopm(coder, (uint32_t)(coder->uncomp_size)); + + // Flush the remaining bytes from the range encoder. + rc_flush(&coder->rc); + + // Copy the remaining bytes to the output buffer. If there + // isn't enough output space, we will copy out the remaining + // bytes on the next call to this function. + if (rc_encode(&coder->rc, out, out_pos, out_size)) { + // This cannot happen with LZMA2. + assert(limit == UINT32_MAX); + + coder->is_flushed = true; + return LZMA_OK; + } return LZMA_STREAM_END; } @@ -414,6 +449,23 @@ } +static lzma_ret +lzma_lzma_set_out_limit( + void *coder_ptr, uint64_t *uncomp_size, uint64_t out_limit) +{ + // Minimum output size is 5 bytes but that cannot hold any output + // so we use 6 bytes. + if (out_limit < 6) + return LZMA_BUF_ERROR; + + lzma_lzma1_encoder *coder = coder_ptr; + coder->out_limit = out_limit; + coder->uncomp_size_ptr = uncomp_size; + coder->use_eopm = false; + return LZMA_OK; +} + + //////////////////// // Initialization // //////////////////// @@ -440,7 +492,8 @@ lz_options->dict_size = options->dict_size; lz_options->after_size = LOOP_INPUT_MAX; lz_options->match_len_max = MATCH_LEN_MAX; - lz_options->nice_len = options->nice_len; + lz_options->nice_len = my_max(mf_get_hash_bytes(options->mf), + options->nice_len); lz_options->match_finder = options->mf; lz_options->depth = options->depth; lz_options->preset_dict = options->preset_dict; @@ -481,7 +534,7 @@ coder->pos_mask = (1U << options->pb) - 1; coder->literal_context_bits = options->lc; - coder->literal_pos_mask = (1U << options->lp) - 1; + coder->literal_mask = literal_mask_calc(options->lc, options->lp); // Range coder rc_reset(&coder->rc); @@ -546,10 +599,13 @@ extern lzma_ret -lzma_lzma_encoder_create(void **coder_ptr, - const lzma_allocator *allocator, - const lzma_options_lzma *options, lzma_lz_options *lz_options) +lzma_lzma_encoder_create(void **coder_ptr, const lzma_allocator *allocator, + lzma_vli id, const lzma_options_lzma *options, + lzma_lz_options *lz_options) { + assert(id == LZMA_FILTER_LZMA1 || id == LZMA_FILTER_LZMA1EXT + || id == LZMA_FILTER_LZMA2); + // Allocate lzma_lzma1_encoder if it wasn't already allocated. if (*coder_ptr == NULL) { *coder_ptr = lzma_alloc(sizeof(lzma_lzma1_encoder), allocator); @@ -559,10 +615,9 @@ lzma_lzma1_encoder *coder = *coder_ptr; - // Set compression mode. We haven't validates the options yet, - // but it's OK here, since nothing bad happens with invalid - // options in the code below, and they will get rejected by - // lzma_lzma_encoder_reset() call at the end of this function. + // Set compression mode. Note that we haven't validated the options + // yet. Invalid options will get rejected by lzma_lzma_encoder_reset() + // call at the end of this function. switch (options->mode) { case LZMA_MODE_FAST: coder->fast_mode = true; @@ -573,6 +628,18 @@ // Set dist_table_size. // Round the dictionary size up to next 2^n. + // + // Currently the maximum encoder dictionary size + // is 1.5 GiB due to lz_encoder.c and here we need + // to be below 2 GiB to make the rounded up value + // fit in an uint32_t and avoid an infinite while-loop + // (and undefined behavior due to a too large shift). + // So do the same check as in LZ encoder, + // limiting to 1.5 GiB. + if (options->dict_size > (UINT32_C(1) << 30) + + (UINT32_C(1) << 29)) + return LZMA_OPTIONS_ERROR; + uint32_t log_size = 0; while ((UINT32_C(1) << log_size) < options->dict_size) ++log_size; @@ -580,10 +647,14 @@ coder->dist_table_size = log_size * 2; // Length encoders' price table size + const uint32_t nice_len = my_max( + mf_get_hash_bytes(options->mf), + options->nice_len); + coder->match_len_encoder.table_size - = options->nice_len + 1 - MATCH_LEN_MIN; + = nice_len + 1 - MATCH_LEN_MIN; coder->rep_len_encoder.table_size - = options->nice_len + 1 - MATCH_LEN_MIN; + = nice_len + 1 - MATCH_LEN_MIN; break; } @@ -598,6 +669,37 @@ coder->is_initialized = options->preset_dict != NULL && options->preset_dict_size > 0; coder->is_flushed = false; + coder->uncomp_size = 0; + coder->uncomp_size_ptr = NULL; + + // Output size limiting is disabled by default. + coder->out_limit = 0; + + // Determine if end marker is wanted: + // - It is never used with LZMA2. + // - It is always used with LZMA_FILTER_LZMA1 (unless + // lzma_lzma_set_out_limit() is called later). + // - LZMA_FILTER_LZMA1EXT has a flag for it in the options. + coder->use_eopm = (id == LZMA_FILTER_LZMA1); + if (id == LZMA_FILTER_LZMA1EXT) { + // Check if unsupported flags are present. + if (options->ext_flags & ~LZMA_LZMA1EXT_ALLOW_EOPM) + return LZMA_OPTIONS_ERROR; + + coder->use_eopm = (options->ext_flags + & LZMA_LZMA1EXT_ALLOW_EOPM) != 0; + + // TODO? As long as there are no filters that change the size + // of the data, it is enough to look at lzma_stream.total_in + // after encoding has been finished to know the uncompressed + // size of the LZMA1 stream. But in the future there could be + // filters that change the size of the data and then total_in + // doesn't work as the LZMA1 stream size might be different + // due to another filter in the chain. The problem is simple + // to solve: Add another flag to ext_flags and then set + // coder->uncomp_size_ptr to the address stored in + // lzma_options_lzma.reserved_ptr2 (or _ptr1). + } set_lz_options(lz_options, options); @@ -607,11 +709,15 @@ static lzma_ret lzma_encoder_init(lzma_lz_encoder *lz, const lzma_allocator *allocator, - const void *options, lzma_lz_options *lz_options) + lzma_vli id, const void *options, lzma_lz_options *lz_options) { + if (options == NULL) + return LZMA_PROG_ERROR; + lz->code = &lzma_encode; + lz->set_out_limit = &lzma_lzma_set_out_limit; return lzma_lzma_encoder_create( - &lz->coder, allocator, options, lz_options); + &lz->coder, allocator, id, options, lz_options); } @@ -658,6 +764,9 @@ extern lzma_ret lzma_lzma_props_encode(const void *options, uint8_t *out) { + if (options == NULL) + return LZMA_PROG_ERROR; + const lzma_options_lzma *const opt = options; if (lzma_lzma_lclppb_encode(opt, out))
diff --git a/Utilities/cmliblzma/liblzma/lzma/lzma_encoder.h b/Utilities/cmliblzma/liblzma/lzma/lzma_encoder.h index 6cfdf22..e8ae807 100644 --- a/Utilities/cmliblzma/liblzma/lzma/lzma_encoder.h +++ b/Utilities/cmliblzma/liblzma/lzma/lzma_encoder.h
@@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file lzma_encoder.h @@ -6,9 +8,6 @@ // Authors: Igor Pavlov // Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef LZMA_LZMA_ENCODER_H @@ -40,7 +39,8 @@ /// Initializes raw LZMA encoder; this is used by LZMA2. extern lzma_ret lzma_lzma_encoder_create( void **coder_ptr, const lzma_allocator *allocator, - const lzma_options_lzma *options, lzma_lz_options *lz_options); + lzma_vli id, const lzma_options_lzma *options, + lzma_lz_options *lz_options); /// Resets an already initialized LZMA encoder; this is used by LZMA2.
diff --git a/Utilities/cmliblzma/liblzma/lzma/lzma_encoder_optimum_fast.c b/Utilities/cmliblzma/liblzma/lzma/lzma_encoder_optimum_fast.c index 6c53d2b..0f063d5 100644 --- a/Utilities/cmliblzma/liblzma/lzma/lzma_encoder_optimum_fast.c +++ b/Utilities/cmliblzma/liblzma/lzma/lzma_encoder_optimum_fast.c
@@ -1,12 +1,11 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file lzma_encoder_optimum_fast.c // // Author: Igor Pavlov // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "lzma_encoder_private.h"
diff --git a/Utilities/cmliblzma/liblzma/lzma/lzma_encoder_optimum_normal.c b/Utilities/cmliblzma/liblzma/lzma/lzma_encoder_optimum_normal.c index 101c8d4..a6c0398 100644 --- a/Utilities/cmliblzma/liblzma/lzma/lzma_encoder_optimum_normal.c +++ b/Utilities/cmliblzma/liblzma/lzma/lzma_encoder_optimum_normal.c
@@ -1,12 +1,11 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file lzma_encoder_optimum_normal.c // // Author: Igor Pavlov // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "lzma_encoder_private.h" @@ -24,7 +23,7 @@ uint32_t match_byte, uint32_t symbol) { const probability *const subcoder = literal_subcoder(coder->literal, - coder->literal_context_bits, coder->literal_pos_mask, + coder->literal_context_bits, coder->literal_mask, pos, prev_byte); uint32_t price = 0;
diff --git a/Utilities/cmliblzma/liblzma/lzma/lzma_encoder_presets.c b/Utilities/cmliblzma/liblzma/lzma/lzma_encoder_presets.c index 711df02..e53483f 100644 --- a/Utilities/cmliblzma/liblzma/lzma/lzma_encoder_presets.c +++ b/Utilities/cmliblzma/liblzma/lzma/lzma_encoder_presets.c
@@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file lzma_encoder_presets.c @@ -6,9 +8,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "common.h"
diff --git a/Utilities/cmliblzma/liblzma/lzma/lzma_encoder_private.h b/Utilities/cmliblzma/liblzma/lzma/lzma_encoder_private.h index 2e34aac..eeea5e9 100644 --- a/Utilities/cmliblzma/liblzma/lzma/lzma_encoder_private.h +++ b/Utilities/cmliblzma/liblzma/lzma/lzma_encoder_private.h
@@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file lzma_encoder_private.h @@ -6,9 +8,6 @@ // Authors: Igor Pavlov // Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef LZMA_LZMA_ENCODER_PRIVATE_H @@ -72,6 +71,18 @@ /// Range encoder lzma_range_encoder rc; + /// Uncompressed size (doesn't include possible preset dictionary) + uint64_t uncomp_size; + + /// If non-zero, produce at most this much output. + /// Some input may then be missing from the output. + uint64_t out_limit; + + /// If the above out_limit is non-zero, *uncomp_size_ptr is set to + /// the amount of uncompressed data that we were able to fit + /// in the output buffer. + uint64_t *uncomp_size_ptr; + /// State lzma_lzma_state state; @@ -99,12 +110,15 @@ /// have been written to the output buffer yet. bool is_flushed; + /// True if end of payload marker will be written. + bool use_eopm; + uint32_t pos_mask; ///< (1 << pos_bits) - 1 uint32_t literal_context_bits; - uint32_t literal_pos_mask; + uint32_t literal_mask; // These are the same as in lzma_decoder.c. See comments there. - probability literal[LITERAL_CODERS_MAX][LITERAL_CODER_SIZE]; + probability literal[LITERAL_CODERS_MAX * LITERAL_CODER_SIZE]; probability is_match[STATES][POS_STATES_MAX]; probability is_rep[STATES]; probability is_rep0[STATES];
diff --git a/Utilities/cmliblzma/liblzma/rangecoder/price.h b/Utilities/cmliblzma/liblzma/rangecoder/price.h index 8ae02ca..cce6bda 100644 --- a/Utilities/cmliblzma/liblzma/rangecoder/price.h +++ b/Utilities/cmliblzma/liblzma/rangecoder/price.h
@@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file price.h @@ -5,9 +7,6 @@ // // Author: Igor Pavlov // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef LZMA_PRICE_H @@ -22,6 +21,7 @@ /// Lookup table for the inline functions defined in this file. +lzma_attr_visibility_hidden extern const uint8_t lzma_rc_prices[RC_PRICE_TABLE_SIZE];
diff --git a/Utilities/cmliblzma/liblzma/rangecoder/price_table.c b/Utilities/cmliblzma/liblzma/rangecoder/price_table.c index ac64bf6..c33433f 100644 --- a/Utilities/cmliblzma/liblzma/rangecoder/price_table.c +++ b/Utilities/cmliblzma/liblzma/rangecoder/price_table.c
@@ -1,4 +1,6 @@ -/* This file has been automatically generated by price_tablegen.c. */ +// SPDX-License-Identifier: 0BSD + +// This file has been generated by price_tablegen.c. #include "range_encoder.h"
diff --git a/Utilities/cmliblzma/liblzma/rangecoder/price_tablegen.c b/Utilities/cmliblzma/liblzma/rangecoder/price_tablegen.c index bf08ce3..4b6ca37 100644 --- a/Utilities/cmliblzma/liblzma/rangecoder/price_tablegen.c +++ b/Utilities/cmliblzma/liblzma/rangecoder/price_tablegen.c
@@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file price_tablegen.c @@ -8,13 +10,15 @@ // Authors: Igor Pavlov // Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include <inttypes.h> #include <stdio.h> + +// Make it compile without common.h. +#define BUILDING_PRICE_TABLEGEN +#define lzma_attr_visibility_hidden + #include "range_common.h" #include "price.h" @@ -54,11 +58,13 @@ static void print_price_table(void) { - printf("/* This file has been automatically generated by " - "price_tablegen.c. */\n\n" - "#include \"range_encoder.h\"\n\n" - "const uint8_t lzma_rc_prices[" - "RC_PRICE_TABLE_SIZE] = {"); + // Split the SPDX string so that it won't accidentally match + // when tools search for the string. + printf("// SPDX" "-License-Identifier" ": 0BSD\n\n" + "// This file has been generated by price_tablegen.c.\n\n" + "#include \"range_encoder.h\"\n\n" + "const uint8_t lzma_rc_prices[" + "RC_PRICE_TABLE_SIZE] = {"); const size_t array_size = sizeof(lzma_rc_prices) / sizeof(lzma_rc_prices[0]);
diff --git a/Utilities/cmliblzma/liblzma/rangecoder/range_common.h b/Utilities/cmliblzma/liblzma/rangecoder/range_common.h index 2c74dc1..ac4dbe1 100644 --- a/Utilities/cmliblzma/liblzma/rangecoder/range_common.h +++ b/Utilities/cmliblzma/liblzma/rangecoder/range_common.h
@@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file range_common.h @@ -6,15 +8,15 @@ // Authors: Igor Pavlov // Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef LZMA_RANGE_COMMON_H #define LZMA_RANGE_COMMON_H -#include "common.h" +// Skip common.h if building price_tablegen.c. +#ifndef BUILDING_PRICE_TABLEGEN +# include "common.h" +#endif /////////////// @@ -66,6 +68,10 @@ /// /// I will be sticking to uint16_t unless some specific architectures /// are *much* faster (20-50 %) with uint32_t. +/// +/// Update in 2024: The branchless C and x86-64 assembly was written so that +/// probability is assumed to be uint16_t. (In contrast, LZMA SDK 23.01 +/// assembly supports both types.) typedef uint16_t probability; #endif
diff --git a/Utilities/cmliblzma/liblzma/rangecoder/range_decoder.h b/Utilities/cmliblzma/liblzma/rangecoder/range_decoder.h index e0b051f..a8aca90 100644 --- a/Utilities/cmliblzma/liblzma/rangecoder/range_decoder.h +++ b/Utilities/cmliblzma/liblzma/rangecoder/range_decoder.h
@@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file range_decoder.h @@ -6,9 +8,6 @@ // Authors: Igor Pavlov // Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef LZMA_RANGE_DECODER_H @@ -17,6 +16,55 @@ #include "range_common.h" +// Choose the range decoder variants to use using a bitmask. +// If no bits are set, only the basic version is used. +// If more than one version is selected for the same feature, +// the last one on the list below is used. +// +// Bitwise-or of the following enable branchless C versions: +// 0x01 normal bittrees +// 0x02 fixed-sized reverse bittrees +// 0x04 variable-sized reverse bittrees (not faster) +// 0x08 matched literal (not faster) +// +// GCC & Clang compatible x86-64 inline assembly: +// 0x010 normal bittrees +// 0x020 fixed-sized reverse bittrees +// 0x040 variable-sized reverse bittrees +// 0x080 matched literal +// 0x100 direct bits +// +// The default can be overridden at build time by defining +// LZMA_RANGE_DECODER_CONFIG to the desired mask. +// +// 2024-02-22: Feedback from benchmarks: +// - Brancless C (0x003) can be better than basic on x86-64 but often it's +// slightly worse on other archs. Since asm is much better on x86-64, +// branchless C is not used at all. +// - With x86-64 asm, there are slight differences between GCC and Clang +// and different processors. Overall 0x1F0 seems to be the best choice. +#ifndef LZMA_RANGE_DECODER_CONFIG +# if defined(__x86_64__) && !defined(__ILP32__) \ + && !defined(__NVCOMPILER) \ + && (defined(__GNUC__) || defined(__clang__)) +# define LZMA_RANGE_DECODER_CONFIG 0x1F0 +# else +# define LZMA_RANGE_DECODER_CONFIG 0 +# endif +#endif + + +// Negative RC_BIT_MODEL_TOTAL but the lowest RC_MOVE_BITS are flipped. +// This is useful for updating probability variables in branchless decoding: +// +// uint32_t decoded_bit = ...; +// probability tmp = RC_BIT_MODEL_OFFSET; +// tmp &= decoded_bit - 1; +// prob -= (prob + tmp) >> RC_MOVE_BITS; +#define RC_BIT_MODEL_OFFSET \ + ((UINT32_C(1) << RC_MOVE_BITS) - 1 - RC_BIT_MODEL_TOTAL) + + typedef struct { uint32_t range; uint32_t code; @@ -50,18 +98,28 @@ /// Makes local copies of range decoder and *in_pos variables. Doing this /// improves speed significantly. The range decoder macros expect also -/// variables `in' and `in_size' to be defined. -#define rc_to_local(range_decoder, in_pos) \ +/// variables 'in' and 'in_size' to be defined. +#define rc_to_local(range_decoder, in_pos, fast_mode_in_required) \ lzma_range_decoder rc = range_decoder; \ - size_t rc_in_pos = (in_pos); \ + const uint8_t *rc_in_ptr = in + (in_pos); \ + const uint8_t *rc_in_end = in + in_size; \ + const uint8_t *rc_in_fast_end \ + = (rc_in_end - rc_in_ptr) <= (fast_mode_in_required) \ + ? rc_in_ptr \ + : rc_in_end - (fast_mode_in_required); \ + (void)rc_in_fast_end; /* Silence a warning with HAVE_SMALL. */ \ uint32_t rc_bound +/// Evaluates to true if there is enough input remaining to use fast mode. +#define rc_is_fast_allowed() (rc_in_ptr < rc_in_fast_end) + + /// Stores the local copes back to the range decoder structure. #define rc_from_local(range_decoder, in_pos) \ do { \ range_decoder = rc; \ - in_pos = rc_in_pos; \ + in_pos = (size_t)(rc_in_ptr - in); \ } while (0) @@ -81,18 +139,30 @@ ((range_decoder).code == 0) -/// Read the next input byte if needed. If more input is needed but there is -/// no more input available, "goto out" is used to jump out of the main -/// decoder loop. -#define rc_normalize(seq) \ +// Read the next input byte if needed. +#define rc_normalize() \ do { \ if (rc.range < RC_TOP_VALUE) { \ - if (unlikely(rc_in_pos == in_size)) { \ + rc.range <<= RC_SHIFT_BITS; \ + rc.code = (rc.code << RC_SHIFT_BITS) | *rc_in_ptr++; \ + } \ +} while (0) + + +/// If more input is needed but there is +/// no more input available, "goto out" is used to jump out of the main +/// decoder loop. The "_safe" macros are used in the Resumable decoder +/// mode in order to save the sequence to continue decoding from that +/// point later. +#define rc_normalize_safe(seq) \ +do { \ + if (rc.range < RC_TOP_VALUE) { \ + if (rc_in_ptr == rc_in_end) { \ coder->sequence = seq; \ goto out; \ } \ rc.range <<= RC_SHIFT_BITS; \ - rc.code = (rc.code << RC_SHIFT_BITS) | in[rc_in_pos++]; \ + rc.code = (rc.code << RC_SHIFT_BITS) | *rc_in_ptr++; \ } \ } while (0) @@ -100,7 +170,7 @@ /// Start decoding a bit. This must be used together with rc_update_0() /// and rc_update_1(): /// -/// rc_if_0(prob, seq) { +/// rc_if_0(prob) { /// rc_update_0(prob); /// // Do something /// } else { @@ -108,18 +178,28 @@ /// // Do something else /// } /// -#define rc_if_0(prob, seq) \ - rc_normalize(seq); \ +#define rc_if_0(prob) \ + rc_normalize(); \ + rc_bound = (rc.range >> RC_BIT_MODEL_TOTAL_BITS) * (prob); \ + if (rc.code < rc_bound) + + +#define rc_if_0_safe(prob, seq) \ + rc_normalize_safe(seq); \ rc_bound = (rc.range >> RC_BIT_MODEL_TOTAL_BITS) * (prob); \ if (rc.code < rc_bound) /// Update the range decoder state and the used probability variable to /// match a decoded bit of 0. +/// +/// The x86-64 assembly uses the commented method but it seems that, +/// at least on x86-64, the first version is slightly faster as C code. #define rc_update_0(prob) \ do { \ rc.range = rc_bound; \ prob += (RC_BIT_MODEL_TOTAL - (prob)) >> RC_MOVE_BITS; \ + /* prob -= ((prob) + RC_BIT_MODEL_OFFSET) >> RC_MOVE_BITS; */ \ } while (0) @@ -137,9 +217,21 @@ /// This macro is used as the last step in bittree reverse decoders since /// those don't use "symbol" for anything else than indexing the probability /// arrays. -#define rc_bit_last(prob, action0, action1, seq) \ +#define rc_bit_last(prob, action0, action1) \ do { \ - rc_if_0(prob, seq) { \ + rc_if_0(prob) { \ + rc_update_0(prob); \ + action0; \ + } else { \ + rc_update_1(prob); \ + action1; \ + } \ +} while (0) + + +#define rc_bit_last_safe(prob, action0, action1, seq) \ +do { \ + rc_if_0_safe(prob, seq) { \ rc_update_0(prob); \ action0; \ } else { \ @@ -151,35 +243,724 @@ /// Decodes one bit, updates "symbol", and runs action0 or action1 depending /// on the decoded bit. -#define rc_bit(prob, action0, action1, seq) \ +#define rc_bit(prob, action0, action1) \ rc_bit_last(prob, \ symbol <<= 1; action0, \ + symbol = (symbol << 1) + 1; action1); + + +#define rc_bit_safe(prob, action0, action1, seq) \ + rc_bit_last_safe(prob, \ + symbol <<= 1; action0, \ symbol = (symbol << 1) + 1; action1, \ seq); +// Unroll fixed-sized bittree decoding. +// +// A compile-time constant in final_add can be used to get rid of the high bit +// from symbol that is used for the array indexing (1U << bittree_bits). +// final_add may also be used to add offset to the result (LZMA length +// decoder does that). +// +// The reason to have final_add here is that in the asm code the addition +// can be done for free: in x86-64 there is SBB instruction with -1 as +// the immediate value, and final_add is combined with that value. +#define rc_bittree_bit(prob) \ + rc_bit(prob, , ) -/// Like rc_bit() but add "case seq:" as a prefix. This makes the unrolled -/// loops more readable because the code isn't littered with "case" -/// statements. On the other hand this also makes it less readable, since -/// spotting the places where the decoder loop may be restarted is less -/// obvious. -#define rc_bit_case(prob, action0, action1, seq) \ - case seq: rc_bit(prob, action0, action1, seq) +#define rc_bittree3(probs, final_add) \ +do { \ + symbol = 1; \ + rc_bittree_bit(probs[symbol]); \ + rc_bittree_bit(probs[symbol]); \ + rc_bittree_bit(probs[symbol]); \ + symbol += (uint32_t)(final_add); \ +} while (0) + +#define rc_bittree6(probs, final_add) \ +do { \ + symbol = 1; \ + rc_bittree_bit(probs[symbol]); \ + rc_bittree_bit(probs[symbol]); \ + rc_bittree_bit(probs[symbol]); \ + rc_bittree_bit(probs[symbol]); \ + rc_bittree_bit(probs[symbol]); \ + rc_bittree_bit(probs[symbol]); \ + symbol += (uint32_t)(final_add); \ +} while (0) + +#define rc_bittree8(probs, final_add) \ +do { \ + symbol = 1; \ + rc_bittree_bit(probs[symbol]); \ + rc_bittree_bit(probs[symbol]); \ + rc_bittree_bit(probs[symbol]); \ + rc_bittree_bit(probs[symbol]); \ + rc_bittree_bit(probs[symbol]); \ + rc_bittree_bit(probs[symbol]); \ + rc_bittree_bit(probs[symbol]); \ + rc_bittree_bit(probs[symbol]); \ + symbol += (uint32_t)(final_add); \ +} while (0) + + +// Fixed-sized reverse bittree +#define rc_bittree_rev4(probs) \ +do { \ + symbol = 0; \ + rc_bit_last(probs[symbol + 1], , symbol += 1); \ + rc_bit_last(probs[symbol + 2], , symbol += 2); \ + rc_bit_last(probs[symbol + 4], , symbol += 4); \ + rc_bit_last(probs[symbol + 8], , symbol += 8); \ +} while (0) + + +// Decode one bit from variable-sized reverse bittree. The loop is done +// in the code that uses this macro. This could be changed if the assembly +// version benefited from having the loop done in assembly but it didn't +// seem so in early 2024. +// +// Also, if the loop was done here, the loop counter would likely be local +// to the macro so that it wouldn't modify yet another input variable. +// If a _safe version of a macro with a loop was done then a modifiable +// input variable couldn't be avoided though. +#define rc_bit_add_if_1(probs, dest, value_to_add_if_1) \ + rc_bit(probs[symbol], \ + , \ + dest += value_to_add_if_1); + + +// Matched literal +#define decode_with_match_bit \ + t_match_byte <<= 1; \ + t_match_bit = t_match_byte & t_offset; \ + t_subcoder_index = t_offset + t_match_bit + symbol; \ + rc_bit(probs[t_subcoder_index], \ + t_offset &= ~t_match_bit, \ + t_offset &= t_match_bit) + +#define rc_matched_literal(probs_base_var, match_byte) \ +do { \ + uint32_t t_match_byte = (match_byte); \ + uint32_t t_match_bit; \ + uint32_t t_subcoder_index; \ + uint32_t t_offset = 0x100; \ + symbol = 1; \ + decode_with_match_bit; \ + decode_with_match_bit; \ + decode_with_match_bit; \ + decode_with_match_bit; \ + decode_with_match_bit; \ + decode_with_match_bit; \ + decode_with_match_bit; \ + decode_with_match_bit; \ +} while (0) /// Decode a bit without using a probability. -#define rc_direct(dest, seq) \ +// +// NOTE: GCC 13 and Clang/LLVM 16 can, at least on x86-64, optimize the bound +// calculation to use an arithmetic right shift so there's no need to provide +// the alternative code which, according to C99/C11/C23 6.3.1.3-p3 isn't +// perfectly portable: rc_bound = (uint32_t)((int32_t)rc.code >> 31); +#define rc_direct(dest, count_var) \ do { \ - rc_normalize(seq); \ + dest = (dest << 1) + 1; \ + rc_normalize(); \ + rc.range >>= 1; \ + rc.code -= rc.range; \ + rc_bound = UINT32_C(0) - (rc.code >> 31); \ + dest += rc_bound; \ + rc.code += rc.range & rc_bound; \ +} while (--count_var > 0) + + + +#define rc_direct_safe(dest, count_var, seq) \ +do { \ + rc_normalize_safe(seq); \ rc.range >>= 1; \ rc.code -= rc.range; \ rc_bound = UINT32_C(0) - (rc.code >> 31); \ rc.code += rc.range & rc_bound; \ dest = (dest << 1) + (rc_bound + 1); \ +} while (--count_var > 0) + + +////////////////// +// Branchless C // +////////////////// + +/// Decode a bit using a branchless method. This reduces the number of +/// mispredicted branches and thus can improve speed. +#define rc_c_bit(prob, action_bit, action_neg) \ +do { \ + probability *p = &(prob); \ + rc_normalize(); \ + rc_bound = (rc.range >> RC_BIT_MODEL_TOTAL_BITS) * *p; \ + uint32_t rc_mask = rc.code >= rc_bound; /* rc_mask = decoded bit */ \ + action_bit; /* action when rc_mask is 0 or 1 */ \ + /* rc_mask becomes 0 if bit is 0 and 0xFFFFFFFF if bit is 1: */ \ + rc_mask = 0U - rc_mask; \ + rc.range &= rc_mask; /* If bit 0: set rc.range = 0 */ \ + rc_bound ^= rc_mask; \ + rc_bound -= rc_mask; /* If bit 1: rc_bound = 0U - rc_bound */ \ + rc.range += rc_bound; \ + rc_bound &= rc_mask; \ + rc.code += rc_bound; \ + action_neg; /* action when rc_mask is 0 or 0xFFFFFFFF */ \ + rc_mask = ~rc_mask; /* If bit 0: all bits are set in rc_mask */ \ + rc_mask &= RC_BIT_MODEL_OFFSET; \ + *p -= (*p + rc_mask) >> RC_MOVE_BITS; \ } while (0) -// NOTE: No macros are provided for bittree decoding. It seems to be simpler -// to just write them open in the code. +// Testing on x86-64 give an impression that only the normal bittrees and +// the fixed-sized reverse bittrees are worth the branchless C code. +// It should be tested on other archs for which there isn't assembly code +// in this file. + +// Using addition in "(symbol << 1) + rc_mask" allows use of x86 LEA +// or RISC-V SH1ADD instructions. Compilers might infer it from +// "(symbol << 1) | rc_mask" too if they see that mask is 0 or 1 but +// the use of addition doesn't require such analysis from compilers. +#if LZMA_RANGE_DECODER_CONFIG & 0x01 +#undef rc_bittree_bit +#define rc_bittree_bit(prob) \ + rc_c_bit(prob, \ + symbol = (symbol << 1) + rc_mask, \ + ) +#endif // LZMA_RANGE_DECODER_CONFIG & 0x01 + +#if LZMA_RANGE_DECODER_CONFIG & 0x02 +#undef rc_bittree_rev4 +#define rc_bittree_rev4(probs) \ +do { \ + symbol = 0; \ + rc_c_bit(probs[symbol + 1], symbol += rc_mask, ); \ + rc_c_bit(probs[symbol + 2], symbol += rc_mask << 1, ); \ + rc_c_bit(probs[symbol + 4], symbol += rc_mask << 2, ); \ + rc_c_bit(probs[symbol + 8], symbol += rc_mask << 3, ); \ +} while (0) +#endif // LZMA_RANGE_DECODER_CONFIG & 0x02 + +#if LZMA_RANGE_DECODER_CONFIG & 0x04 +#undef rc_bit_add_if_1 +#define rc_bit_add_if_1(probs, dest, value_to_add_if_1) \ + rc_c_bit(probs[symbol], \ + symbol = (symbol << 1) + rc_mask, \ + dest += (value_to_add_if_1) & rc_mask) +#endif // LZMA_RANGE_DECODER_CONFIG & 0x04 + + +#if LZMA_RANGE_DECODER_CONFIG & 0x08 +#undef decode_with_match_bit +#define decode_with_match_bit \ + t_match_byte <<= 1; \ + t_match_bit = t_match_byte & t_offset; \ + t_subcoder_index = t_offset + t_match_bit + symbol; \ + rc_c_bit(probs[t_subcoder_index], \ + symbol = (symbol << 1) + rc_mask, \ + t_offset &= ~t_match_bit ^ rc_mask) +#endif // LZMA_RANGE_DECODER_CONFIG & 0x08 + + +//////////// +// x86-64 // +//////////// + +#if LZMA_RANGE_DECODER_CONFIG & 0x1F0 + +// rc_asm_y and rc_asm_n are used as arguments to macros to control which +// strings to include or omit. +#define rc_asm_y(str) str +#define rc_asm_n(str) + +// There are a few possible variations for normalization. +// This is the smallest variant which is also used by LZMA SDK. +// +// - This has partial register write (the MOV from (%[in_ptr])). +// +// - INC saves one byte in code size over ADD. False dependency on +// partial flags from INC shouldn't become a problem on any processor +// because the instructions after normalization don't read the flags +// until SUB which sets all flags. +// +#define rc_asm_normalize \ + "cmp %[top_value], %[range]\n\t" \ + "jae 1f\n\t" \ + "shl %[shift_bits], %[code]\n\t" \ + "mov (%[in_ptr]), %b[code]\n\t" \ + "shl %[shift_bits], %[range]\n\t" \ + "inc %[in_ptr]\n" \ + "1:\n" + +// rc_asm_calc(prob) is roughly equivalent to the C version of rc_if_0(prob)... +// +// rc_bound = (rc.range >> RC_BIT_MODEL_TOTAL_BITS) * (prob); +// if (rc.code < rc_bound) +// +// ...but the bound is stored in "range": +// +// t0 = range; +// range = (range >> RC_BIT_MODEL_TOTAL_BITS) * (prob); +// t0 -= range; +// t1 = code; +// code -= range; +// +// The carry flag (CF) from the last subtraction holds the negation of +// the decoded bit (if CF==0 then the decoded bit is 1). +// The values in t0 and t1 are needed for rc_update_0(prob) and +// rc_update_1(prob). If the bit is 0, rc_update_0(prob)... +// +// rc.range = rc_bound; +// +// ...has already been done but the "code -= range" has to be reverted using +// the old value stored in t1. (Also, prob needs to be updated.) +// +// If the bit is 1, rc_update_1(prob)... +// +// rc.range -= rc_bound; +// rc.code -= rc_bound; +// +// ...is already done for "code" but the value for "range" needs to be taken +// from t0. (Also, prob needs to be updated here as well.) +// +// The assignments from t0 and t1 can be done in a branchless manner with CMOV +// after the instructions from this macro. The CF from SUB tells which moves +// are needed. +#define rc_asm_calc(prob) \ + "mov %[range], %[t0]\n\t" \ + "shr %[bit_model_total_bits], %[range]\n\t" \ + "imul %[" prob "], %[range]\n\t" \ + "sub %[range], %[t0]\n\t" \ + "mov %[code], %[t1]\n\t" \ + "sub %[range], %[code]\n\t" + +// Also, prob needs to be updated: The update math depends on the decoded bit. +// It can be expressed in a few slightly different ways but this is fairly +// convenient here: +// +// prob -= (prob + (bit ? 0 : RC_BIT_MODEL_OFFSET)) >> RC_MOVE_BITS; +// +// To do it in branchless way when the negation of the decoded bit is in CF, +// both "prob" and "prob + RC_BIT_MODEL_OFFSET" are needed. Then the desired +// value can be picked with CMOV. The addition can be done using LEA without +// affecting CF. +// +// (This prob update method is a tiny bit different from LZMA SDK 23.01. +// In the LZMA SDK a single register is reserved solely for a constant to +// be used with CMOV when updating prob. That is fine since there are enough +// free registers to do so. The method used here uses one fewer register, +// which is valuable with inline assembly.) +// +// * * * +// +// In bittree decoding, each (unrolled) loop iteration decodes one bit +// and needs one prob variable. To make it faster, the prob variable of +// the iteration N+1 is loaded during iteration N. There are two possible +// prob variables to choose from for N+1. Both are loaded from memory and +// the correct one is chosen with CMOV using the same CF as is used for +// other things described above. +// +// This preloading/prefetching requires an extra register. To avoid +// useless moves from "preloaded prob register" to "current prob register", +// the macros swap between the two registers for odd and even iterations. +// +// * * * +// +// Finally, the decoded bit has to be stored in "symbol". Since the negation +// of the bit is in CF, this can be done with SBB: symbol -= CF - 1. That is, +// if the decoded bit is 0 (CF==1) the operation is a no-op "symbol -= 0" +// and when bit is 1 (CF==0) the operation is "symbol -= 0 - 1" which is +// the same as "symbol += 1". +// +// The instructions for all things are intertwined for a few reasons: +// - freeing temporary registers for new use +// - not modifying CF too early +// - instruction scheduling +// +// The first and last iterations can cheat a little. For example, +// on the first iteration "symbol" is known to start from 1 so it +// doesn't need to be read; it can even be immediately initialized +// to 2 to prepare for the second iteration of the loop. +// +// * * * +// +// a = number of the current prob variable (0 or 1) +// b = number of the next prob variable (1 or 0) +// *_only = rc_asm_y or _n to include or exclude code marked with them +#define rc_asm_bittree(a, b, first_only, middle_only, last_only) \ + first_only( \ + "movzwl 2(%[probs_base]), %[prob" #a "]\n\t" \ + "mov $2, %[symbol]\n\t" \ + "movzwl 4(%[probs_base]), %[prob" #b "]\n\t" \ + ) \ + middle_only( \ + /* Note the scaling of 4 instead of 2: */ \ + "movzwl (%[probs_base], %q[symbol], 4), %[prob" #b "]\n\t" \ + ) \ + last_only( \ + "add %[symbol], %[symbol]\n\t" \ + ) \ + \ + rc_asm_normalize \ + rc_asm_calc("prob" #a) \ + \ + "cmovae %[t0], %[range]\n\t" \ + \ + first_only( \ + "movzwl 6(%[probs_base]), %[t0]\n\t" \ + "cmovae %[t0], %[prob" #b "]\n\t" \ + ) \ + middle_only( \ + "movzwl 2(%[probs_base], %q[symbol], 4), %[t0]\n\t" \ + "lea (%q[symbol], %q[symbol]), %[symbol]\n\t" \ + "cmovae %[t0], %[prob" #b "]\n\t" \ + ) \ + \ + "lea %c[bit_model_offset](%q[prob" #a "]), %[t0]\n\t" \ + "cmovb %[t1], %[code]\n\t" \ + "mov %[symbol], %[t1]\n\t" \ + "cmovae %[prob" #a "], %[t0]\n\t" \ + \ + first_only( \ + "sbb $-1, %[symbol]\n\t" \ + ) \ + middle_only( \ + "sbb $-1, %[symbol]\n\t" \ + ) \ + last_only( \ + "sbb %[last_sbb], %[symbol]\n\t" \ + ) \ + \ + "shr %[move_bits], %[t0]\n\t" \ + "sub %[t0], %[prob" #a "]\n\t" \ + /* Scaling of 1 instead of 2 because symbol <<= 1. */ \ + "mov %w[prob" #a "], (%[probs_base], %q[t1], 1)\n\t" + +// NOTE: The order of variables in __asm__ can affect speed and code size. +#define rc_asm_bittree_n(probs_base_var, final_add, asm_str) \ +do { \ + uint32_t t0; \ + uint32_t t1; \ + uint32_t t_prob0; \ + uint32_t t_prob1; \ + \ + __asm__( \ + asm_str \ + : \ + [range] "+&r"(rc.range), \ + [code] "+&r"(rc.code), \ + [t0] "=&r"(t0), \ + [t1] "=&r"(t1), \ + [prob0] "=&r"(t_prob0), \ + [prob1] "=&r"(t_prob1), \ + [symbol] "=&r"(symbol), \ + [in_ptr] "+&r"(rc_in_ptr) \ + : \ + [probs_base] "r"(probs_base_var), \ + [last_sbb] "n"(-1 - (final_add)), \ + [top_value] "n"(RC_TOP_VALUE), \ + [shift_bits] "n"(RC_SHIFT_BITS), \ + [bit_model_total_bits] "n"(RC_BIT_MODEL_TOTAL_BITS), \ + [bit_model_offset] "n"(RC_BIT_MODEL_OFFSET), \ + [move_bits] "n"(RC_MOVE_BITS) \ + : \ + "cc", "memory"); \ +} while (0) + + +#if LZMA_RANGE_DECODER_CONFIG & 0x010 +#undef rc_bittree3 +#define rc_bittree3(probs_base_var, final_add) \ + rc_asm_bittree_n(probs_base_var, final_add, \ + rc_asm_bittree(0, 1, rc_asm_y, rc_asm_n, rc_asm_n) \ + rc_asm_bittree(1, 0, rc_asm_n, rc_asm_y, rc_asm_n) \ + rc_asm_bittree(0, 1, rc_asm_n, rc_asm_n, rc_asm_y) \ + ) + +#undef rc_bittree6 +#define rc_bittree6(probs_base_var, final_add) \ + rc_asm_bittree_n(probs_base_var, final_add, \ + rc_asm_bittree(0, 1, rc_asm_y, rc_asm_n, rc_asm_n) \ + rc_asm_bittree(1, 0, rc_asm_n, rc_asm_y, rc_asm_n) \ + rc_asm_bittree(0, 1, rc_asm_n, rc_asm_y, rc_asm_n) \ + rc_asm_bittree(1, 0, rc_asm_n, rc_asm_y, rc_asm_n) \ + rc_asm_bittree(0, 1, rc_asm_n, rc_asm_y, rc_asm_n) \ + rc_asm_bittree(1, 0, rc_asm_n, rc_asm_n, rc_asm_y) \ + ) + +#undef rc_bittree8 +#define rc_bittree8(probs_base_var, final_add) \ + rc_asm_bittree_n(probs_base_var, final_add, \ + rc_asm_bittree(0, 1, rc_asm_y, rc_asm_n, rc_asm_n) \ + rc_asm_bittree(1, 0, rc_asm_n, rc_asm_y, rc_asm_n) \ + rc_asm_bittree(0, 1, rc_asm_n, rc_asm_y, rc_asm_n) \ + rc_asm_bittree(1, 0, rc_asm_n, rc_asm_y, rc_asm_n) \ + rc_asm_bittree(0, 1, rc_asm_n, rc_asm_y, rc_asm_n) \ + rc_asm_bittree(1, 0, rc_asm_n, rc_asm_y, rc_asm_n) \ + rc_asm_bittree(0, 1, rc_asm_n, rc_asm_y, rc_asm_n) \ + rc_asm_bittree(1, 0, rc_asm_n, rc_asm_n, rc_asm_y) \ + ) +#endif // LZMA_RANGE_DECODER_CONFIG & 0x010 + + +// Fixed-sized reverse bittree +// +// This uses the indexing that constructs the final value in symbol directly. +// add = 1, 2, 4, 8 +// dcur = -, 4, 8, 16 +// dnext0 = 4, 8, 16, - +// dnext0 = 6, 12, 24, - +#define rc_asm_bittree_rev(a, b, add, dcur, dnext0, dnext1, \ + first_only, middle_only, last_only) \ + first_only( \ + "movzwl 2(%[probs_base]), %[prob" #a "]\n\t" \ + "xor %[symbol], %[symbol]\n\t" \ + "movzwl 4(%[probs_base]), %[prob" #b "]\n\t" \ + ) \ + middle_only( \ + "movzwl " #dnext0 "(%[probs_base], %q[symbol], 2), " \ + "%[prob" #b "]\n\t" \ + ) \ + \ + rc_asm_normalize \ + rc_asm_calc("prob" #a) \ + \ + "cmovae %[t0], %[range]\n\t" \ + \ + first_only( \ + "movzwl 6(%[probs_base]), %[t0]\n\t" \ + "cmovae %[t0], %[prob" #b "]\n\t" \ + ) \ + middle_only( \ + "movzwl " #dnext1 "(%[probs_base], %q[symbol], 2), %[t0]\n\t" \ + "cmovae %[t0], %[prob" #b "]\n\t" \ + ) \ + \ + "lea " #add "(%q[symbol]), %[t0]\n\t" \ + "cmovb %[t1], %[code]\n\t" \ + middle_only( \ + "mov %[symbol], %[t1]\n\t" \ + ) \ + last_only( \ + "mov %[symbol], %[t1]\n\t" \ + ) \ + "cmovae %[t0], %[symbol]\n\t" \ + "lea %c[bit_model_offset](%q[prob" #a "]), %[t0]\n\t" \ + "cmovae %[prob" #a "], %[t0]\n\t" \ + \ + "shr %[move_bits], %[t0]\n\t" \ + "sub %[t0], %[prob" #a "]\n\t" \ + first_only( \ + "mov %w[prob" #a "], 2(%[probs_base])\n\t" \ + ) \ + middle_only( \ + "mov %w[prob" #a "], " \ + #dcur "(%[probs_base], %q[t1], 2)\n\t" \ + ) \ + last_only( \ + "mov %w[prob" #a "], " \ + #dcur "(%[probs_base], %q[t1], 2)\n\t" \ + ) + +#if LZMA_RANGE_DECODER_CONFIG & 0x020 +#undef rc_bittree_rev4 +#define rc_bittree_rev4(probs_base_var) \ +rc_asm_bittree_n(probs_base_var, 4, \ + rc_asm_bittree_rev(0, 1, 1, -, 4, 6, rc_asm_y, rc_asm_n, rc_asm_n) \ + rc_asm_bittree_rev(1, 0, 2, 4, 8, 12, rc_asm_n, rc_asm_y, rc_asm_n) \ + rc_asm_bittree_rev(0, 1, 4, 8, 16, 24, rc_asm_n, rc_asm_y, rc_asm_n) \ + rc_asm_bittree_rev(1, 0, 8, 16, -, -, rc_asm_n, rc_asm_n, rc_asm_y) \ +) +#endif // LZMA_RANGE_DECODER_CONFIG & 0x020 + + +#if LZMA_RANGE_DECODER_CONFIG & 0x040 +#undef rc_bit_add_if_1 +#define rc_bit_add_if_1(probs_base_var, dest_var, value_to_add_if_1) \ +do { \ + uint32_t t0; \ + uint32_t t1; \ + uint32_t t2 = (value_to_add_if_1); \ + uint32_t t_prob; \ + uint32_t t_index; \ + \ + __asm__( \ + "movzwl (%[probs_base], %q[symbol], 2), %[prob]\n\t" \ + "mov %[symbol], %[index]\n\t" \ + \ + "add %[dest], %[t2]\n\t" \ + "add %[symbol], %[symbol]\n\t" \ + \ + rc_asm_normalize \ + rc_asm_calc("prob") \ + \ + "cmovae %[t0], %[range]\n\t" \ + "lea %c[bit_model_offset](%q[prob]), %[t0]\n\t" \ + "cmovb %[t1], %[code]\n\t" \ + "cmovae %[prob], %[t0]\n\t" \ + \ + "cmovae %[t2], %[dest]\n\t" \ + "sbb $-1, %[symbol]\n\t" \ + \ + "sar %[move_bits], %[t0]\n\t" \ + "sub %[t0], %[prob]\n\t" \ + "mov %w[prob], (%[probs_base], %q[index], 2)" \ + : \ + [range] "+&r"(rc.range), \ + [code] "+&r"(rc.code), \ + [t0] "=&r"(t0), \ + [t1] "=&r"(t1), \ + [prob] "=&r"(t_prob), \ + [index] "=&r"(t_index), \ + [symbol] "+&r"(symbol), \ + [t2] "+&r"(t2), \ + [dest] "+&r"(dest_var), \ + [in_ptr] "+&r"(rc_in_ptr) \ + : \ + [probs_base] "r"(probs_base_var), \ + [top_value] "n"(RC_TOP_VALUE), \ + [shift_bits] "n"(RC_SHIFT_BITS), \ + [bit_model_total_bits] "n"(RC_BIT_MODEL_TOTAL_BITS), \ + [bit_model_offset] "n"(RC_BIT_MODEL_OFFSET), \ + [move_bits] "n"(RC_MOVE_BITS) \ + : \ + "cc", "memory"); \ +} while (0) +#endif // LZMA_RANGE_DECODER_CONFIG & 0x040 + + +// Literal decoding uses a normal 8-bit bittree but literal with match byte +// is more complex in picking the probability variable from the correct +// subtree. This doesn't use preloading/prefetching of the next prob because +// there are four choices instead of two. +// +// FIXME? The first iteration starts with symbol = 1 so it could be optimized +// by a tiny amount. +#define rc_asm_matched_literal(nonlast_only) \ + "add %[offset], %[symbol]\n\t" \ + "and %[offset], %[match_bit]\n\t" \ + "add %[match_bit], %[symbol]\n\t" \ + \ + "movzwl (%[probs_base], %q[symbol], 2), %[prob]\n\t" \ + \ + "add %[symbol], %[symbol]\n\t" \ + \ + nonlast_only( \ + "xor %[match_bit], %[offset]\n\t" \ + "add %[match_byte], %[match_byte]\n\t" \ + ) \ + \ + rc_asm_normalize \ + rc_asm_calc("prob") \ + \ + "cmovae %[t0], %[range]\n\t" \ + "lea %c[bit_model_offset](%q[prob]), %[t0]\n\t" \ + "cmovb %[t1], %[code]\n\t" \ + "mov %[symbol], %[t1]\n\t" \ + "cmovae %[prob], %[t0]\n\t" \ + \ + nonlast_only( \ + "cmovae %[match_bit], %[offset]\n\t" \ + "mov %[match_byte], %[match_bit]\n\t" \ + ) \ + \ + "sbb $-1, %[symbol]\n\t" \ + \ + "shr %[move_bits], %[t0]\n\t" \ + /* Undo symbol += match_bit + offset: */ \ + "and $0x1FF, %[symbol]\n\t" \ + "sub %[t0], %[prob]\n\t" \ + \ + /* Scaling of 1 instead of 2 because symbol <<= 1. */ \ + "mov %w[prob], (%[probs_base], %q[t1], 1)\n\t" + + +#if LZMA_RANGE_DECODER_CONFIG & 0x080 +#undef rc_matched_literal +#define rc_matched_literal(probs_base_var, match_byte_value) \ +do { \ + uint32_t t0; \ + uint32_t t1; \ + uint32_t t_prob; \ + uint32_t t_match_byte = (uint32_t)(match_byte_value) << 1; \ + uint32_t t_match_bit = t_match_byte; \ + uint32_t t_offset = 0x100; \ + symbol = 1; \ + \ + __asm__( \ + rc_asm_matched_literal(rc_asm_y) \ + rc_asm_matched_literal(rc_asm_y) \ + rc_asm_matched_literal(rc_asm_y) \ + rc_asm_matched_literal(rc_asm_y) \ + rc_asm_matched_literal(rc_asm_y) \ + rc_asm_matched_literal(rc_asm_y) \ + rc_asm_matched_literal(rc_asm_y) \ + rc_asm_matched_literal(rc_asm_n) \ + : \ + [range] "+&r"(rc.range), \ + [code] "+&r"(rc.code), \ + [t0] "=&r"(t0), \ + [t1] "=&r"(t1), \ + [prob] "=&r"(t_prob), \ + [match_bit] "+&r"(t_match_bit), \ + [symbol] "+&r"(symbol), \ + [match_byte] "+&r"(t_match_byte), \ + [offset] "+&r"(t_offset), \ + [in_ptr] "+&r"(rc_in_ptr) \ + : \ + [probs_base] "r"(probs_base_var), \ + [top_value] "n"(RC_TOP_VALUE), \ + [shift_bits] "n"(RC_SHIFT_BITS), \ + [bit_model_total_bits] "n"(RC_BIT_MODEL_TOTAL_BITS), \ + [bit_model_offset] "n"(RC_BIT_MODEL_OFFSET), \ + [move_bits] "n"(RC_MOVE_BITS) \ + : \ + "cc", "memory"); \ +} while (0) +#endif // LZMA_RANGE_DECODER_CONFIG & 0x080 + + +// Doing the loop in asm instead of C seems to help a little. +#if LZMA_RANGE_DECODER_CONFIG & 0x100 +#undef rc_direct +#define rc_direct(dest_var, count_var) \ +do { \ + uint32_t t0; \ + uint32_t t1; \ + \ + __asm__( \ + "2:\n\t" \ + "add %[dest], %[dest]\n\t" \ + "lea 1(%q[dest]), %[t1]\n\t" \ + \ + rc_asm_normalize \ + \ + "shr $1, %[range]\n\t" \ + "mov %[code], %[t0]\n\t" \ + "sub %[range], %[code]\n\t" \ + "cmovns %[t1], %[dest]\n\t" \ + "cmovs %[t0], %[code]\n\t" \ + "dec %[count]\n\t" \ + "jnz 2b\n\t" \ + : \ + [range] "+&r"(rc.range), \ + [code] "+&r"(rc.code), \ + [t0] "=&r"(t0), \ + [t1] "=&r"(t1), \ + [dest] "+&r"(dest_var), \ + [count] "+&r"(count_var), \ + [in_ptr] "+&r"(rc_in_ptr) \ + : \ + [top_value] "n"(RC_TOP_VALUE), \ + [shift_bits] "n"(RC_SHIFT_BITS) \ + : \ + "cc", "memory"); \ +} while (0) +#endif // LZMA_RANGE_DECODER_CONFIG & 0x100 + +#endif // x86_64 #endif
diff --git a/Utilities/cmliblzma/liblzma/rangecoder/range_encoder.h b/Utilities/cmliblzma/liblzma/rangecoder/range_encoder.h index 1e1c369..8f62a47 100644 --- a/Utilities/cmliblzma/liblzma/rangecoder/range_encoder.h +++ b/Utilities/cmliblzma/liblzma/rangecoder/range_encoder.h
@@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file range_encoder.h @@ -6,9 +8,6 @@ // Authors: Igor Pavlov // Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef LZMA_RANGE_ENCODER_H @@ -19,9 +18,9 @@ /// Maximum number of symbols that can be put pending into lzma_range_encoder -/// structure between calls to lzma_rc_encode(). For LZMA, 52+5 is enough +/// structure between calls to lzma_rc_encode(). For LZMA, 48+5 is enough /// (match with big distance and length followed by range encoder flush). -#define RC_SYMBOLS_MAX 58 +#define RC_SYMBOLS_MAX 53 typedef struct { @@ -30,6 +29,9 @@ uint32_t range; uint8_t cache; + /// Number of bytes written out by rc_encode() -> rc_shift_low() + uint64_t out_total; + /// Number of symbols in the tables size_t count; @@ -58,12 +60,22 @@ rc->cache_size = 1; rc->range = UINT32_MAX; rc->cache = 0; + rc->out_total = 0; rc->count = 0; rc->pos = 0; } static inline void +rc_forget(lzma_range_encoder *rc) +{ + // This must not be called when rc_encode() is partially done. + assert(rc->pos == 0); + rc->count = 0; +} + + +static inline void rc_bit(lzma_range_encoder *rc, probability *prob, uint32_t bit) { rc->symbols[rc->count] = bit; @@ -132,6 +144,7 @@ out[*out_pos] = rc->cache + (uint8_t)(rc->low >> 32); ++*out_pos; + ++rc->out_total; rc->cache = 0xFF; } while (--rc->cache_size != 0); @@ -146,6 +159,34 @@ } +// NOTE: The last two arguments are uint64_t instead of size_t because in +// the dummy version these refer to the size of the whole range-encoded +// output stream, not just to the currently available output buffer space. +static inline bool +rc_shift_low_dummy(uint64_t *low, uint64_t *cache_size, uint8_t *cache, + uint64_t *out_pos, uint64_t out_size) +{ + if ((uint32_t)(*low) < (uint32_t)(0xFF000000) + || (uint32_t)(*low >> 32) != 0) { + do { + if (*out_pos == out_size) + return true; + + ++*out_pos; + *cache = 0xFF; + + } while (--*cache_size != 0); + + *cache = (*low >> 24) & 0xFF; + } + + ++*cache_size; + *low = (*low & 0x00FFFFFF) << RC_SHIFT_BITS; + + return false; +} + + static inline bool rc_encode(lzma_range_encoder *rc, uint8_t *out, size_t *out_pos, size_t out_size) @@ -222,6 +263,83 @@ } +static inline bool +rc_encode_dummy(const lzma_range_encoder *rc, uint64_t out_limit) +{ + assert(rc->count <= RC_SYMBOLS_MAX); + + uint64_t low = rc->low; + uint64_t cache_size = rc->cache_size; + uint32_t range = rc->range; + uint8_t cache = rc->cache; + uint64_t out_pos = rc->out_total; + + size_t pos = rc->pos; + + while (true) { + // Normalize + if (range < RC_TOP_VALUE) { + if (rc_shift_low_dummy(&low, &cache_size, &cache, + &out_pos, out_limit)) + return true; + + range <<= RC_SHIFT_BITS; + } + + // This check is here because the normalization above must + // be done before flushing the last bytes. + if (pos == rc->count) + break; + + // Encode a bit + switch (rc->symbols[pos]) { + case RC_BIT_0: { + probability prob = *rc->probs[pos]; + range = (range >> RC_BIT_MODEL_TOTAL_BITS) + * prob; + break; + } + + case RC_BIT_1: { + probability prob = *rc->probs[pos]; + const uint32_t bound = prob * (range + >> RC_BIT_MODEL_TOTAL_BITS); + low += bound; + range -= bound; + break; + } + + case RC_DIRECT_0: + range >>= 1; + break; + + case RC_DIRECT_1: + range >>= 1; + low += range; + break; + + case RC_FLUSH: + default: + assert(0); + break; + } + + ++pos; + } + + // Flush the last bytes. This isn't in rc->symbols[] so we do + // it after the above loop to take into account the size of + // the flushing that will be done at the end of the stream. + for (pos = 0; pos < 5; ++pos) { + if (rc_shift_low_dummy(&low, &cache_size, + &cache, &out_pos, out_limit)) + return true; + } + + return false; +} + + static inline uint64_t rc_pending(const lzma_range_encoder *rc) {
diff --git a/Utilities/cmliblzma/liblzma/simple/arm.c b/Utilities/cmliblzma/liblzma/simple/arm.c index ff5073a..58acb2d 100644 --- a/Utilities/cmliblzma/liblzma/simple/arm.c +++ b/Utilities/cmliblzma/liblzma/simple/arm.c
@@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file arm.c @@ -6,9 +8,6 @@ // Authors: Igor Pavlov // Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "simple_private.h" @@ -53,6 +52,7 @@ } +#ifdef HAVE_ENCODER_ARM extern lzma_ret lzma_simple_arm_encoder_init(lzma_next_coder *next, const lzma_allocator *allocator, @@ -60,8 +60,10 @@ { return arm_coder_init(next, allocator, filters, true); } +#endif +#ifdef HAVE_DECODER_ARM extern lzma_ret lzma_simple_arm_decoder_init(lzma_next_coder *next, const lzma_allocator *allocator, @@ -69,3 +71,4 @@ { return arm_coder_init(next, allocator, filters, false); } +#endif
diff --git a/Utilities/cmliblzma/liblzma/simple/arm64.c b/Utilities/cmliblzma/liblzma/simple/arm64.c new file mode 100644 index 0000000..16c2f56 --- /dev/null +++ b/Utilities/cmliblzma/liblzma/simple/arm64.c
@@ -0,0 +1,136 @@ +// SPDX-License-Identifier: 0BSD + +/////////////////////////////////////////////////////////////////////////////// +// +/// \file arm64.c +/// \brief Filter for ARM64 binaries +/// +/// This converts ARM64 relative addresses in the BL and ADRP immediates +/// to absolute values to increase redundancy of ARM64 code. +/// +/// Converting B or ADR instructions was also tested but it's not useful. +/// A majority of the jumps for the B instruction are very small (+/- 0xFF). +/// These are typical for loops and if-statements. Encoding them to their +/// absolute address reduces redundancy since many of the small relative +/// jump values are repeated, but very few of the absolute addresses are. +// +// Authors: Lasse Collin +// Jia Tan +// Igor Pavlov +// +/////////////////////////////////////////////////////////////////////////////// + +#include "simple_private.h" + + +static size_t +arm64_code(void *simple lzma_attribute((__unused__)), + uint32_t now_pos, bool is_encoder, + uint8_t *buffer, size_t size) +{ + size_t i; + + // Clang 14.0.6 on x86-64 makes this four times bigger and 40 % slower + // with auto-vectorization that is enabled by default with -O2. + // Such vectorization bloat happens with -O2 when targeting ARM64 too + // but performance hasn't been tested. +#ifdef __clang__ +# pragma clang loop vectorize(disable) +#endif + for (i = 0; i + 4 <= size; i += 4) { + uint32_t pc = (uint32_t)(now_pos + i); + uint32_t instr = read32le(buffer + i); + + if ((instr >> 26) == 0x25) { + // BL instruction: + // The full 26-bit immediate is converted. + // The range is +/-128 MiB. + // + // Using the full range helps quite a lot with + // big executables. Smaller range would reduce false + // positives in non-code sections of the input though + // so this is a compromise that slightly favors big + // files. With the full range, only six bits of the 32 + // need to match to trigger a conversion. + const uint32_t src = instr; + instr = 0x94000000; + + pc >>= 2; + if (!is_encoder) + pc = 0U - pc; + + instr |= (src + pc) & 0x03FFFFFF; + write32le(buffer + i, instr); + + } else if ((instr & 0x9F000000) == 0x90000000) { + // ADRP instruction: + // Only values in the range +/-512 MiB are converted. + // + // Using less than the full +/-4 GiB range reduces + // false positives on non-code sections of the input + // while being excellent for executables up to 512 MiB. + // The positive effect of ADRP conversion is smaller + // than that of BL but it also doesn't hurt so much in + // non-code sections of input because, with +/-512 MiB + // range, nine bits of 32 need to match to trigger a + // conversion (two 10-bit match choices = 9 bits). + const uint32_t src = ((instr >> 29) & 3) + | ((instr >> 3) & 0x001FFFFC); + + // With the addition only one branch is needed to + // check the +/- range. This is usually false when + // processing ARM64 code so branch prediction will + // handle it well in terms of performance. + // + //if ((src & 0x001E0000) != 0 + // && (src & 0x001E0000) != 0x001E0000) + if ((src + 0x00020000) & 0x001C0000) + continue; + + instr &= 0x9000001F; + + pc >>= 12; + if (!is_encoder) + pc = 0U - pc; + + const uint32_t dest = src + pc; + instr |= (dest & 3) << 29; + instr |= (dest & 0x0003FFFC) << 3; + instr |= (0U - (dest & 0x00020000)) & 0x00E00000; + write32le(buffer + i, instr); + } + } + + return i; +} + + +static lzma_ret +arm64_coder_init(lzma_next_coder *next, const lzma_allocator *allocator, + const lzma_filter_info *filters, bool is_encoder) +{ + return lzma_simple_coder_init(next, allocator, filters, + &arm64_code, 0, 4, 4, is_encoder); +} + + +#ifdef HAVE_ENCODER_ARM64 +extern lzma_ret +lzma_simple_arm64_encoder_init(lzma_next_coder *next, + const lzma_allocator *allocator, + const lzma_filter_info *filters) +{ + return arm64_coder_init(next, allocator, filters, true); +} +#endif + + +#ifdef HAVE_DECODER_ARM64 +extern lzma_ret +lzma_simple_arm64_decoder_init(lzma_next_coder *next, + const lzma_allocator *allocator, + const lzma_filter_info *filters) +{ + return arm64_coder_init(next, allocator, filters, false); +} +#endif
diff --git a/Utilities/cmliblzma/liblzma/simple/armthumb.c b/Utilities/cmliblzma/liblzma/simple/armthumb.c index a8da334..f1eeca9 100644 --- a/Utilities/cmliblzma/liblzma/simple/armthumb.c +++ b/Utilities/cmliblzma/liblzma/simple/armthumb.c
@@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file armthumb.c @@ -6,9 +8,6 @@ // Authors: Igor Pavlov // Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "simple_private.h" @@ -58,6 +57,7 @@ } +#ifdef HAVE_ENCODER_ARMTHUMB extern lzma_ret lzma_simple_armthumb_encoder_init(lzma_next_coder *next, const lzma_allocator *allocator, @@ -65,8 +65,10 @@ { return armthumb_coder_init(next, allocator, filters, true); } +#endif +#ifdef HAVE_DECODER_ARMTHUMB extern lzma_ret lzma_simple_armthumb_decoder_init(lzma_next_coder *next, const lzma_allocator *allocator, @@ -74,3 +76,4 @@ { return armthumb_coder_init(next, allocator, filters, false); } +#endif
diff --git a/Utilities/cmliblzma/liblzma/simple/ia64.c b/Utilities/cmliblzma/liblzma/simple/ia64.c index 6492d0a..5025014 100644 --- a/Utilities/cmliblzma/liblzma/simple/ia64.c +++ b/Utilities/cmliblzma/liblzma/simple/ia64.c
@@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file ia64.c @@ -6,9 +8,6 @@ // Authors: Igor Pavlov // Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "simple_private.h" @@ -94,6 +93,7 @@ } +#ifdef HAVE_ENCODER_IA64 extern lzma_ret lzma_simple_ia64_encoder_init(lzma_next_coder *next, const lzma_allocator *allocator, @@ -101,8 +101,10 @@ { return ia64_coder_init(next, allocator, filters, true); } +#endif +#ifdef HAVE_DECODER_IA64 extern lzma_ret lzma_simple_ia64_decoder_init(lzma_next_coder *next, const lzma_allocator *allocator, @@ -110,3 +112,4 @@ { return ia64_coder_init(next, allocator, filters, false); } +#endif
diff --git a/Utilities/cmliblzma/liblzma/simple/powerpc.c b/Utilities/cmliblzma/liblzma/simple/powerpc.c index 0b60e9b..ba6cfbe 100644 --- a/Utilities/cmliblzma/liblzma/simple/powerpc.c +++ b/Utilities/cmliblzma/liblzma/simple/powerpc.c
@@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file powerpc.c @@ -6,9 +8,6 @@ // Authors: Igor Pavlov // Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "simple_private.h" @@ -58,6 +57,7 @@ } +#ifdef HAVE_ENCODER_POWERPC extern lzma_ret lzma_simple_powerpc_encoder_init(lzma_next_coder *next, const lzma_allocator *allocator, @@ -65,8 +65,10 @@ { return powerpc_coder_init(next, allocator, filters, true); } +#endif +#ifdef HAVE_DECODER_POWERPC extern lzma_ret lzma_simple_powerpc_decoder_init(lzma_next_coder *next, const lzma_allocator *allocator, @@ -74,3 +76,4 @@ { return powerpc_coder_init(next, allocator, filters, false); } +#endif
diff --git a/Utilities/cmliblzma/liblzma/simple/riscv.c b/Utilities/cmliblzma/liblzma/simple/riscv.c new file mode 100644 index 0000000..b18df8b --- /dev/null +++ b/Utilities/cmliblzma/liblzma/simple/riscv.c
@@ -0,0 +1,755 @@ +// SPDX-License-Identifier: 0BSD + +/////////////////////////////////////////////////////////////////////////////// +// +/// \file riscv.c +/// \brief Filter for 32-bit/64-bit little/big endian RISC-V binaries +/// +/// This converts program counter relative addresses in function calls +/// (JAL, AUIPC+JALR), address calculation of functions and global +/// variables (AUIPC+ADDI), loads (AUIPC+load), and stores (AUIPC+store). +/// +/// For AUIPC+inst2 pairs, the paired instruction checking is fairly relaxed. +/// The paired instruction opcode must only have its lowest two bits set, +/// meaning it will convert any paired instruction that is not a 16-bit +/// compressed instruction. This was shown to be enough to keep the number +/// of false matches low while improving code size and speed. +// +// Authors: Lasse Collin +// Jia Tan +// +// Special thanks: +// +// - Chien Wong <m@xv97.com> provided a few early versions of RISC-V +// filter variants along with test files and benchmark results. +// +// - Igor Pavlov helped a lot in the filter design, getting it both +// faster and smaller. The implementation here is still independently +// written, not based on LZMA SDK. +// +/////////////////////////////////////////////////////////////////////////////// + +/* + +RISC-V filtering +================ + + RV32I and RV64I, possibly combined with extensions C, Zfh, F, D, + and Q, are identical enough that the same filter works for both. + + The instruction encoding is always little endian, even on systems + with big endian data access. Thus the same filter works for both + endiannesses. + + The following instructions have program counter relative + (pc-relative) behavior: + +JAL +--- + + JAL is used for function calls (including tail calls) and + unconditional jumps within functions. Jumps within functions + aren't useful to filter because the absolute addresses often + appear only once or at most a few times. Tail calls and jumps + within functions look the same to a simple filter so neither + are filtered, that is, JAL x0 is ignored (the ABI name of the + register x0 is "zero"). + + Almost all calls store the return address to register x1 (ra) + or x5 (t0). To reduce false matches when the filter is applied + to non-code data, only the JAL instructions that use x1 or x5 + are converted. JAL has pc-relative range of +/-1 MiB so longer + calls and jumps need another method (AUIPC+JALR). + +C.J and C.JAL +------------- + + C.J and C.JAL have pc-relative range of +/-2 KiB. + + C.J is for tail calls and jumps within functions and isn't + filtered for the reasons mentioned for JAL x0. + + C.JAL is an RV32C-only instruction. Its encoding overlaps with + RV64C-only C.ADDIW which is a common instruction. So if filtering + C.JAL was useful (it wasn't tested) then a separate filter would + be needed for RV32 and RV64. Also, false positives would be a + significant problem when the filter is applied to non-code data + because C.JAL needs only five bits to match. Thus, this filter + doesn't modify C.JAL instructions. + +BEQ, BNE, BLT, BGE, BLTU, BGEU, C.BEQZ, and C.BNEZ +-------------------------------------------------- + + These are conditional branches with pc-relative range + of +/-4 KiB (+/-256 B for C.*). The absolute addresses often + appear only once and very short distances are the most common, + so filtering these instructions would make compression worse. + +AUIPC with rd != x0 +------------------- + + AUIPC is paired with a second instruction (inst2) to do + pc-relative jumps, calls, loads, stores, and for taking + an address of a symbol. AUIPC has a 20-bit immediate and + the possible inst2 choices have a 12-bit immediate. + + AUIPC stores pc + 20-bit signed immediate to a register. + The immediate encodes a multiple of 4 KiB so AUIPC itself + has a pc-relative range of +/-2 GiB. AUIPC does *NOT* set + the lowest 12 bits of the result to zero! This means that + the 12-bit immediate in inst2 cannot just include the lowest + 12 bits of the absolute address as is; the immediate has to + compensate for the lowest 12 bits that AUIPC copies from the + program counter. This means that a good filter has to convert + not only AUIPC but also the paired inst2. + + A strict filter would focus on filtering the following + AUIPC+inst2 pairs: + + - AUIPC+JALR: Function calls, including tail calls. + + - AUIPC+ADDI: Calculating the address of a function + or a global variable. + + - AUIPC+load/store from the base instruction sets + (RV32I, RV64I) or from the floating point extensions + Zfh, F, D, and Q: + * RV32I: LB, LH, LW, LBU, LHU, SB, SH, SW + * RV64I has also: LD, LWU, SD + * Zfh: FLH, FSH + * F: FLW, FSW + * D: FLD, FSD + * Q: FLQ, FSQ + + NOTE: AUIPC+inst2 can only be a pair if AUIPC's rd specifies + the same register as inst2's rs1. + + Instead of strictly accepting only the above instructions as inst2, + this filter uses a much simpler condition: the lowest two bits of + inst2 must be set, that is, inst2 must not be a 16-bit compressed + instruction. So this will accept all 32-bit and possible future + extended instructions as a pair to AUIPC if the bits in AUIPC's + rd [11:7] match the bits [19:15] in inst2 (the bits that I-type and + S-type instructions use for rs1). Testing showed that this relaxed + condition for inst2 did not consistently or significantly affect + compression ratio but it reduced code size and improved speed. + + Additionally, the paired instruction is always treated as an I-type + instruction. The S-type instructions used by stores (SB, SH, SW, + etc.) place the lowest 5 bits of the immediate in a different + location than I-type instructions. AUIPC+store pairs are less + common than other pairs, and testing showed that the extra + code required to handle S-type instructions was not worth the + compression ratio gained. + + AUIPC+inst2 don't necessarily appear sequentially next to each + other although very often they do. Especially AUIPC+JALR are + sequential as that may allow instruction fusion in processors + (and perhaps help branch prediction as a fused AUIPC+JALR is + a direct branch while JALR alone is an indirect branch). + + Clang 16 can generate code where AUIPC+inst2 is split: + + - AUIPC is outside a loop and inst2 (load/store) is inside + the loop. This way the AUIPC instruction needs to be + executed only once. + + - Load-modify-store may have AUIPC for the load and the same + AUIPC-result is used for the store too. This may get combined + with AUIPC being outside the loop. + + - AUIPC is before a conditional branch and inst2 is hundreds + of bytes away at the branch target. + + - Inner and outer pair: + + auipc a1,0x2f + auipc a2,0x3d + ld a2,-500(a2) + addi a1,a1,-233 + + - Many split pairs with an untaken conditional branch between: + + auipc s9,0x1613 # Pair 1 + auipc s4,0x1613 # Pair 2 + auipc s6,0x1613 # Pair 3 + auipc s10,0x1613 # Pair 4 + beqz a5,a3baae + ld a0,0(a6) + ld a6,246(s9) # Pair 1 + ld a1,250(s4) # Pair 2 + ld a3,254(s6) # Pair 3 + ld a4,258(s10) # Pair 4 + + It's not possible to find all split pairs in a filter like this. + At least in 2024, simple sequential pairs are 99 % of AUIPC uses + so filtering only such pairs gives good results and makes the + filter simpler. However, it's possible that future compilers will + produce different code where sequential pairs aren't as common. + + This filter doesn't convert AUIPC instructions alone because: + + (1) The conversion would be off-by-one (or off-by-4096) half the + time because the lowest 12 bits from inst2 (inst2_imm12) + aren't known. We only know that the absolute address is + pc + AUIPC_imm20 + [-2048, +2047] but there is no way to + know the exact 4096-byte multiple (or 4096 * n + 2048): + there are always two possibilities because AUIPC copies + the 12 lowest bits from pc instead of zeroing them. + + NOTE: The sign-extension of inst2_imm12 adds a tiny bit + of extra complexity to AUIPC math in general but it's not + the reason for this problem. The sign-extension only changes + the relative position of the pc-relative 4096-byte window. + + (2) Matching AUIPC instruction alone requires only seven bits. + When the filter is applied to non-code data, that leads + to many false positives which make compression worse. + As long as most AUIPC+inst2 pairs appear as two consecutive + instructions, converting only such pairs gives better results. + + In assembly, AUIPC+inst2 tend to look like this: + + # Call: + auipc ra, 0x12345 + jalr ra, -42(ra) + + # Tail call: + auipc t1, 0x12345 + jalr zero, -42(t1) + + # Getting the absolute address: + auipc a0, 0x12345 + addi a0, a0, -42 + + # rd of inst2 isn't necessarily the same as rs1 even + # in cases where there is no reason to preserve rs1. + auipc a0, 0x12345 + addi a1, a0, -42 + + As of 2024, 16-bit instructions from the C extension don't + appear as inst2. The RISC-V psABI doesn't list AUIPC+C.* as + a linker relaxation type explicitly but it's not disallowed + either. Usefulness is limited as most of the time the lowest + 12 bits won't fit in a C instruction. This filter doesn't + support AUIPC+C.* combinations because this makes the filter + simpler, there are no test files, and it hopefully will never + be needed anyway. + + (Compare AUIPC to ARM64 where ADRP does set the lowest 12 bits + to zero. The paired instruction has the lowest 12 bits of the + absolute address as is in a zero-extended immediate. Thus the + ARM64 filter doesn't need to care about the instructions that + are paired with ADRP. An off-by-4096 issue can still occur if + the code section isn't aligned with the filter's start offset. + It's not a problem with standalone ELF files but Windows PE + files need start_offset=3072 for best results. Also, a .tar + stores files with 512-byte alignment so most of the time it + won't be the best for ARM64.) + +AUIPC with rd == x0 +------------------- + + AUIPC instructions with rd=x0 are reserved for HINTs in the base + instruction set. Such AUIPC instructions are never filtered. + + As of January 2024, it seems likely that AUIPC with rd=x0 will + be used for landing pads (pseudoinstruction LPAD). LPAD is used + to mark valid targets for indirect jumps (for JALR), for example, + beginnings of functions. The 20-bit immediate in LPAD instruction + is a label, not a pc-relative address. Thus it would be + counterproductive to convert AUIPC instructions with rd=x0. + + Often the next instruction after LPAD won't have rs1=x0 and thus + the filtering would be skipped for that reason alone. However, + it's not good to rely on this. For example, consider a function + that begins like this: + + int foo(int i) + { + if (i <= 234) { + ... + } + + A compiler may generate something like this: + + lpad 0x54321 + li a5, 234 + bgt a0, a5, .L2 + + Converting the pseudoinstructions to raw instructions: + + auipc x0, 0x54321 + addi x15, x0, 234 + blt x15, x10, .L2 + + In this case the filter would undesirably convert the AUIPC+ADDI + pair if the filter didn't explicitly skip AUIPC instructions + that have rd=x0. + +*/ + + +#include "simple_private.h" + + +// This checks two conditions at once: +// - AUIPC rd == inst2 rs1. +// - inst2 opcode has the lowest two bits set. +// +// The 8 bit left shift aligns the rd of AUIPC with the rs1 of inst2. +// By XORing the registers, any non-zero value in those bits indicates the +// registers are not equal and thus not an AUIPC pair. Subtracting 3 from +// inst2 will zero out the first two opcode bits only when they are set. +// The mask tests if any of the register or opcode bits are set (and thus +// not an AUIPC pair). +// +// Alternative expression: (((((auipc) << 8) ^ (inst2)) & 0xF8003) != 3) +#define NOT_AUIPC_PAIR(auipc, inst2) \ + ((((auipc) << 8) ^ ((inst2) - 3)) & 0xF8003) + +// This macro checks multiple conditions: +// (1) AUIPC rd [11:7] == x2 (special rd value). +// (2) AUIPC bits 12 and 13 set (the lowest two opcode bits of packed inst2). +// (3) inst2_rs1 doesn't equal x0 or x2 because the opposite +// conversion is only done when +// auipc_rd != x0 && +// auipc_rd != x2 && +// auipc_rd == inst2_rs1. +// +// The left-hand side takes care of (1) and (2). +// (a) The lowest 7 bits are already known to be AUIPC so subtracting 0x17 +// makes those bits zeros. +// (b) If AUIPC rd equals x2, subtracting 0x100 makes bits [11:7] zeros. +// If rd doesn't equal x2, then there will be at least one non-zero bit +// and the next step (c) is irrelevant. +// (c) If the lowest two opcode bits of the packed inst2 are set in [13:12], +// then subtracting 0x3000 will make those bits zeros. Otherwise there +// will be at least one non-zero bit. +// +// The shift by 18 removes the high bits from the final '>=' comparison and +// ensures that any non-zero result will be larger than any possible result +// from the right-hand side of the comparison. The cast ensures that the +// left-hand side didn't get promoted to a larger type than uint32_t. +// +// On the right-hand side, inst2_rs1 & 0x1D will be non-zero as long as +// inst2_rs1 is not x0 or x2. +// +// The final '>=' comparison will make the expression true if: +// - The subtraction caused any bits to be set (special AUIPC rd value not +// used or inst2 opcode bits not set). (non-zero >= non-zero or 0) +// - The subtraction did not cause any bits to be set but inst2_rs1 was +// x0 or x2. (0 >= 0) +#define NOT_SPECIAL_AUIPC(auipc, inst2_rs1) \ + ((uint32_t)(((auipc) - 0x3117) << 18) >= ((inst2_rs1) & 0x1D)) + + +// The encode and decode functions are split for this filter because of the +// AUIPC+inst2 filtering. This filter design allows a decoder-only +// implementation to be smaller than alternative designs. + +#ifdef HAVE_ENCODER_RISCV +static size_t +riscv_encode(void *simple lzma_attribute((__unused__)), + uint32_t now_pos, + bool is_encoder lzma_attribute((__unused__)), + uint8_t *buffer, size_t size) +{ + // Avoid using i + 8 <= size in the loop condition. + // + // NOTE: If there is a JAL in the last six bytes of the stream, it + // won't be converted. This is intentional to keep the code simpler. + if (size < 8) + return 0; + + size -= 8; + + size_t i; + + // The loop is advanced by 2 bytes every iteration since the + // instruction stream may include 16-bit instructions (C extension). + for (i = 0; i <= size; i += 2) { + uint32_t inst = buffer[i]; + + if (inst == 0xEF) { + // JAL + const uint32_t b1 = buffer[i + 1]; + + // Only filter rd=x1(ra) and rd=x5(t0). + if ((b1 & 0x0D) != 0) + continue; + + // The 20-bit immediate is in four pieces. + // The encoder stores it in big endian form + // since it improves compression slightly. + const uint32_t b2 = buffer[i + 2]; + const uint32_t b3 = buffer[i + 3]; + const uint32_t pc = now_pos + (uint32_t)i; + +// The following chart shows the highest three bytes of JAL, focusing on +// the 20-bit immediate field [31:12]. The first row of numbers is the +// bit position in a 32-bit little endian instruction. The second row of +// numbers shows the order of the immediate field in a J-type instruction. +// The last row is the bit number in each byte. +// +// To determine the amount to shift each bit, subtract the value in +// the last row from the value in the second last row. If the number +// is positive, shift left. If negative, shift right. +// +// For example, at the rightmost side of the chart, the bit 4 in b1 is +// the bit 12 of the address. Thus that bit needs to be shifted left +// by 12 - 4 = 8 bits to put it in the right place in the addr variable. +// +// NOTE: The immediate of a J-type instruction holds bits [20:1] of +// the address. The bit [0] is always 0 and not part of the immediate. +// +// | b3 | b2 | b1 | +// | 31 30 29 28 27 26 25 24 | 23 22 21 20 19 18 17 16 | 15 14 13 12 x x x x | +// | 20 10 9 8 7 6 5 4 | 3 2 1 11 19 18 17 16 | 15 14 13 12 x x x x | +// | 7 6 5 4 3 2 1 0 | 7 6 5 4 3 2 1 0 | 7 6 5 4 x x x x | + + uint32_t addr = ((b1 & 0xF0) << 8) + | ((b2 & 0x0F) << 16) + | ((b2 & 0x10) << 7) + | ((b2 & 0xE0) >> 4) + | ((b3 & 0x7F) << 4) + | ((b3 & 0x80) << 13); + + addr += pc; + + buffer[i + 1] = (uint8_t)((b1 & 0x0F) + | ((addr >> 13) & 0xF0)); + + buffer[i + 2] = (uint8_t)(addr >> 9); + buffer[i + 3] = (uint8_t)(addr >> 1); + + // The "-2" is included because the for-loop will + // always increment by 2. In this case, we want to + // skip an extra 2 bytes since we used 4 bytes + // of input. + i += 4 - 2; + + } else if ((inst & 0x7F) == 0x17) { + // AUIPC + inst |= (uint32_t)buffer[i + 1] << 8; + inst |= (uint32_t)buffer[i + 2] << 16; + inst |= (uint32_t)buffer[i + 3] << 24; + + // Branch based on AUIPC's rd. The bitmask test does + // the same thing as this: + // + // const uint32_t auipc_rd = (inst >> 7) & 0x1F; + // if (auipc_rd != 0 && auipc_rd != 2) { + if (inst & 0xE80) { + // AUIPC's rd doesn't equal x0 or x2. + + // Check if AUIPC+inst2 are a pair. + uint32_t inst2 = read32le(buffer + i + 4); + + if (NOT_AUIPC_PAIR(inst, inst2)) { + // The NOT_AUIPC_PAIR macro allows + // a false AUIPC+AUIPC pair if the + // bits [19:15] (where rs1 would be) + // in the second AUIPC match the rd + // of the first AUIPC. + // + // We must skip enough forward so + // that the first two bytes of the + // second AUIPC cannot get converted. + // Such a conversion could make the + // current pair become a valid pair + // which would desync the decoder. + // + // Skipping six bytes is enough even + // though the above condition looks + // at the lowest four bits of the + // buffer[i + 6] too. This is safe + // because this filter never changes + // those bits if a conversion at + // that position is done. + i += 6 - 2; + continue; + } + + // Convert AUIPC+inst2 to a special format: + // + // - The lowest 7 bits [6:0] retain the + // AUIPC opcode. + // + // - The rd [11:7] is set to x2(sp). x2 is + // used as the stack pointer so AUIPC with + // rd=x2 should be very rare in real-world + // executables. + // + // - The remaining 20 bits [31:12] (that + // normally hold the pc-relative immediate) + // are used to store the lowest 20 bits of + // inst2. That is, the 12-bit immediate of + // inst2 is not included. + // + // - The location of the original inst2 is + // used to store the 32-bit absolute + // address in big endian format. Compared + // to the 20+12-bit split encoding, this + // results in a longer uninterrupted + // sequence of identical common bytes + // when the same address is referred + // with different instruction pairs + // (like AUIPC+LD vs. AUIPC+ADDI) or + // when the occurrences of the same + // pair use different registers. When + // referring to adjacent memory locations + // (like function calls that go via the + // ELF PLT), in big endian order only the + // last 1-2 bytes differ; in little endian + // the differing 1-2 bytes would be in the + // middle of the 8-byte sequence. + // + // When reversing the transformation, the + // original rd of AUIPC can be restored + // from inst2's rs1 as they are required to + // be the same. + + // Arithmetic right shift makes sign extension + // trivial but (1) it's implementation-defined + // behavior (C99/C11/C23 6.5.7-p5) and so is + // (2) casting unsigned to signed (6.3.1.3-p3). + // + // One can check for (1) with + // + // if ((-1 >> 1) == -1) ... + // + // but (2) has to be checked from the + // compiler docs. GCC promises that (1) + // and (2) behave in the common expected + // way and thus + // + // addr += (uint32_t)( + // (int32_t)inst2 >> 20); + // + // does the same as the code below. But since + // the 100 % portable way is only a few bytes + // bigger code and there is no real speed + // difference, let's just use that, especially + // since the decoder doesn't need this at all. + uint32_t addr = inst & 0xFFFFF000; + addr += (inst2 >> 20) + - ((inst2 >> 19) & 0x1000); + + addr += now_pos + (uint32_t)i; + + // Construct the first 32 bits: + // [6:0] AUIPC opcode + // [11:7] Special AUIPC rd = x2 + // [31:12] The lowest 20 bits of inst2 + inst = 0x17 | (2 << 7) | (inst2 << 12); + + write32le(buffer + i, inst); + + // The second 32 bits store the absolute + // address in big endian order. + write32be(buffer + i + 4, addr); + } else { + // AUIPC's rd equals x0 or x2. + // + // x0 indicates a landing pad (LPAD). + // It's always skipped. + // + // AUIPC with rd == x2 is used for the special + // format as explained above. When the input + // contains a byte sequence that matches the + // special format, "fake" decoding must be + // done to keep the filter bijective (that + // is, safe to apply on arbitrary data). + // + // See the "x0 or x2" section in riscv_decode() + // for how the "real" decoding is done. The + // "fake" decoding is a simplified version + // of "real" decoding with the following + // differences (these reduce code size of + // the decoder): + // (1) The lowest 12 bits aren't sign-extended. + // (2) No address conversion is done. + // (3) Big endian format isn't used (the fake + // address is in little endian order). + + // Check if inst matches the special format. + const uint32_t fake_rs1 = inst >> 27; + + if (NOT_SPECIAL_AUIPC(inst, fake_rs1)) { + i += 4 - 2; + continue; + } + + const uint32_t fake_addr = + read32le(buffer + i + 4); + + // Construct the second 32 bits: + // [19:0] Upper 20 bits from AUIPC + // [31:20] The lowest 12 bits of fake_addr + const uint32_t fake_inst2 = (inst >> 12) + | (fake_addr << 20); + + // Construct new first 32 bits from: + // [6:0] AUIPC opcode + // [11:7] Fake AUIPC rd = fake_rs1 + // [31:12] The highest 20 bits of fake_addr + inst = 0x17 | (fake_rs1 << 7) + | (fake_addr & 0xFFFFF000); + + write32le(buffer + i, inst); + write32le(buffer + i + 4, fake_inst2); + } + + i += 8 - 2; + } + } + + return i; +} + + +extern lzma_ret +lzma_simple_riscv_encoder_init(lzma_next_coder *next, + const lzma_allocator *allocator, + const lzma_filter_info *filters) +{ + return lzma_simple_coder_init(next, allocator, filters, + &riscv_encode, 0, 8, 2, true); +} +#endif + + +#ifdef HAVE_DECODER_RISCV +static size_t +riscv_decode(void *simple lzma_attribute((__unused__)), + uint32_t now_pos, + bool is_encoder lzma_attribute((__unused__)), + uint8_t *buffer, size_t size) +{ + if (size < 8) + return 0; + + size -= 8; + + size_t i; + for (i = 0; i <= size; i += 2) { + uint32_t inst = buffer[i]; + + if (inst == 0xEF) { + // JAL + const uint32_t b1 = buffer[i + 1]; + + // Only filter rd=x1(ra) and rd=x5(t0). + if ((b1 & 0x0D) != 0) + continue; + + const uint32_t b2 = buffer[i + 2]; + const uint32_t b3 = buffer[i + 3]; + const uint32_t pc = now_pos + (uint32_t)i; + +// | b3 | b2 | b1 | +// | 31 30 29 28 27 26 25 24 | 23 22 21 20 19 18 17 16 | 15 14 13 12 x x x x | +// | 20 10 9 8 7 6 5 4 | 3 2 1 11 19 18 17 16 | 15 14 13 12 x x x x | +// | 7 6 5 4 3 2 1 0 | 7 6 5 4 3 2 1 0 | 7 6 5 4 x x x x | + + uint32_t addr = ((b1 & 0xF0) << 13) + | (b2 << 9) | (b3 << 1); + + addr -= pc; + + buffer[i + 1] = (uint8_t)((b1 & 0x0F) + | ((addr >> 8) & 0xF0)); + + buffer[i + 2] = (uint8_t)(((addr >> 16) & 0x0F) + | ((addr >> 7) & 0x10) + | ((addr << 4) & 0xE0)); + + buffer[i + 3] = (uint8_t)(((addr >> 4) & 0x7F) + | ((addr >> 13) & 0x80)); + + i += 4 - 2; + + } else if ((inst & 0x7F) == 0x17) { + // AUIPC + uint32_t inst2; + + inst |= (uint32_t)buffer[i + 1] << 8; + inst |= (uint32_t)buffer[i + 2] << 16; + inst |= (uint32_t)buffer[i + 3] << 24; + + if (inst & 0xE80) { + // AUIPC's rd doesn't equal x0 or x2. + + // Check if it is a "fake" AUIPC+inst2 pair. + inst2 = read32le(buffer + i + 4); + + if (NOT_AUIPC_PAIR(inst, inst2)) { + i += 6 - 2; + continue; + } + + // Decode (or more like re-encode) the "fake" + // pair. The "fake" format doesn't do + // sign-extension, address conversion, or + // use big endian. (The use of little endian + // allows sharing the write32le() calls in + // the decoder to reduce code size when + // unaligned access isn't supported.) + uint32_t addr = inst & 0xFFFFF000; + addr += inst2 >> 20; + + inst = 0x17 | (2 << 7) | (inst2 << 12); + inst2 = addr; + } else { + // AUIPC's rd equals x0 or x2. + + // Check if inst matches the special format + // used by the encoder. + const uint32_t inst2_rs1 = inst >> 27; + + if (NOT_SPECIAL_AUIPC(inst, inst2_rs1)) { + i += 4 - 2; + continue; + } + + // Decode the "real" pair. + uint32_t addr = read32be(buffer + i + 4); + + addr -= now_pos + (uint32_t)i; + + // The second instruction: + // - Get the lowest 20 bits from inst. + // - Add the lowest 12 bits of the address + // as the immediate field. + inst2 = (inst >> 12) | (addr << 20); + + // AUIPC: + // - rd is the same as inst2_rs1. + // - The sign extension of the lowest 12 bits + // must be taken into account. + inst = 0x17 | (inst2_rs1 << 7) + | ((addr + 0x800) & 0xFFFFF000); + } + + // Both decoder branches write in little endian order. + write32le(buffer + i, inst); + write32le(buffer + i + 4, inst2); + + i += 8 - 2; + } + } + + return i; +} + + +extern lzma_ret +lzma_simple_riscv_decoder_init(lzma_next_coder *next, + const lzma_allocator *allocator, + const lzma_filter_info *filters) +{ + return lzma_simple_coder_init(next, allocator, filters, + &riscv_decode, 0, 8, 2, false); +} +#endif
diff --git a/Utilities/cmliblzma/liblzma/simple/simple_coder.c b/Utilities/cmliblzma/liblzma/simple/simple_coder.c index 4f499be..5cbfa82 100644 --- a/Utilities/cmliblzma/liblzma/simple/simple_coder.c +++ b/Utilities/cmliblzma/liblzma/simple/simple_coder.c
@@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file simple_coder.c @@ -8,9 +10,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "simple_private.h" @@ -139,9 +138,11 @@ return ret; } - // Filter out[]. + // Filter out[] unless there is nothing to filter. + // This way we avoid null pointer + 0 (undefined behavior) + // when out == NULL. const size_t size = *out_pos - out_start; - const size_t filtered = call_filter( + const size_t filtered = size == 0 ? 0 : call_filter( coder, out + out_start, size); const size_t unfiltered = size - filtered;
diff --git a/Utilities/cmliblzma/liblzma/simple/simple_coder.h b/Utilities/cmliblzma/liblzma/simple/simple_coder.h index 19c2ee0..2b762d5 100644 --- a/Utilities/cmliblzma/liblzma/simple/simple_coder.h +++ b/Utilities/cmliblzma/liblzma/simple/simple_coder.h
@@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file simple_coder.h @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef LZMA_SIMPLE_CODER_H @@ -61,6 +60,15 @@ const lzma_filter_info *filters); +extern lzma_ret lzma_simple_arm64_encoder_init(lzma_next_coder *next, + const lzma_allocator *allocator, + const lzma_filter_info *filters); + +extern lzma_ret lzma_simple_arm64_decoder_init(lzma_next_coder *next, + const lzma_allocator *allocator, + const lzma_filter_info *filters); + + extern lzma_ret lzma_simple_sparc_encoder_init(lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter_info *filters); @@ -69,4 +77,13 @@ const lzma_allocator *allocator, const lzma_filter_info *filters); + +extern lzma_ret lzma_simple_riscv_encoder_init(lzma_next_coder *next, + const lzma_allocator *allocator, + const lzma_filter_info *filters); + +extern lzma_ret lzma_simple_riscv_decoder_init(lzma_next_coder *next, + const lzma_allocator *allocator, + const lzma_filter_info *filters); + #endif
diff --git a/Utilities/cmliblzma/liblzma/simple/simple_decoder.c b/Utilities/cmliblzma/liblzma/simple/simple_decoder.c index dc4d241..d9820ee 100644 --- a/Utilities/cmliblzma/liblzma/simple/simple_decoder.c +++ b/Utilities/cmliblzma/liblzma/simple/simple_decoder.c
@@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file simple_decoder.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "simple_decoder.h"
diff --git a/Utilities/cmliblzma/liblzma/simple/simple_decoder.h b/Utilities/cmliblzma/liblzma/simple/simple_decoder.h index bed8d37..2ae87bb 100644 --- a/Utilities/cmliblzma/liblzma/simple/simple_decoder.h +++ b/Utilities/cmliblzma/liblzma/simple/simple_decoder.h
@@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file simple_decoder.h @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef LZMA_SIMPLE_DECODER_H
diff --git a/Utilities/cmliblzma/liblzma/simple/simple_encoder.c b/Utilities/cmliblzma/liblzma/simple/simple_encoder.c index d2cc03e..d1f3509 100644 --- a/Utilities/cmliblzma/liblzma/simple/simple_encoder.c +++ b/Utilities/cmliblzma/liblzma/simple/simple_encoder.c
@@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file simple_encoder.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "simple_encoder.h"
diff --git a/Utilities/cmliblzma/liblzma/simple/simple_encoder.h b/Utilities/cmliblzma/liblzma/simple/simple_encoder.h index 1cee482..bf5edbb 100644 --- a/Utilities/cmliblzma/liblzma/simple/simple_encoder.h +++ b/Utilities/cmliblzma/liblzma/simple/simple_encoder.h
@@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file simple_encoder.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef LZMA_SIMPLE_ENCODER_H
diff --git a/Utilities/cmliblzma/liblzma/simple/simple_private.h b/Utilities/cmliblzma/liblzma/simple/simple_private.h index 9d2c0fd..7aa360f 100644 --- a/Utilities/cmliblzma/liblzma/simple/simple_private.h +++ b/Utilities/cmliblzma/liblzma/simple/simple_private.h
@@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file simple_private.h @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef LZMA_SIMPLE_PRIVATE_H
diff --git a/Utilities/cmliblzma/liblzma/simple/sparc.c b/Utilities/cmliblzma/liblzma/simple/sparc.c index 74b2655..e8ad285 100644 --- a/Utilities/cmliblzma/liblzma/simple/sparc.c +++ b/Utilities/cmliblzma/liblzma/simple/sparc.c
@@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file sparc.c @@ -6,9 +8,6 @@ // Authors: Igor Pavlov // Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "simple_private.h" @@ -65,6 +64,7 @@ } +#ifdef HAVE_ENCODER_SPARC extern lzma_ret lzma_simple_sparc_encoder_init(lzma_next_coder *next, const lzma_allocator *allocator, @@ -72,8 +72,10 @@ { return sparc_coder_init(next, allocator, filters, true); } +#endif +#ifdef HAVE_DECODER_SPARC extern lzma_ret lzma_simple_sparc_decoder_init(lzma_next_coder *next, const lzma_allocator *allocator, @@ -81,3 +83,4 @@ { return sparc_coder_init(next, allocator, filters, false); } +#endif
diff --git a/Utilities/cmliblzma/liblzma/simple/x86.c b/Utilities/cmliblzma/liblzma/simple/x86.c index b38cebf..b579ac3 100644 --- a/Utilities/cmliblzma/liblzma/simple/x86.c +++ b/Utilities/cmliblzma/liblzma/simple/x86.c
@@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file x86.c @@ -6,9 +8,6 @@ // Authors: Igor Pavlov // Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "simple_private.h" @@ -27,11 +26,7 @@ x86_code(void *simple_ptr, uint32_t now_pos, bool is_encoder, uint8_t *buffer, size_t size) { - static const bool MASK_TO_ALLOWED_STATUS[8] - = { true, true, true, false, true, false, false, false }; - - static const uint32_t MASK_TO_BIT_NUMBER[8] - = { 0, 1, 2, 2, 3, 3, 3, 3 }; + static const uint32_t MASK_TO_BIT_NUMBER[5] = { 0, 1, 2, 2, 3 }; lzma_simple_x86 *simple = simple_ptr; uint32_t prev_mask = simple->prev_mask; @@ -68,9 +63,8 @@ b = buffer[buffer_pos + 4]; - if (Test86MSByte(b) - && MASK_TO_ALLOWED_STATUS[(prev_mask >> 1) & 0x7] - && (prev_mask >> 1) < 0x10) { + if (Test86MSByte(b) && (prev_mask >> 1) <= 4 + && (prev_mask >> 1) != 3) { uint32_t src = ((uint32_t)(b) << 24) | ((uint32_t)(buffer[buffer_pos + 3]) << 16) @@ -141,6 +135,7 @@ } +#ifdef HAVE_ENCODER_X86 extern lzma_ret lzma_simple_x86_encoder_init(lzma_next_coder *next, const lzma_allocator *allocator, @@ -148,8 +143,10 @@ { return x86_coder_init(next, allocator, filters, true); } +#endif +#ifdef HAVE_DECODER_X86 extern lzma_ret lzma_simple_x86_decoder_init(lzma_next_coder *next, const lzma_allocator *allocator, @@ -157,3 +154,4 @@ { return x86_coder_init(next, allocator, filters, false); } +#endif