diff --git a/src/blosc2/LICENSE.txt b/src/blosc2/LICENSE.txt new file mode 100644 index 0000000..076549b --- /dev/null +++ b/src/blosc2/LICENSE.txt @@ -0,0 +1,31 @@ +BSD License + +For Blosc - A blocking, shuffling and lossless compression library + +Copyright (C) 2009-2018 Francesc Alted +Copyright (C) 2019- The Blosc Development Team + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + * Neither the name Francesc Alted nor the names of its contributors may be used + to endorse or promote products derived from this software without specific + prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/src/blosc2/README.rst b/src/blosc2/README.rst new file mode 100644 index 0000000..759780d --- /dev/null +++ b/src/blosc2/README.rst @@ -0,0 +1,253 @@ +======== +C-Blosc2 +======== + +A fast, compressed and persistent data store library for C +========================================================== + + +:Author: The Blosc Development Team +:Contact: blosc@blosc.org +:URL: http://www.blosc.org +:Gitter: |gitter| +:Actions: |actions| +:NumFOCUS: |numfocus| +:Code of Conduct: |Contributor Covenant| + +.. |gitter| image:: https://badges.gitter.im/Blosc/c-blosc.svg + :alt: Join the chat at https://gitter.im/Blosc/c-blosc + :target: https://gitter.im/Blosc/c-blosc?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge + +.. |actions| image:: https://github.com/Blosc/c-blosc2/workflows/CI%20CMake/badge.svg + :target: https://github.com/Blosc/c-blosc2/actions?query=workflow%3A%22CI+CMake%22 + +.. |appveyor| image:: https://ci.appveyor.com/api/projects/status/qiaxywqrouj6nkug/branch/master?svg=true + :target: https://ci.appveyor.com/project/FrancescAlted/c-blosc2/branch/master + +.. |numfocus| image:: https://img.shields.io/badge/powered%20by-NumFOCUS-orange.svg?style=flat&colorA=E1523D&colorB=007D8A + :target: https://numfocus.org + +.. |Contributor Covenant| image:: https://img.shields.io/badge/Contributor%20Covenant-v2.0%20adopted-ff69b4.svg + :target: code_of_conduct.md + + +What is it? +=========== + +`Blosc `_ is a high performance compressor optimized for binary data (i.e. floating point numbers, integers and booleans). It has been designed to transmit data to the processor cache faster than the traditional, non-compressed, direct memory fetch approach via a memcpy() OS call. Blosc main goal is not just to reduce the size of large datasets on-disk or in-memory, but also to accelerate memory-bound computations. + +C-Blosc2 is the new major version of `C-Blosc `_, and tries hard to be backward compatible with both the C-Blosc1 API and its in-memory format. However, the reverse thing is generally not true for the format; buffers generated with C-Blosc2 are not format-compatible with C-Blosc1 (i.e. forward compatibility is not supported). In case you want to ensure full API compatibility with C-Blosc1 API, define the `BLOSC1_COMPAT` symbol. + +See a 3 minutes `introductory video to Blosc2 `_. + + +New features in C-Blosc2 +======================== + +* **64-bit containers:** the first-class container in C-Blosc2 is the `super-chunk` or, for brevity, `schunk`, that is made by smaller chunks which are essentially C-Blosc1 32-bit containers. The super-chunk can be backed or not by another container which is called a `frame` (see later). + +* **More filters:** besides `shuffle` and `bitshuffle` already present in C-Blosc1, C-Blosc2 already implements: + + - `delta`: the stored blocks inside a chunk are diff'ed with respect to first block in the chunk. The idea is that, in some situations, the diff will have more zeros than the original data, leading to better compression. + + - `trunc_prec`: it zeroes the least significant bits of the mantissa of float32 and float64 types. When combined with the `shuffle` or `bitshuffle` filter, this leads to more contiguous zeros, which are compressed better. + +* **A filter pipeline:** the different filters can be pipelined so that the output of one can the input for the other. A possible example is a `delta` followed by `shuffle`, or as described above, `trunc_prec` followed by `bitshuffle`. + +* **Prefilters:** allow to apply user-defined C callbacks **prior** the filter pipeline during compression. See `test_prefilter.c `_ for an example of use. + +* **Postfilters:** allow to apply user-defined C callbacks **after** the filter pipeline during decompression. The combination of prefilters and postfilters could be interesting for supporting e.g. encryption (via prefilters) and decryption (via postfilters). Also, a postfilter alone can be used to produce on-the-flight computation based on existing data (or other metadata, like e.g. coordinates). See `test_postfilter.c `_ for an example of use. + +* **SIMD support for ARM (NEON):** this allows for faster operation on ARM architectures. Only `shuffle` is supported right now, but the idea is to implement `bitshuffle` for NEON too. Thanks to Lucian Marc. + +* **SIMD support for PowerPC (ALTIVEC):** this allows for faster operation on PowerPC architectures. Both `shuffle` and `bitshuffle` are supported; however, this has been done via a transparent mapping from SSE2 into ALTIVEC emulation in GCC 8, so performance could be better (but still, it is already a nice improvement over native C code; see PR https://github.com/Blosc/c-blosc2/pull/59 for details). Thanks to Jerome Kieffer and `ESRF `_ for sponsoring the Blosc team in helping him in this task. + +* **Dictionaries:** when a block is going to be compressed, C-Blosc2 can use a previously made dictionary (stored in the header of the super-chunk) for compressing all the blocks that are part of the chunks. This usually improves the compression ratio, as well as the decompression speed, at the expense of a (small) overhead in compression speed. Currently, it is only supported in the `zstd` codec, but would be nice to extend it to `lz4` and `blosclz` at least. + +* **Contiguous frames:** allow to store super-chunks contiguously, either on-disk or in-memory. When a super-chunk is backed by a frame, instead of storing all the chunks sparsely in-memory, they are serialized inside the frame container. The frame can be stored on-disk too, meaning that persistence of super-chunks is supported. + +* **Sparse frames:** each chunk in a super-chunk is stored in a separate file or different memory area, as well as the metadata. This is allows for more efficient updates/deletes than in contiguous frames (i.e. avoiding 'holes' in monolithic files). The drawback is that it consumes more inodes when on-disk. Thanks to Marta Iborra for this contribution. + +* **Partial chunk reads:** there is support for reading just part of chunks, so avoiding to read the whole thing and then discard the unnecessary data. + +* **Parallel chunk reads:** when several blocks of a chunk are to be read, this is done in parallel by the decompressing machinery. That means that every thread is responsible to read, post-filter and decompress a block by itself, leading to an efficient overlap of I/O and CPU usage that optimizes reads to a maximum. + +* **Meta-layers:** optionally, the user can add meta-data for different uses and in different layers. For example, one may think on providing a meta-layer for `NumPy `_ so that most of the meta-data for it is stored in a meta-layer; then, one can place another meta-layer on top of the latter for adding more high-level info if desired (e.g. geo-spatial, meteorological...). + +* **Variable length meta-layers:** the user may want to add variable-length meta information that can be potentially very large (up to 2 GB). The regular meta-layer described above is very quick to read, but meant to store fixed-length and relatively small meta information. Variable length metalayers are stored in the trailer of a frame, whereas regular meta-layers are in the header. + +* **Efficient support for special values:** large sequences of repeated values can be represented with an efficient, simple and fast run-length representation, without the need to use regular codecs. With that, chunks or super-chunks with values that are the same (zeros, NaNs or any value in general) can be built in constant time, regardless of the size. This can be useful in situations where a lot of zeros (or NaNs) need to be stored (e.g. sparse matrices). + +* **Nice markup for documentation:** we are currently using a combination of Sphinx + Doxygen + Breathe for documenting the C-API. See https://www.blosc.org/c-blosc2/c-blosc2.html. Thanks to Alberto Sabater and Aleix Alcacer for contributing the support for this. + +* **Plugin capabilities for filters and codecs:** we have a plugin register capability inplace so that the info about the new filters and codecs can be persisted and transmitted to different machines. See https://github.com/Blosc/c-blosc2/blob/main/examples/urfilters.c for a self-contained example. Thanks to the NumFOCUS foundation for providing a grant for doing this, and Oscar Griñón and Aleix Alcacer for the implementation. + +* **Pluggable tuning capabilities:** this will allow users with different needs to define an interface so as to better tune different parameters like the codec, the compression level, the filters to use, the blocksize or the shuffle size. Thanks to ironArray for sponsoring us in doing this. + +* **Support for I/O plugins:** so that users can extend the I/O capabilities beyond the current filesystem support. Things like the use of databases or S3 interfaces should be possible by implementing these interfaces. Thanks to ironArray for sponsoring us in doing this. + +* **Python wrapper:** we have a preliminary wrapper in the works. You can have a look at our ongoing efforts in the `python-blosc2 repo `_. Thanks to the Python Software Foundation for providing a grant for doing this. + +* **Security:** we are actively using using the `OSS-Fuzz `_ and `ClusterFuzz `_ for uncovering programming errors in C-Blosc2. Thanks to Google for sponsoring us in doing this, and to Nathan Moinvaziri for most of the work here. + +More info about the `improved capabilities of C-Blosc2 can be found in this talk `_. + +After a long period of testing, C-Blosc2 entered production stage in 2.0.0. The API and format have been frozen, and that means that there is guarantee that your programs will continue to work with future versions of the library, and that next releases will be able to read from persistent storage generated from previous releases (as of 2.0.0). + + +Meta-compression and other advantages over existing compressors +=============================================================== + +C-Blosc2 is not like other compressors: it should rather be called a meta-compressor. This is so because it can use different codecs (libraries that can reduce the size of inputs) and filters (libraries that generally improve compression ratio). At the same time, it can also be called a compressor because it makes an actual use of the several codecs and filters, so it can actually work like so. + +Currently C-Blosc2 comes with support of BloscLZ, a compressor heavily based on `FastLZ `_, `LZ4 and LZ4HC `_, `Zstd `_, and `Zlib, via zlib-ng: `_, as well as a highly optimized (it can use SSE2, AVX2, NEON or ALTIVEC instructions, if available) shuffle and bitshuffle filters (for info on how shuffling works, see slide 17 of http://www.slideshare.net/PyData/blosc-py-data-2014). + +Blosc is in charge of coordinating the codecs and filters so that they can leverage the blocking technique (described above) as +well as multi-threaded execution (if several cores are available) automatically. That makes that every codec and filter +will work at very high speeds, even if it was not initially designed for doing blocking or multi-threading. For example, +Blosc allows you to use the ``LZ4`` codec, but in a multi-threaded way. + +Last but not least, C-Blosc2 comes with an easy-to-use plugin mechanism for codecs and filters, so anyone can inject their own code in the compression pipeline of Blosc2 and reap its benefits (like multi-threading and integration with other filters) for free (see a `self-contained example `_). In addition, we have implemented a centralized plugin system too (see the `docs in the plugins directory `_). + + +Multidimensional containers +=========================== + +As said, C-Blosc2 adds a powerful mechanism for adding different metalayers on top of its containers. `Caterva `_ is a sibling library that adds such a metalayer specifying not only the dimensionality of a dataset, but also the dimensionality of the chunks inside the dataset. In addition, Caterva adds machinery for retrieving arbitrary multi-dimensional slices (aka hyper-slices) out of the multi-dimensional containers in the most efficient way. Hence, Caterva brings the convenience of multi-dimensional containers to your application very easily. For more info, check out the `Caterva documentation `_. + + +Python wrapper +============== + +We are officially supporting (thanks to the Python Software Foundation) a `Python wrapper for Blosc2 `_. Although this is still in early development, it already supports all the features of the venerable `python-blosc ` package. As a bonus, the `python-blosc2` package comes with wheels and binary versions of the C-Blosc2 libraries, so anyone, even non-Python users can install C-Blosc2 binaries easily with: + +.. code-block:: console + + pip install blosc2 + + +Compiling the C-Blosc2 library with CMake +========================================= + +Blosc can be built, tested and installed using `CMake `_. The following procedure describes a typical CMake build. + +Create the build directory inside the sources and move into it: + +.. code-block:: console + + git clone https://github.com/Blosc/c-blosc2 + cd c-blosc2 + mkdir build + cd build + +Now run CMake configuration and optionally specify the installation +directory (e.g. '/usr' or '/usr/local'): + +.. code-block:: console + + cmake -DCMAKE_INSTALL_PREFIX=your_install_prefix_directory .. + +CMake allows to configure Blosc in many different ways, like preferring internal or external sources for compressors or enabling/disabling them. Please note that configuration can also be performed using UI tools provided by CMake (`ccmake` or `cmake-gui`): + +.. code-block:: console + + ccmake .. # run a curses-based interface + cmake-gui .. # run a graphical interface + +Build, test and install Blosc: + +.. code-block:: console + + cmake --build . + ctest + cmake --build . --target install + +The static and dynamic version of the Blosc library, together with header files, will be installed into the specified CMAKE_INSTALL_PREFIX. + +Once you have compiled your Blosc library, you can easily link your apps with it as shown in the `examples/ directory `_. + + +Handling support for codecs (LZ4, LZ4HC, Zstd, Zlib) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +C-Blosc2 comes with full sources for LZ4, LZ4HC, Zstd, and Zlib and in general, you should not worry about not having (or CMake not finding) the libraries in your system because by default the included sources will be automatically compiled and included in the C-Blosc2 library. This means that you can be confident in having a complete support for all the codecs in all the Blosc deployments (unless you are explicitly excluding support for some of them). + +If you want to force Blosc to use external libraries instead of the included compression sources: + +.. code-block:: console + + cmake -DPREFER_EXTERNAL_LZ4=ON .. + +You can also disable support for some compression libraries: + +.. code-block:: console + + cmake -DDEACTIVATE_ZSTD=ON .. + + +Supported platforms +~~~~~~~~~~~~~~~~~~~ + +C-Blosc2 is meant to support all platforms where a C99 compliant C compiler can be found. The ones that are mostly tested are Intel (Linux, Mac OSX and Windows), ARM (Linux, Mac), and PowerPC (Linux) but exotic ones as IBM Blue Gene Q embedded "A2" processor are reported to work too. More on ARM support in `README_ARM.rst`. + +For Windows, you will need at least VS2015 or higher on x86 and x64 targets (i.e. ARM is not supported on Windows). + +For Mac OSX, make sure that you have installed the command line developer tools. You can always install them with: + +.. code-block:: console + + xcode-select --install + +For Mac OSX on arm64 architecture, you need to compile like this: + +.. code-block:: console + + CC="clang -arch arm64" cmake .. + + +Support for the LZ4 optimized version in Intel IPP +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +C-Blosc2 comes with support for a highly optimized version of the LZ4 codec present in Intel IPP. Here it is a way to easily install Intel IPP using Conda(https://docs.conda.io): + +.. code-block:: console + + conda install -c intel ipp-static + +With that, you can enable support for LZ4/IPP (it is disabled by default) with: + +.. code-block:: console + + cmake .. -DDEACTIVATE_IPP=OFF + +In some Intel CPUs LZ4/IPP could be faster than regular LZ4, although in many cases you may experience different compression ratios depending on which version you use. See #313 for some quick and dirty benchmarks. + + +Display error messages +~~~~~~~~~~~~~~~~~~~~~~ + +By default error messages are disabled. To display them, you just need to activate the Blosc tracing machinery by setting +the ``BLOSC_TRACE`` environment variable. + + +Contributing +============ + +If you want to collaborate in this development you are welcome. We need help in the different areas listed at the `ROADMAP `_; also, be sure to read our `DEVELOPING-GUIDE `_ and our `Code of Conduct `_. Blosc is distributed using the `BSD license `_. + + +Tweeter feed +============ + +Follow `@Blosc2 `_ so as to get informed about the latest developments. + + +Acknowledgments +=============== + +See `THANKS document `_. + + +---- + +**Enjoy data!** diff --git a/src/blosc2/THANKS.rst b/src/blosc2/THANKS.rst new file mode 100644 index 0000000..7532356 --- /dev/null +++ b/src/blosc2/THANKS.rst @@ -0,0 +1,33 @@ +Thanks +====== + +* Valentin Haenel did a terrific work implementing the support for the Snappy compression, fixing typos and improving docs and the plotting script. + +* Thibault North, with ideas from Oscar Villellas, contributed a way to call Blosc from different threads in a safe way. Christopher + Speller introduced contexts so that a global lock is not necessary anymore. + +* The CMake support was initially contributed by Thibault North, and Antonio Valentino and Mark Wiebe made great enhancements to it. + +* Christopher Speller also introduced the two new '_ctx' calls to avoid the use of the blosc_init() and blosc_destroy(). + +* Jack Pappas contributed important portability enhancements, specially runtime and cross-platform detection of SSE2/AVX2 as well as high precision timers (HPET) for the benchmark program. + +* @littlezhou implemented the AVX2 version of shuffle routines. + +* Julian Taylor contributed a way to detect AVX2 in runtime and calling the appropriate routines only if the underlying hardware supports it. + +* Lucian Marc provided the support for ARM/NEON for the shuffle filter. + +* Jerome Kieffer contributed support for PowerPC/ALTIVEC for the shuffle/bitshuffle filter. + +* Alberto Sabater, for his great efforts on producing really nice Blosc2 docs, among other aspects. + +* Kiyo Masui for relicensing his bitshuffle project for allowing the inclusion of part of his code in Blosc. + +* Aleix Alcacer for his implementation of mutable super-chunks, multiple variable length metalayers and many other things. + +* Oscar Guiñón for the optimization of reading a (sparse) set of blocks of a chunk in parallel. + +* Nathan Moinvaziri for his outstanding work on the security side of the things via `fuzzer testing `_. + +* Marta Iborra for her implementation of sparse storage for persistent super-chunks. diff --git a/src/blosc2/blosc/CMakeCache.txt b/src/blosc2/blosc/CMakeCache.txt new file mode 100644 index 0000000..340ce52 --- /dev/null +++ b/src/blosc2/blosc/CMakeCache.txt @@ -0,0 +1,385 @@ +# This is the CMakeCache file. +# For build in directory: /home/fangq/space/git/Project/github/zmat/src/blosc2/blosc +# It was generated by CMake: /usr/bin/cmake +# You can edit this file to change values found and used by cmake. +# If you do not want to change any of the values, simply exit the editor. +# If you do want to change a value, simply edit, save, and exit the editor. +# The syntax for the file is as follows: +# KEY:TYPE=VALUE +# KEY is the name of a variable in the cache. +# TYPE is a hint to GUIs for the type of VALUE, DO NOT EDIT TYPE!. +# VALUE is the current value for the KEY. + +######################## +# EXTERNAL cache entries +######################## + +//Path to a program. +CMAKE_ADDR2LINE:FILEPATH=/usr/bin/addr2line + +//Path to a program. +CMAKE_AR:FILEPATH=/usr/bin/ar + +//For backwards compatibility, what version of CMake commands and +// syntax should this version of CMake try to support. +CMAKE_BACKWARDS_COMPATIBILITY:STRING=2.4 + +//Choose the type of build, options are: None Debug Release RelWithDebInfo +// MinSizeRel ... +CMAKE_BUILD_TYPE:STRING= + +//Enable/Disable color output during build. +CMAKE_COLOR_MAKEFILE:BOOL=ON + +//CXX compiler +CMAKE_CXX_COMPILER:FILEPATH=/usr/bin/c++ + +//A wrapper around 'ar' adding the appropriate '--plugin' option +// for the GCC compiler +CMAKE_CXX_COMPILER_AR:FILEPATH=/usr/bin/gcc-ar-11 + +//A wrapper around 'ranlib' adding the appropriate '--plugin' option +// for the GCC compiler +CMAKE_CXX_COMPILER_RANLIB:FILEPATH=/usr/bin/gcc-ranlib-11 + +//Flags used by the CXX compiler during all build types. +CMAKE_CXX_FLAGS:STRING= + +//Flags used by the CXX compiler during DEBUG builds. +CMAKE_CXX_FLAGS_DEBUG:STRING=-g + +//Flags used by the CXX compiler during MINSIZEREL builds. +CMAKE_CXX_FLAGS_MINSIZEREL:STRING=-Os -DNDEBUG + +//Flags used by the CXX compiler during RELEASE builds. +CMAKE_CXX_FLAGS_RELEASE:STRING=-O3 -DNDEBUG + +//Flags used by the CXX compiler during RELWITHDEBINFO builds. +CMAKE_CXX_FLAGS_RELWITHDEBINFO:STRING=-O2 -g -DNDEBUG + +//C compiler +CMAKE_C_COMPILER:FILEPATH=/usr/bin/cc + +//A wrapper around 'ar' adding the appropriate '--plugin' option +// for the GCC compiler +CMAKE_C_COMPILER_AR:FILEPATH=/usr/bin/gcc-ar-11 + +//A wrapper around 'ranlib' adding the appropriate '--plugin' option +// for the GCC compiler +CMAKE_C_COMPILER_RANLIB:FILEPATH=/usr/bin/gcc-ranlib-11 + +//Flags used by the C compiler during all build types. +CMAKE_C_FLAGS:STRING= + +//Flags used by the C compiler during DEBUG builds. +CMAKE_C_FLAGS_DEBUG:STRING=-g + +//Flags used by the C compiler during MINSIZEREL builds. +CMAKE_C_FLAGS_MINSIZEREL:STRING=-Os -DNDEBUG + +//Flags used by the C compiler during RELEASE builds. +CMAKE_C_FLAGS_RELEASE:STRING=-O3 -DNDEBUG + +//Flags used by the C compiler during RELWITHDEBINFO builds. +CMAKE_C_FLAGS_RELWITHDEBINFO:STRING=-O2 -g -DNDEBUG + +//Path to a program. +CMAKE_DLLTOOL:FILEPATH=CMAKE_DLLTOOL-NOTFOUND + +//Flags used by the linker during all build types. +CMAKE_EXE_LINKER_FLAGS:STRING= + +//Flags used by the linker during DEBUG builds. +CMAKE_EXE_LINKER_FLAGS_DEBUG:STRING= + +//Flags used by the linker during MINSIZEREL builds. +CMAKE_EXE_LINKER_FLAGS_MINSIZEREL:STRING= + +//Flags used by the linker during RELEASE builds. +CMAKE_EXE_LINKER_FLAGS_RELEASE:STRING= + +//Flags used by the linker during RELWITHDEBINFO builds. +CMAKE_EXE_LINKER_FLAGS_RELWITHDEBINFO:STRING= + +//Enable/Disable output of compile commands during generation. +CMAKE_EXPORT_COMPILE_COMMANDS:BOOL= + +//Install path prefix, prepended onto install directories. +CMAKE_INSTALL_PREFIX:PATH=/usr/local + +//Path to a program. +CMAKE_LINKER:FILEPATH=/usr/bin/ld + +//Path to a program. +CMAKE_MAKE_PROGRAM:FILEPATH=/usr/bin/gmake + +//Flags used by the linker during the creation of modules during +// all build types. +CMAKE_MODULE_LINKER_FLAGS:STRING= + +//Flags used by the linker during the creation of modules during +// DEBUG builds. +CMAKE_MODULE_LINKER_FLAGS_DEBUG:STRING= + +//Flags used by the linker during the creation of modules during +// MINSIZEREL builds. +CMAKE_MODULE_LINKER_FLAGS_MINSIZEREL:STRING= + +//Flags used by the linker during the creation of modules during +// RELEASE builds. +CMAKE_MODULE_LINKER_FLAGS_RELEASE:STRING= + +//Flags used by the linker during the creation of modules during +// RELWITHDEBINFO builds. +CMAKE_MODULE_LINKER_FLAGS_RELWITHDEBINFO:STRING= + +//Path to a program. +CMAKE_NM:FILEPATH=/usr/bin/nm + +//Path to a program. +CMAKE_OBJCOPY:FILEPATH=/usr/bin/objcopy + +//Path to a program. +CMAKE_OBJDUMP:FILEPATH=/usr/bin/objdump + +//Value Computed by CMake +CMAKE_PROJECT_DESCRIPTION:STATIC= + +//Value Computed by CMake +CMAKE_PROJECT_HOMEPAGE_URL:STATIC= + +//Value Computed by CMake +CMAKE_PROJECT_NAME:STATIC=Project + +//Path to a program. +CMAKE_RANLIB:FILEPATH=/usr/bin/ranlib + +//Path to a program. +CMAKE_READELF:FILEPATH=/usr/bin/readelf + +//Flags used by the linker during the creation of shared libraries +// during all build types. +CMAKE_SHARED_LINKER_FLAGS:STRING= + +//Flags used by the linker during the creation of shared libraries +// during DEBUG builds. +CMAKE_SHARED_LINKER_FLAGS_DEBUG:STRING= + +//Flags used by the linker during the creation of shared libraries +// during MINSIZEREL builds. +CMAKE_SHARED_LINKER_FLAGS_MINSIZEREL:STRING= + +//Flags used by the linker during the creation of shared libraries +// during RELEASE builds. +CMAKE_SHARED_LINKER_FLAGS_RELEASE:STRING= + +//Flags used by the linker during the creation of shared libraries +// during RELWITHDEBINFO builds. +CMAKE_SHARED_LINKER_FLAGS_RELWITHDEBINFO:STRING= + +//If set, runtime paths are not added when installing shared libraries, +// but are added when building. +CMAKE_SKIP_INSTALL_RPATH:BOOL=NO + +//If set, runtime paths are not added when using shared libraries. +CMAKE_SKIP_RPATH:BOOL=NO + +//Flags used by the linker during the creation of static libraries +// during all build types. +CMAKE_STATIC_LINKER_FLAGS:STRING= + +//Flags used by the linker during the creation of static libraries +// during DEBUG builds. +CMAKE_STATIC_LINKER_FLAGS_DEBUG:STRING= + +//Flags used by the linker during the creation of static libraries +// during MINSIZEREL builds. +CMAKE_STATIC_LINKER_FLAGS_MINSIZEREL:STRING= + +//Flags used by the linker during the creation of static libraries +// during RELEASE builds. +CMAKE_STATIC_LINKER_FLAGS_RELEASE:STRING= + +//Flags used by the linker during the creation of static libraries +// during RELWITHDEBINFO builds. +CMAKE_STATIC_LINKER_FLAGS_RELWITHDEBINFO:STRING= + +//Path to a program. +CMAKE_STRIP:FILEPATH=/usr/bin/strip + +//If this value is on, makefiles will be generated without the +// .SILENT directive, and all commands will be echoed to the console +// during the make. This is useful for debugging only. With Visual +// Studio IDE projects all commands are done without /nologo. +CMAKE_VERBOSE_MAKEFILE:BOOL=FALSE + +//Single output directory for building all executables. +EXECUTABLE_OUTPUT_PATH:PATH= + +//Single output directory for building all libraries. +LIBRARY_OUTPUT_PATH:PATH= + +//Value Computed by CMake +Project_BINARY_DIR:STATIC=/home/fangq/space/git/Project/github/zmat/src/blosc2/blosc + +//Value Computed by CMake +Project_IS_TOP_LEVEL:STATIC=ON + +//Value Computed by CMake +Project_SOURCE_DIR:STATIC=/home/fangq/space/git/Project/github/zmat/src/blosc2/blosc + + +######################## +# INTERNAL cache entries +######################## + +//ADVANCED property for variable: CMAKE_ADDR2LINE +CMAKE_ADDR2LINE-ADVANCED:INTERNAL=1 +//ADVANCED property for variable: CMAKE_AR +CMAKE_AR-ADVANCED:INTERNAL=1 +//This is the directory where this CMakeCache.txt was created +CMAKE_CACHEFILE_DIR:INTERNAL=/home/fangq/space/git/Project/github/zmat/src/blosc2/blosc +//Major version of cmake used to create the current loaded cache +CMAKE_CACHE_MAJOR_VERSION:INTERNAL=3 +//Minor version of cmake used to create the current loaded cache +CMAKE_CACHE_MINOR_VERSION:INTERNAL=22 +//Patch version of cmake used to create the current loaded cache +CMAKE_CACHE_PATCH_VERSION:INTERNAL=1 +//ADVANCED property for variable: CMAKE_COLOR_MAKEFILE +CMAKE_COLOR_MAKEFILE-ADVANCED:INTERNAL=1 +//Path to CMake executable. +CMAKE_COMMAND:INTERNAL=/usr/bin/cmake +//Path to cpack program executable. +CMAKE_CPACK_COMMAND:INTERNAL=/usr/bin/cpack +//Path to ctest program executable. +CMAKE_CTEST_COMMAND:INTERNAL=/usr/bin/ctest +//ADVANCED property for variable: CMAKE_CXX_COMPILER +CMAKE_CXX_COMPILER-ADVANCED:INTERNAL=1 +//ADVANCED property for variable: CMAKE_CXX_COMPILER_AR +CMAKE_CXX_COMPILER_AR-ADVANCED:INTERNAL=1 +//ADVANCED property for variable: CMAKE_CXX_COMPILER_RANLIB +CMAKE_CXX_COMPILER_RANLIB-ADVANCED:INTERNAL=1 +//ADVANCED property for variable: CMAKE_CXX_FLAGS +CMAKE_CXX_FLAGS-ADVANCED:INTERNAL=1 +//ADVANCED property for variable: CMAKE_CXX_FLAGS_DEBUG +CMAKE_CXX_FLAGS_DEBUG-ADVANCED:INTERNAL=1 +//ADVANCED property for variable: CMAKE_CXX_FLAGS_MINSIZEREL +CMAKE_CXX_FLAGS_MINSIZEREL-ADVANCED:INTERNAL=1 +//ADVANCED property for variable: CMAKE_CXX_FLAGS_RELEASE +CMAKE_CXX_FLAGS_RELEASE-ADVANCED:INTERNAL=1 +//ADVANCED property for variable: CMAKE_CXX_FLAGS_RELWITHDEBINFO +CMAKE_CXX_FLAGS_RELWITHDEBINFO-ADVANCED:INTERNAL=1 +//ADVANCED property for variable: CMAKE_C_COMPILER +CMAKE_C_COMPILER-ADVANCED:INTERNAL=1 +//ADVANCED property for variable: CMAKE_C_COMPILER_AR +CMAKE_C_COMPILER_AR-ADVANCED:INTERNAL=1 +//ADVANCED property for variable: CMAKE_C_COMPILER_RANLIB +CMAKE_C_COMPILER_RANLIB-ADVANCED:INTERNAL=1 +//ADVANCED property for variable: CMAKE_C_FLAGS +CMAKE_C_FLAGS-ADVANCED:INTERNAL=1 +//ADVANCED property for variable: CMAKE_C_FLAGS_DEBUG +CMAKE_C_FLAGS_DEBUG-ADVANCED:INTERNAL=1 +//ADVANCED property for variable: CMAKE_C_FLAGS_MINSIZEREL +CMAKE_C_FLAGS_MINSIZEREL-ADVANCED:INTERNAL=1 +//ADVANCED property for variable: CMAKE_C_FLAGS_RELEASE +CMAKE_C_FLAGS_RELEASE-ADVANCED:INTERNAL=1 +//ADVANCED property for variable: CMAKE_C_FLAGS_RELWITHDEBINFO +CMAKE_C_FLAGS_RELWITHDEBINFO-ADVANCED:INTERNAL=1 +//ADVANCED property for variable: CMAKE_DLLTOOL +CMAKE_DLLTOOL-ADVANCED:INTERNAL=1 +//Executable file format +CMAKE_EXECUTABLE_FORMAT:INTERNAL=ELF +//ADVANCED property for variable: CMAKE_EXE_LINKER_FLAGS +CMAKE_EXE_LINKER_FLAGS-ADVANCED:INTERNAL=1 +//ADVANCED property for variable: CMAKE_EXE_LINKER_FLAGS_DEBUG +CMAKE_EXE_LINKER_FLAGS_DEBUG-ADVANCED:INTERNAL=1 +//ADVANCED property for variable: CMAKE_EXE_LINKER_FLAGS_MINSIZEREL +CMAKE_EXE_LINKER_FLAGS_MINSIZEREL-ADVANCED:INTERNAL=1 +//ADVANCED property for variable: CMAKE_EXE_LINKER_FLAGS_RELEASE +CMAKE_EXE_LINKER_FLAGS_RELEASE-ADVANCED:INTERNAL=1 +//ADVANCED property for variable: CMAKE_EXE_LINKER_FLAGS_RELWITHDEBINFO +CMAKE_EXE_LINKER_FLAGS_RELWITHDEBINFO-ADVANCED:INTERNAL=1 +//ADVANCED property for variable: CMAKE_EXPORT_COMPILE_COMMANDS +CMAKE_EXPORT_COMPILE_COMMANDS-ADVANCED:INTERNAL=1 +//Name of external makefile project generator. +CMAKE_EXTRA_GENERATOR:INTERNAL= +//Name of generator. +CMAKE_GENERATOR:INTERNAL=Unix Makefiles +//Generator instance identifier. +CMAKE_GENERATOR_INSTANCE:INTERNAL= +//Name of generator platform. +CMAKE_GENERATOR_PLATFORM:INTERNAL= +//Name of generator toolset. +CMAKE_GENERATOR_TOOLSET:INTERNAL= +//Test CMAKE_HAVE_LIBC_PTHREAD +CMAKE_HAVE_LIBC_PTHREAD:INTERNAL=1 +//Have include pthread.h +CMAKE_HAVE_PTHREAD_H:INTERNAL=1 +//Source directory with the top level CMakeLists.txt file for this +// project +CMAKE_HOME_DIRECTORY:INTERNAL=/home/fangq/space/git/Project/github/zmat/src/blosc2/blosc +//Install .so files without execute permission. +CMAKE_INSTALL_SO_NO_EXE:INTERNAL=1 +//ADVANCED property for variable: CMAKE_LINKER +CMAKE_LINKER-ADVANCED:INTERNAL=1 +//ADVANCED property for variable: CMAKE_MAKE_PROGRAM +CMAKE_MAKE_PROGRAM-ADVANCED:INTERNAL=1 +//ADVANCED property for variable: CMAKE_MODULE_LINKER_FLAGS +CMAKE_MODULE_LINKER_FLAGS-ADVANCED:INTERNAL=1 +//ADVANCED property for variable: CMAKE_MODULE_LINKER_FLAGS_DEBUG +CMAKE_MODULE_LINKER_FLAGS_DEBUG-ADVANCED:INTERNAL=1 +//ADVANCED property for variable: CMAKE_MODULE_LINKER_FLAGS_MINSIZEREL +CMAKE_MODULE_LINKER_FLAGS_MINSIZEREL-ADVANCED:INTERNAL=1 +//ADVANCED property for variable: CMAKE_MODULE_LINKER_FLAGS_RELEASE +CMAKE_MODULE_LINKER_FLAGS_RELEASE-ADVANCED:INTERNAL=1 +//ADVANCED property for variable: CMAKE_MODULE_LINKER_FLAGS_RELWITHDEBINFO +CMAKE_MODULE_LINKER_FLAGS_RELWITHDEBINFO-ADVANCED:INTERNAL=1 +//ADVANCED property for variable: CMAKE_NM +CMAKE_NM-ADVANCED:INTERNAL=1 +//number of local generators +CMAKE_NUMBER_OF_MAKEFILES:INTERNAL=1 +//ADVANCED property for variable: CMAKE_OBJCOPY +CMAKE_OBJCOPY-ADVANCED:INTERNAL=1 +//ADVANCED property for variable: CMAKE_OBJDUMP +CMAKE_OBJDUMP-ADVANCED:INTERNAL=1 +//Platform information initialized +CMAKE_PLATFORM_INFO_INITIALIZED:INTERNAL=1 +//ADVANCED property for variable: CMAKE_RANLIB +CMAKE_RANLIB-ADVANCED:INTERNAL=1 +//ADVANCED property for variable: CMAKE_READELF +CMAKE_READELF-ADVANCED:INTERNAL=1 +//Path to CMake installation. +CMAKE_ROOT:INTERNAL=/usr/share/cmake-3.22 +//ADVANCED property for variable: CMAKE_SHARED_LINKER_FLAGS +CMAKE_SHARED_LINKER_FLAGS-ADVANCED:INTERNAL=1 +//ADVANCED property for variable: CMAKE_SHARED_LINKER_FLAGS_DEBUG +CMAKE_SHARED_LINKER_FLAGS_DEBUG-ADVANCED:INTERNAL=1 +//ADVANCED property for variable: CMAKE_SHARED_LINKER_FLAGS_MINSIZEREL +CMAKE_SHARED_LINKER_FLAGS_MINSIZEREL-ADVANCED:INTERNAL=1 +//ADVANCED property for variable: CMAKE_SHARED_LINKER_FLAGS_RELEASE +CMAKE_SHARED_LINKER_FLAGS_RELEASE-ADVANCED:INTERNAL=1 +//ADVANCED property for variable: CMAKE_SHARED_LINKER_FLAGS_RELWITHDEBINFO +CMAKE_SHARED_LINKER_FLAGS_RELWITHDEBINFO-ADVANCED:INTERNAL=1 +//ADVANCED property for variable: CMAKE_SKIP_INSTALL_RPATH +CMAKE_SKIP_INSTALL_RPATH-ADVANCED:INTERNAL=1 +//ADVANCED property for variable: CMAKE_SKIP_RPATH +CMAKE_SKIP_RPATH-ADVANCED:INTERNAL=1 +//ADVANCED property for variable: CMAKE_STATIC_LINKER_FLAGS +CMAKE_STATIC_LINKER_FLAGS-ADVANCED:INTERNAL=1 +//ADVANCED property for variable: CMAKE_STATIC_LINKER_FLAGS_DEBUG +CMAKE_STATIC_LINKER_FLAGS_DEBUG-ADVANCED:INTERNAL=1 +//ADVANCED property for variable: CMAKE_STATIC_LINKER_FLAGS_MINSIZEREL +CMAKE_STATIC_LINKER_FLAGS_MINSIZEREL-ADVANCED:INTERNAL=1 +//ADVANCED property for variable: CMAKE_STATIC_LINKER_FLAGS_RELEASE +CMAKE_STATIC_LINKER_FLAGS_RELEASE-ADVANCED:INTERNAL=1 +//ADVANCED property for variable: CMAKE_STATIC_LINKER_FLAGS_RELWITHDEBINFO +CMAKE_STATIC_LINKER_FLAGS_RELWITHDEBINFO-ADVANCED:INTERNAL=1 +//ADVANCED property for variable: CMAKE_STRIP +CMAKE_STRIP-ADVANCED:INTERNAL=1 +//uname command +CMAKE_UNAME:INTERNAL=/bin/uname +//ADVANCED property for variable: CMAKE_VERBOSE_MAKEFILE +CMAKE_VERBOSE_MAKEFILE-ADVANCED:INTERNAL=1 +//Details about finding Threads +FIND_PACKAGE_MESSAGE_DETAILS_Threads:INTERNAL=[TRUE][v()] + diff --git a/src/blosc2/blosc/CMakeLists.txt b/src/blosc2/blosc/CMakeLists.txt new file mode 100644 index 0000000..441bab6 --- /dev/null +++ b/src/blosc2/blosc/CMakeLists.txt @@ -0,0 +1,297 @@ +# a simple way to detect that we are using CMAKE +add_definitions(-DUSING_CMAKE) + +set(INTERNAL_LIBS ${PROJECT_SOURCE_DIR}/internal-complibs) + +# Hide symbols by default unless they're specifically exported. +# This makes it easier to keep the set of exported symbols the +# same across all compilers/platforms. +set(CMAKE_C_VISIBILITY_PRESET hidden) + +# includes +set(BLOSC_INCLUDE_DIRS ${BLOSC_INCLUDE_DIRS} ${CMAKE_CURRENT_SOURCE_DIR}) +if(LZ4_FOUND) + set(BLOSC_INCLUDE_DIRS ${BLOSC_INCLUDE_DIRS} ${LZ4_INCLUDE_DIR}) +else() + set(LZ4_LOCAL_DIR ${INTERNAL_LIBS}/lz4-1.9.4) + set(BLOSC_INCLUDE_DIRS ${BLOSC_INCLUDE_DIRS} ${LZ4_LOCAL_DIR}) +endif() + +if(NOT DEACTIVATE_ZLIB) + if(ZLIB_NG_FOUND) + set(BLOSC_INCLUDE_DIRS ${BLOSC_INCLUDE_DIRS} ${ZLIB_NG_INCLUDE_DIR}) + elseif(ZLIB_FOUND) + set(BLOSC_INCLUDE_DIRS ${BLOSC_INCLUDE_DIRS} ${ZLIB_INCLUDE_DIR}) + else() + set(ZLIB_LOCAL_DIR ${INTERNAL_LIBS}/${ZLIB_NG_DIR}) + set(BLOSC_INCLUDE_DIRS ${BLOSC_INCLUDE_DIRS} ${ZLIB_LOCAL_DIR}) + endif() +endif() + +if(NOT DEACTIVATE_ZSTD) + if(ZSTD_FOUND) + set(BLOSC_INCLUDE_DIRS ${BLOSC_INCLUDE_DIRS} ${ZSTD_INCLUDE_DIR}) + else() + set(ZSTD_LOCAL_DIR ${INTERNAL_LIBS}/zstd-1.5.2) + set(BLOSC_INCLUDE_DIRS ${BLOSC_INCLUDE_DIRS} ${ZSTD_LOCAL_DIR} + ${ZSTD_LOCAL_DIR}/common) + endif() +endif() + +include_directories(${BLOSC_INCLUDE_DIRS}) + +# library sources +set(SOURCES ${SOURCES} blosc2.c blosclz.c fastcopy.c fastcopy.h schunk.c frame.c stune.c stune.h + context.h delta.c delta.h shuffle-generic.c bitshuffle-generic.c trunc-prec.c trunc-prec.h + timestamp.c sframe.c directories.c blosc2-stdio.c) +if(NOT CMAKE_SYSTEM_PROCESSOR STREQUAL arm64) + if(COMPILER_SUPPORT_SSE2) + message(STATUS "Adding run-time support for SSE2") + set(SOURCES ${SOURCES} shuffle-sse2.c bitshuffle-sse2.c) + endif() + if(COMPILER_SUPPORT_AVX2) + message(STATUS "Adding run-time support for AVX2") + set(SOURCES ${SOURCES} shuffle-avx2.c bitshuffle-avx2.c) + endif() +endif() +if(COMPILER_SUPPORT_NEON) + message(STATUS "Adding run-time support for NEON") + set(SOURCES ${SOURCES} shuffle-neon.c bitshuffle-neon.c) +endif() +if(COMPILER_SUPPORT_ALTIVEC) + message(STATUS "Adding run-time support for ALTIVEC") + set(SOURCES ${SOURCES} shuffle-altivec.c bitshuffle-altivec.c) +endif() +set(SOURCES ${SOURCES} shuffle.c) + +set(version_string ${BLOSC2_VERSION_MAJOR}.${BLOSC2_VERSION_MINOR}.${BLOSC2_VERSION_PATCH}) + +set(CMAKE_THREAD_PREFER_PTHREAD TRUE) # pre 3.1 +set(THREADS_PREFER_PTHREAD_FLAG TRUE) # CMake 3.1+ +if(WIN32) + # try to use the system library + find_package(Threads) + if(NOT Threads_FOUND) + message(STATUS "using the internal pthread library for win32 systems.") + set(SOURCES ${SOURCES} win32/pthread.c) + else() + if(CMAKE_VERSION VERSION_LESS 3.1) + set(LIBS ${LIBS} ${CMAKE_THREAD_LIBS_INIT}) + else() + set(LIBS ${LIBS} Threads::Threads) + endif() + endif() +else() + find_package(Threads REQUIRED) + if(CMAKE_VERSION VERSION_LESS 3.1) + set(LIBS ${LIBS} ${CMAKE_THREAD_LIBS_INIT}) + else() + set(LIBS ${LIBS} Threads::Threads) + endif() +endif() + +if(LZ4_FOUND) + set(LIBS ${LIBS} ${LZ4_LIBRARY}) +else() + file(GLOB LZ4_FILES ${LZ4_LOCAL_DIR}/*.c) + set(SOURCES ${SOURCES} ${LZ4_FILES}) + source_group("LZ4" FILES ${LZ4_FILES}) +endif() + +if(NOT DEACTIVATE_ZLIB) + if(ZLIB_NG_FOUND) + set(LIBS ${LIBS} ${ZLIB_NG_LIBRARY}) + elseif(ZLIB_FOUND) + set(LIBS ${LIBS} ${ZLIB_LIBRARY}) + else() + set(ZLIB_LOCAL_DIR ${INTERNAL_LIBS}/${ZLIB_NG_DIR}) + file(GLOB ZLIB_FILES ${ZLIB_LOCAL_DIR}/*.c) + set(SOURCES ${SOURCES} ${ZLIB_FILES}) + source_group("Zlib" FILES ${ZLIB_FILES}) + endif() +endif() + +if(NOT DEACTIVATE_ZSTD) + if(ZSTD_FOUND) + set(LIBS ${LIBS} ${ZSTD_LIBRARY}) + else() + # Enable assembly code only when not using MSVC *and* x86 is there + if((NOT MSVC) AND COMPILER_SUPPORT_SSE2) # if SSE2 is here, this is an x86 platform + message(STATUS "Adding support for assembly sources in ZSTD") + file(GLOB ZSTD_DECOMPRESS_FILES ${ZSTD_LOCAL_DIR}/decompress/*.S) + else() + message(STATUS "Disabling support for assembly sources in ZSTD") + add_compile_definitions("ZSTD_DISABLE_ASM") + endif() + file(GLOB ZSTD_DECOMPRESS_FILES ${ZSTD_DECOMPRESS_FILES} + ${ZSTD_LOCAL_DIR}/decompress/*.c) + file(GLOB ZSTD_COMMON_FILES ${ZSTD_LOCAL_DIR}/common/*.c) + file(GLOB ZSTD_COMPRESS_FILES ${ZSTD_LOCAL_DIR}/compress/*.c) + file(GLOB ZSTD_DICT_FILES ${ZSTD_LOCAL_DIR}/dictBuilder/*.c) + set(ZSTD_FILES ${ZSTD_COMMON_FILES} ${ZSTD_COMPRESS_FILES} + ${ZSTD_DECOMPRESS_FILES} ${ZSTD_DICT_FILES}) + set(SOURCES ${SOURCES} ${ZSTD_FILES}) + source_group("Zstd" FILES ${ZSTD_FILES}) + endif() +endif() + +if(HAVE_IPP) + set(LIBS ${LIBS} "${IPP_LIBRARIES}") +endif() + +if(UNIX AND NOT APPLE) + set(LIBS ${LIBS} "rt") + set(LIBS ${LIBS} "m") + # set(LIBS ${LIBS} "profiler") +endif() + + +# targets +if(BUILD_SHARED) + add_library(blosc2_shared SHARED ${SOURCES}) + set_target_properties(blosc2_shared PROPERTIES OUTPUT_NAME blosc2) + if(MSVC OR MINGW) + set_target_properties(blosc2_shared PROPERTIES PREFIX lib) + endif() + set_target_properties(blosc2_shared PROPERTIES + VERSION ${version_string} + SOVERSION 2 # Change this when an ABI change happens + ) + set_property( + TARGET blosc2_shared + APPEND PROPERTY COMPILE_DEFINITIONS BLOSC_SHARED_LIBRARY) +endif() + +# Based on the target architecture and hardware features supported +# by the C compiler, set hardware architecture optimization flags +# for specific shuffle implementations. +if(COMPILER_SUPPORT_SSE2) + if(MSVC) + # MSVC targets SSE2 by default on 64-bit configurations, but not 32-bit configurations. + if(${CMAKE_SIZEOF_VOID_P} EQUAL 4) + set_source_files_properties( + shuffle-sse2.c bitshuffle-sse2.c blosclz.c fastcopy.c + PROPERTIES COMPILE_FLAGS "/arch:SSE2") + endif() + else() + set_source_files_properties( + shuffle-sse2.c bitshuffle-sse2.c blosclz.c fastcopy.c + PROPERTIES COMPILE_FLAGS -msse2) + endif() + + # Define a symbol for the shuffle-dispatch implementation + # so it knows SSE2 is supported even though that file is + # compiled without SSE2 support (for portability). + set_property( + SOURCE shuffle.c + APPEND PROPERTY COMPILE_DEFINITIONS SHUFFLE_SSE2_ENABLED) +endif() +if(COMPILER_SUPPORT_AVX2) + if(MSVC) + set_source_files_properties( + shuffle-avx2.c bitshuffle-avx2.c + PROPERTIES COMPILE_FLAGS "/arch:AVX2") + else() + set_source_files_properties( + shuffle-avx2.c bitshuffle-avx2.c + PROPERTIES COMPILE_FLAGS -mavx2) + endif() + + # Define a symbol for the shuffle-dispatch implementation + # so it knows AVX2 is supported even though that file is + # compiled without AVX2 support (for portability). + set_property( + SOURCE shuffle.c + APPEND PROPERTY COMPILE_DEFINITIONS SHUFFLE_AVX2_ENABLED) +endif() +if(COMPILER_SUPPORT_NEON) + set_source_files_properties( + shuffle-neon.c bitshuffle-neon.c + PROPERTIES COMPILE_FLAGS "-flax-vector-conversions") + if(CMAKE_SYSTEM_PROCESSOR STREQUAL armv7l) + # Only armv7l needs special -mfpu=neon flag; aarch64 doesn't. + set_source_files_properties( + shuffle-neon.c bitshuffle-neon.c + PROPERTIES COMPILE_FLAGS "-mfpu=neon -flax-vector-conversions") + endif() + # Define a symbol for the shuffle-dispatch implementation + # so it knows NEON is supported even though that file is + # compiled without NEON support (for portability). + set_property( + SOURCE shuffle.c + APPEND PROPERTY COMPILE_DEFINITIONS SHUFFLE_NEON_ENABLED) +endif() +if(COMPILER_SUPPORT_ALTIVEC) + set_source_files_properties(shuffle-altivec.c bitshuffle-altivec.c + PROPERTIES COMPILE_FLAGS -DNO_WARN_X86_INTRINSICS) + + # Define a symbol for the shuffle-dispatch implementation + # so it knows ALTIVEC is supported even though that file is + # compiled without ALTIVEC support (for portability). + set_property( + SOURCE shuffle.c + APPEND PROPERTY COMPILE_DEFINITIONS SHUFFLE_ALTIVEC_ENABLED) +endif() + +# When the option has been selected to compile the test suite, +# compile an additional version of blosc2_static which exports +# some normally-hidden symbols (to facilitate unit testing). +if(BUILD_TESTS) + add_library(blosc_testing STATIC ${SOURCES}) + set_target_properties(blosc_testing PROPERTIES OUTPUT_NAME blosc_testing) + if(MSVC OR MINGW) + set_target_properties(blosc_testing PROPERTIES PREFIX lib) + endif() + set_property( + TARGET blosc_testing + APPEND PROPERTY COMPILE_DEFINITIONS BLOSC_SHARED_LIBRARY) + set_property( + TARGET blosc_testing + APPEND PROPERTY COMPILE_DEFINITIONS BLOSC_TESTING) +endif() + +if(BUILD_SHARED) + target_link_libraries(blosc2_shared ${LIBS}) + target_include_directories(blosc2_shared PUBLIC ${BLOSC_INCLUDE_DIRS}) +endif() + +if(BUILD_TESTS) + target_link_libraries(blosc_testing ${LIBS}) + target_include_directories(blosc_testing PUBLIC ${BLOSC_INCLUDE_DIRS}) +endif() + +if(BUILD_STATIC) + add_library(blosc2_static STATIC ${SOURCES}) + set_target_properties(blosc2_static PROPERTIES OUTPUT_NAME blosc2) + if(MSVC OR MINGW) + set_target_properties(blosc2_static PROPERTIES PREFIX lib) + endif() + target_link_libraries(blosc2_static ${LIBS}) + target_include_directories(blosc2_static PUBLIC ${BLOSC_INCLUDE_DIRS}) +endif() + +# install +if(BLOSC_INSTALL) + install(FILES ${PROJECT_SOURCE_DIR}/include/blosc2.h DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} COMPONENT DEV) + install(FILES + ${PROJECT_SOURCE_DIR}/include/blosc2/blosc2-export.h + ${PROJECT_SOURCE_DIR}/include/blosc2/blosc2-common.h + ${PROJECT_SOURCE_DIR}/include/blosc2/blosc2-stdio.h + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/blosc2 COMPONENT DEV) + if(BUILD_PLUGINS) + install(FILES + ${PROJECT_SOURCE_DIR}/include/blosc2/filters-registry.h + ${PROJECT_SOURCE_DIR}/include/blosc2/codecs-registry.h + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/blosc2 COMPONENT DEV) + endif() + + if(BUILD_SHARED) + install(TARGETS blosc2_shared + LIBRARY COMPONENT LIB + ARCHIVE COMPONENT DEV + RUNTIME COMPONENT LIB) + endif() + if(BUILD_STATIC) + install(TARGETS blosc2_static COMPONENT DEV) + endif() +endif() diff --git a/src/blosc2/blosc/CTestTestfile.cmake b/src/blosc2/blosc/CTestTestfile.cmake new file mode 100644 index 0000000..5a4b7e1 --- /dev/null +++ b/src/blosc2/blosc/CTestTestfile.cmake @@ -0,0 +1,6 @@ +# CMake generated Testfile for +# Source directory: /home/fangq/space/git/Temp/c-blosc2/blosc +# Build directory: /home/fangq/space/git/Temp/c-blosc2/blosc +# +# This file includes the relevant testing commands required for +# testing this directory and lists subdirectories to be tested as well. diff --git a/src/blosc2/blosc/Makefile b/src/blosc2/blosc/Makefile new file mode 100644 index 0000000..c403724 --- /dev/null +++ b/src/blosc2/blosc/Makefile @@ -0,0 +1,141 @@ +################################################################# +# Makefile for ZMAT +# Qianqian Fang +# 2019/04/30 +################################################################# + +BACKEND ?= + +ROOTDIR ?=.. +ZMATDIR ?=$(ROOTDIR) +LIBDIR ?=$(ROOTDIR)/lib + +MKDIR :=mkdir + +MEX=mex +AR=$(CC) +ECHO := echo + +BINARY:=blosc2 +OUTPUT_DIR=$(ZMATDIR) + +DOXY := doxygen +DOCDIR := $(ZMATDIR)/doc +DOXYCFG=zmat.cfg + +INCLUDEDIRS=-I../../lz4 -I../include + +CUOMPLINK= + +ARCH = $(shell uname -m) +PLATFORM = $(shell uname -s) + +DLLFLAG=-fPIC +OMP=-fopenmp + +CPPOPT=-g -Wall -O3 -msse2 #-g -Wall -std=c99 # -DUSE_OS_TIMER + +OUTPUTFLAG:=-o +OBJSUFFIX=.o +EXESUFFIX=.mex* + +FILES=bitshuffle-sse2 blosc2 blosc2-stdio blosclz delta directories fastcopy \ + frame schunk sframe shuffle shuffle-sse2 stune timestamp trunc-prec + +ifeq ($(findstring CYGWIN,$(PLATFORM)), CYGWIN) + ifeq ($(findstring x86_64,$(ARCH)), x86_64) + LINKOPT=-L"$(CUDA_PATH)/lib/x64" $(CUDART) + else + LINKOPT=-L"$(CUDA_PATH)/lib/Win32" $(CUDART) + endif + INCLUDEDIRS +=-I"$(CUDA_PATH)/lib/include" + CPPOPT =-c -D_CRT_SECURE_NO_DEPRECATE -DWIN32 + OBJSUFFIX=.obj + EXESUFFIX= + DLLFLAG= + OMP=-fopenmp + MEX=cmd /c mex +else ifeq ($(findstring Darwin,$(PLATFORM)), Darwin) + +else + CPPOPT+= + CUCCOPT+=-Xcompiler $(OMP) + ifeq ($(findstring x86_64,$(ARCH)), x86_64) + CPPOPT += + endif +endif + +ifeq ($(MAKECMDGOALS),lib) + AR :=ar + ARFLAGS :=cr + BINARY :=libblosc2.a + AROUTPUT := + LINKOPT := + OUTPUT_DIR :=$(LIBDIR) + ifeq ($(findstring Darwin,$(PLATFORM)), Darwin) + OUTPUTFLAG := + endif +endif + +ifeq ($(MAKECMDGOALS),dll) + OUTPUTFLAG :=-o + BINARY :=libblosc2.so + OUTPUT_DIR :=$(LIBDIR) + + ifeq ($(findstring Darwin,$(PLATFORM)), Darwin) + ARFLAGS :=-shared -Wl,-install_name,$(BINARY).1 -lz + else + ARFLAGS :=-shared -Wl,-soname,$(BINARY).1 -lz + endif +endif + +ifeq ($(MAKECMDGOALS),dll) + BINARY :=libblosc2.so +endif + +dll: CPPOPT +=$(DLLFLAG) +dll: AR :=gcc +dll: ARFLAGS ?=-shared -Wl,-soname,$(BINARY).1 +dll: LINKOPT :=$(LDFLAGS) +dll: AROUTPUT :=-o + +all: lib + +TARGETSUFFIX:=$(suffix $(BINARY)) + +doc: makedocdir + $(DOXY) $(DOXYCFG) + +OBJS := $(addsuffix $(OBJSUFFIX), $(FILES)) + +all dll lib mex oct: $(OUTPUT_DIR)/$(BINARY) + +makedirs: + @if test ! -d $(OUTPUT_DIR); then $(MKDIR) $(OUTPUT_DIR); fi + +makedocdir: + @if test ! -d $(DOCDIR); then $(MKDIR) $(DOCDIR); fi + +$(OUTPUT_DIR)/$(BINARY): makedirs $(OBJS) +$(OUTPUT_DIR)/$(BINARY): $(OBJS) + @$(ECHO) Building $@ + $(AR) $(ARFLAGS) $(OUTPUTFLAG) $@ $(OBJS) $(LINKOPT) $(USERLINKOPT) + +%$(OBJSUFFIX): %.cpp + $(CXX) $(INCLUDEDIRS) $(CPPOPT) -c -o $@ $< + +%$(OBJSUFFIX): %.c + @$(ECHO) Building $@ + $(CC) $(CPPFLAGS) $(CFLAGS) $(INCLUDEDIRS) $(CPPOPT) -c -o $@ $< + +%$(OBJSUFFIX): %.cu + @$(ECHO) Building $@ + $(CUDACC) -c $(CUCCOPT) -o $@ $< + +clean: + -rm -f $(OBJS) $(OUTPUT_DIR)/$(BINARY)$(EXESUFFIX) zmat$(OBJSUFFIX) $(LIBDIR)/* + +.PHONY: all lib dll + +.DEFAULT_GOAL := all + diff --git a/src/blosc2/blosc/bitshuffle-altivec.c b/src/blosc2/blosc/bitshuffle-altivec.c new file mode 100644 index 0000000..60805f5 --- /dev/null +++ b/src/blosc2/blosc/bitshuffle-altivec.c @@ -0,0 +1,592 @@ +/********************************************************************* + Blosc - Blocked Shuffling and Compression Library + + Copyright (C) 2021 The Blosc Developers + https://blosc.org + License: BSD 3-Clause (see LICENSE.txt) + + See LICENSE.txt for details about copyright and rights to use. +**********************************************************************/ + +/********************************************************************* + Bitshuffle - Filter for improving compression of typed binary data. + + Author: Kiyoshi Masui + Website: http://www.github.com/kiyo-masui/bitshuffle + + Note: Adapted for c-blosc by Francesc Alted + Altivec/VSX version by Jerome Kieffer. + + See LICENSES/BITSHUFFLE.txt file for details about copyright and + rights to use. +**********************************************************************/ + + +#include "bitshuffle-generic.h" +#include "bitshuffle-altivec.h" + +/* Make sure ALTIVEC is available for the compilation target and compiler. */ +#if !defined(__ALTIVEC__) + #error ALTIVEC is not supported by the target architecture/platform and/or this compiler. +#endif +#include +#include "transpose-altivec.h" + +/* The next is useful for debugging purposes */ +#if 0 +#include +#include + +static void helper_print(__vector uint8_t v, char* txt){ + printf("%s %02x %02x %02x %02x %02x %02x %02x %02x %02x %02x %02x %02x %02x %02x %02x %02x\n",txt, + v[0], v[1], v[2], v[3], v[4], v[5], v[6], v[7], v[8], v[9], v[10], v[11], v[12], v[13], v[14], v[15]); +} +#endif + + +static inline __vector uint8_t gen_save_mask(size_t offset){ + __vector uint8_t mask; + size_t k; + for (k = 0; k < 16; k++) + mask[k] = (k> 3]; + *oui16 = tmp[4]; + } + } + count = bshuf_trans_bit_byte_remainder(in, out, size, elem_size, + nbyte - nbyte % 16); + return count; +} + + +/* Transpose bits within elements. */ +int64_t bshuf_trans_bit_elem_altivec(void* in, void* out, const size_t size, + const size_t elem_size, void* tmp_buf) { + + int64_t count; + + CHECK_MULT_EIGHT(size); + + count = bshuf_trans_byte_elem_altivec(in, out, size, elem_size, tmp_buf); + CHECK_ERR(count); + // bshuf_trans_bit_byte_altivec / bitshuffle1_altivec + count = bshuf_trans_bit_byte_altivec(out, tmp_buf, size, elem_size); + CHECK_ERR(count); + count = bshuf_trans_bitrow_eight(tmp_buf, out, size, elem_size); + return count; +} + +/* For data organized into a row for each bit (8 * elem_size rows), transpose + * the bytes. */ +int64_t bshuf_trans_byte_bitrow_altivec(void* in, void* out, const size_t size, + const size_t elem_size) { + static const __vector uint8_t epi8_low = (const __vector uint8_t) { + 0x00, 0x10, 0x01, 0x11, 0x02, 0x12, 0x03, 0x13, + 0x04, 0x14, 0x05, 0x15, 0x06, 0x16, 0x07, 0x17}; + static const __vector uint8_t epi8_hi = (const __vector uint8_t) { + 0x08, 0x18, 0x09, 0x19, 0x0a, 0x1a, 0x0b, 0x1b, + 0x0c, 0x1c, 0x0d, 0x1d, 0x0e, 0x1e, 0x0f, 0x1f}; + static const __vector uint8_t epi16_low = (const __vector uint8_t) { + 0x00, 0x01, 0x10, 0x11, 0x02, 0x03, 0x12, 0x13, + 0x04, 0x05, 0x14, 0x15, 0x06, 0x07, 0x16, 0x17}; + static const __vector uint8_t epi16_hi = (const __vector uint8_t) { + 0x08, 0x09, 0x18, 0x19, 0x0a, 0x0b, 0x1a, 0x1b, + 0x0c, 0x0d, 0x1c, 0x1d, 0x0e, 0x0f, 0x1e, 0x1f}; + static const __vector uint8_t epi32_low = (const __vector uint8_t) { + 0x00, 0x01, 0x02, 0x03, 0x10, 0x11, 0x12, 0x13, + 0x04, 0x05, 0x06, 0x07, 0x14, 0x15, 0x16, 0x17}; + static const __vector uint8_t epi32_hi = (const __vector uint8_t) { + 0x08, 0x09, 0x0a, 0x0b, 0x18, 0x19, 0x1a, 0x1b, + 0x0c, 0x0d, 0x0e, 0x0f, 0x1c, 0x1d, 0x1e, 0x1f}; + static const __vector uint8_t epi64_low = (const __vector uint8_t) { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17}; + static const __vector uint8_t epi64_hi = (const __vector uint8_t) { + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f}; + + const uint8_t* in_b = (const uint8_t*)in; + uint8_t* out_b = (uint8_t*)out; + size_t nrows = 8 * elem_size; + size_t nbyte_row = size / 8; + __vector uint8_t xmm0[16], xmm1[16]; + + CHECK_MULT_EIGHT(size); + + // The optimized algorithms can only deal with even values or 1 for elem_size + if ((elem_size > 1) && (elem_size % 2)) { + return bshuf_trans_byte_bitrow_scal(in, out, size, elem_size); + } + + int nvectors = (elem_size == 1) ? 8 : 16; + for (size_t ii = 0; ii + (nvectors - 1) < nrows; ii += nvectors) { + for (size_t jj = 0; jj + 15 < nbyte_row; jj += 16) { // vectors of 16 elements + + if (elem_size == 1) { + for (int k = 0; k < 8; k++) { + xmm0[k] = vec_xl((ii + k) * nbyte_row + jj, in_b); + } + + xmm1[0] = vec_perm(xmm0[0], xmm0[1], epi8_low); + xmm1[1] = vec_perm(xmm0[2], xmm0[3], epi8_low); + xmm1[2] = vec_perm(xmm0[4], xmm0[5], epi8_low); + xmm1[3] = vec_perm(xmm0[6], xmm0[7], epi8_low); + xmm1[4] = vec_perm(xmm0[0], xmm0[1], epi8_hi); + xmm1[5] = vec_perm(xmm0[2], xmm0[3], epi8_hi); + xmm1[6] = vec_perm(xmm0[4], xmm0[5], epi8_hi); + xmm1[7] = vec_perm(xmm0[6], xmm0[7], epi8_hi); + + xmm0[0] = vec_perm(xmm1[0], xmm1[1], epi16_low); + xmm0[1] = vec_perm(xmm1[2], xmm1[3], epi16_low); + xmm0[2] = vec_perm(xmm1[0], xmm1[1], epi16_hi); + xmm0[3] = vec_perm(xmm1[2], xmm1[3], epi16_hi); + xmm0[4] = vec_perm(xmm1[4], xmm1[5], epi16_low); + xmm0[5] = vec_perm(xmm1[6], xmm1[7], epi16_low); + xmm0[6] = vec_perm(xmm1[4], xmm1[5], epi16_hi); + xmm0[7] = vec_perm(xmm1[6], xmm1[7], epi16_hi); + + xmm1[0] = vec_perm(xmm0[0], xmm0[1], epi32_low); + xmm1[1] = vec_perm(xmm0[0], xmm0[1], epi32_hi); + xmm1[2] = vec_perm(xmm0[2], xmm0[3], epi32_low); + xmm1[3] = vec_perm(xmm0[2], xmm0[3], epi32_hi); + xmm1[4] = vec_perm(xmm0[4], xmm0[5], epi32_low); + xmm1[5] = vec_perm(xmm0[4], xmm0[5], epi32_hi); + xmm1[6] = vec_perm(xmm0[6], xmm0[7], epi32_low); + xmm1[7] = vec_perm(xmm0[6], xmm0[7], epi32_hi); + + for (int k = 0; k < 8; k++) { + vec_xst(xmm1[k], (jj + k * 2) * nrows + ii, out_b); + } + + continue; + } + + for (int k = 0; k < 16; k++) { + xmm0[k] = vec_xl((ii + k) * nbyte_row + jj, in_b); + } + + for (int k = 0; k < 16; k += 8) { + xmm1[k + 0] = vec_perm(xmm0[k + 0], xmm0[k + 1], epi8_low); + xmm1[k + 1] = vec_perm(xmm0[k + 2], xmm0[k + 3], epi8_low); + xmm1[k + 2] = vec_perm(xmm0[k + 4], xmm0[k + 5], epi8_low); + xmm1[k + 3] = vec_perm(xmm0[k + 6], xmm0[k + 7], epi8_low); + xmm1[k + 4] = vec_perm(xmm0[k + 0], xmm0[k + 1], epi8_hi); + xmm1[k + 5] = vec_perm(xmm0[k + 2], xmm0[k + 3], epi8_hi); + xmm1[k + 6] = vec_perm(xmm0[k + 4], xmm0[k + 5], epi8_hi); + xmm1[k + 7] = vec_perm(xmm0[k + 6], xmm0[k + 7], epi8_hi); + } + + for (int k = 0; k < 16; k += 8) { + xmm0[k + 0] = vec_perm(xmm1[k + 0], xmm1[k + 1], epi16_low); + xmm0[k + 1] = vec_perm(xmm1[k + 2], xmm1[k + 3], epi16_low); + xmm0[k + 2] = vec_perm(xmm1[k + 0], xmm1[k + 1], epi16_hi); + xmm0[k + 3] = vec_perm(xmm1[k + 2], xmm1[k + 3], epi16_hi); + xmm0[k + 4] = vec_perm(xmm1[k + 4], xmm1[k + 5], epi16_low); + xmm0[k + 5] = vec_perm(xmm1[k + 6], xmm1[k + 7], epi16_low); + xmm0[k + 6] = vec_perm(xmm1[k + 4], xmm1[k + 5], epi16_hi); + xmm0[k + 7] = vec_perm(xmm1[k + 6], xmm1[k + 7], epi16_hi); + } + + for (int k = 0; k < 16; k += 8) { + xmm1[k + 0] = vec_perm(xmm0[k + 0], xmm0[k + 1], epi32_low); + xmm1[k + 1] = vec_perm(xmm0[k + 0], xmm0[k + 1], epi32_hi); + xmm1[k + 2] = vec_perm(xmm0[k + 2], xmm0[k + 3], epi32_low); + xmm1[k + 3] = vec_perm(xmm0[k + 2], xmm0[k + 3], epi32_hi); + xmm1[k + 4] = vec_perm(xmm0[k + 4], xmm0[k + 5], epi32_low); + xmm1[k + 5] = vec_perm(xmm0[k + 4], xmm0[k + 5], epi32_hi); + xmm1[k + 6] = vec_perm(xmm0[k + 6], xmm0[k + 7], epi32_low); + xmm1[k + 7] = vec_perm(xmm0[k + 6], xmm0[k + 7], epi32_hi); + } + + for (int k = 0; k < 8; k += 4) { + xmm0[k * 2 + 0] = vec_perm(xmm1[k + 0], xmm1[k + 8], epi64_low); + xmm0[k * 2 + 1] = vec_perm(xmm1[k + 0], xmm1[k + 8], epi64_hi); + xmm0[k * 2 + 2] = vec_perm(xmm1[k + 1], xmm1[k + 9], epi64_low); + xmm0[k * 2 + 3] = vec_perm(xmm1[k + 1], xmm1[k + 9], epi64_hi); + xmm0[k * 2 + 4] = vec_perm(xmm1[k + 2], xmm1[k + 10], epi64_low); + xmm0[k * 2 + 5] = vec_perm(xmm1[k + 2], xmm1[k + 10], epi64_hi); + xmm0[k * 2 + 6] = vec_perm(xmm1[k + 3], xmm1[k + 11], epi64_low); + xmm0[k * 2 + 7] = vec_perm(xmm1[k + 3], xmm1[k + 11], epi64_hi); + } + + for (int k = 0; k < 16; k++) { + vec_xst(xmm0[k], (jj + k) * nrows + ii, out_b); + } + + } + + // Copy the remainder + for (size_t jj = nbyte_row - nbyte_row % 16; jj < nbyte_row; jj++) { + for (int k = 0; k < nvectors; k++) { + out_b[jj * nrows + ii + k] = in_b[(ii + k) * nbyte_row + jj]; + } + } + + } + + return size * elem_size; +} + + +/* Shuffle bits within the bytes of eight element blocks. */ +int64_t bshuf_shuffle_bit_eightelem_altivec(void* in, void* out, const size_t size, + const size_t elem_size) { + /* With a bit of care, this could be written such that such that it is */ + /* in_buf = out_buf safe. */ + const uint8_t* in_b = (const uint8_t*)in; + uint8_t* out_b = (uint8_t*)out; + size_t nbyte = elem_size * size; + __vector uint8_t masks[8], data; + + CHECK_MULT_EIGHT(size); + + // Generate all 8 needed masks + for (int kk = 0; kk < 8; kk++){ + masks[kk] = make_bitperm_mask(1, kk); + } + + if (elem_size % 2) { + bshuf_shuffle_bit_eightelem_scal(in, out, size, elem_size); + } else { + for (size_t ii = 0; ii + 8 * elem_size - 1 < nbyte; + ii += 8 * elem_size) { + for (size_t jj = 0; jj + 15 < 8 * elem_size; jj += 16) { + data = vec_xl(ii + jj, in_b); + for (size_t kk = 0; kk < 8; kk++) { + __vector uint16_t tmp; + uint16_t* oui16; + tmp = (__vector uint16_t) vec_bperm(data, masks[kk]); + oui16 = (uint16_t*)&out_b[ii + (jj>>3) + kk * elem_size]; + *oui16 = tmp[4]; + } + } + } + } + return size * elem_size; +} + + +/* Untranspose bits within elements. */ +int64_t bshuf_untrans_bit_elem_altivec(void* in, void* out, const size_t size, + const size_t elem_size, void* tmp_buf) { + + int64_t count; + + CHECK_MULT_EIGHT(size); + + count = bshuf_trans_byte_bitrow_altivec(in, tmp_buf, size, elem_size); + CHECK_ERR(count); + count = bshuf_shuffle_bit_eightelem_altivec(tmp_buf, out, size, elem_size); + + return count; +} diff --git a/src/blosc2/blosc/bitshuffle-altivec.h b/src/blosc2/blosc/bitshuffle-altivec.h new file mode 100644 index 0000000..7416c7a --- /dev/null +++ b/src/blosc2/blosc/bitshuffle-altivec.h @@ -0,0 +1,54 @@ +/********************************************************************* + Blosc - Blocked Shuffling and Compression Library + + Copyright (C) 2021 The Blosc Developers + https://blosc.org + License: BSD 3-Clause (see LICENSE.txt) + + See LICENSE.txt for details about copyright and rights to use. +**********************************************************************/ + +/* ALTIVEC-accelerated shuffle/unshuffle routines. */ + +#ifndef BITSHUFFLE_ALTIVEC_H +#define BITSHUFFLE_ALTIVEC_H + +#include "blosc2/blosc2-common.h" + +#ifdef __cplusplus +extern "C" { +#endif + + +BLOSC_NO_EXPORT int64_t + bshuf_trans_byte_elem_altivec(void* in, void* out, const size_t size, + const size_t elem_size, void* tmp_buf); + +BLOSC_NO_EXPORT int64_t + bshuf_trans_byte_bitrow_altivec(void* in, void* out, const size_t size, + const size_t elem_size); + +BLOSC_NO_EXPORT int64_t + bshuf_shuffle_bit_eightelem_altivec(void* in, void* out, const size_t size, + const size_t elem_size); + +/** + ALTIVEC-accelerated bitshuffle routine. +*/ +BLOSC_NO_EXPORT int64_t + bshuf_trans_bit_elem_altivec(void* in, void* out, const size_t size, + const size_t elem_size, void* tmp_buf); + +/** + ALTIVEC-accelerated bitunshuffle routine. +*/ +BLOSC_NO_EXPORT int64_t + bshuf_untrans_bit_elem_altivec(void* in, void* out, const size_t size, + const size_t elem_size, void* tmp_buf); + +#ifdef __cplusplus +} +#endif + + +#endif /* BITSHUFFLE_ALTIVEC_H */ diff --git a/src/blosc2/blosc/bitshuffle-avx2.c b/src/blosc2/blosc/bitshuffle-avx2.c new file mode 100644 index 0000000..2391ecd --- /dev/null +++ b/src/blosc2/blosc/bitshuffle-avx2.c @@ -0,0 +1,257 @@ +/********************************************************************* + Blosc - Blocked Shuffling and Compression Library + + Copyright (C) 2021 The Blosc Developers + https://blosc.org + License: BSD 3-Clause (see LICENSE.txt) + + See LICENSE.txt for details about copyright and rights to use. +**********************************************************************/ + +/********************************************************************* + Bitshuffle - Filter for improving compression of typed binary data. + + Author: Kiyoshi Masui + Website: http://www.github.com/kiyo-masui/bitshuffle + + Note: Adapted for c-blosc by Francesc Alted. + + See LICENSES/BITSHUFFLE.txt file for details about copyright and + rights to use. +**********************************************************************/ + + +#include "bitshuffle-generic.h" +#include "bitshuffle-sse2.h" +#include "bitshuffle-avx2.h" + + +/* Make sure AVX2 is available for the compilation target and compiler. */ +#if !defined(__AVX2__) + #error AVX2 is not supported by the target architecture/platform and/or this compiler. +#endif + +#include + +/* The next is useful for debugging purposes */ +#if 0 +#include +#include + +static void printymm(__m256i ymm0) +{ + uint8_t buf[32]; + + ((__m256i *)buf)[0] = ymm0; + printf("%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x\n", + buf[0], buf[1], buf[2], buf[3], + buf[4], buf[5], buf[6], buf[7], + buf[8], buf[9], buf[10], buf[11], + buf[12], buf[13], buf[14], buf[15], + buf[16], buf[17], buf[18], buf[19], + buf[20], buf[21], buf[22], buf[23], + buf[24], buf[25], buf[26], buf[27], + buf[28], buf[29], buf[30], buf[31]); +} +#endif + + +/* ---- Code that requires AVX2. Intel Haswell (2013) and later. ---- */ + + +/* Transpose bits within bytes. */ +int64_t bshuf_trans_bit_byte_avx2(void* in, void* out, const size_t size, + const size_t elem_size) { + + char* in_b = (char*)in; + char* out_b = (char*)out; + int32_t* out_i32; + + size_t nbyte = elem_size * size; + + int64_t count; + + __m256i ymm; + int32_t bt; + size_t ii, kk; + + for (ii = 0; ii + 31 < nbyte; ii += 32) { + ymm = _mm256_loadu_si256((__m256i*)&in_b[ii]); + for (kk = 0; kk < 8; kk++) { + bt = _mm256_movemask_epi8(ymm); + ymm = _mm256_slli_epi16(ymm, 1); + out_i32 = (int32_t*)&out_b[((7 - kk) * nbyte + ii) / 8]; + *out_i32 = bt; + } + } + count = bshuf_trans_bit_byte_remainder(in, out, size, elem_size, + nbyte - nbyte % 32); + return count; +} + + +/* Transpose bits within elements. */ +int64_t bshuf_trans_bit_elem_avx2(void* in, void* out, const size_t size, + const size_t elem_size, void* tmp_buf) { + + int64_t count; + + CHECK_MULT_EIGHT(size); + + count = bshuf_trans_byte_elem_sse2(in, out, size, elem_size, tmp_buf); + CHECK_ERR(count); + count = bshuf_trans_bit_byte_avx2(out, tmp_buf, size, elem_size); + CHECK_ERR(count); + count = bshuf_trans_bitrow_eight(tmp_buf, out, size, elem_size); + + return count; +} + + +/* For data organized into a row for each bit (8 * elem_size rows), transpose + * the bytes. */ +int64_t bshuf_trans_byte_bitrow_avx2(void* in, void* out, const size_t size, + const size_t elem_size) { + + char* in_b = (char*)in; + char* out_b = (char*)out; + + size_t nrows = 8 * elem_size; + size_t nbyte_row = size / 8; + size_t ii, jj, kk, hh, mm; + + CHECK_MULT_EIGHT(size); + + if (elem_size % 4) + return bshuf_trans_byte_bitrow_sse2(in, out, size, elem_size); + + __m256i ymm_0[8]; + __m256i ymm_1[8]; + __m256i ymm_storeage[8][4]; + + for (jj = 0; jj + 31 < nbyte_row; jj += 32) { + for (ii = 0; ii + 3 < elem_size; ii += 4) { + for (hh = 0; hh < 4; hh++) { + + for (kk = 0; kk < 8; kk++) { + ymm_0[kk] = _mm256_loadu_si256((__m256i*)&in_b[ + (ii * 8 + hh * 8 + kk) * nbyte_row + jj]); + } + + for (kk = 0; kk < 4; kk++) { + ymm_1[kk] = _mm256_unpacklo_epi8(ymm_0[kk * 2], + ymm_0[kk * 2 + 1]); + ymm_1[kk + 4] = _mm256_unpackhi_epi8(ymm_0[kk * 2], + ymm_0[kk * 2 + 1]); + } + + for (kk = 0; kk < 2; kk++) { + for (mm = 0; mm < 2; mm++) { + ymm_0[kk * 4 + mm] = _mm256_unpacklo_epi16( + ymm_1[kk * 4 + mm * 2], + ymm_1[kk * 4 + mm * 2 + 1]); + ymm_0[kk * 4 + mm + 2] = _mm256_unpackhi_epi16( + ymm_1[kk * 4 + mm * 2], + ymm_1[kk * 4 + mm * 2 + 1]); + } + } + + for (kk = 0; kk < 4; kk++) { + ymm_1[kk * 2] = _mm256_unpacklo_epi32(ymm_0[kk * 2], + ymm_0[kk * 2 + 1]); + ymm_1[kk * 2 + 1] = _mm256_unpackhi_epi32(ymm_0[kk * 2], + ymm_0[kk * 2 + 1]); + } + + for (kk = 0; kk < 8; kk++) { + ymm_storeage[kk][hh] = ymm_1[kk]; + } + } + + for (mm = 0; mm < 8; mm++) { + + for (kk = 0; kk < 4; kk++) { + ymm_0[kk] = ymm_storeage[mm][kk]; + } + + ymm_1[0] = _mm256_unpacklo_epi64(ymm_0[0], ymm_0[1]); + ymm_1[1] = _mm256_unpacklo_epi64(ymm_0[2], ymm_0[3]); + ymm_1[2] = _mm256_unpackhi_epi64(ymm_0[0], ymm_0[1]); + ymm_1[3] = _mm256_unpackhi_epi64(ymm_0[2], ymm_0[3]); + + ymm_0[0] = _mm256_permute2x128_si256(ymm_1[0], ymm_1[1], 32); + ymm_0[1] = _mm256_permute2x128_si256(ymm_1[2], ymm_1[3], 32); + ymm_0[2] = _mm256_permute2x128_si256(ymm_1[0], ymm_1[1], 49); + ymm_0[3] = _mm256_permute2x128_si256(ymm_1[2], ymm_1[3], 49); + + _mm256_storeu_si256((__m256i*)&out_b[ + (jj + mm * 2 + 0 * 16) * nrows + ii * 8], ymm_0[0]); + _mm256_storeu_si256((__m256i*)&out_b[ + (jj + mm * 2 + 0 * 16 + 1) * nrows + ii * 8], ymm_0[1]); + _mm256_storeu_si256((__m256i*)&out_b[ + (jj + mm * 2 + 1 * 16) * nrows + ii * 8], ymm_0[2]); + _mm256_storeu_si256((__m256i*)&out_b[ + (jj + mm * 2 + 1 * 16 + 1) * nrows + ii * 8], ymm_0[3]); + } + } + } + for (ii = 0; ii < nrows; ii++) { + for (jj = nbyte_row - nbyte_row % 32; jj < nbyte_row; jj++) { + out_b[jj * nrows + ii] = in_b[ii * nbyte_row + jj]; + } + } + return (int64_t)size * (int64_t)elem_size; +} + + +/* Shuffle bits within the bytes of eight element blocks. */ +int64_t bshuf_shuffle_bit_eightelem_avx2(void* in, void* out, const size_t size, + const size_t elem_size) { + + CHECK_MULT_EIGHT(size); + + /* With a bit of care, this could be written such that such that it is */ + /* in_buf = out_buf safe. */ + char* in_b = (char*)in; + char* out_b = (char*)out; + + size_t nbyte = elem_size * size; + size_t ii, jj, kk, ind; + + __m256i ymm; + int32_t bt; + + if (elem_size % 4) { + return bshuf_shuffle_bit_eightelem_sse2(in, out, size, elem_size); + } else { + for (jj = 0; jj + 31 < 8 * elem_size; jj += 32) { + for (ii = 0; ii + 8 * elem_size - 1 < nbyte; + ii += 8 * elem_size) { + ymm = _mm256_loadu_si256((__m256i*)&in_b[ii + jj]); + for (kk = 0; kk < 8; kk++) { + bt = _mm256_movemask_epi8(ymm); + ymm = _mm256_slli_epi16(ymm, 1); + ind = (ii + jj / 8 + (7 - kk) * elem_size); + *(int32_t*)&out_b[ind] = bt; + } + } + } + } + return (int64_t)size * (int64_t)elem_size; +} + + +/* Untranspose bits within elements. */ +int64_t bshuf_untrans_bit_elem_avx2(void* in, void* out, const size_t size, + const size_t elem_size, void* tmp_buf) { + + int64_t count; + + CHECK_MULT_EIGHT(size); + + count = bshuf_trans_byte_bitrow_avx2(in, tmp_buf, size, elem_size); + CHECK_ERR(count); + count = bshuf_shuffle_bit_eightelem_avx2(tmp_buf, out, size, elem_size); + + return count; +} diff --git a/src/blosc2/blosc/bitshuffle-avx2.h b/src/blosc2/blosc/bitshuffle-avx2.h new file mode 100644 index 0000000..5d9d51b --- /dev/null +++ b/src/blosc2/blosc/bitshuffle-avx2.h @@ -0,0 +1,40 @@ +/********************************************************************* + Blosc - Blocked Shuffling and Compression Library + + Copyright (C) 2021 The Blosc Developers + https://blosc.org + License: BSD 3-Clause (see LICENSE.txt) + + See LICENSE.txt for details about copyright and rights to use. +**********************************************************************/ + +/* AVX2-accelerated shuffle/unshuffle routines. */ + +#ifndef BITSHUFFLE_AVX2_H +#define BITSHUFFLE_AVX2_H + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/** + AVX2-accelerated bitshuffle routine. +*/ +BLOSC_NO_EXPORT int64_t + bshuf_trans_bit_elem_avx2(void* in, void* out, const size_t size, + const size_t elem_size, void* tmp_buf); + +/** + AVX2-accelerated bitunshuffle routine. +*/ +BLOSC_NO_EXPORT int64_t + bshuf_untrans_bit_elem_avx2(void* in, void* out, const size_t size, + const size_t elem_size, void* tmp_buf); + +#ifdef __cplusplus +} +#endif + +#endif /* BITSHUFFLE_AVX2_H */ diff --git a/src/blosc2/blosc/bitshuffle-generic.c b/src/blosc2/blosc/bitshuffle-generic.c new file mode 100644 index 0000000..1840045 --- /dev/null +++ b/src/blosc2/blosc/bitshuffle-generic.c @@ -0,0 +1,231 @@ +/********************************************************************* + Blosc - Blocked Shuffling and Compression Library + + Copyright (C) 2021 The Blosc Developers + https://blosc.org + License: BSD 3-Clause (see LICENSE.txt) + + See LICENSE.txt for details about copyright and rights to use. +**********************************************************************/ + +#include "bitshuffle-generic.h" + +#ifdef _MSC_VER +#pragma warning (push) +#pragma warning (disable: 4146) +#endif + +/* Transpose bytes within elements, starting partway through input. */ +int64_t bshuf_trans_byte_elem_remainder(const void* in, void* out, const size_t size, + const size_t elem_size, const size_t start) { + + char* in_b = (char*) in; + char* out_b = (char*) out; + size_t ii, jj, kk; + + CHECK_MULT_EIGHT(start); + + if (size > start) { + /* ii loop separated into 2 loops so the compiler can unroll */ + /* the inner one. */ + for (ii = start; ii + 7 < size; ii += 8) { + for (jj = 0; jj < elem_size; jj++) { + for (kk = 0; kk < 8; kk++) { + out_b[jj * size + ii + kk] + = in_b[ii * elem_size + kk * elem_size + jj]; + } + } + } + for (ii = size - size % 8; ii < size; ii ++) { + for (jj = 0; jj < elem_size; jj++) { + out_b[jj * size + ii] = in_b[ii * elem_size + jj]; + } + } + } + return (int64_t)size * (int64_t)elem_size; +} + + +/* Transpose bytes within elements. */ +int64_t bshuf_trans_byte_elem_scal(const void* in, void* out, const size_t size, + const size_t elem_size) { + + return bshuf_trans_byte_elem_remainder(in, out, size, elem_size, 0); +} + + +/* Transpose bits within bytes. */ +int64_t bshuf_trans_bit_byte_remainder(const void* in, void* out, const size_t size, + const size_t elem_size, const size_t start_byte) { + + const uint64_t* in_b = (const uint64_t*) in; + uint8_t* out_b = (uint8_t*) out; + + uint64_t x, t; + + size_t ii, kk; + size_t nbyte = elem_size * size; + size_t nbyte_bitrow = nbyte / 8; + + uint64_t e=1; + const int little_endian = *(uint8_t *) &e == 1; + const size_t bit_row_skip = little_endian ? nbyte_bitrow : -nbyte_bitrow; + const size_t bit_row_offset = little_endian ? 0 : 7 * nbyte_bitrow; + + CHECK_MULT_EIGHT(nbyte); + CHECK_MULT_EIGHT(start_byte); + + for (ii = start_byte / 8; ii < nbyte_bitrow; ii ++) { + x = in_b[ii]; + if (little_endian) { + TRANS_BIT_8X8(x, t); + } else { + TRANS_BIT_8X8_BE(x, t); + } + for (kk = 0; kk < 8; kk ++) { + out_b[bit_row_offset + kk * bit_row_skip + ii] = (uint8_t)x; + x = x >> 8; + } + } + return (int64_t)size * (int64_t)elem_size; +} + + +/* Transpose bits within bytes. */ +int64_t bshuf_trans_bit_byte_scal(const void* in, void* out, const size_t size, + const size_t elem_size) { + + return bshuf_trans_bit_byte_remainder(in, out, size, elem_size, 0); +} + + +/* General transpose of an array, optimized for large element sizes. */ +int64_t bshuf_trans_elem(const void* in, void* out, const size_t lda, + const size_t ldb, const size_t elem_size) { + + char* in_b = (char*) in; + char* out_b = (char*) out; + size_t ii, jj; + for (ii = 0; ii < lda; ii++) { + for (jj = 0; jj < ldb; jj++) { + memcpy(&out_b[(jj*lda + ii) * elem_size], + &in_b[(ii*ldb + jj) * elem_size], elem_size); + } + } + return (int64_t)lda * (int64_t)ldb * (int64_t)elem_size; +} + + +/* Transpose rows of shuffled bits (size / 8 bytes) within groups of 8. */ +int64_t bshuf_trans_bitrow_eight(const void* in, void* out, const size_t size, + const size_t elem_size) { + + size_t nbyte_bitrow = size / 8; + + CHECK_MULT_EIGHT(size); + + return bshuf_trans_elem(in, out, 8, elem_size, nbyte_bitrow); +} + + +/* Transpose bits within elements. */ +int64_t bshuf_trans_bit_elem_scal(const void* in, void* out, const size_t size, + const size_t elem_size, void* tmp_buf) { + + int64_t count; + + CHECK_MULT_EIGHT(size); + + count = bshuf_trans_byte_elem_scal(in, out, size, elem_size); + CHECK_ERR(count); + count = bshuf_trans_bit_byte_scal(out, tmp_buf, size, elem_size); + CHECK_ERR(count); + count = bshuf_trans_bitrow_eight(tmp_buf, out, size, elem_size); + + return count; +} + + +/* For data organized into a row for each bit (8 * elem_size rows), transpose + * the bytes. */ +int64_t bshuf_trans_byte_bitrow_scal(const void* in, void* out, const size_t size, + const size_t elem_size) { + char* in_b = (char*) in; + char* out_b = (char*) out; + + size_t nbyte_row = size / 8; + size_t ii, jj, kk; + + CHECK_MULT_EIGHT(size); + + for (jj = 0; jj < elem_size; jj++) { + for (ii = 0; ii < nbyte_row; ii++) { + for (kk = 0; kk < 8; kk++) { + out_b[ii * 8 * elem_size + jj * 8 + kk] = \ + in_b[(jj * 8 + kk) * nbyte_row + ii]; + } + } + } + return (int64_t)size * (int64_t)elem_size; +} + + +/* Shuffle bits within the bytes of eight element blocks. */ +int64_t bshuf_shuffle_bit_eightelem_scal(const void* in, void* out, \ + const size_t size, const size_t elem_size) { + + const char *in_b; + char *out_b; + uint64_t x, t; + size_t ii, jj, kk; + size_t nbyte, out_index; + + uint64_t e=1; + const int little_endian = *(uint8_t *) &e == 1; + const size_t elem_skip = little_endian ? elem_size : -elem_size; + const size_t elem_offset = little_endian ? 0 : 7 * elem_size; + + CHECK_MULT_EIGHT(size); + + in_b = (const char*) in; + out_b = (char*) out; + + nbyte = elem_size * size; + + for (jj = 0; jj < 8 * elem_size; jj += 8) { + for (ii = 0; ii + 8 * elem_size - 1 < nbyte; ii += 8 * elem_size) { + x = *((uint64_t*) &in_b[ii + jj]); + if (little_endian) { + TRANS_BIT_8X8(x, t); + } else { + TRANS_BIT_8X8_BE(x, t); + } + for (kk = 0; kk < 8; kk++) { + out_index = ii + jj / 8 + elem_offset + kk * elem_skip; + *((uint8_t*) &out_b[out_index]) = (uint8_t)x; + x = x >> 8; + } + } + } + return (int64_t)size * (int64_t)elem_size; +} + + +/* Untranspose bits within elements. */ +int64_t bshuf_untrans_bit_elem_scal(const void* in, void* out, const size_t size, + const size_t elem_size, void* tmp_buf) { + + int64_t count; + + CHECK_MULT_EIGHT(size); + + count = bshuf_trans_byte_bitrow_scal(in, tmp_buf, size, elem_size); + CHECK_ERR(count); + count = bshuf_shuffle_bit_eightelem_scal(tmp_buf, out, size, elem_size); + + return count; +} + +#ifdef _MSC_VER +#pragma warning (pop) +#endif diff --git a/src/blosc2/blosc/bitshuffle-generic.h b/src/blosc2/blosc/bitshuffle-generic.h new file mode 100644 index 0000000..49b1be3 --- /dev/null +++ b/src/blosc2/blosc/bitshuffle-generic.h @@ -0,0 +1,166 @@ +/********************************************************************* + Blosc - Blocked Shuffling and Compression Library + + Copyright (C) 2021 The Blosc Developers + https://blosc.org + License: BSD 3-Clause (see LICENSE.txt) + + See LICENSE.txt for details about copyright and rights to use. +**********************************************************************/ + +/* Generic (non-hardware-accelerated) shuffle/unshuffle routines. + These are used when hardware-accelerated functions aren't available + for a particular platform; they are also used by the hardware- + accelerated functions to handle any remaining elements in a block + which isn't a multiple of the hardware's vector size. */ + +#ifndef BITSHUFFLE_GENERIC_H +#define BITSHUFFLE_GENERIC_H + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + + +/* Macros. */ +#define CHECK_MULT_EIGHT(n) if ((n) % 8) return -80; +#define MIN(X,Y) ((X) < (Y) ? (X) : (Y)) +#define MAX(X,Y) ((X) > (Y) ? (X) : (Y)) +#define CHECK_ERR(count) if ((count) < 0) { return count; } + + +/* ---- Worker code not requiring special instruction sets. ---- + * + * The following code does not use any x86 specific vectorized instructions + * and should compile on any machine + * + */ + +/* Transpose 8x8 bit array packed into a single quadword *x*. + * *t* is workspace. */ +#define TRANS_BIT_8X8(x, t) { \ + (t) = ((x) ^ ((x) >> 7)) & 0x00AA00AA00AA00AALL; \ + (x) = (x) ^ (t) ^ ((t) << 7); \ + (t) = ((x) ^ ((x) >> 14)) & 0x0000CCCC0000CCCCLL; \ + (x) = (x) ^ (t) ^ ((t) << 14); \ + (t) = ((x) ^ ((x) >> 28)) & 0x00000000F0F0F0F0LL; \ + (x) = (x) ^ (t) ^ ((t) << 28); \ + } + +/* Transpose 8x8 bit array along the diagonal from upper right + to lower left */ +#define TRANS_BIT_8X8_BE(x, t) { \ + (t) = ((x) ^ ((x) >> 9)) & 0x0055005500550055LL; \ + (x) = (x) ^ (t) ^ ((t) << 9); \ + (t) = ((x) ^ ((x) >> 18)) & 0x0000333300003333LL; \ + (x) = (x) ^ (t) ^ ((t) << 18); \ + (t) = ((x) ^ ((x) >> 36)) & 0x000000000F0F0F0FLL; \ + (x) = (x) ^ (t) ^ ((t) << 36); \ + } + +/* Transpose of an array of arbitrarily typed elements. */ +#define TRANS_ELEM_TYPE(in, out, lda, ldb, type_t) { \ + type_t* in_type = (type_t*) (in); \ + type_t* out_type = (type_t*) (out); \ + size_t ii, jj, kk; \ + for (ii = 0; ii + 7 < (lda); ii += 8) { \ + for (jj = 0; jj < (ldb); jj++) { \ + for (kk = 0; kk < 8; kk++) { \ + out_type[jj*(lda) + ii + kk] = \ + in_type[ii*(ldb) + kk * (ldb) + jj]; \ + } \ + } \ + } \ + for (ii = (lda) - (lda) % 8; ii < (lda); ii ++) { \ + for (jj = 0; jj < (ldb); jj++) { \ + out_type[jj*(lda) + ii] = in_type[ii*(ldb) + jj]; \ + } \ + } \ + } + + +/* Private functions */ +BLOSC_NO_EXPORT int64_t +bshuf_trans_byte_elem_remainder(const void* in, void* out, const size_t size, + const size_t elem_size, const size_t start); + +BLOSC_NO_EXPORT int64_t +bshuf_trans_byte_elem_scal(const void* in, void* out, const size_t size, + const size_t elem_size); + +BLOSC_NO_EXPORT int64_t +bshuf_trans_bit_byte_remainder(const void* in, void* out, const size_t size, + const size_t elem_size, const size_t start_byte); + +BLOSC_NO_EXPORT int64_t +bshuf_trans_elem(const void* in, void* out, const size_t lda, + const size_t ldb, const size_t elem_size); + +BLOSC_NO_EXPORT int64_t +bshuf_trans_bitrow_eight(const void* in, void* out, const size_t size, + const size_t elem_size); + +BLOSC_NO_EXPORT int64_t +bshuf_shuffle_bit_eightelem_scal(const void* in, void* out, + const size_t size, const size_t elem_size); + +BLOSC_NO_EXPORT int64_t +bshuf_trans_byte_bitrow_scal(const void* in, void* out, const size_t size, + const size_t elem_size); + +/* Bitshuffle the data. + * + * Transpose the bits within elements. + * + * Parameters + * ---------- + * in : input buffer, must be of size * elem_size bytes + * out : output buffer, must be of size * elem_size bytes + * size : number of elements in input + * elem_size : element size of typed data + * tmp_buffer : temporary buffer with the same `size` than `in` and `out` + * + * Returns + * ------- + * nothing -- this cannot fail + * + */ + +BLOSC_NO_EXPORT int64_t +bshuf_trans_bit_elem_scal(const void* in, void* out, const size_t size, + const size_t elem_size, void* tmp_buf); + +/* Unshuffle bitshuffled data. + * + * Untranspose the bits within elements. + * + * To properly unshuffle bitshuffled data, *size* and *elem_size* must + * match the parameters used to shuffle the data. + * + * Parameters + * ---------- + * in : input buffer, must be of size * elem_size bytes + * out : output buffer, must be of size * elem_size bytes + * size : number of elements in input + * elem_size : element size of typed data + * tmp_buffer : temporary buffer with the same `size` than `in` and `out` + * + * Returns + * ------- + * nothing -- this cannot fail + * + */ + +BLOSC_NO_EXPORT int64_t +bshuf_untrans_bit_elem_scal(const void* in, void* out, const size_t size, + const size_t elem_size, void* tmp_buf); + + +#ifdef __cplusplus +} +#endif + +#endif /* BITSHUFFLE_GENERIC_H */ diff --git a/src/blosc2/blosc/bitshuffle-neon.c b/src/blosc2/blosc/bitshuffle-neon.c new file mode 100644 index 0000000..be12aac --- /dev/null +++ b/src/blosc2/blosc/bitshuffle-neon.c @@ -0,0 +1,1005 @@ +/********************************************************************* + Blosc - Blocked Shuffling and Compression Library + + Copyright (C) 2021 The Blosc Developers + https://blosc.org + License: BSD 3-Clause (see LICENSE.txt) + + See LICENSE.txt for details about copyright and rights to use. +**********************************************************************/ + +#include "bitshuffle-generic.h" +#include "bitshuffle-neon.h" + +/* Make sure NEON is available for the compilation target and compiler. */ +#if !defined(__ARM_NEON) + #error NEON is not supported by the target architecture/platform and/or this compiler. +#endif + +#include + +/* The next is useful for debugging purposes */ +#if 0 +#include +#include + +static void printmem(uint8_t* buf) +{ + printf("%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,\n", + buf[0], buf[1], buf[2], buf[3], + buf[4], buf[5], buf[6], buf[7], + buf[8], buf[9], buf[10], buf[11], + buf[12], buf[13], buf[14], buf[15]); +} +#endif + +/* Routine optimized for bit-shuffling a buffer for a type size of 1 byte. */ +static void +bitshuffle1_neon(void* src, void* dest, const size_t size, const size_t elem_size) { + + uint16x8_t x0; + size_t i, j, k; + uint8x8_t lo_x, hi_x, lo, hi; + + const int8_t __attribute__ ((aligned (16))) xr[8] = {0, 1, 2, 3, 4, 5, 6, 7}; + uint8x8_t mask_and = vdup_n_u8(0x01); + int8x8_t mask_shift = vld1_s8(xr); + + for (i = 0, k = 0; i < size * elem_size; i += 16, k++) { + /* Load 16-byte groups */ + x0 = vld1q_u8(src + k * 16); + /* Split in 8-bytes grops */ + lo_x = vget_low_u8(x0); + hi_x = vget_high_u8(x0); + for (j = 0; j < 8; j++) { + /* Create mask from the most significant bit of each 8-bit element */ + lo = vand_u8(lo_x, mask_and); + lo = vshl_u8(lo, mask_shift); + hi = vand_u8(hi_x, mask_and); + hi = vshl_u8(hi, mask_shift); + lo = vpadd_u8(lo, lo); + lo = vpadd_u8(lo, lo); + lo = vpadd_u8(lo, lo); + hi = vpadd_u8(hi, hi); + hi = vpadd_u8(hi, hi); + hi = vpadd_u8(hi, hi); + /* Shift packed 8-bit */ + lo_x = vshr_n_u8(lo_x, 1); + hi_x = vshr_n_u8(hi_x, 1); + /* Store the created mask to the destination vector */ + vst1_lane_u8(dest + 2 * k + j * size * elem_size / (8 * elem_size), lo, 0); + vst1_lane_u8(dest + 2 * k + 1 + j * size * elem_size / (8 * elem_size), hi, 0); + } + } +} + +/* Routine optimized for bit-shuffling a buffer for a type size of 2 bytes. */ +static void +bitshuffle2_neon(void* src, void* dest, const size_t size, const size_t elem_size) { + + uint8x16x2_t x0; + size_t i, j, k; + uint8x8_t lo_x[2], hi_x[2], lo[2], hi[2]; + + const int8_t __attribute__ ((aligned (16))) xr[8] = {0, 1, 2, 3, 4, 5, 6, 7}; + uint8x8_t mask_and = vdup_n_u8(0x01); + int8x8_t mask_shift = vld1_s8(xr); + + for (i = 0, k = 0; i < size * elem_size; i += 32, k++) { + /* Load 32-byte groups */ + x0 = vld2q_u8(src + i); + /* Split in 8-bytes grops */ + lo_x[0] = vget_low_u8(x0.val[0]); + hi_x[0] = vget_high_u8(x0.val[0]); + lo_x[1] = vget_low_u8(x0.val[1]); + hi_x[1] = vget_high_u8(x0.val[1]); + for (j = 0; j < 8; j++) { + /* Create mask from the most significant bit of each 8-bit element */ + lo[0] = vand_u8(lo_x[0], mask_and); + lo[0] = vshl_u8(lo[0], mask_shift); + lo[1] = vand_u8(lo_x[1], mask_and); + lo[1] = vshl_u8(lo[1], mask_shift); + + hi[0] = vand_u8(hi_x[0], mask_and); + hi[0] = vshl_u8(hi[0], mask_shift); + hi[1] = vand_u8(hi_x[1], mask_and); + hi[1] = vshl_u8(hi[1], mask_shift); + + lo[0] = vpadd_u8(lo[0], lo[0]); + lo[0] = vpadd_u8(lo[0], lo[0]); + lo[0] = vpadd_u8(lo[0], lo[0]); + lo[1] = vpadd_u8(lo[1], lo[1]); + lo[1] = vpadd_u8(lo[1], lo[1]); + lo[1] = vpadd_u8(lo[1], lo[1]); + + hi[0] = vpadd_u8(hi[0], hi[0]); + hi[0] = vpadd_u8(hi[0], hi[0]); + hi[0] = vpadd_u8(hi[0], hi[0]); + hi[1] = vpadd_u8(hi[1], hi[1]); + hi[1] = vpadd_u8(hi[1], hi[1]); + hi[1] = vpadd_u8(hi[1], hi[1]); + /* Shift packed 8-bit */ + lo_x[0] = vshr_n_u8(lo_x[0], 1); + hi_x[0] = vshr_n_u8(hi_x[0], 1); + lo_x[1] = vshr_n_u8(lo_x[1], 1); + hi_x[1] = vshr_n_u8(hi_x[1], 1); + /* Store the created mask to the destination vector */ + vst1_lane_u8(dest + 2 * k + j * size * elem_size / (8 * elem_size), lo[0], 0); + vst1_lane_u8(dest + 2 * k + j * size * elem_size / (8 * elem_size) + size * elem_size / 2, lo[1], 0); + vst1_lane_u8(dest + 2 * k + 1 + j * size * elem_size / (8 * elem_size), hi[0], 0); + vst1_lane_u8(dest + 2 * k + 1 + j * size * elem_size / (8 * elem_size) + size * elem_size / 2, hi[1], 0); + } + } +} + +/* Routine optimized for bit-shuffling a buffer for a type size of 4 bytes. */ +static void +bitshuffle4_neon(void* src, void* dest, const size_t size, const size_t elem_size) { + uint8x16x4_t x0; + size_t i, j, k; + uint8x8_t lo_x[4], hi_x[4], lo[4], hi[4]; + + const int8_t __attribute__ ((aligned (16))) xr[8] = {0, 1, 2, 3, 4, 5, 6, 7}; + uint8x8_t mask_and = vdup_n_u8(0x01); + int8x8_t mask_shift = vld1_s8(xr); + + for (i = 0, k = 0; i < size * elem_size; i += 64, k++) { + /* Load 64-byte groups */ + x0 = vld4q_u8(src + i); + /* Split in 8-bytes grops */ + lo_x[0] = vget_low_u8(x0.val[0]); + hi_x[0] = vget_high_u8(x0.val[0]); + lo_x[1] = vget_low_u8(x0.val[1]); + hi_x[1] = vget_high_u8(x0.val[1]); + lo_x[2] = vget_low_u8(x0.val[2]); + hi_x[2] = vget_high_u8(x0.val[2]); + lo_x[3] = vget_low_u8(x0.val[3]); + hi_x[3] = vget_high_u8(x0.val[3]); + for (j = 0; j < 8; j++) { + /* Create mask from the most significant bit of each 8-bit element */ + lo[0] = vand_u8(lo_x[0], mask_and); + lo[0] = vshl_u8(lo[0], mask_shift); + lo[1] = vand_u8(lo_x[1], mask_and); + lo[1] = vshl_u8(lo[1], mask_shift); + lo[2] = vand_u8(lo_x[2], mask_and); + lo[2] = vshl_u8(lo[2], mask_shift); + lo[3] = vand_u8(lo_x[3], mask_and); + lo[3] = vshl_u8(lo[3], mask_shift); + + hi[0] = vand_u8(hi_x[0], mask_and); + hi[0] = vshl_u8(hi[0], mask_shift); + hi[1] = vand_u8(hi_x[1], mask_and); + hi[1] = vshl_u8(hi[1], mask_shift); + hi[2] = vand_u8(hi_x[2], mask_and); + hi[2] = vshl_u8(hi[2], mask_shift); + hi[3] = vand_u8(hi_x[3], mask_and); + hi[3] = vshl_u8(hi[3], mask_shift); + + lo[0] = vpadd_u8(lo[0], lo[0]); + lo[0] = vpadd_u8(lo[0], lo[0]); + lo[0] = vpadd_u8(lo[0], lo[0]); + lo[1] = vpadd_u8(lo[1], lo[1]); + lo[1] = vpadd_u8(lo[1], lo[1]); + lo[1] = vpadd_u8(lo[1], lo[1]); + lo[2] = vpadd_u8(lo[2], lo[2]); + lo[2] = vpadd_u8(lo[2], lo[2]); + lo[2] = vpadd_u8(lo[2], lo[2]); + lo[3] = vpadd_u8(lo[3], lo[3]); + lo[3] = vpadd_u8(lo[3], lo[3]); + lo[3] = vpadd_u8(lo[3], lo[3]); + + hi[0] = vpadd_u8(hi[0], hi[0]); + hi[0] = vpadd_u8(hi[0], hi[0]); + hi[0] = vpadd_u8(hi[0], hi[0]); + hi[1] = vpadd_u8(hi[1], hi[1]); + hi[1] = vpadd_u8(hi[1], hi[1]); + hi[1] = vpadd_u8(hi[1], hi[1]); + hi[2] = vpadd_u8(hi[2], hi[2]); + hi[2] = vpadd_u8(hi[2], hi[2]); + hi[2] = vpadd_u8(hi[2], hi[2]); + hi[3] = vpadd_u8(hi[3], hi[3]); + hi[3] = vpadd_u8(hi[3], hi[3]); + hi[3] = vpadd_u8(hi[3], hi[3]); + /* Shift packed 8-bit */ + lo_x[0] = vshr_n_u8(lo_x[0], 1); + hi_x[0] = vshr_n_u8(hi_x[0], 1); + lo_x[1] = vshr_n_u8(lo_x[1], 1); + hi_x[1] = vshr_n_u8(hi_x[1], 1); + lo_x[2] = vshr_n_u8(lo_x[2], 1); + hi_x[2] = vshr_n_u8(hi_x[2], 1); + lo_x[3] = vshr_n_u8(lo_x[3], 1); + hi_x[3] = vshr_n_u8(hi_x[3], 1); + /* Store the created mask to the destination vector */ + vst1_lane_u8(dest + 2 * k + j * size * elem_size / (8 * elem_size) + 0 * size * elem_size / 4, lo[0], 0); + vst1_lane_u8(dest + 2 * k + j * size * elem_size / (8 * elem_size) + 1 * size * elem_size / 4, lo[1], 0); + vst1_lane_u8(dest + 2 * k + j * size * elem_size / (8 * elem_size) + 2 * size * elem_size / 4, lo[2], 0); + vst1_lane_u8(dest + 2 * k + j * size * elem_size / (8 * elem_size) + 3 * size * elem_size / 4, lo[3], 0); + vst1_lane_u8(dest + 2 * k + 1 + j * size * elem_size / (8 * elem_size) + 0 * size * elem_size / 4, hi[0], 0); + vst1_lane_u8(dest + 2 * k + 1 + j * size * elem_size / (8 * elem_size) + 1 * size * elem_size / 4, hi[1], 0); + vst1_lane_u8(dest + 2 * k + 1 + j * size * elem_size / (8 * elem_size) + 2 * size * elem_size / 4, hi[2], 0); + vst1_lane_u8(dest + 2 * k + 1 + j * size * elem_size / (8 * elem_size) + 3 * size * elem_size / 4, hi[3], 0); + } + } +} + +/* Routine optimized for bit-shuffling a buffer for a type size of 8 bytes. */ +static void +bitshuffle8_neon(void* src, void* dest, const size_t size, const size_t elem_size) { + + size_t i, j, k; + uint8x8x2_t r0[4]; + uint16x4x2_t r1[4]; + uint32x2x2_t r2[4]; + + const int8_t __attribute__ ((aligned (16))) xr[8] = {0, 1, 2, 3, 4, 5, 6, 7}; + uint8x8_t mask_and = vdup_n_u8(0x01); + int8x8_t mask_shift = vld1_s8(xr); + + for (i = 0, k = 0; i < size * elem_size; i += 64, k++) { + /* Load and interleave groups of 8 bytes (64 bytes) to the structure r0 */ + r0[0] = vzip_u8(vld1_u8(src + i + 0 * 8), vld1_u8(src + i + 1 * 8)); + r0[1] = vzip_u8(vld1_u8(src + i + 2 * 8), vld1_u8(src + i + 3 * 8)); + r0[2] = vzip_u8(vld1_u8(src + i + 4 * 8), vld1_u8(src + i + 5 * 8)); + r0[3] = vzip_u8(vld1_u8(src + i + 6 * 8), vld1_u8(src + i + 7 * 8)); + /* Interleave 16 bytes */ + r1[0] = vzip_u16(vreinterpret_u16_u8(r0[0].val[0]), vreinterpret_u16_u8(r0[1].val[0])); + r1[1] = vzip_u16(vreinterpret_u16_u8(r0[0].val[1]), vreinterpret_u16_u8(r0[1].val[1])); + r1[2] = vzip_u16(vreinterpret_u16_u8(r0[2].val[0]), vreinterpret_u16_u8(r0[3].val[0])); + r1[3] = vzip_u16(vreinterpret_u16_u8(r0[2].val[1]), vreinterpret_u16_u8(r0[3].val[1])); + /* Interleave 32 bytes */ + r2[0] = vzip_u32(vreinterpret_u32_u16(r1[0].val[0]), vreinterpret_u32_u16(r1[2].val[0])); + r2[1] = vzip_u32(vreinterpret_u32_u16(r1[0].val[1]), vreinterpret_u32_u16(r1[2].val[1])); + r2[2] = vzip_u32(vreinterpret_u32_u16(r1[1].val[0]), vreinterpret_u32_u16(r1[3].val[0])); + r2[3] = vzip_u32(vreinterpret_u32_u16(r1[1].val[1]), vreinterpret_u32_u16(r1[3].val[1])); + for (j = 0; j < 8; j++) { + /* Create mask from the most significant bit of each 8-bit element */ + r0[0].val[0] = vand_u8(vreinterpret_u8_u32(r2[0].val[0]), mask_and); + r0[0].val[0] = vshl_u8(r0[0].val[0], mask_shift); + r0[0].val[1] = vand_u8(vreinterpret_u8_u32(r2[0].val[1]), mask_and); + r0[0].val[1] = vshl_u8(r0[0].val[1], mask_shift); + r0[1].val[0] = vand_u8(vreinterpret_u8_u32(r2[1].val[0]), mask_and); + r0[1].val[0] = vshl_u8(r0[1].val[0], mask_shift); + r0[1].val[1] = vand_u8(vreinterpret_u8_u32(r2[1].val[1]), mask_and); + r0[1].val[1] = vshl_u8(r0[1].val[1], mask_shift); + r0[2].val[0] = vand_u8(vreinterpret_u8_u32(r2[2].val[0]), mask_and); + r0[2].val[0] = vshl_u8(r0[2].val[0], mask_shift); + r0[2].val[1] = vand_u8(vreinterpret_u8_u32(r2[2].val[1]), mask_and); + r0[2].val[1] = vshl_u8(r0[2].val[1], mask_shift); + r0[3].val[0] = vand_u8(vreinterpret_u8_u32(r2[3].val[0]), mask_and); + r0[3].val[0] = vshl_u8(r0[3].val[0], mask_shift); + r0[3].val[1] = vand_u8(vreinterpret_u8_u32(r2[3].val[1]), mask_and); + r0[3].val[1] = vshl_u8(r0[3].val[1], mask_shift); + + r0[0].val[0] = vpadd_u8(r0[0].val[0], r0[0].val[0]); + r0[0].val[0] = vpadd_u8(r0[0].val[0], r0[0].val[0]); + r0[0].val[0] = vpadd_u8(r0[0].val[0], r0[0].val[0]); + r0[0].val[1] = vpadd_u8(r0[0].val[1], r0[0].val[1]); + r0[0].val[1] = vpadd_u8(r0[0].val[1], r0[0].val[1]); + r0[0].val[1] = vpadd_u8(r0[0].val[1], r0[0].val[1]); + r0[1].val[0] = vpadd_u8(r0[1].val[0], r0[1].val[0]); + r0[1].val[0] = vpadd_u8(r0[1].val[0], r0[1].val[0]); + r0[1].val[0] = vpadd_u8(r0[1].val[0], r0[1].val[0]); + r0[1].val[1] = vpadd_u8(r0[1].val[1], r0[1].val[1]); + r0[1].val[1] = vpadd_u8(r0[1].val[1], r0[1].val[1]); + r0[1].val[1] = vpadd_u8(r0[1].val[1], r0[1].val[1]); + r0[2].val[0] = vpadd_u8(r0[2].val[0], r0[2].val[0]); + r0[2].val[0] = vpadd_u8(r0[2].val[0], r0[2].val[0]); + r0[2].val[0] = vpadd_u8(r0[2].val[0], r0[2].val[0]); + r0[2].val[1] = vpadd_u8(r0[2].val[1], r0[2].val[1]); + r0[2].val[1] = vpadd_u8(r0[2].val[1], r0[2].val[1]); + r0[2].val[1] = vpadd_u8(r0[2].val[1], r0[2].val[1]); + r0[3].val[0] = vpadd_u8(r0[3].val[0], r0[3].val[0]); + r0[3].val[0] = vpadd_u8(r0[3].val[0], r0[3].val[0]); + r0[3].val[0] = vpadd_u8(r0[3].val[0], r0[3].val[0]); + r0[3].val[1] = vpadd_u8(r0[3].val[1], r0[3].val[1]); + r0[3].val[1] = vpadd_u8(r0[3].val[1], r0[3].val[1]); + r0[3].val[1] = vpadd_u8(r0[3].val[1], r0[3].val[1]); + /* Shift packed 8-bit */ + r2[0].val[0] = vreinterpret_u8_u32(vshr_n_u8(vreinterpret_u8_u32(r2[0].val[0]), 1)); + r2[0].val[1] = vreinterpret_u8_u32(vshr_n_u8(vreinterpret_u8_u32(r2[0].val[1]), 1)); + r2[1].val[0] = vreinterpret_u8_u32(vshr_n_u8(vreinterpret_u8_u32(r2[1].val[0]), 1)); + r2[1].val[1] = vreinterpret_u8_u32(vshr_n_u8(vreinterpret_u8_u32(r2[1].val[1]), 1)); + r2[2].val[0] = vreinterpret_u8_u32(vshr_n_u8(vreinterpret_u8_u32(r2[2].val[0]), 1)); + r2[2].val[1] = vreinterpret_u8_u32(vshr_n_u8(vreinterpret_u8_u32(r2[2].val[1]), 1)); + r2[3].val[0] = vreinterpret_u8_u32(vshr_n_u8(vreinterpret_u8_u32(r2[3].val[0]), 1)); + r2[3].val[1] = vreinterpret_u8_u32(vshr_n_u8(vreinterpret_u8_u32(r2[3].val[1]), 1)); + /* Store the created mask to the destination vector */ + vst1_lane_u8(dest + k + j * size * elem_size / (8 * elem_size) + 0 * size * elem_size / 8, r0[0].val[0], 0); + vst1_lane_u8(dest + k + j * size * elem_size / (8 * elem_size) + 1 * size * elem_size / 8, r0[0].val[1], 0); + vst1_lane_u8(dest + k + j * size * elem_size / (8 * elem_size) + 2 * size * elem_size / 8, r0[1].val[0], 0); + vst1_lane_u8(dest + k + j * size * elem_size / (8 * elem_size) + 3 * size * elem_size / 8, r0[1].val[1], 0); + vst1_lane_u8(dest + k + j * size * elem_size / (8 * elem_size) + 4 * size * elem_size / 8, r0[2].val[0], 0); + vst1_lane_u8(dest + k + j * size * elem_size / (8 * elem_size) + 5 * size * elem_size / 8, r0[2].val[1], 0); + vst1_lane_u8(dest + k + j * size * elem_size / (8 * elem_size) + 6 * size * elem_size / 8, r0[3].val[0], 0); + vst1_lane_u8(dest + k + j * size * elem_size / (8 * elem_size) + 7 * size * elem_size / 8, r0[3].val[1], 0); + } + } +} + +/* Routine optimized for bit-shuffling a buffer for a type size of 16 bytes. */ +static void +bitshuffle16_neon(void* src, void* dest, const size_t size, const size_t elem_size) { + + size_t i, j, k; + uint8x8x2_t r0[8]; + uint16x4x2_t r1[8]; + uint32x2x2_t r2[8]; + + const int8_t __attribute__ ((aligned (16))) xr[8] = {0, 1, 2, 3, 4, 5, 6, 7}; + uint8x8_t mask_and = vdup_n_u8(0x01); + int8x8_t mask_shift = vld1_s8(xr); + + for (i = 0, k = 0; i < size * elem_size; i += 128, k++) { + /* Load and interleave groups of 16 bytes (128 bytes) to the structure r0 */ + r0[0] = vzip_u8(vld1_u8(src + i + 0 * 8), vld1_u8(src + i + 2 * 8)); + r0[1] = vzip_u8(vld1_u8(src + i + 1 * 8), vld1_u8(src + i + 3 * 8)); + r0[2] = vzip_u8(vld1_u8(src + i + 4 * 8), vld1_u8(src + i + 6 * 8)); + r0[3] = vzip_u8(vld1_u8(src + i + 5 * 8), vld1_u8(src + i + 7 * 8)); + r0[4] = vzip_u8(vld1_u8(src + i + 8 * 8), vld1_u8(src + i + 10 * 8)); + r0[5] = vzip_u8(vld1_u8(src + i + 9 * 8), vld1_u8(src + i + 11 * 8)); + r0[6] = vzip_u8(vld1_u8(src + i + 12 * 8), vld1_u8(src + i + 14 * 8)); + r0[7] = vzip_u8(vld1_u8(src + i + 13 * 8), vld1_u8(src + i + 15 * 8)); + /* Interleave 16 bytes */ + r1[0] = vzip_u16(vreinterpret_u16_u8(r0[0].val[0]), vreinterpret_u16_u8(r0[2].val[0])); + r1[1] = vzip_u16(vreinterpret_u16_u8(r0[0].val[1]), vreinterpret_u16_u8(r0[2].val[1])); + r1[2] = vzip_u16(vreinterpret_u16_u8(r0[1].val[0]), vreinterpret_u16_u8(r0[3].val[0])); + r1[3] = vzip_u16(vreinterpret_u16_u8(r0[1].val[1]), vreinterpret_u16_u8(r0[3].val[1])); + r1[4] = vzip_u16(vreinterpret_u16_u8(r0[4].val[0]), vreinterpret_u16_u8(r0[6].val[0])); + r1[5] = vzip_u16(vreinterpret_u16_u8(r0[4].val[1]), vreinterpret_u16_u8(r0[6].val[1])); + r1[6] = vzip_u16(vreinterpret_u16_u8(r0[5].val[0]), vreinterpret_u16_u8(r0[7].val[0])); + r1[7] = vzip_u16(vreinterpret_u16_u8(r0[5].val[1]), vreinterpret_u16_u8(r0[7].val[1])); + /* Interleave 32 bytes */ + r2[0] = vzip_u32(vreinterpret_u32_u16(r1[0].val[0]), vreinterpret_u32_u16(r1[4].val[0])); + r2[1] = vzip_u32(vreinterpret_u32_u16(r1[0].val[1]), vreinterpret_u32_u16(r1[4].val[1])); + r2[2] = vzip_u32(vreinterpret_u32_u16(r1[1].val[0]), vreinterpret_u32_u16(r1[5].val[0])); + r2[3] = vzip_u32(vreinterpret_u32_u16(r1[1].val[1]), vreinterpret_u32_u16(r1[5].val[1])); + r2[4] = vzip_u32(vreinterpret_u32_u16(r1[2].val[0]), vreinterpret_u32_u16(r1[6].val[0])); + r2[5] = vzip_u32(vreinterpret_u32_u16(r1[2].val[1]), vreinterpret_u32_u16(r1[6].val[1])); + r2[6] = vzip_u32(vreinterpret_u32_u16(r1[3].val[0]), vreinterpret_u32_u16(r1[7].val[0])); + r2[7] = vzip_u32(vreinterpret_u32_u16(r1[3].val[1]), vreinterpret_u32_u16(r1[7].val[1])); + for (j = 0; j < 8; j++) { + /* Create mask from the most significant bit of each 8-bit element */ + r0[0].val[0] = vand_u8(vreinterpret_u8_u32(r2[0].val[0]), mask_and); + r0[0].val[0] = vshl_u8(r0[0].val[0], mask_shift); + r0[0].val[1] = vand_u8(vreinterpret_u8_u32(r2[0].val[1]), mask_and); + r0[0].val[1] = vshl_u8(r0[0].val[1], mask_shift); + r0[1].val[0] = vand_u8(vreinterpret_u8_u32(r2[1].val[0]), mask_and); + r0[1].val[0] = vshl_u8(r0[1].val[0], mask_shift); + r0[1].val[1] = vand_u8(vreinterpret_u8_u32(r2[1].val[1]), mask_and); + r0[1].val[1] = vshl_u8(r0[1].val[1], mask_shift); + r0[2].val[0] = vand_u8(vreinterpret_u8_u32(r2[2].val[0]), mask_and); + r0[2].val[0] = vshl_u8(r0[2].val[0], mask_shift); + r0[2].val[1] = vand_u8(vreinterpret_u8_u32(r2[2].val[1]), mask_and); + r0[2].val[1] = vshl_u8(r0[2].val[1], mask_shift); + r0[3].val[0] = vand_u8(vreinterpret_u8_u32(r2[3].val[0]), mask_and); + r0[3].val[0] = vshl_u8(r0[3].val[0], mask_shift); + r0[3].val[1] = vand_u8(vreinterpret_u8_u32(r2[3].val[1]), mask_and); + r0[3].val[1] = vshl_u8(r0[3].val[1], mask_shift); + r0[4].val[0] = vand_u8(vreinterpret_u8_u32(r2[4].val[0]), mask_and); + r0[4].val[0] = vshl_u8(r0[4].val[0], mask_shift); + r0[4].val[1] = vand_u8(vreinterpret_u8_u32(r2[4].val[1]), mask_and); + r0[4].val[1] = vshl_u8(r0[4].val[1], mask_shift); + r0[5].val[0] = vand_u8(vreinterpret_u8_u32(r2[5].val[0]), mask_and); + r0[5].val[0] = vshl_u8(r0[5].val[0], mask_shift); + r0[5].val[1] = vand_u8(vreinterpret_u8_u32(r2[5].val[1]), mask_and); + r0[5].val[1] = vshl_u8(r0[5].val[1], mask_shift); + r0[6].val[0] = vand_u8(vreinterpret_u8_u32(r2[6].val[0]), mask_and); + r0[6].val[0] = vshl_u8(r0[6].val[0], mask_shift); + r0[6].val[1] = vand_u8(vreinterpret_u8_u32(r2[6].val[1]), mask_and); + r0[6].val[1] = vshl_u8(r0[6].val[1], mask_shift); + r0[7].val[0] = vand_u8(vreinterpret_u8_u32(r2[7].val[0]), mask_and); + r0[7].val[0] = vshl_u8(r0[7].val[0], mask_shift); + r0[7].val[1] = vand_u8(vreinterpret_u8_u32(r2[7].val[1]), mask_and); + r0[7].val[1] = vshl_u8(r0[7].val[1], mask_shift); + + r0[0].val[0] = vpadd_u8(r0[0].val[0], r0[0].val[0]); + r0[0].val[0] = vpadd_u8(r0[0].val[0], r0[0].val[0]); + r0[0].val[0] = vpadd_u8(r0[0].val[0], r0[0].val[0]); + r0[0].val[1] = vpadd_u8(r0[0].val[1], r0[0].val[1]); + r0[0].val[1] = vpadd_u8(r0[0].val[1], r0[0].val[1]); + r0[0].val[1] = vpadd_u8(r0[0].val[1], r0[0].val[1]); + r0[1].val[0] = vpadd_u8(r0[1].val[0], r0[1].val[0]); + r0[1].val[0] = vpadd_u8(r0[1].val[0], r0[1].val[0]); + r0[1].val[0] = vpadd_u8(r0[1].val[0], r0[1].val[0]); + r0[1].val[1] = vpadd_u8(r0[1].val[1], r0[1].val[1]); + r0[1].val[1] = vpadd_u8(r0[1].val[1], r0[1].val[1]); + r0[1].val[1] = vpadd_u8(r0[1].val[1], r0[1].val[1]); + r0[2].val[0] = vpadd_u8(r0[2].val[0], r0[2].val[0]); + r0[2].val[0] = vpadd_u8(r0[2].val[0], r0[2].val[0]); + r0[2].val[0] = vpadd_u8(r0[2].val[0], r0[2].val[0]); + r0[2].val[1] = vpadd_u8(r0[2].val[1], r0[2].val[1]); + r0[2].val[1] = vpadd_u8(r0[2].val[1], r0[2].val[1]); + r0[2].val[1] = vpadd_u8(r0[2].val[1], r0[2].val[1]); + r0[3].val[0] = vpadd_u8(r0[3].val[0], r0[3].val[0]); + r0[3].val[0] = vpadd_u8(r0[3].val[0], r0[3].val[0]); + r0[3].val[0] = vpadd_u8(r0[3].val[0], r0[3].val[0]); + r0[3].val[1] = vpadd_u8(r0[3].val[1], r0[3].val[1]); + r0[3].val[1] = vpadd_u8(r0[3].val[1], r0[3].val[1]); + r0[3].val[1] = vpadd_u8(r0[3].val[1], r0[3].val[1]); + r0[4].val[0] = vpadd_u8(r0[4].val[0], r0[4].val[0]); + r0[4].val[0] = vpadd_u8(r0[4].val[0], r0[4].val[0]); + r0[4].val[0] = vpadd_u8(r0[4].val[0], r0[4].val[0]); + r0[4].val[1] = vpadd_u8(r0[4].val[1], r0[4].val[1]); + r0[4].val[1] = vpadd_u8(r0[4].val[1], r0[4].val[1]); + r0[4].val[1] = vpadd_u8(r0[4].val[1], r0[4].val[1]); + r0[5].val[0] = vpadd_u8(r0[5].val[0], r0[5].val[0]); + r0[5].val[0] = vpadd_u8(r0[5].val[0], r0[5].val[0]); + r0[5].val[0] = vpadd_u8(r0[5].val[0], r0[5].val[0]); + r0[5].val[1] = vpadd_u8(r0[5].val[1], r0[5].val[1]); + r0[5].val[1] = vpadd_u8(r0[5].val[1], r0[5].val[1]); + r0[5].val[1] = vpadd_u8(r0[5].val[1], r0[5].val[1]); + r0[6].val[0] = vpadd_u8(r0[6].val[0], r0[6].val[0]); + r0[6].val[0] = vpadd_u8(r0[6].val[0], r0[6].val[0]); + r0[6].val[0] = vpadd_u8(r0[6].val[0], r0[6].val[0]); + r0[6].val[1] = vpadd_u8(r0[6].val[1], r0[6].val[1]); + r0[6].val[1] = vpadd_u8(r0[6].val[1], r0[6].val[1]); + r0[6].val[1] = vpadd_u8(r0[6].val[1], r0[6].val[1]); + r0[7].val[0] = vpadd_u8(r0[7].val[0], r0[7].val[0]); + r0[7].val[0] = vpadd_u8(r0[7].val[0], r0[7].val[0]); + r0[7].val[0] = vpadd_u8(r0[7].val[0], r0[7].val[0]); + r0[7].val[1] = vpadd_u8(r0[7].val[1], r0[7].val[1]); + r0[7].val[1] = vpadd_u8(r0[7].val[1], r0[7].val[1]); + r0[7].val[1] = vpadd_u8(r0[7].val[1], r0[7].val[1]); + /* Shift packed 8-bit */ + r2[0].val[0] = vreinterpret_u8_u32(vshr_n_u8(vreinterpret_u8_u32(r2[0].val[0]), 1)); + r2[0].val[1] = vreinterpret_u8_u32(vshr_n_u8(vreinterpret_u8_u32(r2[0].val[1]), 1)); + r2[1].val[0] = vreinterpret_u8_u32(vshr_n_u8(vreinterpret_u8_u32(r2[1].val[0]), 1)); + r2[1].val[1] = vreinterpret_u8_u32(vshr_n_u8(vreinterpret_u8_u32(r2[1].val[1]), 1)); + r2[2].val[0] = vreinterpret_u8_u32(vshr_n_u8(vreinterpret_u8_u32(r2[2].val[0]), 1)); + r2[2].val[1] = vreinterpret_u8_u32(vshr_n_u8(vreinterpret_u8_u32(r2[2].val[1]), 1)); + r2[3].val[0] = vreinterpret_u8_u32(vshr_n_u8(vreinterpret_u8_u32(r2[3].val[0]), 1)); + r2[3].val[1] = vreinterpret_u8_u32(vshr_n_u8(vreinterpret_u8_u32(r2[3].val[1]), 1)); + r2[4].val[0] = vreinterpret_u8_u32(vshr_n_u8(vreinterpret_u8_u32(r2[4].val[0]), 1)); + r2[4].val[1] = vreinterpret_u8_u32(vshr_n_u8(vreinterpret_u8_u32(r2[4].val[1]), 1)); + r2[5].val[0] = vreinterpret_u8_u32(vshr_n_u8(vreinterpret_u8_u32(r2[5].val[0]), 1)); + r2[5].val[1] = vreinterpret_u8_u32(vshr_n_u8(vreinterpret_u8_u32(r2[5].val[1]), 1)); + r2[6].val[0] = vreinterpret_u8_u32(vshr_n_u8(vreinterpret_u8_u32(r2[6].val[0]), 1)); + r2[6].val[1] = vreinterpret_u8_u32(vshr_n_u8(vreinterpret_u8_u32(r2[6].val[1]), 1)); + r2[7].val[0] = vreinterpret_u8_u32(vshr_n_u8(vreinterpret_u8_u32(r2[7].val[0]), 1)); + r2[7].val[1] = vreinterpret_u8_u32(vshr_n_u8(vreinterpret_u8_u32(r2[7].val[1]), 1)); + /* Store the created mask to the destination vector */ + vst1_lane_u8(dest + k + j * size * elem_size / (8 * elem_size) + 0 * size * elem_size / 16, r0[0].val[0], 0); + vst1_lane_u8(dest + k + j * size * elem_size / (8 * elem_size) + 1 * size * elem_size / 16, r0[0].val[1], 0); + vst1_lane_u8(dest + k + j * size * elem_size / (8 * elem_size) + 2 * size * elem_size / 16, r0[1].val[0], 0); + vst1_lane_u8(dest + k + j * size * elem_size / (8 * elem_size) + 3 * size * elem_size / 16, r0[1].val[1], 0); + vst1_lane_u8(dest + k + j * size * elem_size / (8 * elem_size) + 4 * size * elem_size / 16, r0[2].val[0], 0); + vst1_lane_u8(dest + k + j * size * elem_size / (8 * elem_size) + 5 * size * elem_size / 16, r0[2].val[1], 0); + vst1_lane_u8(dest + k + j * size * elem_size / (8 * elem_size) + 6 * size * elem_size / 16, r0[3].val[0], 0); + vst1_lane_u8(dest + k + j * size * elem_size / (8 * elem_size) + 7 * size * elem_size / 16, r0[3].val[1], 0); + vst1_lane_u8(dest + k + j * size * elem_size / (8 * elem_size) + 8 * size * elem_size / 16, r0[4].val[0], 0); + vst1_lane_u8(dest + k + j * size * elem_size / (8 * elem_size) + 9 * size * elem_size / 16, r0[4].val[1], 0); + vst1_lane_u8(dest + k + j * size * elem_size / (8 * elem_size) + 10 * size * elem_size / 16, r0[5].val[0], 0); + vst1_lane_u8(dest + k + j * size * elem_size / (8 * elem_size) + 11 * size * elem_size / 16, r0[5].val[1], 0); + vst1_lane_u8(dest + k + j * size * elem_size / (8 * elem_size) + 12 * size * elem_size / 16, r0[6].val[0], 0); + vst1_lane_u8(dest + k + j * size * elem_size / (8 * elem_size) + 13 * size * elem_size / 16, r0[6].val[1], 0); + vst1_lane_u8(dest + k + j * size * elem_size / (8 * elem_size) + 14 * size * elem_size / 16, r0[7].val[0], 0); + vst1_lane_u8(dest + k + j * size * elem_size / (8 * elem_size) + 15 * size * elem_size / 16, r0[7].val[1], 0); + } + } +} + +/* Routine optimized for bit-unshuffling a buffer for a type size of 1 byte. */ +static void +bitunshuffle1_neon(void* _src, void* dest, const size_t size, const size_t elem_size) { + + uint8x8_t lo_x, hi_x, lo, hi; + size_t i, j, k; + uint8_t* src = _src; + + const int8_t __attribute__ ((aligned (16))) xr[8] = {0, 1, 2, 3, 4, 5, 6, 7}; + uint8x8_t mask_and = vdup_n_u8(0x01); + int8x8_t mask_shift = vld1_s8(xr); + + for (i = 0, k = 0; i < size * elem_size; i += 16, k++) { + for (j = 0; j < 8; j++) { + /* Load lanes */ + lo_x[j] = src[2 * k + 0 + j * size * elem_size / (8 * elem_size)]; + hi_x[j] = src[2 * k + 1 + j * size * elem_size / (8 * elem_size)]; + } + for (j = 0; j < 8; j++) { + /* Create mask from the most significant bit of each 8-bit element */ + lo = vand_u8(lo_x, mask_and); + lo = vshl_u8(lo, mask_shift); + hi = vand_u8(hi_x, mask_and); + hi = vshl_u8(hi, mask_shift); + lo = vpadd_u8(lo, lo); + lo = vpadd_u8(lo, lo); + lo = vpadd_u8(lo, lo); + hi = vpadd_u8(hi, hi); + hi = vpadd_u8(hi, hi); + hi = vpadd_u8(hi, hi); + /* Shift packed 8-bit */ + lo_x = vshr_n_u8(lo_x, 1); + hi_x = vshr_n_u8(hi_x, 1); + /* Store the created mask to the destination vector */ + vst1_lane_u8(dest + j + i, lo, 0); + vst1_lane_u8(dest + j + i + 8, hi, 0); + } + } +} + +/* Routine optimized for bit-unshuffling a buffer for a type size of 2 byte. */ +static void +bitunshuffle2_neon(void* _src, void* dest, const size_t size, const size_t elem_size) { + + size_t i, j, k; + uint8x8_t lo_x[2], hi_x[2], lo[2], hi[2]; + uint8_t* src = _src; + + const int8_t __attribute__ ((aligned (16))) xr[8] = {0, 1, 2, 3, 4, 5, 6, 7}; + uint8x8_t mask_and = vdup_n_u8(0x01); + int8x8_t mask_shift = vld1_s8(xr); + + for (i = 0, k = 0; i < size * elem_size; i += 32, k++) { + for (j = 0; j < 8; j++) { + /* Load lanes */ + lo_x[0][j] = src[2 * k + j * size * elem_size / (8 * elem_size)]; + lo_x[1][j] = src[2 * k + j * size * elem_size / (8 * elem_size) + size * elem_size / 2]; + hi_x[0][j] = src[2 * k + 1 + j * size * elem_size / (8 * elem_size)]; + hi_x[1][j] = src[2 * k + 1 + j * size * elem_size / (8 * elem_size) + size * elem_size / 2]; + } + for (j = 0; j < 8; j++) { + /* Create mask from the most significant bit of each 8-bit element */ + lo[0] = vand_u8(lo_x[0], mask_and); + lo[0] = vshl_u8(lo[0], mask_shift); + lo[1] = vand_u8(lo_x[1], mask_and); + lo[1] = vshl_u8(lo[1], mask_shift); + + hi[0] = vand_u8(hi_x[0], mask_and); + hi[0] = vshl_u8(hi[0], mask_shift); + hi[1] = vand_u8(hi_x[1], mask_and); + hi[1] = vshl_u8(hi[1], mask_shift); + + lo[0] = vpadd_u8(lo[0], lo[0]); + lo[0] = vpadd_u8(lo[0], lo[0]); + lo[0] = vpadd_u8(lo[0], lo[0]); + lo[1] = vpadd_u8(lo[1], lo[1]); + lo[1] = vpadd_u8(lo[1], lo[1]); + lo[1] = vpadd_u8(lo[1], lo[1]); + + hi[0] = vpadd_u8(hi[0], hi[0]); + hi[0] = vpadd_u8(hi[0], hi[0]); + hi[0] = vpadd_u8(hi[0], hi[0]); + hi[1] = vpadd_u8(hi[1], hi[1]); + hi[1] = vpadd_u8(hi[1], hi[1]); + hi[1] = vpadd_u8(hi[1], hi[1]); + /* Shift packed 8-bit */ + lo_x[0] = vshr_n_u8(lo_x[0], 1); + hi_x[0] = vshr_n_u8(hi_x[0], 1); + lo_x[1] = vshr_n_u8(lo_x[1], 1); + hi_x[1] = vshr_n_u8(hi_x[1], 1); + /* Store the created mask to the destination vector */ + vst1_lane_u8(dest + 2 * j + i, lo[0], 0); + vst1_lane_u8(dest + 2 * j + 1 + i, lo[1], 0); + vst1_lane_u8(dest + 2 * j + i + 16, hi[0], 0); + vst1_lane_u8(dest + 2 * j + 1 + i + 16, hi[1], 0); + } + } +} + +/* Routine optimized for bit-unshuffling a buffer for a type size of 4 byte. */ +static void +bitunshuffle4_neon(void* _src, void* dest, const size_t size, const size_t elem_size) { + size_t i, j, k; + uint8x8_t lo_x[4], hi_x[4], lo[4], hi[4]; + uint8_t* src = _src; + + const int8_t __attribute__ ((aligned (16))) xr[8] = {0, 1, 2, 3, 4, 5, 6, 7}; + uint8x8_t mask_and = vdup_n_u8(0x01); + int8x8_t mask_shift = vld1_s8(xr); + + for (i = 0, k = 0; i < size * elem_size; i += 64, k++) { + for (j = 0; j < 8; j++) { + /* Load lanes */ + lo_x[0][j] = src[2 * k + j * size * elem_size / (8 * elem_size) + 0 * size * elem_size / 4]; + hi_x[0][j] = src[2 * k + 1 + j * size * elem_size / (8 * elem_size) + 0 * size * elem_size / 4]; + lo_x[1][j] = src[2 * k + j * size * elem_size / (8 * elem_size) + 1 * size * elem_size / 4]; + hi_x[1][j] = src[2 * k + 1 + j * size * elem_size / (8 * elem_size) + 1 * size * elem_size / 4]; + lo_x[2][j] = src[2 * k + j * size * elem_size / (8 * elem_size) + 2 * size * elem_size / 4]; + hi_x[2][j] = src[2 * k + 1 + j * size * elem_size / (8 * elem_size) + 2 * size * elem_size / 4]; + lo_x[3][j] = src[2 * k + j * size * elem_size / (8 * elem_size) + 3 * size * elem_size / 4]; + hi_x[3][j] = src[2 * k + 1 + j * size * elem_size / (8 * elem_size) + 3 * size * elem_size / 4]; + } + + for (j = 0; j < 8; j++) { + /* Create mask from the most significant bit of each 8-bit element */ + lo[0] = vand_u8(lo_x[0], mask_and); + lo[0] = vshl_u8(lo[0], mask_shift); + lo[1] = vand_u8(lo_x[1], mask_and); + lo[1] = vshl_u8(lo[1], mask_shift); + lo[2] = vand_u8(lo_x[2], mask_and); + lo[2] = vshl_u8(lo[2], mask_shift); + lo[3] = vand_u8(lo_x[3], mask_and); + lo[3] = vshl_u8(lo[3], mask_shift); + + hi[0] = vand_u8(hi_x[0], mask_and); + hi[0] = vshl_u8(hi[0], mask_shift); + hi[1] = vand_u8(hi_x[1], mask_and); + hi[1] = vshl_u8(hi[1], mask_shift); + hi[2] = vand_u8(hi_x[2], mask_and); + hi[2] = vshl_u8(hi[2], mask_shift); + hi[3] = vand_u8(hi_x[3], mask_and); + hi[3] = vshl_u8(hi[3], mask_shift); + + lo[0] = vpadd_u8(lo[0], lo[0]); + lo[0] = vpadd_u8(lo[0], lo[0]); + lo[0] = vpadd_u8(lo[0], lo[0]); + lo[1] = vpadd_u8(lo[1], lo[1]); + lo[1] = vpadd_u8(lo[1], lo[1]); + lo[1] = vpadd_u8(lo[1], lo[1]); + lo[2] = vpadd_u8(lo[2], lo[2]); + lo[2] = vpadd_u8(lo[2], lo[2]); + lo[2] = vpadd_u8(lo[2], lo[2]); + lo[3] = vpadd_u8(lo[3], lo[3]); + lo[3] = vpadd_u8(lo[3], lo[3]); + lo[3] = vpadd_u8(lo[3], lo[3]); + + hi[0] = vpadd_u8(hi[0], hi[0]); + hi[0] = vpadd_u8(hi[0], hi[0]); + hi[0] = vpadd_u8(hi[0], hi[0]); + hi[1] = vpadd_u8(hi[1], hi[1]); + hi[1] = vpadd_u8(hi[1], hi[1]); + hi[1] = vpadd_u8(hi[1], hi[1]); + hi[2] = vpadd_u8(hi[2], hi[2]); + hi[2] = vpadd_u8(hi[2], hi[2]); + hi[2] = vpadd_u8(hi[2], hi[2]); + hi[3] = vpadd_u8(hi[3], hi[3]); + hi[3] = vpadd_u8(hi[3], hi[3]); + hi[3] = vpadd_u8(hi[3], hi[3]); + /* Shift packed 8-bit */ + lo_x[0] = vshr_n_u8(lo_x[0], 1); + hi_x[0] = vshr_n_u8(hi_x[0], 1); + lo_x[1] = vshr_n_u8(lo_x[1], 1); + hi_x[1] = vshr_n_u8(hi_x[1], 1); + lo_x[2] = vshr_n_u8(lo_x[2], 1); + hi_x[2] = vshr_n_u8(hi_x[2], 1); + lo_x[3] = vshr_n_u8(lo_x[3], 1); + hi_x[3] = vshr_n_u8(hi_x[3], 1); + /* Store the created mask to the destination vector */ + vst1_lane_u8(dest + 4 * j + i, lo[0], 0); + vst1_lane_u8(dest + 4 * j + 1 + i, lo[1], 0); + vst1_lane_u8(dest + 4 * j + 2 + i, lo[2], 0); + vst1_lane_u8(dest + 4 * j + 3 + i, lo[3], 0); + vst1_lane_u8(dest + 4 * j + i + 32, hi[0], 0); + vst1_lane_u8(dest + 4 * j + 1 + i + 32, hi[1], 0); + vst1_lane_u8(dest + 4 * j + 2 + i + 32, hi[2], 0); + vst1_lane_u8(dest + 4 * j + 3 + i + 32, hi[3], 0); + } + } +} + +/* Routine optimized for bit-unshuffling a buffer for a type size of 8 byte. */ +static void +bitunshuffle8_neon(void* _src, void* dest, const size_t size, const size_t elem_size) { + + size_t i, j, k; + uint8x8x2_t r0[4], r1[4]; + uint8_t* src = _src; + + const int8_t __attribute__ ((aligned (16))) xr[8] = {0, 1, 2, 3, 4, 5, 6, 7}; + uint8x8_t mask_and = vdup_n_u8(0x01); + int8x8_t mask_shift = vld1_s8(xr); + + for (i = 0, k = 0; i < size * elem_size; i += 64, k++) { + for (j = 0; j < 8; j++) { + /* Load lanes */ + r0[0].val[0][j] = src[k + j * size * elem_size / (8 * elem_size) + 0 * size * elem_size / 8]; + r0[0].val[1][j] = src[k + j * size * elem_size / (8 * elem_size) + 1 * size * elem_size / 8]; + r0[1].val[0][j] = src[k + j * size * elem_size / (8 * elem_size) + 2 * size * elem_size / 8]; + r0[1].val[1][j] = src[k + j * size * elem_size / (8 * elem_size) + 3 * size * elem_size / 8]; + r0[2].val[0][j] = src[k + j * size * elem_size / (8 * elem_size) + 4 * size * elem_size / 8]; + r0[2].val[1][j] = src[k + j * size * elem_size / (8 * elem_size) + 5 * size * elem_size / 8]; + r0[3].val[0][j] = src[k + j * size * elem_size / (8 * elem_size) + 6 * size * elem_size / 8]; + r0[3].val[1][j] = src[k + j * size * elem_size / (8 * elem_size) + 7 * size * elem_size / 8]; + } + for (j = 0; j < 8; j++) { + /* Create mask from the most significant bit of each 8-bit element */ + r1[0].val[0] = vand_u8(r0[0].val[0], mask_and); + r1[0].val[0] = vshl_u8(r1[0].val[0], mask_shift); + r1[0].val[1] = vand_u8(r0[0].val[1], mask_and); + r1[0].val[1] = vshl_u8(r1[0].val[1], mask_shift); + r1[1].val[0] = vand_u8(r0[1].val[0], mask_and); + r1[1].val[0] = vshl_u8(r1[1].val[0], mask_shift); + r1[1].val[1] = vand_u8(r0[1].val[1], mask_and); + r1[1].val[1] = vshl_u8(r1[1].val[1], mask_shift); + r1[2].val[0] = vand_u8(r0[2].val[0], mask_and); + r1[2].val[0] = vshl_u8(r1[2].val[0], mask_shift); + r1[2].val[1] = vand_u8(r0[2].val[1], mask_and); + r1[2].val[1] = vshl_u8(r1[2].val[1], mask_shift); + r1[3].val[0] = vand_u8(r0[3].val[0], mask_and); + r1[3].val[0] = vshl_u8(r1[3].val[0], mask_shift); + r1[3].val[1] = vand_u8(r0[3].val[1], mask_and); + r1[3].val[1] = vshl_u8(r1[3].val[1], mask_shift); + + r1[0].val[0] = vpadd_u8(r1[0].val[0], r1[0].val[0]); + r1[0].val[0] = vpadd_u8(r1[0].val[0], r1[0].val[0]); + r1[0].val[0] = vpadd_u8(r1[0].val[0], r1[0].val[0]); + r1[0].val[1] = vpadd_u8(r1[0].val[1], r1[0].val[1]); + r1[0].val[1] = vpadd_u8(r1[0].val[1], r1[0].val[1]); + r1[0].val[1] = vpadd_u8(r1[0].val[1], r1[0].val[1]); + r1[1].val[0] = vpadd_u8(r1[1].val[0], r1[1].val[0]); + r1[1].val[0] = vpadd_u8(r1[1].val[0], r1[1].val[0]); + r1[1].val[0] = vpadd_u8(r1[1].val[0], r1[1].val[0]); + r1[1].val[1] = vpadd_u8(r1[1].val[1], r1[1].val[1]); + r1[1].val[1] = vpadd_u8(r1[1].val[1], r1[1].val[1]); + r1[1].val[1] = vpadd_u8(r1[1].val[1], r1[1].val[1]); + r1[2].val[0] = vpadd_u8(r1[2].val[0], r1[2].val[0]); + r1[2].val[0] = vpadd_u8(r1[2].val[0], r1[2].val[0]); + r1[2].val[0] = vpadd_u8(r1[2].val[0], r1[2].val[0]); + r1[2].val[1] = vpadd_u8(r1[2].val[1], r1[2].val[1]); + r1[2].val[1] = vpadd_u8(r1[2].val[1], r1[2].val[1]); + r1[2].val[1] = vpadd_u8(r1[2].val[1], r1[2].val[1]); + r1[3].val[0] = vpadd_u8(r1[3].val[0], r1[3].val[0]); + r1[3].val[0] = vpadd_u8(r1[3].val[0], r1[3].val[0]); + r1[3].val[0] = vpadd_u8(r1[3].val[0], r1[3].val[0]); + r1[3].val[1] = vpadd_u8(r1[3].val[1], r1[3].val[1]); + r1[3].val[1] = vpadd_u8(r1[3].val[1], r1[3].val[1]); + r1[3].val[1] = vpadd_u8(r1[3].val[1], r1[3].val[1]); + /* Shift packed 8-bit */ + r0[0].val[0] = vshr_n_u8(r0[0].val[0], 1); + r0[0].val[1] = vshr_n_u8(r0[0].val[1], 1); + r0[1].val[0] = vshr_n_u8(r0[1].val[0], 1); + r0[1].val[1] = vshr_n_u8(r0[1].val[1], 1); + r0[2].val[0] = vshr_n_u8(r0[2].val[0], 1); + r0[2].val[1] = vshr_n_u8(r0[2].val[1], 1); + r0[3].val[0] = vshr_n_u8(r0[3].val[0], 1); + r0[3].val[1] = vshr_n_u8(r0[3].val[1], 1); + /* Store the created mask to the destination vector */ + vst1_lane_u8(dest + 8 * j + 0 + i, r1[0].val[0], 0); + vst1_lane_u8(dest + 8 * j + 1 + i, r1[0].val[1], 0); + vst1_lane_u8(dest + 8 * j + 2 + i, r1[1].val[0], 0); + vst1_lane_u8(dest + 8 * j + 3 + i, r1[1].val[1], 0); + vst1_lane_u8(dest + 8 * j + 4 + i, r1[2].val[0], 0); + vst1_lane_u8(dest + 8 * j + 5 + i, r1[2].val[1], 0); + vst1_lane_u8(dest + 8 * j + 6 + i, r1[3].val[0], 0); + vst1_lane_u8(dest + 8 * j + 7 + i, r1[3].val[1], 0); + } + } +} + +/* Routine optimized for bit-unshuffling a buffer for a type size of 16 byte. */ +static void +bitunshuffle16_neon(void* _src, void* dest, const size_t size, const size_t elem_size) { + + size_t i, j, k; + uint8x8x2_t r0[8], r1[8]; + uint8_t* src = _src; + + const int8_t __attribute__ ((aligned (16))) xr[8] = {0, 1, 2, 3, 4, 5, 6, 7}; + uint8x8_t mask_and = vdup_n_u8(0x01); + int8x8_t mask_shift = vld1_s8(xr); + + for (i = 0, k = 0; i < size * elem_size; i += 128, k++) { + for (j = 0; j < 8; j++) { + /* Load lanes */ + r0[0].val[0][j] = src[k + j * size * elem_size / (8 * elem_size) + 0 * size * elem_size / 16]; + r0[0].val[1][j] = src[k + j * size * elem_size / (8 * elem_size) + 1 * size * elem_size / 16]; + r0[1].val[0][j] = src[k + j * size * elem_size / (8 * elem_size) + 2 * size * elem_size / 16]; + r0[1].val[1][j] = src[k + j * size * elem_size / (8 * elem_size) + 3 * size * elem_size / 16]; + r0[2].val[0][j] = src[k + j * size * elem_size / (8 * elem_size) + 4 * size * elem_size / 16]; + r0[2].val[1][j] = src[k + j * size * elem_size / (8 * elem_size) + 5 * size * elem_size / 16]; + r0[3].val[0][j] = src[k + j * size * elem_size / (8 * elem_size) + 6 * size * elem_size / 16]; + r0[3].val[1][j] = src[k + j * size * elem_size / (8 * elem_size) + 7 * size * elem_size / 16]; + r0[4].val[0][j] = src[k + j * size * elem_size / (8 * elem_size) + 8 * size * elem_size / 16]; + r0[4].val[1][j] = src[k + j * size * elem_size / (8 * elem_size) + 9 * size * elem_size / 16]; + r0[5].val[0][j] = src[k + j * size * elem_size / (8 * elem_size) + 10 * size * elem_size / 16]; + r0[5].val[1][j] = src[k + j * size * elem_size / (8 * elem_size) + 11 * size * elem_size / 16]; + r0[6].val[0][j] = src[k + j * size * elem_size / (8 * elem_size) + 12 * size * elem_size / 16]; + r0[6].val[1][j] = src[k + j * size * elem_size / (8 * elem_size) + 13 * size * elem_size / 16]; + r0[7].val[0][j] = src[k + j * size * elem_size / (8 * elem_size) + 14 * size * elem_size / 16]; + r0[7].val[1][j] = src[k + j * size * elem_size / (8 * elem_size) + 15 * size * elem_size / 16]; + } + for (j = 0; j < 8; j++) { + /* Create mask from the most significant bit of each 8-bit element */ + r1[0].val[0] = vand_u8(r0[0].val[0], mask_and); + r1[0].val[0] = vshl_u8(r1[0].val[0], mask_shift); + r1[0].val[1] = vand_u8(r0[0].val[1], mask_and); + r1[0].val[1] = vshl_u8(r1[0].val[1], mask_shift); + r1[1].val[0] = vand_u8(r0[1].val[0], mask_and); + r1[1].val[0] = vshl_u8(r1[1].val[0], mask_shift); + r1[1].val[1] = vand_u8(r0[1].val[1], mask_and); + r1[1].val[1] = vshl_u8(r1[1].val[1], mask_shift); + r1[2].val[0] = vand_u8(r0[2].val[0], mask_and); + r1[2].val[0] = vshl_u8(r1[2].val[0], mask_shift); + r1[2].val[1] = vand_u8(r0[2].val[1], mask_and); + r1[2].val[1] = vshl_u8(r1[2].val[1], mask_shift); + r1[3].val[0] = vand_u8(r0[3].val[0], mask_and); + r1[3].val[0] = vshl_u8(r1[3].val[0], mask_shift); + r1[3].val[1] = vand_u8(r0[3].val[1], mask_and); + r1[3].val[1] = vshl_u8(r1[3].val[1], mask_shift); + r1[4].val[0] = vand_u8(r0[4].val[0], mask_and); + r1[4].val[0] = vshl_u8(r1[4].val[0], mask_shift); + r1[4].val[1] = vand_u8(r0[4].val[1], mask_and); + r1[4].val[1] = vshl_u8(r1[4].val[1], mask_shift); + r1[5].val[0] = vand_u8(r0[5].val[0], mask_and); + r1[5].val[0] = vshl_u8(r1[5].val[0], mask_shift); + r1[5].val[1] = vand_u8(r0[5].val[1], mask_and); + r1[5].val[1] = vshl_u8(r1[5].val[1], mask_shift); + r1[6].val[0] = vand_u8(r0[6].val[0], mask_and); + r1[6].val[0] = vshl_u8(r1[6].val[0], mask_shift); + r1[6].val[1] = vand_u8(r0[6].val[1], mask_and); + r1[6].val[1] = vshl_u8(r1[6].val[1], mask_shift); + r1[7].val[0] = vand_u8(r0[7].val[0], mask_and); + r1[7].val[0] = vshl_u8(r1[7].val[0], mask_shift); + r1[7].val[1] = vand_u8(r0[7].val[1], mask_and); + r1[7].val[1] = vshl_u8(r1[7].val[1], mask_shift); + + r1[0].val[0] = vpadd_u8(r1[0].val[0], r1[0].val[0]); + r1[0].val[0] = vpadd_u8(r1[0].val[0], r1[0].val[0]); + r1[0].val[0] = vpadd_u8(r1[0].val[0], r1[0].val[0]); + r1[0].val[1] = vpadd_u8(r1[0].val[1], r1[0].val[1]); + r1[0].val[1] = vpadd_u8(r1[0].val[1], r1[0].val[1]); + r1[0].val[1] = vpadd_u8(r1[0].val[1], r1[0].val[1]); + r1[1].val[0] = vpadd_u8(r1[1].val[0], r1[1].val[0]); + r1[1].val[0] = vpadd_u8(r1[1].val[0], r1[1].val[0]); + r1[1].val[0] = vpadd_u8(r1[1].val[0], r1[1].val[0]); + r1[1].val[1] = vpadd_u8(r1[1].val[1], r1[1].val[1]); + r1[1].val[1] = vpadd_u8(r1[1].val[1], r1[1].val[1]); + r1[1].val[1] = vpadd_u8(r1[1].val[1], r1[1].val[1]); + r1[2].val[0] = vpadd_u8(r1[2].val[0], r1[2].val[0]); + r1[2].val[0] = vpadd_u8(r1[2].val[0], r1[2].val[0]); + r1[2].val[0] = vpadd_u8(r1[2].val[0], r1[2].val[0]); + r1[2].val[1] = vpadd_u8(r1[2].val[1], r1[2].val[1]); + r1[2].val[1] = vpadd_u8(r1[2].val[1], r1[2].val[1]); + r1[2].val[1] = vpadd_u8(r1[2].val[1], r1[2].val[1]); + r1[3].val[0] = vpadd_u8(r1[3].val[0], r1[3].val[0]); + r1[3].val[0] = vpadd_u8(r1[3].val[0], r1[3].val[0]); + r1[3].val[0] = vpadd_u8(r1[3].val[0], r1[3].val[0]); + r1[3].val[1] = vpadd_u8(r1[3].val[1], r1[3].val[1]); + r1[3].val[1] = vpadd_u8(r1[3].val[1], r1[3].val[1]); + r1[3].val[1] = vpadd_u8(r1[3].val[1], r1[3].val[1]); + r1[4].val[0] = vpadd_u8(r1[4].val[0], r1[4].val[0]); + r1[4].val[0] = vpadd_u8(r1[4].val[0], r1[4].val[0]); + r1[4].val[0] = vpadd_u8(r1[4].val[0], r1[4].val[0]); + r1[4].val[1] = vpadd_u8(r1[4].val[1], r1[4].val[1]); + r1[4].val[1] = vpadd_u8(r1[4].val[1], r1[4].val[1]); + r1[4].val[1] = vpadd_u8(r1[4].val[1], r1[4].val[1]); + r1[5].val[0] = vpadd_u8(r1[5].val[0], r1[5].val[0]); + r1[5].val[0] = vpadd_u8(r1[5].val[0], r1[5].val[0]); + r1[5].val[0] = vpadd_u8(r1[5].val[0], r1[5].val[0]); + r1[5].val[1] = vpadd_u8(r1[5].val[1], r1[5].val[1]); + r1[5].val[1] = vpadd_u8(r1[5].val[1], r1[5].val[1]); + r1[5].val[1] = vpadd_u8(r1[5].val[1], r1[5].val[1]); + r1[6].val[0] = vpadd_u8(r1[6].val[0], r1[6].val[0]); + r1[6].val[0] = vpadd_u8(r1[6].val[0], r1[6].val[0]); + r1[6].val[0] = vpadd_u8(r1[6].val[0], r1[6].val[0]); + r1[6].val[1] = vpadd_u8(r1[6].val[1], r1[6].val[1]); + r1[6].val[1] = vpadd_u8(r1[6].val[1], r1[6].val[1]); + r1[6].val[1] = vpadd_u8(r1[6].val[1], r1[6].val[1]); + r1[7].val[0] = vpadd_u8(r1[7].val[0], r1[7].val[0]); + r1[7].val[0] = vpadd_u8(r1[7].val[0], r1[7].val[0]); + r1[7].val[0] = vpadd_u8(r1[7].val[0], r1[7].val[0]); + r1[7].val[1] = vpadd_u8(r1[7].val[1], r1[7].val[1]); + r1[7].val[1] = vpadd_u8(r1[7].val[1], r1[7].val[1]); + r1[7].val[1] = vpadd_u8(r1[7].val[1], r1[7].val[1]); + /* Shift packed 8-bit */ + r0[0].val[0] = vshr_n_u8(r0[0].val[0], 1); + r0[0].val[1] = vshr_n_u8(r0[0].val[1], 1); + r0[1].val[0] = vshr_n_u8(r0[1].val[0], 1); + r0[1].val[1] = vshr_n_u8(r0[1].val[1], 1); + r0[2].val[0] = vshr_n_u8(r0[2].val[0], 1); + r0[2].val[1] = vshr_n_u8(r0[2].val[1], 1); + r0[3].val[0] = vshr_n_u8(r0[3].val[0], 1); + r0[3].val[1] = vshr_n_u8(r0[3].val[1], 1); + r0[4].val[0] = vshr_n_u8(r0[4].val[0], 1); + r0[4].val[1] = vshr_n_u8(r0[4].val[1], 1); + r0[5].val[0] = vshr_n_u8(r0[5].val[0], 1); + r0[5].val[1] = vshr_n_u8(r0[5].val[1], 1); + r0[6].val[0] = vshr_n_u8(r0[6].val[0], 1); + r0[6].val[1] = vshr_n_u8(r0[6].val[1], 1); + r0[7].val[0] = vshr_n_u8(r0[7].val[0], 1); + r0[7].val[1] = vshr_n_u8(r0[7].val[1], 1); + /* Store the created mask to the destination vector */ + vst1_lane_u8(dest + 16 * j + 0 + i, r1[0].val[0], 0); + vst1_lane_u8(dest + 16 * j + 1 + i, r1[0].val[1], 0); + vst1_lane_u8(dest + 16 * j + 2 + i, r1[1].val[0], 0); + vst1_lane_u8(dest + 16 * j + 3 + i, r1[1].val[1], 0); + vst1_lane_u8(dest + 16 * j + 4 + i, r1[2].val[0], 0); + vst1_lane_u8(dest + 16 * j + 5 + i, r1[2].val[1], 0); + vst1_lane_u8(dest + 16 * j + 6 + i, r1[3].val[0], 0); + vst1_lane_u8(dest + 16 * j + 7 + i, r1[3].val[1], 0); + vst1_lane_u8(dest + 16 * j + 8 + i, r1[4].val[0], 0); + vst1_lane_u8(dest + 16 * j + 9 + i, r1[4].val[1], 0); + vst1_lane_u8(dest + 16 * j + 10 + i, r1[5].val[0], 0); + vst1_lane_u8(dest + 16 * j + 11 + i, r1[5].val[1], 0); + vst1_lane_u8(dest + 16 * j + 12 + i, r1[6].val[0], 0); + vst1_lane_u8(dest + 16 * j + 13 + i, r1[6].val[1], 0); + vst1_lane_u8(dest + 16 * j + 14 + i, r1[7].val[0], 0); + vst1_lane_u8(dest + 16 * j + 15 + i, r1[7].val[1], 0); + } + } +} + +/* Shuffle a block. This can never fail. */ +int64_t +bitshuffle_neon(void* _src, void* _dest, const size_t size, + const size_t elem_size, void* tmp_buf) { + size_t vectorized_chunk_size = 0; + int64_t count; + if (elem_size == 1 || elem_size == 2 || elem_size == 4) { + vectorized_chunk_size = elem_size * 16; + } else if (elem_size == 8 || elem_size == 16) { + vectorized_chunk_size = elem_size * 8; + } + + /* If the block size is too small to be vectorized, + use the generic implementation. */ + if (size * elem_size < vectorized_chunk_size) { + count = bshuf_trans_bit_elem_scal((void*)_src, (void*)_dest, size, elem_size, tmp_buf); + return count; + } + + /* Optimized bitshuffle implementations */ + switch (elem_size) { + case 1: + bitshuffle1_neon(_src, _dest, size, elem_size); + break; + case 2: + bitshuffle2_neon(_src, _dest, size, elem_size); + break; + case 4: + bitshuffle4_neon(_src, _dest, size, elem_size); + break; + case 8: + bitshuffle8_neon(_src, _dest, size, elem_size); + break; + case 16: + bitshuffle16_neon(_src, _dest, size, elem_size); + break; + default: + /* Non-optimized bitshuffle */ + count = bshuf_trans_bit_elem_scal((void*)_src, (void*)_dest, size, elem_size, tmp_buf); + /* The non-optimized function covers the whole buffer, + so we're done processing here. */ + return count; + } + + return (int64_t)size * (int64_t)elem_size; +} + +/* Bitunshuffle a block. This can never fail. */ +int64_t +bitunshuffle_neon(void* _src, void* _dest, const size_t size, + const size_t elem_size, void* tmp_buf) { + size_t vectorized_chunk_size = 0; + int64_t count; + if (size * elem_size == 1 || size * elem_size == 2 || size * elem_size == 4) { + vectorized_chunk_size = size * elem_size * 16; + } else if (size * elem_size == 8 || size * elem_size == 16) { + vectorized_chunk_size = size * elem_size * 8; + } + + /* If the block size is too small to be vectorized, + use the generic implementation. */ + if (size * elem_size < vectorized_chunk_size) { + count = bshuf_untrans_bit_elem_scal((void*)_src, (void*)_dest, size, elem_size, tmp_buf); + return count; + } + + /* Optimized bitunshuffle implementations */ + switch (elem_size) { + case 1: + bitunshuffle1_neon(_src, _dest, size, elem_size); + break; + case 2: + bitunshuffle2_neon(_src, _dest, size, elem_size); + break; + case 4: + bitunshuffle4_neon(_src, _dest, size, elem_size); + break; + case 8: + bitunshuffle8_neon(_src, _dest, size, elem_size); + break; + case 16: + bitunshuffle16_neon(_src, _dest, size, elem_size); + break; + default: + /* Non-optimized bitunshuffle */ + count = bshuf_untrans_bit_elem_scal((void*)_src, (void*)_dest, size, elem_size, tmp_buf); + /* The non-optimized function covers the whole buffer, + so we're done processing here. */ + return count; + } + + return (int64_t)size * (int64_t)elem_size; +} diff --git a/src/blosc2/blosc/bitshuffle-neon.h b/src/blosc2/blosc/bitshuffle-neon.h new file mode 100644 index 0000000..4b08e99 --- /dev/null +++ b/src/blosc2/blosc/bitshuffle-neon.h @@ -0,0 +1,40 @@ +/********************************************************************* + Blosc - Blocked Shuffling and Compression Library + + Copyright (C) 2021 The Blosc Developers + https://blosc.org + License: BSD 3-Clause (see LICENSE.txt) + + Note: Adapted for NEON by Lucian Marc. + + See LICENSE.txt for details about copyright and rights to use. +**********************************************************************/ + +/* NEON-accelerated bitshuffle/bitunshuffle routines. */ + +#ifndef BITSHUFFLE_NEON_H +#define BITSHUFFLE_NEON_H + +#include "blosc2/blosc2-common.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/** + NEON-accelerated bitshuffle routine. +*/ +BLOSC_NO_EXPORT int64_t bitshuffle_neon(void* _src, void* _dest, const size_t blocksize, + const size_t bytesoftype, void* tmp_buf); + +/** + NEON-accelerated bitunshuffle routine. +*/ +BLOSC_NO_EXPORT int64_t bitunshuffle_neon(void* _src, void* _dest, const size_t blocksize, + const size_t bytesoftype, void* tmp_buf); + +#ifdef __cplusplus +} +#endif + +#endif /* BITSHUFFLE_NEON_H */ diff --git a/src/blosc2/blosc/bitshuffle-sse2.c b/src/blosc2/blosc/bitshuffle-sse2.c new file mode 100644 index 0000000..1c55e23 --- /dev/null +++ b/src/blosc2/blosc/bitshuffle-sse2.c @@ -0,0 +1,476 @@ +/********************************************************************* + Blosc - Blocked Shuffling and Compression Library + + Copyright (C) 2021 The Blosc Developers + https://blosc.org + License: BSD 3-Clause (see LICENSE.txt) + + See LICENSE.txt for details about copyright and rights to use. +**********************************************************************/ + +/********************************************************************* + Bitshuffle - Filter for improving compression of typed binary data. + + Author: Kiyoshi Masui + Website: http://www.github.com/kiyo-masui/bitshuffle + + Note: Adapted for c-blosc by Francesc Alted. + + See LICENSES/BITSHUFFLE.txt file for details about copyright and + rights to use. +**********************************************************************/ + + +#include "bitshuffle-generic.h" +#include "bitshuffle-sse2.h" + +/* Make sure SSE2 is available for the compilation target and compiler. */ +#if !defined(__SSE2__) + #error SSE2 is not supported by the target architecture/platform and/or this compiler. +#endif + +#include + +/* The next is useful for debugging purposes */ +#if 0 +#include +#include + + +static void printxmm(__m128i xmm0) +{ + uint8_t buf[32]; + + ((__m128i *)buf)[0] = xmm0; + printf("%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x\n", + buf[0], buf[1], buf[2], buf[3], + buf[4], buf[5], buf[6], buf[7], + buf[8], buf[9], buf[10], buf[11], + buf[12], buf[13], buf[14], buf[15]); +} +#endif + + +/* ---- Worker code that requires SSE2. Intel Petium 4 (2000) and later. ---- */ + +/* Transpose bytes within elements for 16 bit elements. */ +int64_t bshuf_trans_byte_elem_SSE_16(void* in, void* out, const size_t size) { + + char* in_b = (char*)in; + char* out_b = (char*)out; + __m128i a0, b0, a1, b1; + size_t ii; + + for (ii = 0; ii + 15 < size; ii += 16) { + a0 = _mm_loadu_si128((__m128i*)&in_b[2 * ii + 0 * 16]); + b0 = _mm_loadu_si128((__m128i*)&in_b[2 * ii + 1 * 16]); + + a1 = _mm_unpacklo_epi8(a0, b0); + b1 = _mm_unpackhi_epi8(a0, b0); + + a0 = _mm_unpacklo_epi8(a1, b1); + b0 = _mm_unpackhi_epi8(a1, b1); + + a1 = _mm_unpacklo_epi8(a0, b0); + b1 = _mm_unpackhi_epi8(a0, b0); + + a0 = _mm_unpacklo_epi8(a1, b1); + b0 = _mm_unpackhi_epi8(a1, b1); + + _mm_storeu_si128((__m128i*)&out_b[0 * size + ii], a0); + _mm_storeu_si128((__m128i*)&out_b[1 * size + ii], b0); + } + return bshuf_trans_byte_elem_remainder(in, out, size, 2, + size - size % 16); +} + + +/* Transpose bytes within elements for 32 bit elements. */ +int64_t bshuf_trans_byte_elem_SSE_32(void* in, void* out, const size_t size) { + + char* in_b = (char*)in; + char* out_b = (char*)out; + __m128i a0, b0, c0, d0, a1, b1, c1, d1; + size_t ii; + + for (ii = 0; ii + 15 < size; ii += 16) { + a0 = _mm_loadu_si128((__m128i*)&in_b[4 * ii + 0 * 16]); + b0 = _mm_loadu_si128((__m128i*)&in_b[4 * ii + 1 * 16]); + c0 = _mm_loadu_si128((__m128i*)&in_b[4 * ii + 2 * 16]); + d0 = _mm_loadu_si128((__m128i*)&in_b[4 * ii + 3 * 16]); + + a1 = _mm_unpacklo_epi8(a0, b0); + b1 = _mm_unpackhi_epi8(a0, b0); + c1 = _mm_unpacklo_epi8(c0, d0); + d1 = _mm_unpackhi_epi8(c0, d0); + + a0 = _mm_unpacklo_epi8(a1, b1); + b0 = _mm_unpackhi_epi8(a1, b1); + c0 = _mm_unpacklo_epi8(c1, d1); + d0 = _mm_unpackhi_epi8(c1, d1); + + a1 = _mm_unpacklo_epi8(a0, b0); + b1 = _mm_unpackhi_epi8(a0, b0); + c1 = _mm_unpacklo_epi8(c0, d0); + d1 = _mm_unpackhi_epi8(c0, d0); + + a0 = _mm_unpacklo_epi64(a1, c1); + b0 = _mm_unpackhi_epi64(a1, c1); + c0 = _mm_unpacklo_epi64(b1, d1); + d0 = _mm_unpackhi_epi64(b1, d1); + + _mm_storeu_si128((__m128i*)&out_b[0 * size + ii], a0); + _mm_storeu_si128((__m128i*)&out_b[1 * size + ii], b0); + _mm_storeu_si128((__m128i*)&out_b[2 * size + ii], c0); + _mm_storeu_si128((__m128i*)&out_b[3 * size + ii], d0); + } + return bshuf_trans_byte_elem_remainder(in, out, size, 4, + size - size % 16); +} + + +/* Transpose bytes within elements for 64 bit elements. */ +int64_t bshuf_trans_byte_elem_SSE_64(void* in, void* out, const size_t size) { + + char* in_b = (char*)in; + char* out_b = (char*)out; + __m128i a0, b0, c0, d0, e0, f0, g0, h0; + __m128i a1, b1, c1, d1, e1, f1, g1, h1; + size_t ii; + + for (ii = 0; ii + 15 < size; ii += 16) { + a0 = _mm_loadu_si128((__m128i*)&in_b[8 * ii + 0 * 16]); + b0 = _mm_loadu_si128((__m128i*)&in_b[8 * ii + 1 * 16]); + c0 = _mm_loadu_si128((__m128i*)&in_b[8 * ii + 2 * 16]); + d0 = _mm_loadu_si128((__m128i*)&in_b[8 * ii + 3 * 16]); + e0 = _mm_loadu_si128((__m128i*)&in_b[8 * ii + 4 * 16]); + f0 = _mm_loadu_si128((__m128i*)&in_b[8 * ii + 5 * 16]); + g0 = _mm_loadu_si128((__m128i*)&in_b[8 * ii + 6 * 16]); + h0 = _mm_loadu_si128((__m128i*)&in_b[8 * ii + 7 * 16]); + + a1 = _mm_unpacklo_epi8(a0, b0); + b1 = _mm_unpackhi_epi8(a0, b0); + c1 = _mm_unpacklo_epi8(c0, d0); + d1 = _mm_unpackhi_epi8(c0, d0); + e1 = _mm_unpacklo_epi8(e0, f0); + f1 = _mm_unpackhi_epi8(e0, f0); + g1 = _mm_unpacklo_epi8(g0, h0); + h1 = _mm_unpackhi_epi8(g0, h0); + + a0 = _mm_unpacklo_epi8(a1, b1); + b0 = _mm_unpackhi_epi8(a1, b1); + c0 = _mm_unpacklo_epi8(c1, d1); + d0 = _mm_unpackhi_epi8(c1, d1); + e0 = _mm_unpacklo_epi8(e1, f1); + f0 = _mm_unpackhi_epi8(e1, f1); + g0 = _mm_unpacklo_epi8(g1, h1); + h0 = _mm_unpackhi_epi8(g1, h1); + + a1 = _mm_unpacklo_epi32(a0, c0); + b1 = _mm_unpackhi_epi32(a0, c0); + c1 = _mm_unpacklo_epi32(b0, d0); + d1 = _mm_unpackhi_epi32(b0, d0); + e1 = _mm_unpacklo_epi32(e0, g0); + f1 = _mm_unpackhi_epi32(e0, g0); + g1 = _mm_unpacklo_epi32(f0, h0); + h1 = _mm_unpackhi_epi32(f0, h0); + + a0 = _mm_unpacklo_epi64(a1, e1); + b0 = _mm_unpackhi_epi64(a1, e1); + c0 = _mm_unpacklo_epi64(b1, f1); + d0 = _mm_unpackhi_epi64(b1, f1); + e0 = _mm_unpacklo_epi64(c1, g1); + f0 = _mm_unpackhi_epi64(c1, g1); + g0 = _mm_unpacklo_epi64(d1, h1); + h0 = _mm_unpackhi_epi64(d1, h1); + + _mm_storeu_si128((__m128i*)&out_b[0 * size + ii], a0); + _mm_storeu_si128((__m128i*)&out_b[1 * size + ii], b0); + _mm_storeu_si128((__m128i*)&out_b[2 * size + ii], c0); + _mm_storeu_si128((__m128i*)&out_b[3 * size + ii], d0); + _mm_storeu_si128((__m128i*)&out_b[4 * size + ii], e0); + _mm_storeu_si128((__m128i*)&out_b[5 * size + ii], f0); + _mm_storeu_si128((__m128i*)&out_b[6 * size + ii], g0); + _mm_storeu_si128((__m128i*)&out_b[7 * size + ii], h0); + } + return bshuf_trans_byte_elem_remainder(in, out, size, 8, + size - size % 16); +} + + +/* Memory copy with bshuf call signature. */ +int64_t bshuf_copy(void* in, void* out, const size_t size, + const size_t elem_size) { + + char* in_b = (char*)in; + char* out_b = (char*)out; + + memcpy(out_b, in_b, size * elem_size); + return (int64_t)size * (int64_t)elem_size; +} + + +/* Transpose bytes within elements using best SSE algorithm available. */ +int64_t bshuf_trans_byte_elem_sse2(void* in, void* out, const size_t size, + const size_t elem_size, void* tmp_buf) { + + int64_t count; + + /* Trivial cases: power of 2 bytes. */ + switch (elem_size) { + case 1: + count = bshuf_copy(in, out, size, elem_size); + return count; + case 2: + count = bshuf_trans_byte_elem_SSE_16(in, out, size); + return count; + case 4: + count = bshuf_trans_byte_elem_SSE_32(in, out, size); + return count; + case 8: + count = bshuf_trans_byte_elem_SSE_64(in, out, size); + return count; + } + + /* Worst case: odd number of bytes. Turns out that this is faster for */ + /* (odd * 2) byte elements as well (hence % 4). */ + if (elem_size % 4) { + count = bshuf_trans_byte_elem_scal(in, out, size, elem_size); + return count; + } + + /* Multiple of power of 2: transpose hierarchically. */ + { + size_t nchunk_elem; + + if ((elem_size % 8) == 0) { + nchunk_elem = elem_size / 8; + TRANS_ELEM_TYPE(in, out, size, nchunk_elem, int64_t); + count = bshuf_trans_byte_elem_SSE_64(out, tmp_buf, + size * nchunk_elem); + bshuf_trans_elem(tmp_buf, out, 8, nchunk_elem, size); + } else if ((elem_size % 4) == 0) { + nchunk_elem = elem_size / 4; + TRANS_ELEM_TYPE(in, out, size, nchunk_elem, int32_t); + count = bshuf_trans_byte_elem_SSE_32(out, tmp_buf, + size * nchunk_elem); + bshuf_trans_elem(tmp_buf, out, 4, nchunk_elem, size); + } else { + /* Not used since scalar algorithm is faster. */ + nchunk_elem = elem_size / 2; + TRANS_ELEM_TYPE(in, out, size, nchunk_elem, int16_t); + count = bshuf_trans_byte_elem_SSE_16(out, tmp_buf, + size * nchunk_elem); + bshuf_trans_elem(tmp_buf, out, 2, nchunk_elem, size); + } + + return count; + } +} + + +/* Transpose bits within bytes. */ +int64_t bshuf_trans_bit_byte_sse2(void* in, void* out, const size_t size, + const size_t elem_size) { + + char* in_b = (char*)in; + char* out_b = (char*)out; + uint16_t* out_ui16; + int64_t count; + size_t nbyte = elem_size * size; + __m128i xmm; + int32_t bt; + size_t ii, kk; + + CHECK_MULT_EIGHT(nbyte); + + for (ii = 0; ii + 15 < nbyte; ii += 16) { + xmm = _mm_loadu_si128((__m128i*)&in_b[ii]); + for (kk = 0; kk < 8; kk++) { + bt = _mm_movemask_epi8(xmm); + xmm = _mm_slli_epi16(xmm, 1); + out_ui16 = (uint16_t*)&out_b[((7 - kk) * nbyte + ii) / 8]; + *out_ui16 = (uint16_t)bt; + } + } + count = bshuf_trans_bit_byte_remainder(in, out, size, elem_size, + nbyte - nbyte % 16); + return count; +} + + +/* Transpose bits within elements. */ +int64_t bshuf_trans_bit_elem_sse2(void* in, void* out, const size_t size, + const size_t elem_size, void* tmp_buf) { + + int64_t count; + + CHECK_MULT_EIGHT(size); + + count = bshuf_trans_byte_elem_sse2(in, out, size, elem_size, tmp_buf); + CHECK_ERR(count); + count = bshuf_trans_bit_byte_sse2(out, tmp_buf, size, elem_size); + CHECK_ERR(count); + count = bshuf_trans_bitrow_eight(tmp_buf, out, size, elem_size); + + return count; +} + + +/* For data organized into a row for each bit (8 * elem_size rows), transpose + * the bytes. */ +int64_t bshuf_trans_byte_bitrow_sse2(void* in, void* out, const size_t size, + const size_t elem_size) { + + char* in_b = (char*)in; + char* out_b = (char*)out; + size_t nrows = 8 * elem_size; + size_t nbyte_row = size / 8; + size_t ii, jj; + + __m128i a0, b0, c0, d0, e0, f0, g0, h0; + __m128i a1, b1, c1, d1, e1, f1, g1, h1; + __m128* as, * bs, * cs, * ds, * es, * fs, * gs, * hs; + + CHECK_MULT_EIGHT(size); + + for (ii = 0; ii + 7 < nrows; ii += 8) { + for (jj = 0; jj + 15 < nbyte_row; jj += 16) { + a0 = _mm_loadu_si128((__m128i*)&in_b[(ii + 0) * nbyte_row + jj]); + b0 = _mm_loadu_si128((__m128i*)&in_b[(ii + 1) * nbyte_row + jj]); + c0 = _mm_loadu_si128((__m128i*)&in_b[(ii + 2) * nbyte_row + jj]); + d0 = _mm_loadu_si128((__m128i*)&in_b[(ii + 3) * nbyte_row + jj]); + e0 = _mm_loadu_si128((__m128i*)&in_b[(ii + 4) * nbyte_row + jj]); + f0 = _mm_loadu_si128((__m128i*)&in_b[(ii + 5) * nbyte_row + jj]); + g0 = _mm_loadu_si128((__m128i*)&in_b[(ii + 6) * nbyte_row + jj]); + h0 = _mm_loadu_si128((__m128i*)&in_b[(ii + 7) * nbyte_row + jj]); + + + a1 = _mm_unpacklo_epi8(a0, b0); + b1 = _mm_unpacklo_epi8(c0, d0); + c1 = _mm_unpacklo_epi8(e0, f0); + d1 = _mm_unpacklo_epi8(g0, h0); + e1 = _mm_unpackhi_epi8(a0, b0); + f1 = _mm_unpackhi_epi8(c0, d0); + g1 = _mm_unpackhi_epi8(e0, f0); + h1 = _mm_unpackhi_epi8(g0, h0); + + + a0 = _mm_unpacklo_epi16(a1, b1); + b0 = _mm_unpacklo_epi16(c1, d1); + c0 = _mm_unpackhi_epi16(a1, b1); + d0 = _mm_unpackhi_epi16(c1, d1); + + e0 = _mm_unpacklo_epi16(e1, f1); + f0 = _mm_unpacklo_epi16(g1, h1); + g0 = _mm_unpackhi_epi16(e1, f1); + h0 = _mm_unpackhi_epi16(g1, h1); + + + a1 = _mm_unpacklo_epi32(a0, b0); + b1 = _mm_unpackhi_epi32(a0, b0); + + c1 = _mm_unpacklo_epi32(c0, d0); + d1 = _mm_unpackhi_epi32(c0, d0); + + e1 = _mm_unpacklo_epi32(e0, f0); + f1 = _mm_unpackhi_epi32(e0, f0); + + g1 = _mm_unpacklo_epi32(g0, h0); + h1 = _mm_unpackhi_epi32(g0, h0); + + /* We don't have a storeh instruction for integers, so interpret */ + /* as a float. Have a storel (_mm_storel_epi64). */ + as = (__m128*)&a1; + bs = (__m128*)&b1; + cs = (__m128*)&c1; + ds = (__m128*)&d1; + es = (__m128*)&e1; + fs = (__m128*)&f1; + gs = (__m128*)&g1; + hs = (__m128*)&h1; + + _mm_storel_pi((__m64*)&out_b[(jj + 0) * nrows + ii], *as); + _mm_storel_pi((__m64*)&out_b[(jj + 2) * nrows + ii], *bs); + _mm_storel_pi((__m64*)&out_b[(jj + 4) * nrows + ii], *cs); + _mm_storel_pi((__m64*)&out_b[(jj + 6) * nrows + ii], *ds); + _mm_storel_pi((__m64*)&out_b[(jj + 8) * nrows + ii], *es); + _mm_storel_pi((__m64*)&out_b[(jj + 10) * nrows + ii], *fs); + _mm_storel_pi((__m64*)&out_b[(jj + 12) * nrows + ii], *gs); + _mm_storel_pi((__m64*)&out_b[(jj + 14) * nrows + ii], *hs); + + _mm_storeh_pi((__m64*)&out_b[(jj + 1) * nrows + ii], *as); + _mm_storeh_pi((__m64*)&out_b[(jj + 3) * nrows + ii], *bs); + _mm_storeh_pi((__m64*)&out_b[(jj + 5) * nrows + ii], *cs); + _mm_storeh_pi((__m64*)&out_b[(jj + 7) * nrows + ii], *ds); + _mm_storeh_pi((__m64*)&out_b[(jj + 9) * nrows + ii], *es); + _mm_storeh_pi((__m64*)&out_b[(jj + 11) * nrows + ii], *fs); + _mm_storeh_pi((__m64*)&out_b[(jj + 13) * nrows + ii], *gs); + _mm_storeh_pi((__m64*)&out_b[(jj + 15) * nrows + ii], *hs); + } + for (jj = nbyte_row - nbyte_row % 16; jj < nbyte_row; jj++) { + out_b[jj * nrows + ii + 0] = in_b[(ii + 0) * nbyte_row + jj]; + out_b[jj * nrows + ii + 1] = in_b[(ii + 1) * nbyte_row + jj]; + out_b[jj * nrows + ii + 2] = in_b[(ii + 2) * nbyte_row + jj]; + out_b[jj * nrows + ii + 3] = in_b[(ii + 3) * nbyte_row + jj]; + out_b[jj * nrows + ii + 4] = in_b[(ii + 4) * nbyte_row + jj]; + out_b[jj * nrows + ii + 5] = in_b[(ii + 5) * nbyte_row + jj]; + out_b[jj * nrows + ii + 6] = in_b[(ii + 6) * nbyte_row + jj]; + out_b[jj * nrows + ii + 7] = in_b[(ii + 7) * nbyte_row + jj]; + } + } + return (int64_t)size * (int64_t)elem_size; +} + + +/* Shuffle bits within the bytes of eight element blocks. */ +int64_t bshuf_shuffle_bit_eightelem_sse2(void* in, void* out, const size_t size, + const size_t elem_size) { + /* With a bit of care, this could be written such that such that it is */ + /* in_buf = out_buf safe. */ + char* in_b = (char*)in; + uint16_t* out_ui16 = (uint16_t*)out; + + size_t nbyte = elem_size * size; + + __m128i xmm; + int32_t bt; + size_t ii, jj, kk; + size_t ind; + + CHECK_MULT_EIGHT(size); + + if (elem_size % 2) { + bshuf_shuffle_bit_eightelem_scal(in, out, size, elem_size); + } else { + for (ii = 0; ii + 8 * elem_size - 1 < nbyte; + ii += 8 * elem_size) { + for (jj = 0; jj + 15 < 8 * elem_size; jj += 16) { + xmm = _mm_loadu_si128((__m128i*)&in_b[ii + jj]); + for (kk = 0; kk < 8; kk++) { + bt = _mm_movemask_epi8(xmm); + xmm = _mm_slli_epi16(xmm, 1); + ind = (ii + jj / 8 + (7 - kk) * elem_size); + out_ui16[ind / 2] = (uint16_t)bt; + } + } + } + } + return (int64_t)size * (int64_t)elem_size; +} + + +/* Untranspose bits within elements. */ +int64_t bshuf_untrans_bit_elem_sse2(void* in, void* out, const size_t size, + const size_t elem_size, void* tmp_buf) { + + int64_t count; + + CHECK_MULT_EIGHT(size); + + count = bshuf_trans_byte_bitrow_sse2(in, tmp_buf, size, elem_size); + CHECK_ERR(count); + count = bshuf_shuffle_bit_eightelem_sse2(tmp_buf, out, size, elem_size); + + return count; +} diff --git a/src/blosc2/blosc/bitshuffle-sse2.h b/src/blosc2/blosc/bitshuffle-sse2.h new file mode 100644 index 0000000..7e90798 --- /dev/null +++ b/src/blosc2/blosc/bitshuffle-sse2.h @@ -0,0 +1,54 @@ +/********************************************************************* + Blosc - Blocked Shuffling and Compression Library + + Copyright (C) 2021 The Blosc Developers + https://blosc.org + License: BSD 3-Clause (see LICENSE.txt) + + See LICENSE.txt for details about copyright and rights to use. +**********************************************************************/ + +/* SSE2-accelerated shuffle/unshuffle routines. */ + +#ifndef BITSHUFFLE_SSE2_H +#define BITSHUFFLE_SSE2_H + +#include "blosc2/blosc2-common.h" + +#ifdef __cplusplus +extern "C" { +#endif + + +BLOSC_NO_EXPORT int64_t + bshuf_trans_byte_elem_sse2(void* in, void* out, const size_t size, + const size_t elem_size, void* tmp_buf); + +BLOSC_NO_EXPORT int64_t + bshuf_trans_byte_bitrow_sse2(void* in, void* out, const size_t size, + const size_t elem_size); + +BLOSC_NO_EXPORT int64_t + bshuf_shuffle_bit_eightelem_sse2(void* in, void* out, const size_t size, + const size_t elem_size); + +/** + SSE2-accelerated bitshuffle routine. +*/ +BLOSC_NO_EXPORT int64_t + bshuf_trans_bit_elem_sse2(void* in, void* out, const size_t size, + const size_t elem_size, void* tmp_buf); + +/** + SSE2-accelerated bitunshuffle routine. +*/ +BLOSC_NO_EXPORT int64_t + bshuf_untrans_bit_elem_sse2(void* in, void* out, const size_t size, + const size_t elem_size, void* tmp_buf); + +#ifdef __cplusplus +} +#endif + + +#endif /* BITSHUFFLE_SSE2_H */ diff --git a/src/blosc2/blosc/blosc-private.h b/src/blosc2/blosc/blosc-private.h new file mode 100644 index 0000000..73e21c3 --- /dev/null +++ b/src/blosc2/blosc/blosc-private.h @@ -0,0 +1,182 @@ +/********************************************************************* + Blosc - Blocked Shuffling and Compression Library + + Copyright (C) 2021 The Blosc Developers + https://blosc.org + License: BSD 3-Clause (see LICENSE.txt) + + See LICENSE.txt for details about copyright and rights to use. +**********************************************************************/ + + +#ifndef IARRAY_BLOSC_PRIVATE_H +#define IARRAY_BLOSC_PRIVATE_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include "stdbool.h" +#include "blosc2.h" +#include "blosc2/blosc2-common.h" + +/********************************************************************* + + Utility functions meant to be used internally. + +*********************************************************************/ + +#define to_little(dest, src, itemsize) endian_handler(true, dest, src, itemsize) +#define from_little(dest, src, itemsize) endian_handler(true, dest, src, itemsize) +#define to_big(dest, src, itemsize) endian_handler(false, dest, src, itemsize) +#define from_big(dest, src, itemsize) endian_handler(false, dest, src, itemsize) + +#define BLOSC_ERROR_NULL(pointer, rc) \ + do { \ + if ((pointer) == NULL) { \ + BLOSC_TRACE_ERROR("Pointer is NULL"); \ + return rc; \ + } \ + } while (0) + + +// Return true if platform is little endian; else false +static bool is_little_endian(void) { + static const int i = 1; + char* p = (char*)&i; + + if (p[0] == 1) { + return true; + } + else { + return false; + } +} + + +static inline void endian_handler(bool little, void *dest, const void *pa, int size) +{ + bool little_endian = is_little_endian(); + if (little_endian == little) { + memcpy(dest, pa, size); + } + else { + uint8_t* pa_ = (uint8_t*)pa; + uint8_t pa2_[8]; + switch (size) { + case 8: + pa2_[0] = pa_[7]; + pa2_[1] = pa_[6]; + pa2_[2] = pa_[5]; + pa2_[3] = pa_[4]; + pa2_[4] = pa_[3]; + pa2_[5] = pa_[2]; + pa2_[6] = pa_[1]; + pa2_[7] = pa_[0]; + break; + case 4: + pa2_[0] = pa_[3]; + pa2_[1] = pa_[2]; + pa2_[2] = pa_[1]; + pa2_[3] = pa_[0]; + break; + case 2: + pa2_[0] = pa_[1]; + pa2_[1] = pa_[0]; + break; + case 1: + pa2_[0] = pa_[0]; + break; + default: + BLOSC_TRACE_ERROR("Unhandled size: %d.", size); + } + memcpy(dest, pa2_, size); + } +} + +/* Copy 4 bytes from @p *pa to int32_t, changing endianness if necessary. */ +static inline int32_t sw32_(const void* pa) { + int32_t idest; + + bool little_endian = is_little_endian(); + if (little_endian) { + idest = *(int32_t *)pa; + } + else { +#if defined (__GNUC__) + return __builtin_bswap32(*(unsigned int *)pa); +#elif defined (_MSC_VER) /* Visual Studio */ + return _byteswap_ulong(*(unsigned int *)pa); +#else + uint8_t *dest = (uint8_t *)&idest; + dest[0] = pa_[3]; + dest[1] = pa_[2]; + dest[2] = pa_[1]; + dest[3] = pa_[0]; +#endif + } + return idest; +} + +/* Copy 4 bytes from int32_t to @p *dest, changing endianness if necessary. */ +static inline void _sw32(void* dest, int32_t a) { + uint8_t* dest_ = (uint8_t*)dest; + uint8_t* pa = (uint8_t*)&a; + + bool little_endian = is_little_endian(); + if (little_endian) { + *(int32_t *)dest_ = a; + } + else { +#if defined (__GNUC__) + *(int32_t *)dest_ = __builtin_bswap32(*(unsigned int *)pa); +#elif defined (_MSC_VER) /* Visual Studio */ + *(int32_t *)dest_ = _byteswap_ulong(*(unsigned int *)pa); +#else + dest_[0] = pa[3]; + dest_[1] = pa[2]; + dest_[2] = pa[1]; + dest_[3] = pa[0]; +#endif + } +} + +/* Reverse swap bits in a 32-bit integer */ +static inline int32_t bswap32_(int32_t a) { +#if defined (__GNUC__) + return __builtin_bswap32(a); + +#elif defined (_MSC_VER) /* Visual Studio */ + return _byteswap_ulong(a); +#else + a = ((a & 0x000000FF) << 24) | + ((a & 0x0000FF00) << 8) | + ((a & 0x00FF0000) >> 8) | + ((a & 0xFF000000) >> 24); + return a; +#endif +} + +/** + * @brief Register a filter in Blosc. + * + * @param filter The filter to register. + * + * @return 0 if succeeds. Else a negative code is returned. + */ +int register_filter_private(blosc2_filter *filter); + +/** + * @brief Register a codec in Blosc. + * + * @param codec The codec to register. + * + * @return 0 if succeeds. Else a negative code is returned. + */ +int register_codec_private(blosc2_codec *codec); + +#ifdef __cplusplus +} +#endif + +#endif //IARRAY_BLOSC_PRIVATE_H diff --git a/src/blosc2/blosc/blosc2-stdio.c b/src/blosc2/blosc/blosc2-stdio.c new file mode 100644 index 0000000..23f34af --- /dev/null +++ b/src/blosc2/blosc/blosc2-stdio.c @@ -0,0 +1,76 @@ +/********************************************************************* + Blosc - Blocked Shuffling and Compression Library + + Copyright (C) 2021 The Blosc Developers + https://blosc.org + License: BSD 3-Clause (see LICENSE.txt) + + See LICENSE.txt for details about copyright and rights to use. +**********************************************************************/ + + +#include "blosc2/blosc2-stdio.h" +#include "blosc2.h" + +void *blosc2_stdio_open(const char *urlpath, const char *mode, void *params) { + BLOSC_UNUSED_PARAM(params); + FILE *file = fopen(urlpath, mode); + if (file == NULL) + return NULL; + blosc2_stdio_file *my_fp = malloc(sizeof(blosc2_stdio_file)); + my_fp->file = file; + return my_fp; +} + +int blosc2_stdio_close(void *stream) { + blosc2_stdio_file *my_fp = (blosc2_stdio_file *) stream; + int err = fclose(my_fp->file); + free(my_fp); + return err; +} + +int64_t blosc2_stdio_tell(void *stream) { + blosc2_stdio_file *my_fp = (blosc2_stdio_file *) stream; + int64_t pos; +#if defined(_MSC_VER) && (_MSC_VER >= 1400) + pos = _ftelli64(my_fp->file); +#else + pos = (int64_t)ftell(my_fp->file); +#endif + return pos; +} + +int blosc2_stdio_seek(void *stream, int64_t offset, int whence) { + blosc2_stdio_file *my_fp = (blosc2_stdio_file *) stream; + int rc; +#if defined(_MSC_VER) && (_MSC_VER >= 1400) + rc = _fseeki64(my_fp->file, offset, whence); +#else + rc = fseek(my_fp->file, (long) offset, whence); +#endif + return rc; +} + +int64_t blosc2_stdio_write(const void *ptr, int64_t size, int64_t nitems, void *stream) { + blosc2_stdio_file *my_fp = (blosc2_stdio_file *) stream; + + size_t nitems_ = fwrite(ptr, (size_t) size, (size_t) nitems, my_fp->file); + return (int64_t) nitems_; +} + +int64_t blosc2_stdio_read(void *ptr, int64_t size, int64_t nitems, void *stream) { + blosc2_stdio_file *my_fp = (blosc2_stdio_file *) stream; + size_t nitems_ = fread(ptr, (size_t) size, (size_t) nitems, my_fp->file); + return (int64_t) nitems_; +} + +int blosc2_stdio_truncate(void *stream, int64_t size) { + blosc2_stdio_file *my_fp = (blosc2_stdio_file *) stream; + int rc; +#if defined(_MSC_VER) && (_MSC_VER >= 1400) + rc = _chsize_s(_fileno(my_fp->file), size); +#else + rc = ftruncate(fileno(my_fp->file), size); +#endif + return rc; +} diff --git a/src/blosc2/blosc/blosc2.c b/src/blosc2/blosc/blosc2.c new file mode 100644 index 0000000..c666d6c --- /dev/null +++ b/src/blosc2/blosc/blosc2.c @@ -0,0 +1,4139 @@ +/********************************************************************* + Blosc - Blocked Shuffling and Compression Library + + Copyright (C) 2021 The Blosc Developers + https://blosc.org + License: BSD 3-Clause (see LICENSE.txt) + + See LICENSE.txt for details about copyright and rights to use. +**********************************************************************/ + + +#include +#include +#include +#include +#include +#include +#include + +#include "blosc2.h" +#include "blosc-private.h" +#include "../plugins/codecs/zfp/blosc2-zfp.h" +#include "frame.h" + + +#if defined(USING_CMAKE) + #include "config.h" +#endif /* USING_CMAKE */ +#include "context.h" + +#include "shuffle.h" +#include "delta.h" +#include "trunc-prec.h" +#include "blosclz.h" +#include "stune.h" +#include "blosc2/codecs-registry.h" +#include "blosc2/filters-registry.h" + +#include "lz4.h" +#include "lz4hc.h" +#ifdef HAVE_IPP + #include + #include +#endif +#if defined(HAVE_ZLIB_NG) +#ifdef ZLIB_COMPAT + #include "zlib.h" +#else + #include "zlib-ng.h" +#endif +#elif defined(HAVE_ZLIB) + #include "zlib.h" +#endif /* HAVE_MINIZ */ +#if defined(HAVE_ZSTD) + #include "zstd.h" + #include "zstd_errors.h" + // #include "cover.h" // for experimenting with fast cover training for building dicts + #include "zdict.h" +#endif /* HAVE_ZSTD */ + + +#if defined(_WIN32) && !defined(__MINGW32__) + #include + #include + #include + #define getpid _getpid +#endif /* _WIN32 */ + +#if defined(_WIN32) && !defined(__GNUC__) + #include "win32/pthread.c" +#endif + +/* Synchronization variables */ + +/* Global context for non-contextual API */ +static blosc2_context* g_global_context; +static pthread_mutex_t global_comp_mutex; +static int g_compressor = BLOSC_BLOSCLZ; +static int g_delta = 0; +/* The default splitmode */ +static int32_t g_splitmode = BLOSC_FORWARD_COMPAT_SPLIT; +/* the compressor to use by default */ +static int16_t g_nthreads = 1; +static int32_t g_force_blocksize = 0; +static int g_initlib = 0; +static blosc2_schunk* g_schunk = NULL; /* the pointer to super-chunk */ + +blosc2_codec g_codecs[256] = {0}; +uint8_t g_ncodecs = 0; + +static blosc2_filter g_filters[256] = {0}; +static uint64_t g_nfilters = 0; + +static blosc2_io_cb g_io[256] = {0}; +static uint64_t g_nio = 0; + + +// Forward declarations +int init_threadpool(blosc2_context *context); +int release_threadpool(blosc2_context *context); + +/* Macros for synchronization */ + +/* Wait until all threads are initialized */ +#ifdef BLOSC_POSIX_BARRIERS +#define WAIT_INIT(RET_VAL, CONTEXT_PTR) \ + rc = pthread_barrier_wait(&(CONTEXT_PTR)->barr_init); \ + if (rc != 0 && rc != PTHREAD_BARRIER_SERIAL_THREAD) { \ + BLOSC_TRACE_ERROR("Could not wait on barrier (init): %d", rc); \ + return((RET_VAL)); \ + } +#else +#define WAIT_INIT(RET_VAL, CONTEXT_PTR) \ + pthread_mutex_lock(&(CONTEXT_PTR)->count_threads_mutex); \ + if ((CONTEXT_PTR)->count_threads < (CONTEXT_PTR)->nthreads) { \ + (CONTEXT_PTR)->count_threads++; \ + pthread_cond_wait(&(CONTEXT_PTR)->count_threads_cv, \ + &(CONTEXT_PTR)->count_threads_mutex); \ + } \ + else { \ + pthread_cond_broadcast(&(CONTEXT_PTR)->count_threads_cv); \ + } \ + pthread_mutex_unlock(&(CONTEXT_PTR)->count_threads_mutex); +#endif + +/* Wait for all threads to finish */ +#ifdef BLOSC_POSIX_BARRIERS +#define WAIT_FINISH(RET_VAL, CONTEXT_PTR) \ + rc = pthread_barrier_wait(&(CONTEXT_PTR)->barr_finish); \ + if (rc != 0 && rc != PTHREAD_BARRIER_SERIAL_THREAD) { \ + BLOSC_TRACE_ERROR("Could not wait on barrier (finish)"); \ + return((RET_VAL)); \ + } +#else +#define WAIT_FINISH(RET_VAL, CONTEXT_PTR) \ + pthread_mutex_lock(&(CONTEXT_PTR)->count_threads_mutex); \ + if ((CONTEXT_PTR)->count_threads > 0) { \ + (CONTEXT_PTR)->count_threads--; \ + pthread_cond_wait(&(CONTEXT_PTR)->count_threads_cv, \ + &(CONTEXT_PTR)->count_threads_mutex); \ + } \ + else { \ + pthread_cond_broadcast(&(CONTEXT_PTR)->count_threads_cv); \ + } \ + pthread_mutex_unlock(&(CONTEXT_PTR)->count_threads_mutex); +#endif + + +/* global variable to change threading backend from Blosc-managed to caller-managed */ +static blosc_threads_callback threads_callback = 0; +static void *threads_callback_data = 0; + +/* non-threadsafe function should be called before any other Blosc function in + order to change how threads are managed */ +void blosc2_set_threads_callback(blosc_threads_callback callback, void *callback_data) +{ + threads_callback = callback; + threads_callback_data = callback_data; +} + + +/* A function for aligned malloc that is portable */ +static uint8_t* my_malloc(size_t size) { + void* block = NULL; + int res = 0; + +/* Do an alignment to 32 bytes because AVX2 is supported */ +#if defined(_WIN32) + /* A (void *) cast needed for avoiding a warning with MINGW :-/ */ + block = (void *)_aligned_malloc(size, 32); +#elif _POSIX_C_SOURCE >= 200112L || _XOPEN_SOURCE >= 600 + /* Platform does have an implementation of posix_memalign */ + res = posix_memalign(&block, 32, size); +#else + block = malloc(size); +#endif /* _WIN32 */ + + if (block == NULL || res != 0) { + BLOSC_TRACE_ERROR("Error allocating memory!"); + return NULL; + } + + return (uint8_t*)block; +} + + +/* Release memory booked by my_malloc */ +static void my_free(void* block) { +#if defined(_WIN32) + _aligned_free(block); +#else + free(block); +#endif /* _WIN32 */ +} + + +/* + * Conversion routines between compressor and compression libraries + */ + +/* Return the library code associated with the compressor name */ +static int compname_to_clibcode(const char* compname) { + if (strcmp(compname, BLOSC_BLOSCLZ_COMPNAME) == 0) + return BLOSC_BLOSCLZ_LIB; + if (strcmp(compname, BLOSC_LZ4_COMPNAME) == 0) + return BLOSC_LZ4_LIB; + if (strcmp(compname, BLOSC_LZ4HC_COMPNAME) == 0) + return BLOSC_LZ4_LIB; + if (strcmp(compname, BLOSC_ZLIB_COMPNAME) == 0) + return BLOSC_ZLIB_LIB; + if (strcmp(compname, BLOSC_ZSTD_COMPNAME) == 0) + return BLOSC_ZSTD_LIB; + for (int i = 0; i < g_ncodecs; ++i) { + if (strcmp(compname, g_codecs[i].compname) == 0) + return g_codecs[i].complib; + } + return BLOSC2_ERROR_NOT_FOUND; +} + +/* Return the library name associated with the compressor code */ +static const char* clibcode_to_clibname(int clibcode) { + if (clibcode == BLOSC_BLOSCLZ_LIB) return BLOSC_BLOSCLZ_LIBNAME; + if (clibcode == BLOSC_LZ4_LIB) return BLOSC_LZ4_LIBNAME; + if (clibcode == BLOSC_ZLIB_LIB) return BLOSC_ZLIB_LIBNAME; + if (clibcode == BLOSC_ZSTD_LIB) return BLOSC_ZSTD_LIBNAME; + for (int i = 0; i < g_ncodecs; ++i) { + if (clibcode == g_codecs[i].complib) + return g_codecs[i].compname; + } + return NULL; /* should never happen */ +} + + +/* + * Conversion routines between compressor names and compressor codes + */ + +/* Get the compressor name associated with the compressor code */ +int blosc2_compcode_to_compname(int compcode, const char** compname) { + int code = -1; /* -1 means non-existent compressor code */ + const char* name = NULL; + + /* Map the compressor code */ + if (compcode == BLOSC_BLOSCLZ) + name = BLOSC_BLOSCLZ_COMPNAME; + else if (compcode == BLOSC_LZ4) + name = BLOSC_LZ4_COMPNAME; + else if (compcode == BLOSC_LZ4HC) + name = BLOSC_LZ4HC_COMPNAME; + else if (compcode == BLOSC_ZLIB) + name = BLOSC_ZLIB_COMPNAME; + else if (compcode == BLOSC_ZSTD) + name = BLOSC_ZSTD_COMPNAME; + else { + for (int i = 0; i < g_ncodecs; ++i) { + if (compcode == g_codecs[i].compcode) { + name = g_codecs[i].compname; + break; + } + } + } + + *compname = name; + + /* Guess if there is support for this code */ + if (compcode == BLOSC_BLOSCLZ) + code = BLOSC_BLOSCLZ; + else if (compcode == BLOSC_LZ4) + code = BLOSC_LZ4; + else if (compcode == BLOSC_LZ4HC) + code = BLOSC_LZ4HC; +#if defined(HAVE_ZLIB) + else if (compcode == BLOSC_ZLIB) + code = BLOSC_ZLIB; +#endif /* HAVE_ZLIB */ +#if defined(HAVE_ZSTD) + else if (compcode == BLOSC_ZSTD) + code = BLOSC_ZSTD; +#endif /* HAVE_ZSTD */ + else if (compcode >= BLOSC_LAST_CODEC) + code = compcode; + return code; +} + +/* Get the compressor code for the compressor name. -1 if it is not available */ +int blosc2_compname_to_compcode(const char* compname) { + int code = -1; /* -1 means non-existent compressor code */ + + if (strcmp(compname, BLOSC_BLOSCLZ_COMPNAME) == 0) { + code = BLOSC_BLOSCLZ; + } + else if (strcmp(compname, BLOSC_LZ4_COMPNAME) == 0) { + code = BLOSC_LZ4; + } + else if (strcmp(compname, BLOSC_LZ4HC_COMPNAME) == 0) { + code = BLOSC_LZ4HC; + } +#if defined(HAVE_ZLIB) + else if (strcmp(compname, BLOSC_ZLIB_COMPNAME) == 0) { + code = BLOSC_ZLIB; + } +#endif /* HAVE_ZLIB */ +#if defined(HAVE_ZSTD) + else if (strcmp(compname, BLOSC_ZSTD_COMPNAME) == 0) { + code = BLOSC_ZSTD; + } +#endif /* HAVE_ZSTD */ + else{ + for (int i = 0; i < g_ncodecs; ++i) { + if (strcmp(compname, g_codecs[i].compname) == 0) { + code = g_codecs[i].compcode; + break; + } + } + } + return code; +} + + +/* Convert compressor code to blosc compressor format code */ +static int compcode_to_compformat(int compcode) { + switch (compcode) { + case BLOSC_BLOSCLZ: return BLOSC_BLOSCLZ_FORMAT; + case BLOSC_LZ4: return BLOSC_LZ4_FORMAT; + case BLOSC_LZ4HC: return BLOSC_LZ4HC_FORMAT; + +#if defined(HAVE_ZLIB) + case BLOSC_ZLIB: return BLOSC_ZLIB_FORMAT; +#endif /* HAVE_ZLIB */ + +#if defined(HAVE_ZSTD) + case BLOSC_ZSTD: return BLOSC_ZSTD_FORMAT; + break; +#endif /* HAVE_ZSTD */ + default: + return BLOSC_UDCODEC_FORMAT; + } + return -1; +} + + +/* Convert compressor code to blosc compressor format version */ +static int compcode_to_compversion(int compcode) { + /* Write compressor format */ + switch (compcode) { + case BLOSC_BLOSCLZ: return BLOSC_BLOSCLZ_VERSION_FORMAT; + case BLOSC_LZ4: return BLOSC_LZ4_VERSION_FORMAT; + case BLOSC_LZ4HC: return BLOSC_LZ4HC_VERSION_FORMAT; + +#if defined(HAVE_ZLIB) + case BLOSC_ZLIB: return BLOSC_ZLIB_VERSION_FORMAT; + break; +#endif /* HAVE_ZLIB */ + +#if defined(HAVE_ZSTD) + case BLOSC_ZSTD: return BLOSC_ZSTD_VERSION_FORMAT; + break; +#endif /* HAVE_ZSTD */ + default: + for (int i = 0; i < g_ncodecs; ++i) { + if (compcode == g_codecs[i].compcode) { + return g_codecs[i].compver; + } + } + } + return -1; +} + + +static int lz4_wrap_compress(const char* input, size_t input_length, + char* output, size_t maxout, int accel, void* hash_table) { + BLOSC_UNUSED_PARAM(accel); + int cbytes; +#ifdef HAVE_IPP + if (hash_table == NULL) { + return BLOSC2_ERROR_INVALID_PARAM; // the hash table should always be initialized + } + int outlen = (int)maxout; + int inlen = (int)input_length; + // I have not found any function that uses `accel` like in `LZ4_compress_fast`, but + // the IPP LZ4Safe call does a pretty good job on compressing well, so let's use it + IppStatus status = ippsEncodeLZ4Safe_8u((const Ipp8u*)input, &inlen, + (Ipp8u*)output, &outlen, (Ipp8u*)hash_table); + if (status == ippStsDstSizeLessExpected) { + return 0; // we cannot compress in required outlen + } + else if (status != ippStsNoErr) { + return BLOSC2_ERROR_FAILURE; // an unexpected error happened + } + cbytes = outlen; +#else + BLOSC_UNUSED_PARAM(hash_table); + accel = 1; // deactivate acceleration to match IPP behaviour + cbytes = LZ4_compress_fast(input, output, (int)input_length, (int)maxout, accel); +#endif + return cbytes; +} + + +static int lz4hc_wrap_compress(const char* input, size_t input_length, + char* output, size_t maxout, int clevel) { + int cbytes; + if (input_length > (size_t)(UINT32_C(2) << 30)) + return BLOSC2_ERROR_2GB_LIMIT; + /* clevel for lz4hc goes up to 12, at least in LZ4 1.7.5 + * but levels larger than 9 do not buy much compression. */ + cbytes = LZ4_compress_HC(input, output, (int)input_length, (int)maxout, + clevel); + return cbytes; +} + + +static int lz4_wrap_decompress(const char* input, size_t compressed_length, + char* output, size_t maxout) { + int nbytes; +#ifdef HAVE_IPP + int outlen = (int)maxout; + int inlen = (int)compressed_length; + IppStatus status; + status = ippsDecodeLZ4_8u((const Ipp8u*)input, inlen, (Ipp8u*)output, &outlen); + //status = ippsDecodeLZ4Dict_8u((const Ipp8u*)input, &inlen, (Ipp8u*)output, 0, &outlen, NULL, 1 << 16); + nbytes = (status == ippStsNoErr) ? outlen : -outlen; +#else + nbytes = LZ4_decompress_safe(input, output, (int)compressed_length, (int)maxout); +#endif + if (nbytes != (int)maxout) { + return 0; + } + return (int)maxout; +} + +#if defined(HAVE_ZLIB) +/* zlib is not very respectful with sharing name space with others. + Fortunately, its names do not collide with those already in blosc. */ +static int zlib_wrap_compress(const char* input, size_t input_length, + char* output, size_t maxout, int clevel) { + int status; +#if defined(HAVE_ZLIB_NG) && ! defined(ZLIB_COMPAT) + size_t cl = maxout; + status = zng_compress2( + (uint8_t*)output, &cl, (uint8_t*)input, input_length, clevel); +#else + uLongf cl = (uLongf)maxout; + status = compress2( + (Bytef*)output, &cl, (Bytef*)input, (uLong)input_length, clevel); +#endif + if (status != Z_OK) { + return 0; + } + return (int)cl; +} + +static int zlib_wrap_decompress(const char* input, size_t compressed_length, + char* output, size_t maxout) { + int status; +#if defined(HAVE_ZLIB_NG) && ! defined(ZLIB_COMPAT) + size_t ul = maxout; + status = zng_uncompress( + (uint8_t*)output, &ul, (uint8_t*)input, compressed_length); +#else + uLongf ul = (uLongf)maxout; + status = uncompress( + (Bytef*)output, &ul, (Bytef*)input, (uLong)compressed_length); +#endif + if (status != Z_OK) { + return 0; + } + return (int)ul; +} +#endif /* HAVE_ZLIB */ + + +#if defined(HAVE_ZSTD) +static int zstd_wrap_compress(struct thread_context* thread_context, + const char* input, size_t input_length, + char* output, size_t maxout, int clevel) { + size_t code; + blosc2_context* context = thread_context->parent_context; + + clevel = (clevel < 9) ? clevel * 2 - 1 : ZSTD_maxCLevel(); + /* Make the level 8 close enough to maxCLevel */ + if (clevel == 8) clevel = ZSTD_maxCLevel() - 2; + + if (thread_context->zstd_cctx == NULL) { + thread_context->zstd_cctx = ZSTD_createCCtx(); + } + + if (context->use_dict) { + assert(context->dict_cdict != NULL); + code = ZSTD_compress_usingCDict( + thread_context->zstd_cctx, (void*)output, maxout, (void*)input, + input_length, context->dict_cdict); + } else { + code = ZSTD_compressCCtx(thread_context->zstd_cctx, + (void*)output, maxout, (void*)input, input_length, clevel); + } + if (ZSTD_isError(code) != ZSTD_error_no_error) { + // Do not print anything because blosc will just memcpy this buffer + // fprintf(stderr, "Error in ZSTD compression: '%s'. Giving up.\n", + // ZDICT_getErrorName(code)); + return 0; + } + return (int)code; +} + +static int zstd_wrap_decompress(struct thread_context* thread_context, + const char* input, size_t compressed_length, + char* output, size_t maxout) { + size_t code; + blosc2_context* context = thread_context->parent_context; + + if (thread_context->zstd_dctx == NULL) { + thread_context->zstd_dctx = ZSTD_createDCtx(); + } + + if (context->use_dict) { + assert(context->dict_ddict != NULL); + code = ZSTD_decompress_usingDDict( + thread_context->zstd_dctx, (void*)output, maxout, (void*)input, + compressed_length, context->dict_ddict); + } else { + code = ZSTD_decompressDCtx(thread_context->zstd_dctx, + (void*)output, maxout, (void*)input, compressed_length); + } + if (ZSTD_isError(code) != ZSTD_error_no_error) { + BLOSC_TRACE_ERROR("Error in ZSTD decompression: '%s'. Giving up.", + ZDICT_getErrorName(code)); + return 0; + } + return (int)code; +} +#endif /* HAVE_ZSTD */ + +/* Compute acceleration for blosclz */ +static int get_accel(const blosc2_context* context) { + int clevel = context->clevel; + + if (context->compcode == BLOSC_LZ4) { + /* This acceleration setting based on discussions held in: + * https://groups.google.com/forum/#!topic/lz4c/zosy90P8MQw + */ + return (10 - clevel); + } + return 1; +} + + +int do_nothing(uint8_t filter, char cmode) { + if (cmode == 'c') { + return (filter == BLOSC_NOFILTER); + } else { + // TRUNC_PREC do not have to be applied during decompression + return ((filter == BLOSC_NOFILTER) || (filter == BLOSC_TRUNC_PREC)); + } +} + + +int next_filter(const uint8_t* filters, int current_filter, char cmode) { + for (int i = current_filter - 1; i >= 0; i--) { + if (!do_nothing(filters[i], cmode)) { + return filters[i]; + } + } + return BLOSC_NOFILTER; +} + + +int last_filter(const uint8_t* filters, char cmode) { + int last_index = -1; + for (int i = BLOSC2_MAX_FILTERS - 1; i >= 0; i--) { + if (!do_nothing(filters[i], cmode)) { + last_index = i; + } + } + return last_index; +} + + +/* Convert filter pipeline to filter flags */ +static uint8_t filters_to_flags(const uint8_t* filters) { + uint8_t flags = 0; + + for (int i = 0; i < BLOSC2_MAX_FILTERS; i++) { + switch (filters[i]) { + case BLOSC_SHUFFLE: + flags |= BLOSC_DOSHUFFLE; + break; + case BLOSC_BITSHUFFLE: + flags |= BLOSC_DOBITSHUFFLE; + break; + case BLOSC_DELTA: + flags |= BLOSC_DODELTA; + break; + default : + break; + } + } + return flags; +} + + +/* Convert filter flags to filter pipeline */ +static void flags_to_filters(const uint8_t flags, uint8_t* filters) { + /* Initialize the filter pipeline */ + memset(filters, 0, BLOSC2_MAX_FILTERS); + /* Fill the filter pipeline */ + if (flags & BLOSC_DOSHUFFLE) + filters[BLOSC2_MAX_FILTERS - 1] = BLOSC_SHUFFLE; + if (flags & BLOSC_DOBITSHUFFLE) + filters[BLOSC2_MAX_FILTERS - 1] = BLOSC_BITSHUFFLE; + if (flags & BLOSC_DODELTA) + filters[BLOSC2_MAX_FILTERS - 2] = BLOSC_DELTA; +} + + +/* Get filter flags from header flags */ +static uint8_t get_filter_flags(const uint8_t header_flags, + const int32_t typesize) { + uint8_t flags = 0; + + if ((header_flags & BLOSC_DOSHUFFLE) && (typesize > 1)) { + flags |= BLOSC_DOSHUFFLE; + } + if (header_flags & BLOSC_DOBITSHUFFLE) { + flags |= BLOSC_DOBITSHUFFLE; + } + if (header_flags & BLOSC_DODELTA) { + flags |= BLOSC_DODELTA; + } + if (header_flags & BLOSC_MEMCPYED) { + flags |= BLOSC_MEMCPYED; + } + return flags; +} + +typedef struct blosc_header_s { + uint8_t version; + uint8_t versionlz; + uint8_t flags; + uint8_t typesize; + int32_t nbytes; + int32_t blocksize; + int32_t cbytes; + // Extended Blosc2 header + uint8_t filter_codes[BLOSC2_MAX_FILTERS]; + uint8_t udcompcode; + uint8_t compcode_meta; + uint8_t filter_meta[BLOSC2_MAX_FILTERS]; + uint8_t reserved2; + uint8_t blosc2_flags; +} blosc_header; + + +int read_chunk_header(const uint8_t* src, int32_t srcsize, bool extended_header, blosc_header* header) +{ + memset(header, 0, sizeof(blosc_header)); + + if (srcsize < BLOSC_MIN_HEADER_LENGTH) { + BLOSC_TRACE_ERROR("Not enough space to read Blosc header."); + return BLOSC2_ERROR_READ_BUFFER; + } + + memcpy(header, src, BLOSC_MIN_HEADER_LENGTH); + + bool little_endian = is_little_endian(); + + if (!little_endian) { + header->nbytes = bswap32_(header->nbytes); + header->blocksize = bswap32_(header->blocksize); + header->cbytes = bswap32_(header->cbytes); + } + + if (header->version > BLOSC2_VERSION_FORMAT) { + /* Version from future */ + return BLOSC2_ERROR_VERSION_SUPPORT; + } + if (header->cbytes < BLOSC_MIN_HEADER_LENGTH) { + BLOSC_TRACE_ERROR("`cbytes` is too small to read min header."); + return BLOSC2_ERROR_INVALID_HEADER; + } + if (header->blocksize <= 0 || (header->nbytes > 0 && (header->blocksize > header->nbytes))) { + BLOSC_TRACE_ERROR("`blocksize` is zero or greater than uncompressed size"); + return BLOSC2_ERROR_INVALID_HEADER; + } + if (header->blocksize > BLOSC2_MAXBLOCKSIZE) { + BLOSC_TRACE_ERROR("`blocksize` greater than maximum allowed"); + return BLOSC2_ERROR_INVALID_HEADER; + } + if (header->typesize == 0) { + BLOSC_TRACE_ERROR("`typesize` is zero."); + return BLOSC2_ERROR_INVALID_HEADER; + } + + /* Read extended header if it is wanted */ + if ((extended_header) && (header->flags & BLOSC_DOSHUFFLE) && (header->flags & BLOSC_DOBITSHUFFLE)) { + if (header->cbytes < BLOSC_EXTENDED_HEADER_LENGTH) { + BLOSC_TRACE_ERROR("`cbytes` is too small to read extended header."); + return BLOSC2_ERROR_INVALID_HEADER; + } + if (srcsize < BLOSC_EXTENDED_HEADER_LENGTH) { + BLOSC_TRACE_ERROR("Not enough space to read Blosc extended header."); + return BLOSC2_ERROR_READ_BUFFER; + } + + memcpy((uint8_t *)header + BLOSC_MIN_HEADER_LENGTH, src + BLOSC_MIN_HEADER_LENGTH, + BLOSC_EXTENDED_HEADER_LENGTH - BLOSC_MIN_HEADER_LENGTH); + + int32_t special_type = (header->blosc2_flags >> 4) & BLOSC2_SPECIAL_MASK; + if (special_type != 0) { + if (header->nbytes % header->typesize != 0) { + BLOSC_TRACE_ERROR("`nbytes` is not a multiple of typesize"); + return BLOSC2_ERROR_INVALID_HEADER; + } + if (special_type == BLOSC2_SPECIAL_VALUE) { + if (header->cbytes < BLOSC_EXTENDED_HEADER_LENGTH + header->typesize) { + BLOSC_TRACE_ERROR("`cbytes` is too small for run length encoding"); + return BLOSC2_ERROR_READ_BUFFER; + } + } + } + // The number of filters depends on the version of the header. Blosc2 alpha series + // did not initialize filters to zero beyond the max supported. + if (header->version == BLOSC2_VERSION_FORMAT_ALPHA) { + header->filter_codes[5] = 0; + header->filter_meta[5] = 0; + } + } + else { + flags_to_filters(header->flags, header->filter_codes); + } + return 0; +} + +static inline void blosc2_calculate_blocks(blosc2_context* context) { + /* Compute number of blocks in buffer */ + context->nblocks = context->sourcesize / context->blocksize; + context->leftover = context->sourcesize % context->blocksize; + context->nblocks = (context->leftover > 0) ? + (context->nblocks + 1) : context->nblocks; +} + +static int blosc2_initialize_context_from_header(blosc2_context* context, blosc_header* header) { + context->header_flags = header->flags; + context->typesize = header->typesize; + context->sourcesize = header->nbytes; + context->blocksize = header->blocksize; + context->blosc2_flags = header->blosc2_flags; + context->compcode = header->flags >> 5; + if (context->compcode == BLOSC_UDCODEC_FORMAT) { + context->compcode = header->udcompcode; + } + blosc2_calculate_blocks(context); + + bool is_lazy = false; + if ((context->header_flags & BLOSC_DOSHUFFLE) && + (context->header_flags & BLOSC_DOBITSHUFFLE)) { + /* Extended header */ + context->header_overhead = BLOSC_EXTENDED_HEADER_LENGTH; + + memcpy(context->filters, header->filter_codes, BLOSC2_MAX_FILTERS); + memcpy(context->filters_meta, header->filter_meta, BLOSC2_MAX_FILTERS); + context->compcode_meta = header->compcode_meta; + + context->filter_flags = filters_to_flags(header->filter_codes); + context->special_type = (header->blosc2_flags >> 4) & BLOSC2_SPECIAL_MASK; + + is_lazy = (context->blosc2_flags & 0x08u); + } + else { + context->header_overhead = BLOSC_MIN_HEADER_LENGTH; + context->filter_flags = get_filter_flags(context->header_flags, context->typesize); + flags_to_filters(context->header_flags, context->filters); + } + + // Some checks for malformed headers + if (!is_lazy && header->cbytes > context->srcsize) { + return BLOSC2_ERROR_INVALID_HEADER; + } + + return 0; +} + + +static int blosc2_intialize_header_from_context(blosc2_context* context, blosc_header* header, bool extended_header) { + memset(header, 0, sizeof(blosc_header)); + + header->version = BLOSC2_VERSION_FORMAT; + header->versionlz = compcode_to_compversion(context->compcode); + header->flags = context->header_flags; + header->typesize = (uint8_t)context->typesize; + header->nbytes = (int32_t)context->sourcesize; + header->blocksize = (int32_t)context->blocksize; + + int little_endian = is_little_endian(); + if (!little_endian) { + header->nbytes = bswap32_(header->nbytes); + header->blocksize = bswap32_(header->blocksize); + // cbytes written after compression + } + + if (extended_header) { + /* Store filter pipeline info at the end of the header */ + for (int i = 0; i < BLOSC2_MAX_FILTERS; i++) { + header->filter_codes[i] = context->filters[i]; + header->filter_meta[i] = context->filters_meta[i]; + } + header->udcompcode = context->compcode; + header->compcode_meta = context->compcode_meta; + + if (!little_endian) { + header->blosc2_flags |= BLOSC2_BIGENDIAN; + } + if (context->use_dict) { + header->blosc2_flags |= BLOSC2_USEDICT; + } + if (context->blosc2_flags & BLOSC2_INSTR_CODEC) { + header->blosc2_flags |= BLOSC2_INSTR_CODEC; + } + } + + return 0; +} + + +uint8_t* pipeline_forward(struct thread_context* thread_context, const int32_t bsize, + const uint8_t* src, const int32_t offset, + uint8_t* dest, uint8_t* tmp, uint8_t* tmp2) { + blosc2_context* context = thread_context->parent_context; + uint8_t* _src = (uint8_t*)src + offset; + uint8_t* _tmp = tmp; + uint8_t* _dest = dest; + int32_t typesize = context->typesize; + uint8_t* filters = context->filters; + uint8_t* filters_meta = context->filters_meta; + bool memcpyed = context->header_flags & (uint8_t)BLOSC_MEMCPYED; + + /* Prefilter function */ + if (context->prefilter != NULL) { + /* Set unwritten values to zero */ + memset(_dest, 0, bsize); + // Create new prefilter parameters for this block (must be private for each thread) + blosc2_prefilter_params preparams; + memcpy(&preparams, context->preparams, sizeof(preparams)); + preparams.in = _src; + preparams.out = _dest; + preparams.out_size = bsize; + preparams.out_typesize = typesize; + preparams.out_offset = offset; + preparams.nblock = offset / context->blocksize; + preparams.nchunk = context->schunk != NULL ? context->schunk->current_nchunk : -1; + preparams.tid = thread_context->tid; + preparams.ttmp = thread_context->tmp; + preparams.ttmp_nbytes = thread_context->tmp_nbytes; + preparams.ctx = context; + + if (context->prefilter(&preparams) != 0) { + BLOSC_TRACE_ERROR("Execution of prefilter function failed"); + return NULL; + } + + if (memcpyed) { + // No more filters are required + return _dest; + } + // Cycle buffers + _src = _dest; + _dest = _tmp; + _tmp = _src; + } + + /* Process the filter pipeline */ + for (int i = 0; i < BLOSC2_MAX_FILTERS; i++) { + int rc = BLOSC2_ERROR_SUCCESS; + if (filters[i] <= BLOSC2_DEFINED_FILTERS_STOP) { + switch (filters[i]) { + case BLOSC_SHUFFLE: + for (int j = 0; j <= filters_meta[i]; j++) { + shuffle(typesize, bsize, _src, _dest); + // Cycle filters when required + if (j < filters_meta[i]) { + _src = _dest; + _dest = _tmp; + _tmp = _src; + } + } + break; + case BLOSC_BITSHUFFLE: + if (bitshuffle(typesize, bsize, _src, _dest, tmp2) < 0) { + return NULL; + } + break; + case BLOSC_DELTA: + delta_encoder(src, offset, bsize, typesize, _src, _dest); + break; + case BLOSC_TRUNC_PREC: + if (truncate_precision(filters_meta[i], typesize, bsize, _src, _dest) < 0) { + return NULL; + } + break; + default: + if (filters[i] != BLOSC_NOFILTER) { + BLOSC_TRACE_ERROR("Filter %d not handled during compression\n", filters[i]); + return NULL; + } + } + } + else { + // Look for the filters_meta in user filters and run it + for (uint64_t j = 0; j < g_nfilters; ++j) { + if (g_filters[j].id == filters[i]) { + if (g_filters[j].forward != NULL) { + blosc2_cparams cparams; + blosc2_ctx_get_cparams(context, &cparams); + rc = g_filters[j].forward(_src, _dest, bsize, filters_meta[i], &cparams); + } else { + BLOSC_TRACE_ERROR("Forward function is NULL"); + return NULL; + } + if (rc != BLOSC2_ERROR_SUCCESS) { + BLOSC_TRACE_ERROR("User-defined filter %d failed during compression\n", filters[i]); + return NULL; + } + goto urfiltersuccess; + } + } + BLOSC_TRACE_ERROR("User-defined filter %d not found during compression\n", filters[i]); + return NULL; + + urfiltersuccess:; + + } + + // Cycle buffers when required + if (filters[i] != BLOSC_NOFILTER) { + _src = _dest; + _dest = _tmp; + _tmp = _src; + } + } + return _src; +} + + +// Optimized version for detecting runs. It compares 8 bytes values wherever possible. +static bool get_run(const uint8_t* ip, const uint8_t* ip_bound) { + uint8_t x = *ip; + int64_t value, value2; + /* Broadcast the value for every byte in a 64-bit register */ + memset(&value, x, 8); + while (ip < (ip_bound - 8)) { +#if defined(BLOSC_STRICT_ALIGN) + memcpy(&value2, ip, 8); +#else + value2 = *(int64_t*)ip; +#endif + if (value != value2) { + // Values differ. We don't have a run. + return false; + } + else { + ip += 8; + } + } + /* Look into the remainder */ + while ((ip < ip_bound) && (*ip == x)) ip++; + return ip == ip_bound ? true : false; +} + + +/* Shuffle & compress a single block */ +static int blosc_c(struct thread_context* thread_context, int32_t bsize, + int32_t leftoverblock, int32_t ntbytes, int32_t destsize, + const uint8_t* src, const int32_t offset, uint8_t* dest, + uint8_t* tmp, uint8_t* tmp2) { + blosc2_context* context = thread_context->parent_context; + int dont_split = (context->header_flags & 0x10) >> 4; + int dict_training = context->use_dict && context->dict_cdict == NULL; + int32_t j, neblock, nstreams; + int32_t cbytes; /* number of compressed bytes in split */ + int32_t ctbytes = 0; /* number of compressed bytes in block */ + int32_t maxout; + int32_t typesize = context->typesize; + const char* compname; + int accel; + const uint8_t* _src; + uint8_t *_tmp = tmp, *_tmp2 = tmp2; + uint8_t *_tmp3 = thread_context->tmp4; + int last_filter_index = last_filter(context->filters, 'c'); + bool memcpyed = context->header_flags & (uint8_t)BLOSC_MEMCPYED; + bool instr_codec = context->blosc2_flags & BLOSC2_INSTR_CODEC; + blosc_timestamp_t last, current; + float filter_time = 0.f; + + if (instr_codec) { + blosc_set_timestamp(&last); + } + + // See whether we have a run here + if (last_filter_index >= 0 || context->prefilter != NULL) { + /* Apply the filter pipeline just for the prefilter */ + if (memcpyed && context->prefilter != NULL) { + // We only need the prefilter output + _src = pipeline_forward(thread_context, bsize, src, offset, dest, _tmp2, _tmp3); + if (_src == NULL) { + return BLOSC2_ERROR_FILTER_PIPELINE; + } + return bsize; + } + /* Apply regular filter pipeline */ + _src = pipeline_forward(thread_context, bsize, src, offset, _tmp, _tmp2, _tmp3); + if (_src == NULL) { + return BLOSC2_ERROR_FILTER_PIPELINE; + } + } else { + _src = src + offset; + } + + if (instr_codec) { + blosc_set_timestamp(¤t); + filter_time = (float) blosc_elapsed_secs(last, current); + last = current; + } + + assert(context->clevel > 0); + + /* Calculate acceleration for different compressors */ + accel = get_accel(context); + + /* The number of compressed data streams for this block */ + if (!dont_split && !leftoverblock && !dict_training) { + nstreams = (int32_t)typesize; + } + else { + nstreams = 1; + } + neblock = bsize / nstreams; + for (j = 0; j < nstreams; j++) { + if (instr_codec) { + blosc_set_timestamp(&last); + } + if (!dict_training) { + dest += sizeof(int32_t); + ntbytes += sizeof(int32_t); + ctbytes += sizeof(int32_t); + + const uint8_t *ip = (uint8_t *) _src + j * neblock; + const uint8_t *ipbound = (uint8_t *) _src + (j + 1) * neblock; + + if (context->header_overhead == BLOSC_EXTENDED_HEADER_LENGTH && get_run(ip, ipbound)) { + // A run + int32_t value = _src[j * neblock]; + if (ntbytes > destsize) { + return 0; /* Non-compressible data */ + } + + if (instr_codec) { + blosc_set_timestamp(¤t); + int32_t instr_size = sizeof(blosc2_instr); + ntbytes += instr_size; + ctbytes += instr_size; + if (ntbytes > destsize) { + return 0; /* Non-compressible data */ + } + _sw32(dest - 4, instr_size); + blosc2_instr *desti = (blosc2_instr *)dest; + memset(desti, 0, sizeof(blosc2_instr)); + // Special values have an overhead of about 1 int32 + int32_t ssize = value == 0 ? sizeof(int32_t) : sizeof(int32_t) + 1; + desti->cratio = (float) neblock / (float) ssize; + float ctime = (float) blosc_elapsed_secs(last, current); + desti->cspeed = (float) neblock / ctime; + desti->filter_speed = (float) neblock / filter_time; + desti->flags[0] = 1; // mark a runlen + dest += instr_size; + continue; + } + + // Encode the repeated byte in the first (LSB) byte of the length of the split. + _sw32(dest - 4, -value); // write the value in two's complement + if (value > 0) { + // Mark encoding as a run-length (== 0 is always a 0's run) + ntbytes += 1; + ctbytes += 1; + if (ntbytes > destsize) { + return 0; /* Non-compressible data */ + } + // Set MSB bit (sign) to 1 (not really necessary here, but for demonstration purposes) + // dest[-1] |= 0x80; + dest[0] = 0x1; // set run-length bit (0) in token + dest += 1; + } + continue; + } + } + + maxout = neblock; + if (ntbytes + maxout > destsize) { + /* avoid buffer * overrun */ + maxout = destsize - ntbytes; + if (maxout <= 0) { + return 0; /* non-compressible block */ + } + } + if (dict_training) { + // We are in the build dict state, so don't compress + // TODO: copy only a percentage for sampling + memcpy(dest, _src + j * neblock, (unsigned int)neblock); + cbytes = (int32_t)neblock; + } + else if (context->compcode == BLOSC_BLOSCLZ) { + cbytes = blosclz_compress(context->clevel, _src + j * neblock, + (int)neblock, dest, maxout, context); + } + else if (context->compcode == BLOSC_LZ4) { + void *hash_table = NULL; + #ifdef HAVE_IPP + hash_table = (void*)thread_context->lz4_hash_table; + #endif + cbytes = lz4_wrap_compress((char*)_src + j * neblock, (size_t)neblock, + (char*)dest, (size_t)maxout, accel, hash_table); + } + else if (context->compcode == BLOSC_LZ4HC) { + cbytes = lz4hc_wrap_compress((char*)_src + j * neblock, (size_t)neblock, + (char*)dest, (size_t)maxout, context->clevel); + } + #if defined(HAVE_ZLIB) + else if (context->compcode == BLOSC_ZLIB) { + cbytes = zlib_wrap_compress((char*)_src + j * neblock, (size_t)neblock, + (char*)dest, (size_t)maxout, context->clevel); + } + #endif /* HAVE_ZLIB */ + #if defined(HAVE_ZSTD) + else if (context->compcode == BLOSC_ZSTD) { + cbytes = zstd_wrap_compress(thread_context, + (char*)_src + j * neblock, (size_t)neblock, + (char*)dest, (size_t)maxout, context->clevel); + } + #endif /* HAVE_ZSTD */ + else if (context->compcode > BLOSC2_DEFINED_CODECS_STOP) { + for (int i = 0; i < g_ncodecs; ++i) { + if (g_codecs[i].compcode == context->compcode) { + blosc2_cparams cparams; + blosc2_ctx_get_cparams(context, &cparams); + cbytes = g_codecs[i].encoder(_src + j * neblock, + neblock, + dest, + maxout, + context->compcode_meta, + &cparams, + context->src); + goto urcodecsuccess; + } + } + BLOSC_TRACE_ERROR("User-defined compressor codec %d not found during compression", context->compcode); + return BLOSC2_ERROR_CODEC_SUPPORT; + urcodecsuccess: + ; + } else { + blosc2_compcode_to_compname(context->compcode, &compname); + BLOSC_TRACE_ERROR("Blosc has not been compiled with '%s' compression support." + "Please use one having it.", compname); + return BLOSC2_ERROR_CODEC_SUPPORT; + } + + if (cbytes > maxout) { + /* Buffer overrun caused by compression (should never happen) */ + return BLOSC2_ERROR_WRITE_BUFFER; + } + if (cbytes < 0) { + /* cbytes should never be negative */ + return BLOSC2_ERROR_DATA; + } + if (cbytes == 0) { + // When cbytes is 0, the compressor has not been able to compress anything + cbytes = neblock; + } + + if (instr_codec) { + blosc_set_timestamp(¤t); + int32_t instr_size = sizeof(blosc2_instr); + ntbytes += instr_size; + ctbytes += instr_size; + if (ntbytes > destsize) { + return 0; /* Non-compressible data */ + } + _sw32(dest - 4, instr_size); + float ctime = (float)blosc_elapsed_secs(last, current); + blosc2_instr *desti = (blosc2_instr *)dest; + memset(desti, 0, sizeof(blosc2_instr)); + // cratio is computed having into account 1 additional int (csize) + desti->cratio = (float)neblock / (float)(cbytes + sizeof(int32_t)); + desti->cspeed = (float)neblock / ctime; + desti->filter_speed = (float) neblock / filter_time; + dest += instr_size; + continue; + } + + if (!dict_training) { + if (cbytes == neblock) { + /* The compressor has been unable to compress data at all. */ + /* Before doing the copy, check that we are not running into a + buffer overflow. */ + if ((ntbytes + neblock) > destsize) { + return 0; /* Non-compressible data */ + } + memcpy(dest, _src + j * neblock, (unsigned int)neblock); + cbytes = neblock; + } + _sw32(dest - 4, cbytes); + } + dest += cbytes; + ntbytes += cbytes; + ctbytes += cbytes; + } /* Closes j < nstreams */ + + //printf("c%d", ctbytes); + return ctbytes; +} + + +/* Process the filter pipeline (decompression mode) */ +int pipeline_backward(struct thread_context* thread_context, const int32_t bsize, uint8_t* dest, + const int32_t offset, uint8_t* src, uint8_t* tmp, + uint8_t* tmp2, int last_filter_index, int32_t nblock) { + blosc2_context* context = thread_context->parent_context; + int32_t typesize = context->typesize; + uint8_t* filters = context->filters; + uint8_t* filters_meta = context->filters_meta; + uint8_t* _src = src; + uint8_t* _dest = tmp; + uint8_t* _tmp = tmp2; + int errcode = 0; + + for (int i = BLOSC2_MAX_FILTERS - 1; i >= 0; i--) { + // Delta filter requires the whole chunk ready + int last_copy_filter = (last_filter_index == i) || (next_filter(filters, i, 'd') == BLOSC_DELTA); + if (last_copy_filter && context->postfilter == NULL) { + _dest = dest + offset; + } + int rc = BLOSC2_ERROR_SUCCESS; + if (filters[i] <= BLOSC2_DEFINED_FILTERS_STOP) { + switch (filters[i]) { + case BLOSC_SHUFFLE: + for (int j = 0; j <= filters_meta[i]; j++) { + unshuffle(typesize, bsize, _src, _dest); + // Cycle filters when required + if (j < filters_meta[i]) { + _src = _dest; + _dest = _tmp; + _tmp = _src; + } + // Check whether we have to copy the intermediate _dest buffer to final destination + if (last_copy_filter && (filters_meta[i] % 2) == 1 && j == filters_meta[i]) { + memcpy(dest + offset, _dest, (unsigned int) bsize); + } + } + break; + case BLOSC_BITSHUFFLE: + if (bitunshuffle(typesize, bsize, _src, _dest, _tmp, context->src[BLOSC2_CHUNK_VERSION]) < 0) { + return BLOSC2_ERROR_FILTER_PIPELINE; + } + break; + case BLOSC_DELTA: + if (context->nthreads == 1) { + /* Serial mode */ + delta_decoder(dest, offset, bsize, typesize, _dest); + } else { + /* Force the thread in charge of the block 0 to go first */ + pthread_mutex_lock(&context->delta_mutex); + if (context->dref_not_init) { + if (offset != 0) { + pthread_cond_wait(&context->delta_cv, &context->delta_mutex); + } else { + delta_decoder(dest, offset, bsize, typesize, _dest); + context->dref_not_init = 0; + pthread_cond_broadcast(&context->delta_cv); + } + } + pthread_mutex_unlock(&context->delta_mutex); + if (offset != 0) { + delta_decoder(dest, offset, bsize, typesize, _dest); + } + } + break; + case BLOSC_TRUNC_PREC: + // TRUNC_PREC filter does not need to be undone + break; + default: + if (filters[i] != BLOSC_NOFILTER) { + BLOSC_TRACE_ERROR("Filter %d not handled during decompression.", + filters[i]); + errcode = -1; + } + } + } else { + // Look for the filters_meta in user filters and run it + for (uint64_t j = 0; j < g_nfilters; ++j) { + if (g_filters[j].id == filters[i]) { + if (g_filters[j].backward != NULL) { + blosc2_dparams dparams; + blosc2_ctx_get_dparams(context, &dparams); + rc = g_filters[j].backward(_src, _dest, bsize, filters_meta[i], &dparams); + } else { + BLOSC_TRACE_ERROR("Backward function is NULL"); + return BLOSC2_ERROR_FILTER_PIPELINE; + } + if (rc != BLOSC2_ERROR_SUCCESS) { + BLOSC_TRACE_ERROR("User-defined filter %d failed during decompression.", filters[i]); + return rc; + } + goto urfiltersuccess; + } + } + BLOSC_TRACE_ERROR("User-defined filter %d not found during decompression.", filters[i]); + return BLOSC2_ERROR_FILTER_PIPELINE; + urfiltersuccess:; + } + + // Cycle buffers when required + if ((filters[i] != BLOSC_NOFILTER) && (filters[i] != BLOSC_TRUNC_PREC)) { + _src = _dest; + _dest = _tmp; + _tmp = _src; + } + if (last_filter_index == i) { + break; + } + } + + /* Postfilter function */ + if (context->postfilter != NULL) { + // Create new postfilter parameters for this block (must be private for each thread) + blosc2_postfilter_params postparams; + memcpy(&postparams, context->postparams, sizeof(postparams)); + postparams.in = _src; + postparams.out = dest + offset; + postparams.size = bsize; + postparams.typesize = typesize; + postparams.offset = nblock * context->blocksize; + postparams.nchunk = context->schunk != NULL ? context->schunk->current_nchunk : -1; + postparams.nblock = nblock; + postparams.tid = thread_context->tid; + postparams.ttmp = thread_context->tmp; + postparams.ttmp_nbytes = thread_context->tmp_nbytes; + postparams.ctx = context; + + if (context->postfilter(&postparams) != 0) { + BLOSC_TRACE_ERROR("Execution of postfilter function failed"); + return BLOSC2_ERROR_POSTFILTER; + } + } + + return errcode; +} + + +static int32_t set_nans(int32_t typesize, uint8_t* dest, int32_t destsize) { + // destsize can only be a multiple of typesize + if (destsize % typesize != 0) { + return -1; + } + int32_t nitems = destsize / typesize; + if (nitems == 0) { + return 0; + } + + if (typesize == 4) { + float* dest_ = (float*)dest; + float val = nanf(""); + for (int i = 0; i < nitems; i++) { + dest_[i] = val; + } + return nitems; + } + else if (typesize == 8) { + double* dest_ = (double*)dest; + double val = nan(""); + for (int i = 0; i < nitems; i++) { + dest_[i] = val; + } + return nitems; + } + + BLOSC_TRACE_ERROR("Unsupported typesize for NaN"); + return BLOSC2_ERROR_DATA; +} + + +static int32_t set_values(int32_t typesize, const uint8_t* src, uint8_t* dest, int32_t destsize) { + // destsize can only be a multiple of typesize + int64_t val8; + int64_t* dest8; + int32_t val4; + int32_t* dest4; + int16_t val2; + int16_t* dest2; + int8_t val1; + int8_t* dest1; + + if (destsize % typesize != 0) { + return -1; + } + int32_t nitems = destsize / typesize; + if (nitems == 0) { + return 0; + } + + switch (typesize) { + case 8: + val8 = ((int64_t*)(src + BLOSC_EXTENDED_HEADER_LENGTH))[0]; + dest8 = (int64_t*)dest; + for (int i = 0; i < nitems; i++) { + dest8[i] = val8; + } + break; + case 4: + val4 = ((int32_t*)(src + BLOSC_EXTENDED_HEADER_LENGTH))[0]; + dest4 = (int32_t*)dest; + for (int i = 0; i < nitems; i++) { + dest4[i] = val4; + } + break; + case 2: + val2 = ((int16_t*)(src + BLOSC_EXTENDED_HEADER_LENGTH))[0]; + dest2 = (int16_t*)dest; + for (int i = 0; i < nitems; i++) { + dest2[i] = val2; + } + break; + case 1: + val1 = ((int8_t*)(src + BLOSC_EXTENDED_HEADER_LENGTH))[0]; + dest1 = (int8_t*)dest; + for (int i = 0; i < nitems; i++) { + dest1[i] = val1; + } + break; + default: + for (int i = 0; i < nitems; i++) { + memcpy(dest + i * typesize, src + BLOSC_EXTENDED_HEADER_LENGTH, typesize); + } + } + + return nitems; +} + + +/* Decompress & unshuffle a single block */ +static int blosc_d( + struct thread_context* thread_context, int32_t bsize, + int32_t leftoverblock, bool memcpyed, const uint8_t* src, int32_t srcsize, int32_t src_offset, + int32_t nblock, uint8_t* dest, int32_t dest_offset, uint8_t* tmp, uint8_t* tmp2) { + blosc2_context* context = thread_context->parent_context; + uint8_t* filters = context->filters; + uint8_t *tmp3 = thread_context->tmp4; + int32_t compformat = (context->header_flags & (uint8_t)0xe0) >> 5u; + int dont_split = (context->header_flags & 0x10) >> 4; + int32_t chunk_nbytes; + int32_t chunk_cbytes; + int nstreams; + int32_t neblock; + int32_t nbytes; /* number of decompressed bytes in split */ + int32_t cbytes; /* number of compressed bytes in split */ + int32_t ctbytes = 0; /* number of compressed bytes in block */ + int32_t ntbytes = 0; /* number of uncompressed bytes in block */ + uint8_t* _dest; + int32_t typesize = context->typesize; + bool instr_codec = context->blosc2_flags & BLOSC2_INSTR_CODEC; + const char* compname; + int rc; + + if (context->block_maskout != NULL && context->block_maskout[nblock]) { + // Do not decompress, but act as if we successfully decompressed everything + return bsize; + } + + rc = blosc2_cbuffer_sizes(src, &chunk_nbytes, &chunk_cbytes, NULL); + if (rc < 0) { + return rc; + } + + // In some situations (lazychunks) the context can arrive uninitialized + // (but BITSHUFFLE needs it for accessing the format of the chunk) + if (context->src == NULL) { + context->src = src; + } + + // Chunks with special values cannot be lazy + bool is_lazy = ((context->header_overhead == BLOSC_EXTENDED_HEADER_LENGTH) && + (context->blosc2_flags & 0x08u) && !context->special_type); + if (is_lazy) { + // The chunk is on disk, so just lazily load the block + if (context->schunk == NULL) { + BLOSC_TRACE_ERROR("Lazy chunk needs an associated super-chunk."); + return BLOSC2_ERROR_INVALID_PARAM; + } + if (context->schunk->frame == NULL) { + BLOSC_TRACE_ERROR("Lazy chunk needs an associated frame."); + return BLOSC2_ERROR_INVALID_PARAM; + } + blosc2_frame_s* frame = (blosc2_frame_s*)context->schunk->frame; + char* urlpath = frame->urlpath; + size_t trailer_offset = BLOSC_EXTENDED_HEADER_LENGTH + context->nblocks * sizeof(int32_t); + int32_t nchunk; + int64_t chunk_offset; + // The nchunk and the offset of the current chunk are in the trailer + nchunk = *(int32_t*)(src + trailer_offset); + chunk_offset = *(int64_t*)(src + trailer_offset + sizeof(int32_t)); + // Get the csize of the nblock + int32_t *block_csizes = (int32_t *)(src + trailer_offset + sizeof(int32_t) + sizeof(int64_t)); + int32_t block_csize = block_csizes[nblock]; + // Read the lazy block on disk + void* fp = NULL; + blosc2_io_cb *io_cb = blosc2_get_io_cb(context->schunk->storage->io->id); + if (io_cb == NULL) { + BLOSC_TRACE_ERROR("Error getting the input/output API"); + return BLOSC2_ERROR_PLUGIN_IO; + } + + if (frame->sframe) { + // The chunk is not in the frame + char* chunkpath = malloc(strlen(frame->urlpath) + 1 + 8 + strlen(".chunk") + 1); + BLOSC_ERROR_NULL(chunkpath, BLOSC2_ERROR_MEMORY_ALLOC); + sprintf(chunkpath, "%s/%08X.chunk", frame->urlpath, nchunk); + fp = io_cb->open(chunkpath, "rb", context->schunk->storage->io->params); + BLOSC_ERROR_NULL(fp, BLOSC2_ERROR_FILE_OPEN); + free(chunkpath); + // The offset of the block is src_offset + io_cb->seek(fp, src_offset, SEEK_SET); + } + else { + fp = io_cb->open(urlpath, "rb", context->schunk->storage->io->params); + BLOSC_ERROR_NULL(fp, BLOSC2_ERROR_FILE_OPEN); + // The offset of the block is src_offset + io_cb->seek(fp, frame->file_offset + chunk_offset + src_offset, SEEK_SET); + } + // We can make use of tmp3 because it will be used after src is not needed anymore + int64_t rbytes = io_cb->read(tmp3, 1, block_csize, fp); + io_cb->close(fp); + if ((int32_t)rbytes != block_csize) { + BLOSC_TRACE_ERROR("Cannot read the (lazy) block out of the fileframe."); + return BLOSC2_ERROR_READ_BUFFER; + } + src = tmp3; + src_offset = 0; + srcsize = block_csize; + } + + // If the chunk is memcpyed, we just have to copy the block to dest and return + if (memcpyed) { + int bsize_ = leftoverblock ? chunk_nbytes % context->blocksize : bsize; + if (!context->special_type) { + if (chunk_nbytes + context->header_overhead != chunk_cbytes) { + return BLOSC2_ERROR_WRITE_BUFFER; + } + if (chunk_cbytes < context->header_overhead + (nblock * context->blocksize) + bsize_) { + /* Not enough input to copy block */ + return BLOSC2_ERROR_READ_BUFFER; + } + } + if (!is_lazy) { + src += context->header_overhead + nblock * context->blocksize; + } + _dest = dest + dest_offset; + if (context->postfilter != NULL) { + // We are making use of a postfilter, so use a temp for destination + _dest = tmp; + } + rc = 0; + switch (context->special_type) { + case BLOSC2_SPECIAL_VALUE: + // All repeated values + rc = set_values(context->typesize, context->src, _dest, bsize_); + if (rc < 0) { + BLOSC_TRACE_ERROR("set_values failed"); + return BLOSC2_ERROR_DATA; + } + break; + case BLOSC2_SPECIAL_NAN: + rc = set_nans(context->typesize, _dest, bsize_); + if (rc < 0) { + BLOSC_TRACE_ERROR("set_nans failed"); + return BLOSC2_ERROR_DATA; + } + break; + case BLOSC2_SPECIAL_ZERO: + memset(_dest, 0, bsize_); + break; + case BLOSC2_SPECIAL_UNINIT: + // We do nothing here + break; + default: + memcpy(_dest, src, bsize_); + } + if (context->postfilter != NULL) { + // Create new postfilter parameters for this block (must be private for each thread) + blosc2_postfilter_params postparams; + memcpy(&postparams, context->postparams, sizeof(postparams)); + postparams.in = tmp; + postparams.out = dest + dest_offset; + postparams.size = bsize; + postparams.typesize = typesize; + postparams.offset = nblock * context->blocksize; + postparams.nchunk = context->schunk != NULL ? context->schunk->current_nchunk : -1; + postparams.nblock = nblock; + postparams.tid = thread_context->tid; + postparams.ttmp = thread_context->tmp; + postparams.ttmp_nbytes = thread_context->tmp_nbytes; + postparams.ctx = context; + + // Execute the postfilter (the processed block will be copied to dest) + if (context->postfilter(&postparams) != 0) { + BLOSC_TRACE_ERROR("Execution of postfilter function failed"); + return BLOSC2_ERROR_POSTFILTER; + } + } + thread_context->zfp_cell_nitems = 0; + + return bsize_; + } + + if (!is_lazy && (src_offset <= 0 || src_offset >= srcsize)) { + /* Invalid block src offset encountered */ + return BLOSC2_ERROR_DATA; + } + + src += src_offset; + srcsize -= src_offset; + + int last_filter_index = last_filter(filters, 'd'); + if (instr_codec) { + // If instrumented, we don't want to run the filters + _dest = dest + dest_offset; + } + else if (((last_filter_index >= 0) && + (next_filter(filters, BLOSC2_MAX_FILTERS, 'd') != BLOSC_DELTA)) || + context->postfilter != NULL) { + // We are making use of some filter, so use a temp for destination + _dest = tmp; + } + else { + // If no filters, or only DELTA in pipeline + _dest = dest + dest_offset; + } + + /* The number of compressed data streams for this block */ + if (!dont_split && !leftoverblock && !context->use_dict) { + // We don't want to split when in a training dict state + nstreams = (int32_t)typesize; + } + else { + nstreams = 1; + } + + neblock = bsize / nstreams; + if (neblock == 0) { + /* Not enough space to output bytes */ + return -1; + } + for (int j = 0; j < nstreams; j++) { + if (srcsize < (signed)sizeof(int32_t)) { + /* Not enough input to read compressed size */ + return BLOSC2_ERROR_READ_BUFFER; + } + srcsize -= sizeof(int32_t); + cbytes = sw32_(src); /* amount of compressed bytes */ + if (cbytes > 0) { + if (srcsize < cbytes) { + /* Not enough input to read compressed bytes */ + return BLOSC2_ERROR_READ_BUFFER; + } + srcsize -= cbytes; + } + src += sizeof(int32_t); + ctbytes += (signed)sizeof(int32_t); + + /* Uncompress */ + if (cbytes == 0) { + // A run of 0's + memset(_dest, 0, (unsigned int)neblock); + nbytes = neblock; + } + else if (cbytes < 0) { + // A negative number means some encoding depending on the token that comes next + uint8_t token; + + if (srcsize < (signed)sizeof(uint8_t)) { + // Not enough input to read token */ + return BLOSC2_ERROR_READ_BUFFER; + } + srcsize -= sizeof(uint8_t); + + token = src[0]; + src += 1; + ctbytes += 1; + + if (token & 0x1) { + // A run of bytes that are different than 0 + if (cbytes < -255) { + // Runs can only encode a byte + return BLOSC2_ERROR_RUN_LENGTH; + } + uint8_t value = -cbytes; + memset(_dest, value, (unsigned int)neblock); + } else { + BLOSC_TRACE_ERROR("Invalid or unsupported compressed stream token value - %d", token); + return BLOSC2_ERROR_RUN_LENGTH; + } + nbytes = neblock; + cbytes = 0; // everything is encoded in the cbytes token + } + else if (cbytes == neblock) { + memcpy(_dest, src, (unsigned int)neblock); + nbytes = (int32_t)neblock; + } + else { + if (compformat == BLOSC_BLOSCLZ_FORMAT) { + nbytes = blosclz_decompress(src, cbytes, _dest, (int)neblock); + } + else if (compformat == BLOSC_LZ4_FORMAT) { + nbytes = lz4_wrap_decompress((char*)src, (size_t)cbytes, + (char*)_dest, (size_t)neblock); + } + #if defined(HAVE_ZLIB) + else if (compformat == BLOSC_ZLIB_FORMAT) { + nbytes = zlib_wrap_decompress((char*)src, (size_t)cbytes, + (char*)_dest, (size_t)neblock); + } + #endif /* HAVE_ZLIB */ + #if defined(HAVE_ZSTD) + else if (compformat == BLOSC_ZSTD_FORMAT) { + nbytes = zstd_wrap_decompress(thread_context, + (char*)src, (size_t)cbytes, + (char*)_dest, (size_t)neblock); + } + #endif /* HAVE_ZSTD */ + else if (compformat == BLOSC_UDCODEC_FORMAT) { + bool getcell = false; + +#if defined(HAVE_PLUGINS) + if ((context->compcode == BLOSC_CODEC_ZFP_FIXED_RATE) && + (thread_context->zfp_cell_nitems > 0)) { + nbytes = zfp_getcell(thread_context, src, cbytes, _dest, neblock); + if (nbytes < 0) { + return BLOSC2_ERROR_DATA; + } + if (nbytes == thread_context->zfp_cell_nitems * typesize) { + getcell = true; + } + } +#endif /* HAVE_PLUGINS */ + if (!getcell) { + thread_context->zfp_cell_nitems = 0; + for (int i = 0; i < g_ncodecs; ++i) { + if (g_codecs[i].compcode == context->compcode) { + blosc2_dparams dparams; + blosc2_ctx_get_dparams(context, &dparams); + nbytes = g_codecs[i].decoder(src, + cbytes, + _dest, + neblock, + context->compcode_meta, + &dparams, + context->src); + goto urcodecsuccess; + } + } + BLOSC_TRACE_ERROR("User-defined compressor codec %d not found during decompression", context->compcode); + return BLOSC2_ERROR_CODEC_SUPPORT; + } + urcodecsuccess: + ; + } + else { + compname = clibcode_to_clibname(compformat); + BLOSC_TRACE_ERROR( + "Blosc has not been compiled with decompression " + "support for '%s' format. " + "Please recompile for adding this support.", compname); + return BLOSC2_ERROR_CODEC_SUPPORT; + } + + /* Check that decompressed bytes number is correct */ + if ((nbytes != neblock) && (thread_context->zfp_cell_nitems == 0)) { + return BLOSC2_ERROR_DATA; + } + + } + src += cbytes; + ctbytes += cbytes; + _dest += nbytes; + ntbytes += nbytes; + } /* Closes j < nstreams */ + + if (!instr_codec) { + if (last_filter_index >= 0 || context->postfilter != NULL) { + /* Apply regular filter pipeline */ + int errcode = pipeline_backward(thread_context, bsize, dest, dest_offset, tmp, tmp2, tmp3, + last_filter_index, nblock); + if (errcode < 0) + return errcode; + } + } + + /* Return the number of uncompressed bytes */ + return (int)ntbytes; +} + + +/* Serial version for compression/decompression */ +static int serial_blosc(struct thread_context* thread_context) { + blosc2_context* context = thread_context->parent_context; + int32_t j, bsize, leftoverblock; + int32_t cbytes; + int32_t ntbytes = (int32_t)context->output_bytes; + int32_t* bstarts = context->bstarts; + uint8_t* tmp = thread_context->tmp; + uint8_t* tmp2 = thread_context->tmp2; + int dict_training = context->use_dict && (context->dict_cdict == NULL); + bool memcpyed = context->header_flags & (uint8_t)BLOSC_MEMCPYED; + if (!context->do_compress && context->special_type) { + // Fake a runlen as if it was a memcpyed chunk + memcpyed = true; + } + + for (j = 0; j < context->nblocks; j++) { + if (context->do_compress && !memcpyed && !dict_training) { + _sw32(bstarts + j, ntbytes); + } + bsize = context->blocksize; + leftoverblock = 0; + if ((j == context->nblocks - 1) && (context->leftover > 0)) { + bsize = context->leftover; + leftoverblock = 1; + } + if (context->do_compress) { + if (memcpyed && !context->prefilter) { + /* We want to memcpy only */ + memcpy(context->dest + context->header_overhead + j * context->blocksize, + context->src + j * context->blocksize, (unsigned int)bsize); + cbytes = (int32_t)bsize; + } + else { + /* Regular compression */ + cbytes = blosc_c(thread_context, bsize, leftoverblock, ntbytes, + context->destsize, context->src, j * context->blocksize, + context->dest + ntbytes, tmp, tmp2); + if (cbytes == 0) { + ntbytes = 0; /* uncompressible data */ + break; + } + } + } + else { + /* Regular decompression */ + // If memcpyed we don't have a bstarts section (because it is not needed) + int32_t src_offset = memcpyed ? + context->header_overhead + j * context->blocksize : sw32_(bstarts + j); + cbytes = blosc_d(thread_context, bsize, leftoverblock, memcpyed, + context->src, context->srcsize, src_offset, j, + context->dest, j * context->blocksize, tmp, tmp2); + } + + if (cbytes < 0) { + ntbytes = cbytes; /* error in blosc_c or blosc_d */ + break; + } + ntbytes += cbytes; + } + + return ntbytes; +} + +static void t_blosc_do_job(void *ctxt); + +/* Threaded version for compression/decompression */ +static int parallel_blosc(blosc2_context* context) { +#ifdef BLOSC_POSIX_BARRIERS + int rc; +#endif + /* Set sentinels */ + context->thread_giveup_code = 1; + context->thread_nblock = -1; + + if (threads_callback) { + threads_callback(threads_callback_data, t_blosc_do_job, + context->nthreads, sizeof(struct thread_context), (void*) context->thread_contexts); + } + else { + /* Synchronization point for all threads (wait for initialization) */ + WAIT_INIT(-1, context); + + /* Synchronization point for all threads (wait for finalization) */ + WAIT_FINISH(-1, context); + } + + if (context->thread_giveup_code <= 0) { + /* Compression/decompression gave up. Return error code. */ + return context->thread_giveup_code; + } + + /* Return the total bytes (de-)compressed in threads */ + return (int)context->output_bytes; +} + +/* initialize a thread_context that has already been allocated */ +static int init_thread_context(struct thread_context* thread_context, blosc2_context* context, int32_t tid) +{ + int32_t ebsize; + + thread_context->parent_context = context; + thread_context->tid = tid; + + ebsize = context->blocksize + context->typesize * (signed)sizeof(int32_t); + thread_context->tmp_nbytes = (size_t)4 * ebsize; + thread_context->tmp = my_malloc(thread_context->tmp_nbytes); + BLOSC_ERROR_NULL(thread_context->tmp, BLOSC2_ERROR_MEMORY_ALLOC); + thread_context->tmp2 = thread_context->tmp + ebsize; + thread_context->tmp3 = thread_context->tmp2 + ebsize; + thread_context->tmp4 = thread_context->tmp3 + ebsize; + thread_context->tmp_blocksize = context->blocksize; + thread_context->zfp_cell_nitems = 0; + thread_context->zfp_cell_start = 0; + #if defined(HAVE_ZSTD) + thread_context->zstd_cctx = NULL; + thread_context->zstd_dctx = NULL; + #endif + + /* Create the hash table for LZ4 in case we are using IPP */ +#ifdef HAVE_IPP + IppStatus status; + int inlen = thread_context->tmp_blocksize > 0 ? thread_context->tmp_blocksize : 1 << 16; + int hash_size = 0; + status = ippsEncodeLZ4HashTableGetSize_8u(&hash_size); + if (status != ippStsNoErr) { + BLOSC_TRACE_ERROR("Error in ippsEncodeLZ4HashTableGetSize_8u."); + } + Ipp8u *hash_table = ippsMalloc_8u(hash_size); + status = ippsEncodeLZ4HashTableInit_8u(hash_table, inlen); + if (status != ippStsNoErr) { + BLOSC_TRACE_ERROR("Error in ippsEncodeLZ4HashTableInit_8u."); + } + thread_context->lz4_hash_table = hash_table; +#endif + return 0; +} + +static struct thread_context* +create_thread_context(blosc2_context* context, int32_t tid) { + struct thread_context* thread_context; + thread_context = (struct thread_context*)my_malloc(sizeof(struct thread_context)); + BLOSC_ERROR_NULL(thread_context, NULL); + int rc = init_thread_context(thread_context, context, tid); + if (rc < 0) { + return NULL; + } + return thread_context; +} + +/* free members of thread_context, but not thread_context itself */ +static void destroy_thread_context(struct thread_context* thread_context) { + my_free(thread_context->tmp); +#if defined(HAVE_ZSTD) + if (thread_context->zstd_cctx != NULL) { + ZSTD_freeCCtx(thread_context->zstd_cctx); + } + if (thread_context->zstd_dctx != NULL) { + ZSTD_freeDCtx(thread_context->zstd_dctx); + } +#endif +#ifdef HAVE_IPP + if (thread_context->lz4_hash_table != NULL) { + ippsFree(thread_context->lz4_hash_table); + } +#endif +} + +void free_thread_context(struct thread_context* thread_context) { + destroy_thread_context(thread_context); + my_free(thread_context); +} + + +int check_nthreads(blosc2_context* context) { + if (context->nthreads <= 0) { + BLOSC_TRACE_ERROR("nthreads must be a positive integer."); + return BLOSC2_ERROR_INVALID_PARAM; + } + + if (context->new_nthreads != context->nthreads) { + if (context->nthreads > 1) { + release_threadpool(context); + } + context->nthreads = context->new_nthreads; + } + if (context->new_nthreads > 1 && context->threads_started == 0) { + init_threadpool(context); + } + + return context->nthreads; +} + +/* Do the compression or decompression of the buffer depending on the + global params. */ +static int do_job(blosc2_context* context) { + int32_t ntbytes; + + /* Set sentinels */ + context->dref_not_init = 1; + + /* Check whether we need to restart threads */ + check_nthreads(context); + + /* Run the serial version when nthreads is 1 or when the buffers are + not larger than blocksize */ + if (context->nthreads == 1 || (context->sourcesize / context->blocksize) <= 1) { + /* The context for this 'thread' has no been initialized yet */ + if (context->serial_context == NULL) { + context->serial_context = create_thread_context(context, 0); + } + else if (context->blocksize != context->serial_context->tmp_blocksize) { + free_thread_context(context->serial_context); + context->serial_context = create_thread_context(context, 0); + } + BLOSC_ERROR_NULL(context->serial_context, BLOSC2_ERROR_THREAD_CREATE); + ntbytes = serial_blosc(context->serial_context); + } + else { + ntbytes = parallel_blosc(context); + } + + return ntbytes; +} + + +static int initialize_context_compression( + blosc2_context* context, const void* src, int32_t srcsize, void* dest, + int32_t destsize, int clevel, uint8_t const *filters, + uint8_t const *filters_meta, int32_t typesize, int compressor, + int32_t blocksize, int16_t new_nthreads, int16_t nthreads, + int32_t splitmode, + blosc2_btune *udbtune, void *btune_config, + blosc2_schunk* schunk) { + + /* Set parameters */ + context->do_compress = 1; + context->src = (const uint8_t*)src; + context->srcsize = srcsize; + context->dest = (uint8_t*)dest; + context->output_bytes = 0; + context->destsize = destsize; + context->sourcesize = srcsize; + context->typesize = (int32_t)typesize; + context->filter_flags = filters_to_flags(filters); + for (int i = 0; i < BLOSC2_MAX_FILTERS; i++) { + context->filters[i] = filters[i]; + context->filters_meta[i] = filters_meta[i]; + } + context->compcode = compressor; + context->nthreads = nthreads; + context->new_nthreads = new_nthreads; + context->end_threads = 0; + context->clevel = clevel; + context->schunk = schunk; + context->btune = btune_config; + context->udbtune = udbtune; + context->splitmode = splitmode; + /* Tune some compression parameters */ + context->blocksize = (int32_t)blocksize; + if (context->btune != NULL) { + context->udbtune->btune_next_cparams(context); + } else { + context->udbtune->btune_next_blocksize(context); + } + + char* envvar = getenv("BLOSC_WARN"); + int64_t warnlvl = 0; + if (envvar != NULL) { + warnlvl = strtol(envvar, NULL, 10); + } + + /* Check buffer size limits */ + if (srcsize > BLOSC2_MAX_BUFFERSIZE) { + if (warnlvl > 0) { + BLOSC_TRACE_ERROR("Input buffer size cannot exceed %d bytes.", + BLOSC2_MAX_BUFFERSIZE); + } + return 0; + } + + if (destsize < BLOSC2_MAX_OVERHEAD) { + if (warnlvl > 0) { + BLOSC_TRACE_ERROR("Output buffer size should be larger than %d bytes.", + BLOSC2_MAX_OVERHEAD); + } + return 0; + } + + /* Compression level */ + if (clevel < 0 || clevel > 9) { + /* If clevel not in 0..9, print an error */ + BLOSC_TRACE_ERROR("`clevel` parameter must be between 0 and 9!."); + return BLOSC2_ERROR_CODEC_PARAM; + } + + /* Check typesize limits */ + if (context->typesize > BLOSC_MAX_TYPESIZE) { + /* If typesize is too large, treat buffer as an 1-byte stream. */ + context->typesize = 1; + } + + blosc2_calculate_blocks(context); + + return 1; +} + + +static int initialize_context_decompression(blosc2_context* context, blosc_header* header, const void* src, + int32_t srcsize, void* dest, int32_t destsize) { + int32_t bstarts_end; + + context->do_compress = 0; + context->src = (const uint8_t*)src; + context->srcsize = srcsize; + context->dest = (uint8_t*)dest; + context->destsize = destsize; + context->output_bytes = 0; + context->end_threads = 0; + + int rc = blosc2_initialize_context_from_header(context, header); + if (rc < 0) { + return rc; + } + + /* Check that we have enough space to decompress */ + if (context->sourcesize > (int32_t)context->destsize) { + return BLOSC2_ERROR_WRITE_BUFFER; + } + + if (context->block_maskout != NULL && context->block_maskout_nitems != context->nblocks) { + BLOSC_TRACE_ERROR("The number of items in block_maskout (%d) must match the number" + " of blocks in chunk (%d).", + context->block_maskout_nitems, context->nblocks); + return BLOSC2_ERROR_DATA; + } + + context->special_type = (header->blosc2_flags >> 4) & BLOSC2_SPECIAL_MASK; + if (context->special_type > BLOSC2_SPECIAL_LASTID) { + BLOSC_TRACE_ERROR("Unknown special values ID (%d) ", + context->special_type); + return BLOSC2_ERROR_DATA; + } + + int memcpyed = (context->header_flags & (uint8_t) BLOSC_MEMCPYED); + if (memcpyed && (header->cbytes != header->nbytes + context->header_overhead)) { + BLOSC_TRACE_ERROR("Wrong header info for this memcpyed chunk"); + return BLOSC2_ERROR_DATA; + } + + if ((header->nbytes == 0) && (header->cbytes == context->header_overhead) && + !context->special_type) { + // A compressed buffer with only a header can only contain a zero-length buffer + return 0; + } + + context->bstarts = (int32_t *) (context->src + context->header_overhead); + bstarts_end = context->header_overhead; + if (!context->special_type && !memcpyed) { + /* If chunk is not special or a memcpyed, we do have a bstarts section */ + bstarts_end = (int32_t)(context->header_overhead + (context->nblocks * sizeof(int32_t))); + } + + if (srcsize < bstarts_end) { + BLOSC_TRACE_ERROR("`bstarts` exceeds length of source buffer."); + return BLOSC2_ERROR_READ_BUFFER; + } + srcsize -= bstarts_end; + + /* Read optional dictionary if flag set */ + if (context->blosc2_flags & BLOSC2_USEDICT) { +#if defined(HAVE_ZSTD) + context->use_dict = 1; + if (context->dict_ddict != NULL) { + // Free the existing dictionary (probably from another chunk) + ZSTD_freeDDict(context->dict_ddict); + } + // The trained dictionary is after the bstarts block + if (srcsize < (signed)sizeof(int32_t)) { + BLOSC_TRACE_ERROR("Not enough space to read size of dictionary."); + return BLOSC2_ERROR_READ_BUFFER; + } + srcsize -= sizeof(int32_t); + // Read dictionary size + context->dict_size = sw32_(context->src + bstarts_end); + if (context->dict_size <= 0 || context->dict_size > BLOSC2_MAXDICTSIZE) { + BLOSC_TRACE_ERROR("Dictionary size is smaller than minimum or larger than maximum allowed."); + return BLOSC2_ERROR_CODEC_DICT; + } + if (srcsize < (int32_t)context->dict_size) { + BLOSC_TRACE_ERROR("Not enough space to read entire dictionary."); + return BLOSC2_ERROR_READ_BUFFER; + } + srcsize -= context->dict_size; + // Read dictionary + context->dict_buffer = (void*)(context->src + bstarts_end + sizeof(int32_t)); + context->dict_ddict = ZSTD_createDDict(context->dict_buffer, context->dict_size); +#endif // HAVE_ZSTD + } + + return 0; +} + +static int write_compression_header(blosc2_context* context, bool extended_header) { + blosc_header header; + int dont_split; + int dict_training = context->use_dict && (context->dict_cdict == NULL); + + context->header_flags = 0; + + if (context->clevel == 0) { + /* Compression level 0 means buffer to be memcpy'ed */ + context->header_flags |= (uint8_t)BLOSC_MEMCPYED; + } + if (context->sourcesize < BLOSC_MIN_BUFFERSIZE) { + /* Buffer is too small. Try memcpy'ing. */ + context->header_flags |= (uint8_t)BLOSC_MEMCPYED; + } + + bool memcpyed = context->header_flags & (uint8_t)BLOSC_MEMCPYED; + if (extended_header) { + /* Indicate that we are building an extended header */ + context->header_overhead = BLOSC_EXTENDED_HEADER_LENGTH; + context->header_flags |= (BLOSC_DOSHUFFLE | BLOSC_DOBITSHUFFLE); + /* Store filter pipeline info at the end of the header */ + if (dict_training || memcpyed) { + context->bstarts = NULL; + context->output_bytes = context->header_overhead; + } else { + context->bstarts = (int32_t*)(context->dest + context->header_overhead); + context->output_bytes = context->header_overhead + (int32_t)sizeof(int32_t) * context->nblocks; + } + } else { + // Regular header + context->header_overhead = BLOSC_MIN_HEADER_LENGTH; + if (memcpyed) { + context->bstarts = NULL; + context->output_bytes = context->header_overhead; + } else { + context->bstarts = (int32_t *) (context->dest + context->header_overhead); + context->output_bytes = context->header_overhead + (int32_t)sizeof(int32_t) * context->nblocks; + } + } + + // when memcpyed bit is set, there is no point in dealing with others + if (!memcpyed) { + if (context->filter_flags & BLOSC_DOSHUFFLE) { + /* Byte-shuffle is active */ + context->header_flags |= BLOSC_DOSHUFFLE; + } + + if (context->filter_flags & BLOSC_DOBITSHUFFLE) { + /* Bit-shuffle is active */ + context->header_flags |= BLOSC_DOBITSHUFFLE; + } + + if (context->filter_flags & BLOSC_DODELTA) { + /* Delta is active */ + context->header_flags |= BLOSC_DODELTA; + } + + dont_split = !split_block(context, context->typesize, + context->blocksize); + + /* dont_split is in bit 4 */ + context->header_flags |= dont_split << 4; + /* codec starts at bit 5 */ + uint8_t compformat = compcode_to_compformat(context->compcode); + context->header_flags |= compformat << 5; + } + + // Create blosc header and store to dest + blosc2_intialize_header_from_context(context, &header, extended_header); + + memcpy(context->dest, &header, (extended_header) ? + BLOSC_EXTENDED_HEADER_LENGTH : BLOSC_MIN_HEADER_LENGTH); + + return 1; +} + + +static int blosc_compress_context(blosc2_context* context) { + int ntbytes = 0; + blosc_timestamp_t last, current; + bool memcpyed = context->header_flags & (uint8_t)BLOSC_MEMCPYED; + + blosc_set_timestamp(&last); + + if (!memcpyed) { + /* Do the actual compression */ + ntbytes = do_job(context); + if (ntbytes < 0) { + return ntbytes; + } + if (ntbytes == 0) { + // Try out with a memcpy later on (last chance for fitting src buffer in dest). + context->header_flags |= (uint8_t)BLOSC_MEMCPYED; + memcpyed = true; + } + } + + int dont_split = (context->header_flags & 0x10) >> 4; + int nstreams = context->nblocks; + if (!dont_split) { + // When splitting, the number of streams is computed differently + if (context->leftover) { + nstreams = (context->nblocks - 1) * context->typesize + 1; + } + else { + nstreams *= context->typesize; + } + } + + if (memcpyed) { + if (context->sourcesize + context->header_overhead > context->destsize) { + /* We are exceeding maximum output size */ + ntbytes = 0; + } + else { + context->output_bytes = context->header_overhead; + ntbytes = do_job(context); + if (ntbytes < 0) { + return ntbytes; + } + // Success! update the memcpy bit in header + context->dest[BLOSC2_CHUNK_FLAGS] = context->header_flags; + // and clear the memcpy bit in context (for next reuse) + context->header_flags &= ~(uint8_t)BLOSC_MEMCPYED; + } + } + else { + // Check whether we have a run for the whole chunk + int start_csizes = context->header_overhead + 4 * context->nblocks; + if (ntbytes == (int)(start_csizes + nstreams * sizeof(int32_t))) { + // The streams are all zero runs (by construction). Encode it... + context->dest[BLOSC2_CHUNK_BLOSC2_FLAGS] |= BLOSC2_SPECIAL_ZERO << 4; + // ...and assign the new chunk length + ntbytes = context->header_overhead; + } + } + + /* Set the number of compressed bytes in header */ + _sw32(context->dest + BLOSC2_CHUNK_CBYTES, ntbytes); + if (context->blosc2_flags & BLOSC2_INSTR_CODEC) { + dont_split = (context->header_flags & 0x10) >> 4; + int32_t blocksize = dont_split ? (int32_t)sizeof(blosc2_instr) : (int32_t)sizeof(blosc2_instr) * context->typesize; + _sw32(context->dest + BLOSC2_CHUNK_NBYTES, nstreams * (int32_t)sizeof(blosc2_instr)); + _sw32(context->dest + BLOSC2_CHUNK_BLOCKSIZE, blocksize); + } + + /* Set the number of bytes in dest buffer (might be useful for btune) */ + context->destsize = ntbytes; + + if (context->btune != NULL) { + blosc_set_timestamp(¤t); + double ctime = blosc_elapsed_secs(last, current); + context->udbtune->btune_update(context, ctime); + } + + return ntbytes; +} + + +/* The public secure routine for compression with context. */ +int blosc2_compress_ctx(blosc2_context* context, const void* src, int32_t srcsize, + void* dest, int32_t destsize) { + int error, cbytes; + + if (context->do_compress != 1) { + BLOSC_TRACE_ERROR("Context is not meant for compression. Giving up."); + return BLOSC2_ERROR_INVALID_PARAM; + } + + error = initialize_context_compression( + context, src, srcsize, dest, destsize, + context->clevel, context->filters, context->filters_meta, + context->typesize, context->compcode, context->blocksize, + context->new_nthreads, context->nthreads, context->splitmode, + context->udbtune, context->btune, context->schunk); + if (error <= 0) { + return error; + } + + /* Write the extended header */ + error = write_compression_header(context, true); + if (error < 0) { + return error; + } + + cbytes = blosc_compress_context(context); + if (cbytes < 0) { + return cbytes; + } + + if (context->use_dict && context->dict_cdict == NULL) { + + if (context->compcode != BLOSC_ZSTD) { + const char* compname; + compname = clibcode_to_clibname(context->compcode); + BLOSC_TRACE_ERROR("Codec %s does not support dicts. Giving up.", + compname); + return BLOSC2_ERROR_CODEC_DICT; + } + +#ifdef HAVE_ZSTD + // Build the dictionary out of the filters outcome and compress with it + int32_t dict_maxsize = BLOSC2_MAXDICTSIZE; + // Do not make the dict more than 5% larger than uncompressed buffer + if (dict_maxsize > srcsize / 20) { + dict_maxsize = srcsize / 20; + } + void* samples_buffer = context->dest + context->header_overhead; + unsigned nblocks = 8; // the minimum that accepts zstd as of 1.4.0 + unsigned sample_fraction = 1; // 1 allows to use most of the chunk for training + size_t sample_size = context->sourcesize / nblocks / sample_fraction; + + // Populate the samples sizes for training the dictionary + size_t* samples_sizes = malloc(nblocks * sizeof(void*)); + BLOSC_ERROR_NULL(samples_sizes, BLOSC2_ERROR_MEMORY_ALLOC); + for (size_t i = 0; i < nblocks; i++) { + samples_sizes[i] = sample_size; + } + + // Train from samples + void* dict_buffer = malloc(dict_maxsize); + BLOSC_ERROR_NULL(dict_buffer, BLOSC2_ERROR_MEMORY_ALLOC); + int32_t dict_actual_size = (int32_t)ZDICT_trainFromBuffer(dict_buffer, dict_maxsize, samples_buffer, samples_sizes, nblocks); + + // TODO: experiment with parameters of low-level fast cover algorithm + // Note that this API is still unstable. See: https://github.com/facebook/zstd/issues/1599 + // ZDICT_fastCover_params_t fast_cover_params; + // memset(&fast_cover_params, 0, sizeof(fast_cover_params)); + // fast_cover_params.d = nblocks; + // fast_cover_params.steps = 4; + // fast_cover_params.zParams.compressionLevel = context->clevel; + //size_t dict_actual_size = ZDICT_optimizeTrainFromBuffer_fastCover(dict_buffer, dict_maxsize, samples_buffer, samples_sizes, nblocks, &fast_cover_params); + + if (ZDICT_isError(dict_actual_size) != ZSTD_error_no_error) { + BLOSC_TRACE_ERROR("Error in ZDICT_trainFromBuffer(): '%s'." + " Giving up.", ZDICT_getErrorName(dict_actual_size)); + return BLOSC2_ERROR_CODEC_DICT; + } + assert(dict_actual_size > 0); + free(samples_sizes); + + // Update bytes counter and pointers to bstarts for the new compressed buffer + context->bstarts = (int32_t*)(context->dest + context->header_overhead); + context->output_bytes = context->header_overhead + (int32_t)sizeof(int32_t) * context->nblocks; + /* Write the size of trained dict at the end of bstarts */ + _sw32(context->dest + context->output_bytes, (int32_t)dict_actual_size); + context->output_bytes += sizeof(int32_t); + /* Write the trained dict afterwards */ + context->dict_buffer = context->dest + context->output_bytes; + memcpy(context->dict_buffer, dict_buffer, (unsigned int)dict_actual_size); + context->dict_cdict = ZSTD_createCDict(dict_buffer, dict_actual_size, 1); // TODO: use get_accel() + free(dict_buffer); // the dictionary is copied in the header now + context->output_bytes += (int32_t)dict_actual_size; + context->dict_size = dict_actual_size; + + /* Compress with dict */ + cbytes = blosc_compress_context(context); + + // Invalidate the dictionary for compressing other chunks using the same context + context->dict_buffer = NULL; + ZSTD_freeCDict(context->dict_cdict); + context->dict_cdict = NULL; +#endif // HAVE_ZSTD + } + + return cbytes; +} + + +void build_filters(const int doshuffle, const int delta, + const size_t typesize, uint8_t* filters) { + + /* Fill the end part of the filter pipeline */ + if ((doshuffle == BLOSC_SHUFFLE) && (typesize > 1)) + filters[BLOSC2_MAX_FILTERS - 1] = BLOSC_SHUFFLE; + if (doshuffle == BLOSC_BITSHUFFLE) + filters[BLOSC2_MAX_FILTERS - 1] = BLOSC_BITSHUFFLE; + if (delta) + filters[BLOSC2_MAX_FILTERS - 2] = BLOSC_DELTA; +} + +/* The public secure routine for compression. */ +int blosc2_compress(int clevel, int doshuffle, int32_t typesize, + const void* src, int32_t srcsize, void* dest, int32_t destsize) { + int error; + int result; + char* envvar; + + /* Check whether the library should be initialized */ + if (!g_initlib) blosc2_init(); + + /* Check for a BLOSC_CLEVEL environment variable */ + envvar = getenv("BLOSC_CLEVEL"); + if (envvar != NULL) { + long value; + value = strtol(envvar, NULL, 10); + if ((value != EINVAL) && (value >= 0)) { + clevel = (int)value; + } + } + + /* Check for a BLOSC_SHUFFLE environment variable */ + envvar = getenv("BLOSC_SHUFFLE"); + if (envvar != NULL) { + if (strcmp(envvar, "NOSHUFFLE") == 0) { + doshuffle = BLOSC_NOSHUFFLE; + } + if (strcmp(envvar, "SHUFFLE") == 0) { + doshuffle = BLOSC_SHUFFLE; + } + if (strcmp(envvar, "BITSHUFFLE") == 0) { + doshuffle = BLOSC_BITSHUFFLE; + } + } + + /* Check for a BLOSC_DELTA environment variable */ + envvar = getenv("BLOSC_DELTA"); + if (envvar != NULL) { + if (strcmp(envvar, "1") == 0) { + blosc2_set_delta(1); + } else { + blosc2_set_delta(0); + } + } + + /* Check for a BLOSC_TYPESIZE environment variable */ + envvar = getenv("BLOSC_TYPESIZE"); + if (envvar != NULL) { + long value; + value = strtol(envvar, NULL, 10); + if ((value != EINVAL) && (value > 0)) { + typesize = (int32_t)value; + } + } + + /* Check for a BLOSC_COMPRESSOR environment variable */ + envvar = getenv("BLOSC_COMPRESSOR"); + if (envvar != NULL) { + result = blosc1_set_compressor(envvar); + if (result < 0) { return result; } + } + + /* Check for a BLOSC_COMPRESSOR environment variable */ + envvar = getenv("BLOSC_BLOCKSIZE"); + if (envvar != NULL) { + long blocksize; + blocksize = strtol(envvar, NULL, 10); + if ((blocksize != EINVAL) && (blocksize > 0)) { + blosc1_set_blocksize((size_t) blocksize); + } + } + + /* Check for a BLOSC_NTHREADS environment variable */ + envvar = getenv("BLOSC_NTHREADS"); + if (envvar != NULL) { + long nthreads; + nthreads = strtol(envvar, NULL, 10); + if ((nthreads != EINVAL) && (nthreads > 0)) { + result = blosc2_set_nthreads((int16_t) nthreads); + if (result < 0) { return result; } + } + } + + /* Check for a BLOSC_NOLOCK environment variable. It is important + that this should be the last env var so that it can take the + previous ones into account */ + envvar = getenv("BLOSC_NOLOCK"); + if (envvar != NULL) { + // TODO: here is the only place that returns an extended header from + // a blosc1_compress() call. This should probably be fixed. + const char *compname; + blosc2_context *cctx; + blosc2_cparams cparams = BLOSC2_CPARAMS_DEFAULTS; + + blosc2_compcode_to_compname(g_compressor, &compname); + /* Create a context for compression */ + build_filters(doshuffle, g_delta, typesize, cparams.filters); + // TODO: cparams can be shared in a multithreaded environment. do a copy! + cparams.typesize = (uint8_t)typesize; + cparams.compcode = (uint8_t)g_compressor; + cparams.clevel = (uint8_t)clevel; + cparams.nthreads = g_nthreads; + cparams.splitmode = g_splitmode; + cctx = blosc2_create_cctx(cparams); + /* Do the actual compression */ + result = blosc2_compress_ctx(cctx, src, srcsize, dest, destsize); + /* Release context resources */ + blosc2_free_ctx(cctx); + return result; + } + + pthread_mutex_lock(&global_comp_mutex); + + /* Initialize a context compression */ + uint8_t* filters = calloc(1, BLOSC2_MAX_FILTERS); + BLOSC_ERROR_NULL(filters, BLOSC2_ERROR_MEMORY_ALLOC); + uint8_t* filters_meta = calloc(1, BLOSC2_MAX_FILTERS); + BLOSC_ERROR_NULL(filters_meta, BLOSC2_ERROR_MEMORY_ALLOC); + build_filters(doshuffle, g_delta, typesize, filters); + error = initialize_context_compression( + g_global_context, src, srcsize, dest, destsize, clevel, filters, + filters_meta, (int32_t)typesize, g_compressor, g_force_blocksize, g_nthreads, g_nthreads, + g_splitmode, &BTUNE_DEFAULTS, NULL, g_schunk); + free(filters); + free(filters_meta); + if (error <= 0) { + pthread_mutex_unlock(&global_comp_mutex); + return error; + } + + envvar = getenv("BLOSC_BLOSC1_COMPAT"); + if (envvar != NULL) { + /* Write chunk header without extended header (Blosc1 compatibility mode) */ + error = write_compression_header(g_global_context, false); + } + else { + error = write_compression_header(g_global_context, true); + } + if (error < 0) { + pthread_mutex_unlock(&global_comp_mutex); + return error; + } + + result = blosc_compress_context(g_global_context); + + pthread_mutex_unlock(&global_comp_mutex); + + return result; +} + + +/* The public routine for compression. */ +int blosc1_compress(int clevel, int doshuffle, size_t typesize, size_t nbytes, + const void* src, void* dest, size_t destsize) { + return blosc2_compress(clevel, doshuffle, (int32_t)typesize, src, (int32_t)nbytes, dest, (int32_t)destsize); +} + + + +static int blosc_run_decompression_with_context(blosc2_context* context, const void* src, int32_t srcsize, + void* dest, int32_t destsize) { + blosc_header header; + int32_t ntbytes; + int rc; + + rc = read_chunk_header(src, srcsize, true, &header); + if (rc < 0) { + return rc; + } + + if (header.nbytes > destsize) { + // Not enough space for writing into the destination + return BLOSC2_ERROR_WRITE_BUFFER; + } + + rc = initialize_context_decompression(context, &header, src, srcsize, dest, destsize); + if (rc < 0) { + return rc; + } + + /* Do the actual decompression */ + ntbytes = do_job(context); + if (ntbytes < 0) { + return ntbytes; + } + + assert(ntbytes <= (int32_t)destsize); + return ntbytes; +} + + +/* The public secure routine for decompression with context. */ +int blosc2_decompress_ctx(blosc2_context* context, const void* src, int32_t srcsize, + void* dest, int32_t destsize) { + int result; + + if (context->do_compress != 0) { + BLOSC_TRACE_ERROR("Context is not meant for decompression. Giving up."); + return BLOSC2_ERROR_INVALID_PARAM; + } + + result = blosc_run_decompression_with_context(context, src, srcsize, dest, destsize); + + // Reset a possible block_maskout + if (context->block_maskout != NULL) { + free(context->block_maskout); + context->block_maskout = NULL; + } + context->block_maskout_nitems = 0; + + return result; +} + + +/* The public secure routine for decompression. */ +int blosc2_decompress(const void* src, int32_t srcsize, void* dest, int32_t destsize) { + int result; + char* envvar; + long nthreads; + blosc2_context *dctx; + blosc2_dparams dparams = BLOSC2_DPARAMS_DEFAULTS; + + /* Check whether the library should be initialized */ + if (!g_initlib) blosc2_init(); + + /* Check for a BLOSC_NTHREADS environment variable */ + envvar = getenv("BLOSC_NTHREADS"); + if (envvar != NULL) { + nthreads = strtol(envvar, NULL, 10); + if ((nthreads != EINVAL) && (nthreads > 0)) { + result = blosc2_set_nthreads((int16_t) nthreads); + if (result < 0) { return result; } + } + } + + /* Check for a BLOSC_NOLOCK environment variable. It is important + that this should be the last env var so that it can take the + previous ones into account */ + envvar = getenv("BLOSC_NOLOCK"); + if (envvar != NULL) { + dparams.nthreads = g_nthreads; + dctx = blosc2_create_dctx(dparams); + result = blosc2_decompress_ctx(dctx, src, srcsize, dest, destsize); + blosc2_free_ctx(dctx); + return result; + } + + pthread_mutex_lock(&global_comp_mutex); + + result = blosc_run_decompression_with_context( + g_global_context, src, srcsize, dest, destsize); + + pthread_mutex_unlock(&global_comp_mutex); + + return result; +} + + +/* The public routine for decompression. */ +int blosc1_decompress(const void* src, void* dest, size_t destsize) { + return blosc2_decompress(src, INT32_MAX, dest, (int32_t)destsize); +} + + +/* Specific routine optimized for decompression a small number of + items out of a compressed chunk. This does not use threads because + it would affect negatively to performance. */ +int _blosc_getitem(blosc2_context* context, blosc_header* header, const void* src, int32_t srcsize, + int start, int nitems, void* dest, int32_t destsize) { + uint8_t* _src = (uint8_t*)(src); /* current pos for source buffer */ + uint8_t* _dest = (uint8_t*)(dest); + int32_t ntbytes = 0; /* the number of uncompressed bytes */ + int32_t bsize, bsize2, ebsize, leftoverblock; + int32_t startb, stopb; + int32_t stop = start + nitems; + int j, rc; + + if (nitems == 0) { + // We have nothing to do + return 0; + } + if (nitems * header->typesize > destsize) { + BLOSC_TRACE_ERROR("`nitems`*`typesize` out of dest bounds."); + return BLOSC2_ERROR_WRITE_BUFFER; + } + + context->bstarts = (int32_t*)(_src + context->header_overhead); + + /* Check region boundaries */ + if ((start < 0) || (start * header->typesize > header->nbytes)) { + BLOSC_TRACE_ERROR("`start` out of bounds."); + return BLOSC2_ERROR_INVALID_PARAM; + } + + if ((stop < 0) || (stop * header->typesize > header->nbytes)) { + BLOSC_TRACE_ERROR("`start`+`nitems` out of bounds."); + return BLOSC2_ERROR_INVALID_PARAM; + } + + int chunk_memcpy = header->flags & 0x1; + if (!context->special_type && !chunk_memcpy && + ((uint8_t *)(_src + srcsize) < (uint8_t *)(context->bstarts + context->nblocks))) { + BLOSC_TRACE_ERROR("`bstarts` out of bounds."); + return BLOSC2_ERROR_READ_BUFFER; + } + + bool memcpyed = header->flags & (uint8_t)BLOSC_MEMCPYED; + if (context->special_type) { + // Fake a runlen as if its a memcpyed chunk + memcpyed = true; + } + + bool is_lazy = ((context->header_overhead == BLOSC_EXTENDED_HEADER_LENGTH) && + (context->blosc2_flags & 0x08u) && !context->special_type); + if (memcpyed && !is_lazy && !context->postfilter) { + // Short-circuit for (non-lazy) memcpyed or special values + ntbytes = nitems * header->typesize; + switch (context->special_type) { + case BLOSC2_SPECIAL_VALUE: + // All repeated values + rc = set_values(context->typesize, _src, _dest, ntbytes); + if (rc < 0) { + BLOSC_TRACE_ERROR("set_values failed"); + return BLOSC2_ERROR_DATA; + } + break; + case BLOSC2_SPECIAL_NAN: + rc = set_nans(context->typesize, _dest, ntbytes); + if (rc < 0) { + BLOSC_TRACE_ERROR("set_nans failed"); + return BLOSC2_ERROR_DATA; + } + break; + case BLOSC2_SPECIAL_ZERO: + memset(_dest, 0, ntbytes); + break; + case BLOSC2_SPECIAL_UNINIT: + // We do nothing here + break; + case BLOSC2_NO_SPECIAL: + _src += context->header_overhead + start * context->typesize; + memcpy(_dest, _src, ntbytes); + break; + default: + BLOSC_TRACE_ERROR("Unhandled special value case"); + return -1; + } + return ntbytes; + } + + ebsize = header->blocksize + header->typesize * (signed)sizeof(int32_t); + struct thread_context* scontext = context->serial_context; + /* Resize the temporaries in serial context if needed */ + if (header->blocksize > scontext->tmp_blocksize) { + my_free(scontext->tmp); + scontext->tmp_nbytes = (size_t)4 * ebsize; + scontext->tmp = my_malloc(scontext->tmp_nbytes); + BLOSC_ERROR_NULL(scontext->tmp, BLOSC2_ERROR_MEMORY_ALLOC); + scontext->tmp2 = scontext->tmp + ebsize; + scontext->tmp3 = scontext->tmp2 + ebsize; + scontext->tmp4 = scontext->tmp3 + ebsize; + scontext->tmp_blocksize = (int32_t)header->blocksize; + } + + for (j = 0; j < context->nblocks; j++) { + bsize = header->blocksize; + leftoverblock = 0; + if ((j == context->nblocks - 1) && (context->leftover > 0)) { + bsize = context->leftover; + leftoverblock = 1; + } + + /* Compute start & stop for each block */ + startb = start * header->typesize - j * header->blocksize; + stopb = stop * header->typesize - j * header->blocksize; + if (stopb <= 0) { + // We can exit as soon as this block is beyond stop + break; + } + if (startb >= header->blocksize) { + continue; + } + if (startb < 0) { + startb = 0; + } + if (stopb > header->blocksize) { + stopb = header->blocksize; + } + bsize2 = stopb - startb; + +#if defined(HAVE_PLUGINS) + if (context->compcode == BLOSC_CODEC_ZFP_FIXED_RATE) { + scontext->zfp_cell_start = startb / context->typesize; + scontext->zfp_cell_nitems = nitems; + } +#endif /* HAVE_PLUGINS */ + + /* Do the actual data copy */ + // Regular decompression. Put results in tmp2. + // If the block is aligned and the worst case fits in destination, let's avoid a copy + bool get_single_block = ((startb == 0) && (bsize == nitems * header->typesize)); + uint8_t* tmp2 = get_single_block ? dest : scontext->tmp2; + + // If memcpyed we don't have a bstarts section (because it is not needed) + int32_t src_offset = memcpyed ? + context->header_overhead + j * bsize : sw32_(context->bstarts + j); + + int32_t cbytes = blosc_d(context->serial_context, bsize, leftoverblock, memcpyed, + src, srcsize, src_offset, j, + tmp2, 0, scontext->tmp, scontext->tmp3); + if (cbytes < 0) { + ntbytes = cbytes; + break; + } + if (scontext->zfp_cell_nitems > 0) { + if (cbytes == bsize2) { + memcpy((uint8_t *) dest, tmp2, (unsigned int) bsize2); + } else if (cbytes == context->blocksize) { + memcpy((uint8_t *) dest, tmp2 + scontext->zfp_cell_start * context->typesize, (unsigned int) bsize2); + cbytes = bsize2; + } + } else if (!get_single_block) { + /* Copy to destination */ + memcpy((uint8_t *) dest + ntbytes, tmp2 + startb, (unsigned int) bsize2); + } + ntbytes += bsize2; + } + + scontext->zfp_cell_nitems = 0; + + return ntbytes; +} + +int blosc2_getitem(const void* src, int32_t srcsize, int start, int nitems, void* dest, int32_t destsize) { + blosc2_context context; + int result; + + /* Minimally populate the context */ + memset(&context, 0, sizeof(blosc2_context)); + + context.schunk = g_schunk; + context.nthreads = 1; // force a serial decompression; fixes #95 + + /* Call the actual getitem function */ + result = blosc2_getitem_ctx(&context, src, srcsize, start, nitems, dest, destsize); + + /* Release resources */ + if (context.serial_context != NULL) { + free_thread_context(context.serial_context); + } + return result; +} + +/* Specific routine optimized for decompression a small number of + items out of a compressed chunk. Public non-contextual API. */ +int blosc1_getitem(const void* src, int start, int nitems, void* dest) { + return blosc2_getitem(src, INT32_MAX, start, nitems, dest, INT32_MAX); +} + +int blosc2_getitem_ctx(blosc2_context* context, const void* src, int32_t srcsize, + int start, int nitems, void* dest, int32_t destsize) { + blosc_header header; + int result; + + /* Minimally populate the context */ + result = read_chunk_header((uint8_t *) src, srcsize, true, &header); + if (result < 0) { + return result; + } + + context->src = src; + context->srcsize = srcsize; + context->dest = dest; + context->destsize = destsize; + + result = blosc2_initialize_context_from_header(context, &header); + if (result < 0) { + return result; + } + + if (context->serial_context == NULL) { + context->serial_context = create_thread_context(context, 0); + } + BLOSC_ERROR_NULL(context->serial_context, BLOSC2_ERROR_THREAD_CREATE); + /* Call the actual getitem function */ + result = _blosc_getitem(context, &header, src, srcsize, start, nitems, dest, destsize); + + return result; +} + +/* execute single compression/decompression job for a single thread_context */ +static void t_blosc_do_job(void *ctxt) +{ + struct thread_context* thcontext = (struct thread_context*)ctxt; + blosc2_context* context = thcontext->parent_context; + int32_t cbytes; + int32_t ntdest; + int32_t tblocks; /* number of blocks per thread */ + int32_t tblock; /* limit block on a thread */ + int32_t nblock_; /* private copy of nblock */ + int32_t bsize; + int32_t leftoverblock; + /* Parameters for threads */ + int32_t blocksize; + int32_t ebsize; + int32_t srcsize; + bool compress = context->do_compress != 0; + int32_t maxbytes; + int32_t nblocks; + int32_t leftover; + int32_t leftover2; + int32_t* bstarts; + const uint8_t* src; + uint8_t* dest; + uint8_t* tmp; + uint8_t* tmp2; + uint8_t* tmp3; + + /* Get parameters for this thread before entering the main loop */ + blocksize = context->blocksize; + ebsize = blocksize + context->typesize * (int32_t)sizeof(int32_t); + maxbytes = context->destsize; + nblocks = context->nblocks; + leftover = context->leftover; + bstarts = context->bstarts; + src = context->src; + srcsize = context->srcsize; + dest = context->dest; + + /* Resize the temporaries if needed */ + if (blocksize > thcontext->tmp_blocksize) { + my_free(thcontext->tmp); + thcontext->tmp_nbytes = (size_t) 4 * ebsize; + thcontext->tmp = my_malloc(thcontext->tmp_nbytes); + thcontext->tmp2 = thcontext->tmp + ebsize; + thcontext->tmp3 = thcontext->tmp2 + ebsize; + thcontext->tmp4 = thcontext->tmp3 + ebsize; + thcontext->tmp_blocksize = blocksize; + } + + tmp = thcontext->tmp; + tmp2 = thcontext->tmp2; + tmp3 = thcontext->tmp3; + + // Determine whether we can do a static distribution of workload among different threads + bool memcpyed = context->header_flags & (uint8_t)BLOSC_MEMCPYED; + if (!context->do_compress && context->special_type) { + // Fake a runlen as if its a memcpyed chunk + memcpyed = true; + } + + bool static_schedule = (!compress || memcpyed) && context->block_maskout == NULL; + if (static_schedule) { + /* Blocks per thread */ + tblocks = nblocks / context->nthreads; + leftover2 = nblocks % context->nthreads; + tblocks = (leftover2 > 0) ? tblocks + 1 : tblocks; + nblock_ = thcontext->tid * tblocks; + tblock = nblock_ + tblocks; + if (tblock > nblocks) { + tblock = nblocks; + } + } + else { + // Use dynamic schedule via a queue. Get the next block. + pthread_mutex_lock(&context->count_mutex); + context->thread_nblock++; + nblock_ = context->thread_nblock; + pthread_mutex_unlock(&context->count_mutex); + tblock = nblocks; + } + + /* Loop over blocks */ + leftoverblock = 0; + while ((nblock_ < tblock) && (context->thread_giveup_code > 0)) { + bsize = blocksize; + if (nblock_ == (nblocks - 1) && (leftover > 0)) { + bsize = leftover; + leftoverblock = 1; + } + if (compress) { + if (memcpyed) { + if (!context->prefilter) { + /* We want to memcpy only */ + memcpy(dest + context->header_overhead + nblock_ * blocksize, + src + nblock_ * blocksize, (unsigned int) bsize); + cbytes = (int32_t) bsize; + } + else { + /* Only the prefilter has to be executed, and this is done in blosc_c(). + * However, no further actions are needed, so we can put the result + * directly in dest. */ + cbytes = blosc_c(thcontext, bsize, leftoverblock, 0, + ebsize, src, nblock_ * blocksize, + dest + context->header_overhead + nblock_ * blocksize, + tmp, tmp3); + } + } + else { + /* Regular compression */ + cbytes = blosc_c(thcontext, bsize, leftoverblock, 0, + ebsize, src, nblock_ * blocksize, tmp2, tmp, tmp3); + } + } + else { + /* Regular decompression */ + if (context->special_type == BLOSC2_NO_SPECIAL && !memcpyed && + (srcsize < (int32_t)(context->header_overhead + (sizeof(int32_t) * nblocks)))) { + /* Not enough input to read all `bstarts` */ + cbytes = -1; + } + else { + // If memcpyed we don't have a bstarts section (because it is not needed) + int32_t src_offset = memcpyed ? + context->header_overhead + nblock_ * blocksize : sw32_(bstarts + nblock_); + cbytes = blosc_d(thcontext, bsize, leftoverblock, memcpyed, + src, srcsize, src_offset, nblock_, + dest, nblock_ * blocksize, tmp, tmp2); + } + } + + /* Check whether current thread has to giveup */ + if (context->thread_giveup_code <= 0) { + break; + } + + /* Check results for the compressed/decompressed block */ + if (cbytes < 0) { /* compr/decompr failure */ + /* Set giveup_code error */ + pthread_mutex_lock(&context->count_mutex); + context->thread_giveup_code = cbytes; + pthread_mutex_unlock(&context->count_mutex); + break; + } + + if (compress && !memcpyed) { + /* Start critical section */ + pthread_mutex_lock(&context->count_mutex); + ntdest = context->output_bytes; + // Note: do not use a typical local dict_training variable here + // because it is probably cached from previous calls if the number of + // threads does not change (the usual thing). + if (!(context->use_dict && context->dict_cdict == NULL)) { + _sw32(bstarts + nblock_, (int32_t) ntdest); + } + + if ((cbytes == 0) || (ntdest + cbytes > maxbytes)) { + context->thread_giveup_code = 0; /* uncompressible buf */ + pthread_mutex_unlock(&context->count_mutex); + break; + } + context->thread_nblock++; + nblock_ = context->thread_nblock; + context->output_bytes += cbytes; + pthread_mutex_unlock(&context->count_mutex); + /* End of critical section */ + + /* Copy the compressed buffer to destination */ + memcpy(dest + ntdest, tmp2, (unsigned int) cbytes); + } + else if (static_schedule) { + nblock_++; + } + else { + pthread_mutex_lock(&context->count_mutex); + context->thread_nblock++; + nblock_ = context->thread_nblock; + context->output_bytes += cbytes; + pthread_mutex_unlock(&context->count_mutex); + } + + } /* closes while (nblock_) */ + + if (static_schedule) { + pthread_mutex_lock(&context->count_mutex); + context->output_bytes = context->sourcesize; + if (compress) { + context->output_bytes += context->header_overhead; + } + pthread_mutex_unlock(&context->count_mutex); + } + +} + +/* Decompress & unshuffle several blocks in a single thread */ +static void* t_blosc(void* ctxt) { + struct thread_context* thcontext = (struct thread_context*)ctxt; + blosc2_context* context = thcontext->parent_context; +#ifdef BLOSC_POSIX_BARRIERS + int rc; +#endif + + while (1) { + /* Synchronization point for all threads (wait for initialization) */ + WAIT_INIT(NULL, context); + + if (context->end_threads) { + break; + } + + t_blosc_do_job(ctxt); + + /* Meeting point for all threads (wait for finalization) */ + WAIT_FINISH(NULL, context); + } + + /* Cleanup our working space and context */ + free_thread_context(thcontext); + + return (NULL); +} + + +int init_threadpool(blosc2_context *context) { + int32_t tid; + int rc2; + + /* Initialize mutex and condition variable objects */ + pthread_mutex_init(&context->count_mutex, NULL); + pthread_mutex_init(&context->delta_mutex, NULL); + pthread_mutex_init(&context->nchunk_mutex, NULL); + pthread_cond_init(&context->delta_cv, NULL); + + /* Set context thread sentinels */ + context->thread_giveup_code = 1; + context->thread_nblock = -1; + + /* Barrier initialization */ +#ifdef BLOSC_POSIX_BARRIERS + pthread_barrier_init(&context->barr_init, NULL, context->nthreads + 1); + pthread_barrier_init(&context->barr_finish, NULL, context->nthreads + 1); +#else + pthread_mutex_init(&context->count_threads_mutex, NULL); + pthread_cond_init(&context->count_threads_cv, NULL); + context->count_threads = 0; /* Reset threads counter */ +#endif + + if (threads_callback) { + /* Create thread contexts to store data for callback threads */ + context->thread_contexts = (struct thread_context *)my_malloc( + context->nthreads * sizeof(struct thread_context)); + BLOSC_ERROR_NULL(context->thread_contexts, BLOSC2_ERROR_MEMORY_ALLOC); + for (tid = 0; tid < context->nthreads; tid++) + init_thread_context(context->thread_contexts + tid, context, tid); + } + else { + #if !defined(_WIN32) + /* Initialize and set thread detached attribute */ + pthread_attr_init(&context->ct_attr); + pthread_attr_setdetachstate(&context->ct_attr, PTHREAD_CREATE_JOINABLE); + #endif + + /* Make space for thread handlers */ + context->threads = (pthread_t*)my_malloc( + context->nthreads * sizeof(pthread_t)); + BLOSC_ERROR_NULL(context->threads, BLOSC2_ERROR_MEMORY_ALLOC); + /* Finally, create the threads */ + for (tid = 0; tid < context->nthreads; tid++) { + /* Create a thread context (will destroy when finished) */ + struct thread_context *thread_context = create_thread_context(context, tid); + BLOSC_ERROR_NULL(thread_context, BLOSC2_ERROR_THREAD_CREATE); + #if !defined(_WIN32) + rc2 = pthread_create(&context->threads[tid], &context->ct_attr, t_blosc, + (void*)thread_context); + #else + rc2 = pthread_create(&context->threads[tid], NULL, t_blosc, + (void *)thread_context); + #endif + if (rc2) { + BLOSC_TRACE_ERROR("Return code from pthread_create() is %d.\n" + "\tError detail: %s\n", rc2, strerror(rc2)); + return BLOSC2_ERROR_THREAD_CREATE; + } + } + } + + /* We have now started/initialized the threads */ + context->threads_started = context->nthreads; + context->new_nthreads = context->nthreads; + + return 0; +} + +int16_t blosc2_get_nthreads(void) +{ + return g_nthreads; +} + +int16_t blosc2_set_nthreads(int16_t nthreads) { + int16_t ret = g_nthreads; /* the previous number of threads */ + + /* Check whether the library should be initialized */ + if (!g_initlib) blosc2_init(); + + if (nthreads != ret) { + g_nthreads = nthreads; + g_global_context->new_nthreads = nthreads; + check_nthreads(g_global_context); + } + + return ret; +} + + +const char* blosc1_get_compressor(void) +{ + const char* compname; + blosc2_compcode_to_compname(g_compressor, &compname); + + return compname; +} + +int blosc1_set_compressor(const char* compname) { + int code = blosc2_compname_to_compcode(compname); + if (code >= BLOSC_LAST_CODEC) { + BLOSC_TRACE_ERROR("User defined codecs cannot be set here. Use Blosc2 mechanism instead."); + return -1; + } + g_compressor = code; + + /* Check whether the library should be initialized */ + if (!g_initlib) blosc2_init(); + + return code; +} + +void blosc2_set_delta(int dodelta) { + + g_delta = dodelta; + + /* Check whether the library should be initialized */ + if (!g_initlib) blosc2_init(); + +} + +const char* blosc2_list_compressors(void) { + static int compressors_list_done = 0; + static char ret[256]; + + if (compressors_list_done) return ret; + ret[0] = '\0'; + strcat(ret, BLOSC_BLOSCLZ_COMPNAME); + strcat(ret, ","); + strcat(ret, BLOSC_LZ4_COMPNAME); + strcat(ret, ","); + strcat(ret, BLOSC_LZ4HC_COMPNAME); +#if defined(HAVE_ZLIB) + strcat(ret, ","); + strcat(ret, BLOSC_ZLIB_COMPNAME); +#endif /* HAVE_ZLIB */ +#if defined(HAVE_ZSTD) + strcat(ret, ","); + strcat(ret, BLOSC_ZSTD_COMPNAME); +#endif /* HAVE_ZSTD */ + compressors_list_done = 1; + return ret; +} + + +const char* blosc2_get_version_string(void) { + return BLOSC2_VERSION_STRING; +} + + +int blosc2_get_complib_info(const char* compname, char** complib, char** version) { + int clibcode; + const char* clibname; + const char* clibversion = "unknown"; + char sbuffer[256]; + + clibcode = compname_to_clibcode(compname); + clibname = clibcode_to_clibname(clibcode); + + /* complib version */ + if (clibcode == BLOSC_BLOSCLZ_LIB) { + clibversion = BLOSCLZ_VERSION_STRING; + } + else if (clibcode == BLOSC_LZ4_LIB) { + sprintf(sbuffer, "%d.%d.%d", + LZ4_VERSION_MAJOR, LZ4_VERSION_MINOR, LZ4_VERSION_RELEASE); + clibversion = sbuffer; + } +#if defined(HAVE_ZLIB) + else if (clibcode == BLOSC_ZLIB_LIB) { +#ifdef ZLIB_COMPAT + clibversion = ZLIB_VERSION; +#elif defined(HAVE_ZLIB_NG) + clibversion = ZLIBNG_VERSION; +#else + clibversion = ZLIB_VERSION; +#endif + } +#endif /* HAVE_ZLIB */ +#if defined(HAVE_ZSTD) + else if (clibcode == BLOSC_ZSTD_LIB) { + sprintf(sbuffer, "%d.%d.%d", + ZSTD_VERSION_MAJOR, ZSTD_VERSION_MINOR, ZSTD_VERSION_RELEASE); + clibversion = sbuffer; + } +#endif /* HAVE_ZSTD */ + +#ifdef _MSC_VER + *complib = _strdup(clibname); + *version = _strdup(clibversion); +#else + *complib = strdup(clibname); + *version = strdup(clibversion); +#endif + return clibcode; +} + +/* Return `nbytes`, `cbytes` and `blocksize` from a compressed buffer. */ +void blosc1_cbuffer_sizes(const void* cbuffer, size_t* nbytes, size_t* cbytes, size_t* blocksize) { + int32_t nbytes32, cbytes32, blocksize32; + blosc2_cbuffer_sizes(cbuffer, &nbytes32, &cbytes32, &blocksize32); + *nbytes = nbytes32; + *cbytes = cbytes32; + *blocksize = blocksize32; +} + +int blosc2_cbuffer_sizes(const void* cbuffer, int32_t* nbytes, int32_t* cbytes, int32_t* blocksize) { + blosc_header header; + int rc = read_chunk_header((uint8_t *) cbuffer, BLOSC_MIN_HEADER_LENGTH, false, &header); + if (rc < 0) { + /* Return zeros if error reading header */ + memset(&header, 0, sizeof(header)); + } + + /* Read the interesting values */ + if (nbytes != NULL) + *nbytes = header.nbytes; + if (cbytes != NULL) + *cbytes = header.cbytes; + if (blocksize != NULL) + *blocksize = header.blocksize; + return rc; +} + +int blosc1_cbuffer_validate(const void* cbuffer, size_t cbytes, size_t* nbytes) { + int32_t header_cbytes; + int32_t header_nbytes; + if (cbytes < BLOSC_MIN_HEADER_LENGTH) { + /* Compressed data should contain enough space for header */ + *nbytes = 0; + return BLOSC2_ERROR_WRITE_BUFFER; + } + int rc = blosc2_cbuffer_sizes(cbuffer, &header_nbytes, &header_cbytes, NULL); + if (rc < 0) { + *nbytes = 0; + return rc; + } + *nbytes = header_nbytes; + if (header_cbytes != (int32_t)cbytes) { + /* Compressed size from header does not match `cbytes` */ + *nbytes = 0; + return BLOSC2_ERROR_INVALID_HEADER; + } + if (*nbytes > BLOSC2_MAX_BUFFERSIZE) { + /* Uncompressed size is larger than allowed */ + *nbytes = 0; + return BLOSC2_ERROR_MEMORY_ALLOC; + } + return 0; +} + +/* Return `typesize` and `flags` from a compressed buffer. */ +void blosc1_cbuffer_metainfo(const void* cbuffer, size_t* typesize, int* flags) { + blosc_header header; + int rc = read_chunk_header((uint8_t *) cbuffer, BLOSC_MIN_HEADER_LENGTH, false, &header); + if (rc < 0) { + *typesize = *flags = 0; + return; + } + + /* Read the interesting values */ + *flags = header.flags; + *typesize = header.typesize; +} + + +/* Return version information from a compressed buffer. */ +void blosc2_cbuffer_versions(const void* cbuffer, int* version, int* versionlz) { + blosc_header header; + int rc = read_chunk_header((uint8_t *) cbuffer, BLOSC_MIN_HEADER_LENGTH, false, &header); + if (rc < 0) { + *version = *versionlz = 0; + return; + } + + /* Read the version info */ + *version = header.version; + *versionlz = header.versionlz; +} + + +/* Return the compressor library/format used in a compressed buffer. */ +const char* blosc2_cbuffer_complib(const void* cbuffer) { + blosc_header header; + int clibcode; + const char* complib; + int rc = read_chunk_header((uint8_t *) cbuffer, BLOSC_MIN_HEADER_LENGTH, false, &header); + if (rc < 0) { + return NULL; + } + + /* Read the compressor format/library info */ + clibcode = (header.flags & 0xe0) >> 5; + complib = clibcode_to_clibname(clibcode); + return complib; +} + + +/* Get the internal blocksize to be used during compression. 0 means + that an automatic blocksize is computed internally. */ +int blosc1_get_blocksize(void) +{ + return (int)g_force_blocksize; +} + + +/* Force the use of a specific blocksize. If 0, an automatic + blocksize will be used (the default). */ +void blosc1_set_blocksize(size_t blocksize) { + g_force_blocksize = (int32_t)blocksize; +} + + +/* Force the use of a specific split mode. */ +void blosc1_set_splitmode(int mode) +{ + g_splitmode = mode; +} + + +/* Set pointer to super-chunk. If NULL, no super-chunk will be + reachable (the default). */ +void blosc_set_schunk(blosc2_schunk* schunk) { + g_schunk = schunk; + g_global_context->schunk = schunk; +} + +blosc2_io *blosc2_io_global = NULL; + +void blosc2_init(void) { + /* Return if Blosc is already initialized */ + if (g_initlib) return; + + g_ncodecs = 0; + g_nfilters = 0; + +#if defined(HAVE_PLUGINS) + #include "blosc2/blosc2-common.h" + #include "blosc2/blosc2-stdio.h" + register_codecs(); + register_filters(); +#endif + pthread_mutex_init(&global_comp_mutex, NULL); + /* Create a global context */ + g_global_context = (blosc2_context*)my_malloc(sizeof(blosc2_context)); + memset(g_global_context, 0, sizeof(blosc2_context)); + g_global_context->nthreads = g_nthreads; + g_global_context->new_nthreads = g_nthreads; + g_initlib = 1; +} + + +int blosc2_free_resources(void) { + /* Return if Blosc is not initialized */ + if (!g_initlib) return BLOSC2_ERROR_FAILURE; + + return release_threadpool(g_global_context); +} + + +void blosc2_destroy(void) { + /* Return if Blosc is not initialized */ + if (!g_initlib) return; + + blosc2_free_resources(); + g_initlib = 0; + blosc2_free_ctx(g_global_context); + + pthread_mutex_destroy(&global_comp_mutex); + +} + + +int release_threadpool(blosc2_context *context) { + int32_t t; + void* status; + int rc; + + if (context->threads_started > 0) { + if (threads_callback) { + /* free context data for user-managed threads */ + for (t=0; tthreads_started; t++) + destroy_thread_context(context->thread_contexts + t); + my_free(context->thread_contexts); + } + else { + /* Tell all existing threads to finish */ + context->end_threads = 1; + WAIT_INIT(-1, context); + + /* Join exiting threads */ + for (t = 0; t < context->threads_started; t++) { + rc = pthread_join(context->threads[t], &status); + if (rc) { + BLOSC_TRACE_ERROR("Return code from pthread_join() is %d\n" + "\tError detail: %s.", rc, strerror(rc)); + } + } + + /* Thread attributes */ + #if !defined(_WIN32) + pthread_attr_destroy(&context->ct_attr); + #endif + + /* Release thread handlers */ + my_free(context->threads); + } + + /* Release mutex and condition variable objects */ + pthread_mutex_destroy(&context->count_mutex); + pthread_mutex_destroy(&context->delta_mutex); + pthread_mutex_destroy(&context->nchunk_mutex); + pthread_cond_destroy(&context->delta_cv); + + /* Barriers */ + #ifdef BLOSC_POSIX_BARRIERS + pthread_barrier_destroy(&context->barr_init); + pthread_barrier_destroy(&context->barr_finish); + #else + pthread_mutex_destroy(&context->count_threads_mutex); + pthread_cond_destroy(&context->count_threads_cv); + context->count_threads = 0; /* Reset threads counter */ + #endif + + /* Reset flags and counters */ + context->end_threads = 0; + context->threads_started = 0; + } + + + return 0; +} + + +/* Contexts */ + +/* Create a context for compression */ +blosc2_context* blosc2_create_cctx(blosc2_cparams cparams) { + blosc2_context* context = (blosc2_context*)my_malloc(sizeof(blosc2_context)); + BLOSC_ERROR_NULL(context, NULL); + + /* Populate the context, using zeros as default values */ + memset(context, 0, sizeof(blosc2_context)); + context->do_compress = 1; /* meant for compression */ + context->compcode = cparams.compcode; + context->compcode_meta = cparams.compcode_meta; + context->clevel = cparams.clevel; + context->use_dict = cparams.use_dict; + if (cparams.instr_codec) { + context->blosc2_flags = BLOSC2_INSTR_CODEC; + } + context->typesize = cparams.typesize; + for (int i = 0; i < BLOSC2_MAX_FILTERS; i++) { + context->filters[i] = cparams.filters[i]; + context->filters_meta[i] = cparams.filters_meta[i]; + + if (context->filters[i] >= BLOSC_LAST_FILTER && context->filters[i] <= BLOSC2_DEFINED_FILTERS_STOP) { + BLOSC_TRACE_ERROR("filter (%d) is not yet defined", + context->filters[i]); + free(context); + return NULL; + } + if (context->filters[i] > BLOSC_LAST_REGISTERED_FILTER && context->filters[i] <= BLOSC2_GLOBAL_REGISTERED_FILTERS_STOP) { + BLOSC_TRACE_ERROR("filter (%d) is not yet defined", + context->filters[i]); + free(context); + return NULL; + } + } + + context->nthreads = cparams.nthreads; + context->new_nthreads = context->nthreads; + context->blocksize = cparams.blocksize; + context->splitmode = cparams.splitmode; + context->threads_started = 0; + context->schunk = cparams.schunk; + + if (cparams.prefilter != NULL) { + context->prefilter = cparams.prefilter; + context->preparams = (blosc2_prefilter_params*)my_malloc(sizeof(blosc2_prefilter_params)); + BLOSC_ERROR_NULL(context->preparams, NULL); + memcpy(context->preparams, cparams.preparams, sizeof(blosc2_prefilter_params)); + } + + if (cparams.udbtune == NULL) { + context->udbtune = &BTUNE_DEFAULTS; + } else { + context->udbtune = cparams.udbtune; + } + + return context; +} + +/* Create a context for decompression */ +blosc2_context* blosc2_create_dctx(blosc2_dparams dparams) { + blosc2_context* context = (blosc2_context*)my_malloc(sizeof(blosc2_context)); + BLOSC_ERROR_NULL(context, NULL); + + /* Populate the context, using zeros as default values */ + memset(context, 0, sizeof(blosc2_context)); + context->do_compress = 0; /* Meant for decompression */ + context->nthreads = dparams.nthreads; + context->new_nthreads = context->nthreads; + context->threads_started = 0; + context->block_maskout = NULL; + context->block_maskout_nitems = 0; + context->schunk = dparams.schunk; + + if (dparams.postfilter != NULL) { + context->postfilter = dparams.postfilter; + context->postparams = (blosc2_postfilter_params*)my_malloc(sizeof(blosc2_postfilter_params)); + BLOSC_ERROR_NULL(context->postparams, NULL); + memcpy(context->postparams, dparams.postparams, sizeof(blosc2_postfilter_params)); + } + + return context; +} + + +void blosc2_free_ctx(blosc2_context* context) { + release_threadpool(context); + if (context->serial_context != NULL) { + free_thread_context(context->serial_context); + } + if (context->dict_cdict != NULL) { +#ifdef HAVE_ZSTD + ZSTD_freeCDict(context->dict_cdict); +#endif + } + if (context->dict_ddict != NULL) { +#ifdef HAVE_ZSTD + ZSTD_freeDDict(context->dict_ddict); +#endif + } + if (context->btune != NULL) { + context->udbtune->btune_free(context); + } + if (context->prefilter != NULL) { + my_free(context->preparams); + } + if (context->postfilter != NULL) { + my_free(context->postparams); + } + + if (context->block_maskout != NULL) { + free(context->block_maskout); + } + my_free(context); +} + + +int blosc2_ctx_get_cparams(blosc2_context *ctx, blosc2_cparams *cparams) { + cparams->compcode = ctx->compcode; + cparams->compcode_meta = ctx->compcode_meta; + cparams->clevel = ctx->clevel; + cparams->use_dict = ctx->use_dict; + cparams->instr_codec = ctx->blosc2_flags & BLOSC2_INSTR_CODEC; + cparams->typesize = ctx->typesize; + cparams->nthreads = ctx->nthreads; + cparams->blocksize = ctx->blocksize; + cparams->splitmode = ctx->splitmode; + cparams->schunk = ctx->schunk; + for (int i = 0; i < BLOSC2_MAX_FILTERS; ++i) { + cparams->filters[i] = ctx->filters[i]; + cparams->filters_meta[i] = ctx->filters_meta[i]; + } + cparams->prefilter = ctx->prefilter; + cparams->preparams = ctx->preparams; + cparams->udbtune = ctx->udbtune; + + return BLOSC2_ERROR_SUCCESS; +} + + +int blosc2_ctx_get_dparams(blosc2_context *ctx, blosc2_dparams *dparams) { + dparams->nthreads = ctx->nthreads; + dparams->schunk = ctx->schunk; + dparams->postfilter = ctx->postfilter; + dparams->postparams = ctx->postparams; + + return BLOSC2_ERROR_SUCCESS; +} + + +/* Set a maskout in decompression context */ +int blosc2_set_maskout(blosc2_context *ctx, bool *maskout, int nblocks) { + + if (ctx->block_maskout != NULL) { + // Get rid of a possible mask here + free(ctx->block_maskout); + } + + bool *maskout_ = malloc(nblocks); + BLOSC_ERROR_NULL(maskout_, BLOSC2_ERROR_MEMORY_ALLOC); + memcpy(maskout_, maskout, nblocks); + ctx->block_maskout = maskout_; + ctx->block_maskout_nitems = nblocks; + + return 0; +} + + +/* Create a chunk made of zeros */ +int blosc2_chunk_zeros(blosc2_cparams cparams, const int32_t nbytes, void* dest, int32_t destsize) { + if (destsize < BLOSC_EXTENDED_HEADER_LENGTH) { + BLOSC_TRACE_ERROR("dest buffer is not long enough"); + return BLOSC2_ERROR_DATA; + } + + if (nbytes % cparams.typesize) { + BLOSC_TRACE_ERROR("nbytes must be a multiple of typesize"); + return BLOSC2_ERROR_DATA; + } + + blosc_header header; + blosc2_context* context = blosc2_create_cctx(cparams); + + int error = initialize_context_compression( + context, NULL, nbytes, dest, destsize, + context->clevel, context->filters, context->filters_meta, + context->typesize, context->compcode, context->blocksize, + context->new_nthreads, context->nthreads, context->splitmode, + context->udbtune, context->btune, context->schunk); + if (error <= 0) { + blosc2_free_ctx(context); + return error; + } + + memset(&header, 0, sizeof(header)); + header.version = BLOSC2_VERSION_FORMAT; + header.versionlz = BLOSC_BLOSCLZ_VERSION_FORMAT; + header.flags = BLOSC_DOSHUFFLE | BLOSC_DOBITSHUFFLE; // extended header + header.typesize = context->typesize; + header.nbytes = (int32_t)nbytes; + header.blocksize = context->blocksize; + header.cbytes = BLOSC_EXTENDED_HEADER_LENGTH; + header.blosc2_flags = BLOSC2_SPECIAL_ZERO << 4; // mark chunk as all zeros + memcpy((uint8_t *)dest, &header, sizeof(header)); + + blosc2_free_ctx(context); + + return BLOSC_EXTENDED_HEADER_LENGTH; +} + + +/* Create a chunk made of uninitialized values */ +int blosc2_chunk_uninit(blosc2_cparams cparams, const int32_t nbytes, void* dest, int32_t destsize) { + if (destsize < BLOSC_EXTENDED_HEADER_LENGTH) { + BLOSC_TRACE_ERROR("dest buffer is not long enough"); + return BLOSC2_ERROR_DATA; + } + + if (nbytes % cparams.typesize) { + BLOSC_TRACE_ERROR("nbytes must be a multiple of typesize"); + return BLOSC2_ERROR_DATA; + } + + blosc_header header; + blosc2_context* context = blosc2_create_cctx(cparams); + int error = initialize_context_compression( + context, NULL, nbytes, dest, destsize, + context->clevel, context->filters, context->filters_meta, + context->typesize, context->compcode, context->blocksize, + context->new_nthreads, context->nthreads, context->splitmode, + context->udbtune, context->btune, context->schunk); + if (error <= 0) { + blosc2_free_ctx(context); + return error; + } + + memset(&header, 0, sizeof(header)); + header.version = BLOSC2_VERSION_FORMAT; + header.versionlz = BLOSC_BLOSCLZ_VERSION_FORMAT; + header.flags = BLOSC_DOSHUFFLE | BLOSC_DOBITSHUFFLE; // extended header + header.typesize = context->typesize; + header.nbytes = (int32_t)nbytes; + header.blocksize = context->blocksize; + header.cbytes = BLOSC_EXTENDED_HEADER_LENGTH; + header.blosc2_flags = BLOSC2_SPECIAL_UNINIT << 4; // mark chunk as uninitialized + memcpy((uint8_t *)dest, &header, sizeof(header)); + + blosc2_free_ctx(context); + + return BLOSC_EXTENDED_HEADER_LENGTH; +} + + +/* Create a chunk made of nans */ +int blosc2_chunk_nans(blosc2_cparams cparams, const int32_t nbytes, void* dest, int32_t destsize) { + if (destsize < BLOSC_EXTENDED_HEADER_LENGTH) { + BLOSC_TRACE_ERROR("dest buffer is not long enough"); + return BLOSC2_ERROR_DATA; + } + + if (nbytes % cparams.typesize) { + BLOSC_TRACE_ERROR("nbytes must be a multiple of typesize"); + return BLOSC2_ERROR_DATA; + } + + blosc_header header; + blosc2_context* context = blosc2_create_cctx(cparams); + + int error = initialize_context_compression( + context, NULL, nbytes, dest, destsize, + context->clevel, context->filters, context->filters_meta, + context->typesize, context->compcode, context->blocksize, + context->new_nthreads, context->nthreads, context->splitmode, + context->udbtune, context->btune, context->schunk); + if (error <= 0) { + blosc2_free_ctx(context); + return error; + } + + memset(&header, 0, sizeof(header)); + header.version = BLOSC2_VERSION_FORMAT; + header.versionlz = BLOSC_BLOSCLZ_VERSION_FORMAT; + header.flags = BLOSC_DOSHUFFLE | BLOSC_DOBITSHUFFLE; // extended header + header.typesize = context->typesize; + header.nbytes = (int32_t)nbytes; + header.blocksize = context->blocksize; + header.cbytes = BLOSC_EXTENDED_HEADER_LENGTH; + header.blosc2_flags = BLOSC2_SPECIAL_NAN << 4; // mark chunk as all NaNs + memcpy((uint8_t *)dest, &header, sizeof(header)); + + blosc2_free_ctx(context); + + return BLOSC_EXTENDED_HEADER_LENGTH; +} + + +/* Create a chunk made of repeated values */ +int blosc2_chunk_repeatval(blosc2_cparams cparams, const int32_t nbytes, + void* dest, int32_t destsize, void* repeatval) { + uint8_t typesize = cparams.typesize; + if (destsize < BLOSC_EXTENDED_HEADER_LENGTH + typesize) { + BLOSC_TRACE_ERROR("dest buffer is not long enough"); + return BLOSC2_ERROR_DATA; + } + + if (nbytes % cparams.typesize) { + BLOSC_TRACE_ERROR("nbytes must be a multiple of typesize"); + return BLOSC2_ERROR_DATA; + } + + blosc_header header; + blosc2_context* context = blosc2_create_cctx(cparams); + + int error = initialize_context_compression( + context, NULL, nbytes, dest, destsize, + context->clevel, context->filters, context->filters_meta, + context->typesize, context->compcode, context->blocksize, + context->new_nthreads, context->nthreads, context->splitmode, + context->udbtune, context->btune, context->schunk); + if (error <= 0) { + blosc2_free_ctx(context); + return error; + } + + memset(&header, 0, sizeof(header)); + header.version = BLOSC2_VERSION_FORMAT; + header.versionlz = BLOSC_BLOSCLZ_VERSION_FORMAT; + header.flags = BLOSC_DOSHUFFLE | BLOSC_DOBITSHUFFLE; // extended header + header.typesize = (uint8_t)typesize; + header.nbytes = (int32_t)nbytes; + header.blocksize = context->blocksize; + header.cbytes = BLOSC_EXTENDED_HEADER_LENGTH + (int32_t)typesize; + header.blosc2_flags = BLOSC2_SPECIAL_VALUE << 4; // mark chunk as all repeated value + memcpy((uint8_t *)dest, &header, sizeof(header)); + memcpy((uint8_t *)dest + sizeof(header), repeatval, typesize); + + blosc2_free_ctx(context); + + return BLOSC_EXTENDED_HEADER_LENGTH + (uint8_t)typesize; +} + + +/* Register filters */ + +int register_filter_private(blosc2_filter *filter) { + BLOSC_ERROR_NULL(filter, BLOSC2_ERROR_INVALID_PARAM); + if (g_nfilters == UINT8_MAX) { + BLOSC_TRACE_ERROR("Can not register more filters"); + return BLOSC2_ERROR_CODEC_SUPPORT; + } + if (filter->id < BLOSC2_GLOBAL_REGISTERED_FILTERS_START) { + BLOSC_TRACE_ERROR("The id must be greater or equal than %d", BLOSC2_GLOBAL_REGISTERED_FILTERS_START); + return BLOSC2_ERROR_FAILURE; + } + /* This condition can never be fulfilled + if (filter->id > BLOSC2_USER_REGISTERED_FILTERS_STOP) { + BLOSC_TRACE_ERROR("The id must be less than or equal to %d", BLOSC2_USER_REGISTERED_FILTERS_STOP); + return BLOSC2_ERROR_FAILURE; + } + */ + + // Check if the filter is already registered + for (uint64_t i = 0; i < g_nfilters; ++i) { + if (g_filters[i].id == filter->id) { + BLOSC_TRACE_ERROR("The filter is already registered!"); + return BLOSC2_ERROR_FAILURE; + } + } + + blosc2_filter *filter_new = &g_filters[g_nfilters++]; + memcpy(filter_new, filter, sizeof(blosc2_filter)); + + return BLOSC2_ERROR_SUCCESS; +} + + +int blosc2_register_filter(blosc2_filter *filter) { + if (filter->id < BLOSC2_USER_REGISTERED_FILTERS_START) { + BLOSC_TRACE_ERROR("The id must be greater or equal to %d", BLOSC2_USER_REGISTERED_FILTERS_START); + return BLOSC2_ERROR_FAILURE; + } + + return register_filter_private(filter); +} + + +/* Register codecs */ + +int register_codec_private(blosc2_codec *codec) { + BLOSC_ERROR_NULL(codec, BLOSC2_ERROR_INVALID_PARAM); + if (g_ncodecs == UINT8_MAX) { + BLOSC_TRACE_ERROR("Can not register more codecs"); + return BLOSC2_ERROR_CODEC_SUPPORT; + } + if (codec->compcode < BLOSC2_GLOBAL_REGISTERED_CODECS_START) { + BLOSC_TRACE_ERROR("The id must be greater or equal than %d", BLOSC2_GLOBAL_REGISTERED_CODECS_START); + return BLOSC2_ERROR_FAILURE; + } + /* This condition can never be fulfilled + if (codec->compcode > BLOSC2_USER_REGISTERED_CODECS_STOP) { + BLOSC_TRACE_ERROR("The id must be lower or equal to %d", BLOSC2_USER_REGISTERED_CODECS_STOP); + return BLOSC2_ERROR_FAILURE; + } + */ + + // Check if the code is already registered + for (int i = 0; i < g_ncodecs; ++i) { + if (g_codecs[i].compcode == codec->compcode) { + BLOSC_TRACE_ERROR("The codec is already registered!"); + return BLOSC2_ERROR_CODEC_PARAM; + } + } + + blosc2_codec *codec_new = &g_codecs[g_ncodecs++]; + memcpy(codec_new, codec, sizeof(blosc2_codec)); + + return BLOSC2_ERROR_SUCCESS; +} + + +int blosc2_register_codec(blosc2_codec *codec) { + if (codec->compcode < BLOSC2_USER_REGISTERED_CODECS_START) { + BLOSC_TRACE_ERROR("The compcode must be greater or equal than %d", BLOSC2_USER_REGISTERED_CODECS_START); + return BLOSC2_ERROR_CODEC_PARAM; + } + + return register_codec_private(codec); +} + + +int _blosc2_register_io_cb(const blosc2_io_cb *io) { + + // Check if the io is already registered + for (uint64_t i = 0; i < g_nio; ++i) { + if (g_io[i].id == io->id) { + BLOSC_TRACE_ERROR("The codec is already registered!"); + return BLOSC2_ERROR_PLUGIN_IO; + } + } + + blosc2_io_cb *io_new = &g_io[g_nio++]; + memcpy(io_new, io, sizeof(blosc2_io_cb)); + + return BLOSC2_ERROR_SUCCESS; +} + +int blosc2_register_io_cb(const blosc2_io_cb *io) { + BLOSC_ERROR_NULL(io, BLOSC2_ERROR_INVALID_PARAM); + if (g_nio == UINT8_MAX) { + BLOSC_TRACE_ERROR("Can not register more codecs"); + return BLOSC2_ERROR_PLUGIN_IO; + } + + if (io->id < BLOSC2_IO_REGISTERED) { + BLOSC_TRACE_ERROR("The compcode must be greater or equal than %d", BLOSC2_IO_REGISTERED); + return BLOSC2_ERROR_PLUGIN_IO; + } + + return _blosc2_register_io_cb(io); +} + +blosc2_io_cb *blosc2_get_io_cb(uint8_t id) { + for (uint64_t i = 0; i < g_nio; ++i) { + if (g_io[i].id == id) { + return &g_io[i]; + } + } + if (id == BLOSC2_IO_FILESYSTEM) { + if (_blosc2_register_io_cb(&BLOSC2_IO_CB_DEFAULTS) < 0) { + BLOSC_TRACE_ERROR("Error registering the default IO API"); + return NULL; + } + return blosc2_get_io_cb(id); + } + return NULL; +} + +void blosc2_unidim_to_multidim(uint8_t ndim, int64_t *shape, int64_t i, int64_t *index) { + int64_t strides[BLOSC2_MAX_DIM]; + if (ndim == 0) { + return; + } + strides[ndim - 1] = 1; + for (int j = ndim - 2; j >= 0; --j) { + strides[j] = shape[j + 1] * strides[j + 1]; + } + + index[0] = i / strides[0]; + for (int j = 1; j < ndim; ++j) { + index[j] = (i % strides[j - 1]) / strides[j]; + } +} + +void blosc2_multidim_to_unidim(const int64_t *index, int8_t ndim, const int64_t *strides, int64_t *i) { + *i = 0; + for (int j = 0; j < ndim; ++j) { + *i += index[j] * strides[j]; + } +} diff --git a/src/blosc2/blosc/blosclz.c b/src/blosc2/blosc/blosclz.c new file mode 100644 index 0000000..f5be769 --- /dev/null +++ b/src/blosc2/blosc/blosclz.c @@ -0,0 +1,790 @@ +/********************************************************************* + Blosc - Blocked Shuffling and Compression Library + + Copyright (C) 2021 The Blosc Developers + https://blosc.org + License: BSD 3-Clause (see LICENSE.txt) + + See LICENSE.txt for details about copyright and rights to use. +**********************************************************************/ + +/********************************************************************* + The code in this file is heavily based on FastLZ, a lightning-fast + lossless compression library. See LICENSES/FASTLZ.txt for details. +**********************************************************************/ + + +#include +#include +#include "blosclz.h" +#include "fastcopy.h" +#include "blosc2/blosc2-common.h" + + +/* + * Give hints to the compiler for branch prediction optimization. + */ +#if defined(__GNUC__) && (__GNUC__ > 2) +#define BLOSCLZ_LIKELY(c) (__builtin_expect((c), 1)) +#define BLOSCLZ_UNLIKELY(c) (__builtin_expect((c), 0)) +#else +#define BLOSCLZ_LIKELY(c) (c) +#define BLOSCLZ_UNLIKELY(c) (c) +#endif + +/* + * Use inlined functions for supported systems. + */ +#if defined(_MSC_VER) && !defined(__cplusplus) /* Visual Studio */ +#define inline __inline /* Visual C is not C99, but supports some kind of inline */ +#endif + +#define MAX_COPY 32U +#define MAX_DISTANCE 8191 +#define MAX_FARDISTANCE (65535 + MAX_DISTANCE - 1) + +#ifdef BLOSC_STRICT_ALIGN + #define BLOSCLZ_READU16(p) ((p)[0] | (p)[1]<<8) + #define BLOSCLZ_READU32(p) ((p)[0] | (p)[1]<<8 | (p)[2]<<16 | (p)[3]<<24) +#else + #define BLOSCLZ_READU16(p) *((const uint16_t*)(p)) + #define BLOSCLZ_READU32(p) *((const uint32_t*)(p)) +#endif + +#define HASH_LOG (14U) +#define HASH_LOG2 (12U) + +// This is used in LZ4 and seems to work pretty well here too +#define HASH_FUNCTION(v, s, h) { \ + (v) = ((s) * 2654435761U) >> (32U - (h)); \ +} + + +#if defined(__AVX2__) +static uint8_t *get_run_32(uint8_t *ip, const uint8_t *ip_bound, const uint8_t *ref) { + uint8_t x = ip[-1]; + + while (ip < (ip_bound - (sizeof(__m256i)))) { + __m256i value, value2, cmp; + /* Broadcast the value for every byte in a 256-bit register */ + memset(&value, x, sizeof(__m256i)); + value2 = _mm256_loadu_si256((__m256i *)ref); + cmp = _mm256_cmpeq_epi64(value, value2); + if ((unsigned)_mm256_movemask_epi8(cmp) != 0xFFFFFFFF) { + /* Return the byte that starts to differ */ + while (*ref++ == x) ip++; + return ip; + } + else { + ip += sizeof(__m256i); + ref += sizeof(__m256i); + } + } + /* Look into the remainder */ + while ((ip < ip_bound) && (*ref++ == x)) ip++; + return ip; +} +#endif + +#if defined(__SSE2__) +uint8_t *get_run_16(uint8_t *ip, const uint8_t *ip_bound, const uint8_t *ref) { + uint8_t x = ip[-1]; + + while (ip < (ip_bound - sizeof(__m128i))) { + __m128i value, value2, cmp; + /* Broadcast the value for every byte in a 128-bit register */ + memset(&value, x, sizeof(__m128i)); + value2 = _mm_loadu_si128((__m128i *)ref); + cmp = _mm_cmpeq_epi32(value, value2); + if (_mm_movemask_epi8(cmp) != 0xFFFF) { + /* Return the byte that starts to differ */ + while (*ref++ == x) ip++; + return ip; + } + else { + ip += sizeof(__m128i); + ref += sizeof(__m128i); + } + } + /* Look into the remainder */ + while ((ip < ip_bound) && (*ref++ == x)) ip++; + return ip; +} + +#endif + + +static uint8_t *get_run(uint8_t *ip, const uint8_t *ip_bound, const uint8_t *ref) { + uint8_t x = ip[-1]; + int64_t value, value2; + /* Broadcast the value for every byte in a 64-bit register */ + memset(&value, x, 8); + /* safe because the outer check against ip limit */ + while (ip < (ip_bound - sizeof(int64_t))) { +#if defined(BLOSC_STRICT_ALIGN) + memcpy(&value2, ref, 8); +#else + value2 = ((int64_t*)ref)[0]; +#endif + if (value != value2) { + /* Return the byte that starts to differ */ + while (*ref++ == x) ip++; + return ip; + } + else { + ip += 8; + ref += 8; + } + } + /* Look into the remainder */ + while ((ip < ip_bound) && (*ref++ == x)) ip++; + return ip; +} + + +/* Return the byte that starts to differ */ +uint8_t *get_match(uint8_t *ip, const uint8_t *ip_bound, const uint8_t *ref) { +#if !defined(BLOSC_STRICT_ALIGN) + while (ip < (ip_bound - sizeof(int64_t))) { + if (*(int64_t*)ref != *(int64_t*)ip) { + /* Return the byte that starts to differ */ + while (*ref++ == *ip++) {} + return ip; + } + else { + ip += sizeof(int64_t); + ref += sizeof(int64_t); + } + } +#endif + /* Look into the remainder */ + while ((ip < ip_bound) && (*ref++ == *ip++)) {} + return ip; +} + + +#if defined(__SSE2__) +static uint8_t *get_match_16(uint8_t *ip, const uint8_t *ip_bound, const uint8_t *ref) { + __m128i value, value2, cmp; + + while (ip < (ip_bound - sizeof(__m128i))) { + value = _mm_loadu_si128((__m128i *) ip); + value2 = _mm_loadu_si128((__m128i *) ref); + cmp = _mm_cmpeq_epi32(value, value2); + if (_mm_movemask_epi8(cmp) != 0xFFFF) { + /* Return the byte that starts to differ */ + while (*ref++ == *ip++) {} + return ip; + } + else { + ip += sizeof(__m128i); + ref += sizeof(__m128i); + } + } + /* Look into the remainder */ + while ((ip < ip_bound) && (*ref++ == *ip++)) {} + return ip; +} +#endif + + +#if defined(__AVX2__) +static uint8_t *get_match_32(uint8_t *ip, const uint8_t *ip_bound, const uint8_t *ref) { + + while (ip < (ip_bound - sizeof(__m256i))) { + __m256i value, value2, cmp; + value = _mm256_loadu_si256((__m256i *) ip); + value2 = _mm256_loadu_si256((__m256i *)ref); + cmp = _mm256_cmpeq_epi64(value, value2); + if ((unsigned)_mm256_movemask_epi8(cmp) != 0xFFFFFFFF) { + /* Return the byte that starts to differ */ + while (*ref++ == *ip++) {} + return ip; + } + else { + ip += sizeof(__m256i); + ref += sizeof(__m256i); + } + } + /* Look into the remainder */ + while ((ip < ip_bound) && (*ref++ == *ip++)) {} + return ip; +} +#endif + + +static uint8_t* get_run_or_match(uint8_t* ip, uint8_t* ip_bound, const uint8_t* ref, bool run) { + if (BLOSCLZ_UNLIKELY(run)) { +#if defined(__AVX2__) + // Extensive experiments on AMD Ryzen3 say that regular get_run is faster + // ip = get_run_32(ip, ip_bound, ref); + ip = get_run(ip, ip_bound, ref); +#elif defined(__SSE2__) + // Extensive experiments on AMD Ryzen3 say that regular get_run is faster + // ip = get_run_16(ip, ip_bound, ref); + ip = get_run(ip, ip_bound, ref); +#else + ip = get_run(ip, ip_bound, ref); +#endif + } + else { +#if defined(__AVX2__) + // Extensive experiments on AMD Ryzen3 say that regular get_match_16 is faster + // ip = get_match_32(ip, ip_bound, ref); + ip = get_match_16(ip, ip_bound, ref); +#elif defined(__SSE2__) + ip = get_match_16(ip, ip_bound, ref); +#else + ip = get_match(ip, ip_bound, ref); +#endif + } + + return ip; +} + + +#define LITERAL(ip, op, op_limit, anchor, copy) { \ + if (BLOSCLZ_UNLIKELY((op) + 2 > (op_limit))) \ + goto out; \ + *(op)++ = *(anchor)++; \ + (ip) = (anchor); \ + (copy)++; \ + if (BLOSCLZ_UNLIKELY((copy) == MAX_COPY)) { \ + (copy) = 0; \ + *(op)++ = MAX_COPY-1; \ + } \ +} + +#define LITERAL2(ip, anchor, copy) { \ + oc++; (anchor)++; \ + (ip) = (anchor); \ + (copy)++; \ + if (BLOSCLZ_UNLIKELY((copy) == MAX_COPY)) { \ + (copy) = 0; \ + oc++; \ + } \ +} + +#define MATCH_SHORT(op, op_limit, len, distance) { \ + if (BLOSCLZ_UNLIKELY((op) + 2 > (op_limit))) \ + goto out; \ + *(op)++ = (uint8_t)(((len) << 5U) + ((distance) >> 8U));\ + *(op)++ = (uint8_t)(((distance) & 255U)); \ +} + +#define MATCH_LONG(op, op_limit, len, distance) { \ + if (BLOSCLZ_UNLIKELY((op) + 1 > (op_limit))) \ + goto out; \ + *(op)++ = (uint8_t)((7U << 5U) + ((distance) >> 8U)); \ + for ((len) -= 7; (len) >= 255; (len) -= 255) { \ + if (BLOSCLZ_UNLIKELY((op) + 1 > (op_limit))) \ + goto out; \ + *(op)++ = 255; \ + } \ + if (BLOSCLZ_UNLIKELY((op) + 2 > (op_limit))) \ + goto out; \ + *(op)++ = (uint8_t)(len); \ + *(op)++ = (uint8_t)(((distance) & 255U)); \ +} + +#define MATCH_SHORT_FAR(op, op_limit, len, distance) { \ + if (BLOSCLZ_UNLIKELY((op) + 4 > (op_limit))) \ + goto out; \ + *(op)++ = (uint8_t)(((len) << 5U) + 31); \ + *(op)++ = 255; \ + *(op)++ = (uint8_t)((distance) >> 8U); \ + *(op)++ = (uint8_t)((distance) & 255U); \ +} + +#define MATCH_LONG_FAR(op, op_limit, len, distance) { \ + if (BLOSCLZ_UNLIKELY((op) + 1 > (op_limit))) \ + goto out; \ + *(op)++ = (7U << 5U) + 31; \ + for ((len) -= 7; (len) >= 255; (len) -= 255) { \ + if (BLOSCLZ_UNLIKELY((op) + 1 > (op_limit))) \ + goto out; \ + *(op)++ = 255; \ + } \ + if (BLOSCLZ_UNLIKELY((op) + 4 > (op_limit))) \ + goto out; \ + *(op)++ = (uint8_t)(len); \ + *(op)++ = 255; \ + *(op)++ = (uint8_t)((distance) >> 8U); \ + *(op)++ = (uint8_t)((distance) & 255U); \ +} + + +// Get a guess for the compressed size of a buffer +static double get_cratio(uint8_t* ibase, int maxlen, int minlen, int ipshift) { + uint8_t* ip = ibase; + int32_t oc = 0; + const uint16_t hashlen = (1U << (uint8_t)HASH_LOG2); + uint16_t htab[1U << (uint8_t)HASH_LOG2]; + uint32_t hval; + uint32_t seq; + uint8_t copy; + // Make a tradeoff between testing too much and too little + uint16_t limit = (maxlen > hashlen) ? hashlen : maxlen; + uint8_t* ip_bound = ibase + limit - 1; + uint8_t* ip_limit = ibase + limit - 12; + + // Initialize the hash table to distances of 0 + memset(htab, 0, hashlen * sizeof(uint16_t)); + + /* we start with literal copy */ + copy = 4; + oc += 5; + + /* main loop */ + while (BLOSCLZ_LIKELY(ip < ip_limit)) { + const uint8_t* ref; + unsigned distance; + uint8_t* anchor = ip; /* comparison starting-point */ + + /* find potential match */ + seq = BLOSCLZ_READU32(ip); + HASH_FUNCTION(hval, seq, HASH_LOG2) + ref = ibase + htab[hval]; + + /* calculate distance to the match */ + distance = (unsigned int)(anchor - ref); + + /* update hash table */ + htab[hval] = (uint16_t) (anchor - ibase); + + if (distance == 0 || (distance >= MAX_FARDISTANCE)) { + LITERAL2(ip, anchor, copy) + continue; + } + + /* is this a match? check the first 4 bytes */ + if (BLOSCLZ_READU32(ref) == BLOSCLZ_READU32(ip)) { + ref += 4; + } + else { + /* no luck, copy as a literal */ + LITERAL2(ip, anchor, copy) + continue; + } + + /* last matched byte */ + ip = anchor + 4; + + /* distance is biased */ + distance--; + + /* get runs or matches; zero distance means a run */ + ip = get_run_or_match(ip, ip_bound, ref, !distance); + + ip -= ipshift; + int len = (int)(ip - anchor); + if (len < minlen) { + LITERAL2(ip, anchor, copy) + continue; + } + + /* if we haven't copied anything, adjust the output counter */ + if (!copy) + oc--; + /* reset literal counter */ + copy = 0; + + /* encode the match */ + if (distance < MAX_DISTANCE) { + if (len >= 7) { + oc += ((len - 7) / 255) + 1; + } + oc += 2; + } + else { + /* far away, but not yet in the another galaxy... */ + if (len >= 7) { + oc += ((len - 7) / 255) + 1; + } + oc += 4; + } + + /* update the hash at match boundary */ + seq = BLOSCLZ_READU32(ip); + HASH_FUNCTION(hval, seq, HASH_LOG2) + htab[hval] = (uint16_t)(ip++ - ibase); + ip++; + /* assuming literal copy */ + oc++; + } + + double ic = (double)(ip - ibase); + return ic / (double)oc; +} + + +int blosclz_compress(const int clevel, const void* input, int length, + void* output, int maxout, blosc2_context* ctx) { + uint8_t* ibase = (uint8_t*)input; + + // Experiments say that checking 1/4 of the buffer is enough to figure out approx cratio + int maxlen = length / 4; + // Start probing somewhere inside the buffer + int shift = length - maxlen; + // Actual entropy probing! + double cratio = get_cratio(ibase + shift, maxlen, 3, 3); + // discard probes with small compression ratios (too expensive) + double cratio_[10] = {0, 2, 1.5, 1.2, 1.2, 1.2, 1.2, 1.15, 1.1, 1.0}; + if (cratio < cratio_[clevel]) { + goto out; + } + + /* When we go back in a match (shift), we obtain quite different compression properties. + * It looks like 4 is more useful in combination with bitshuffle and small typesizes + * Fallback to 4 because it provides more consistent results for large cratios. + * + * In this block we also check cratios for the beginning of the buffers and + * eventually discard those that are small (take too long to decompress). + * This process is called _entropy probing_. + */ + unsigned ipshift = 4; + // Compute optimal shift and minimum lengths for encoding + // Use 4 by default, except for low entropy data, where we should do a best effort + unsigned minlen = 4; + // BloscLZ works better with splits mostly, so when data is not split, do a best effort + const int split_block = !((ctx->header_flags & 0x10) >> 4); + // Why using cratio < 4 is based in experiments with low and high entropy + if (!split_block || cratio < 4) { + ipshift = 3; + minlen = 3; + } + else { + minlen = 4; + } + + uint8_t hashlog_[10] = {0, HASH_LOG - 2, HASH_LOG - 1, HASH_LOG, HASH_LOG, + HASH_LOG, HASH_LOG, HASH_LOG, HASH_LOG, HASH_LOG}; + uint8_t hashlog = hashlog_[clevel]; + + uint8_t* ip = ibase; + uint8_t* ip_bound = ibase + length - 1; + uint8_t* ip_limit = ibase + length - 12; + uint8_t* op = (uint8_t*)output; + const uint8_t* op_limit = op + maxout; + uint32_t seq; + uint8_t copy; + uint32_t hval; + + /* input and output buffer cannot be less than 16 and 66 bytes or we can get into trouble */ + if (length < 16 || maxout < 66) { + return 0; + } + + // Initialize the hash table + uint32_t htab[1U << (uint8_t)HASH_LOG]; + memset(htab, 0, (1U << hashlog) * sizeof(uint32_t)); + + /* we start with literal copy */ + copy = 4; + *op++ = MAX_COPY - 1; + *op++ = *ip++; + *op++ = *ip++; + *op++ = *ip++; + *op++ = *ip++; + + /* main loop */ + while (BLOSCLZ_LIKELY(ip < ip_limit)) { + const uint8_t* ref; + unsigned distance; + uint8_t* anchor = ip; /* comparison starting-point */ + + /* find potential match */ + seq = BLOSCLZ_READU32(ip); + HASH_FUNCTION(hval, seq, hashlog) + ref = ibase + htab[hval]; + + /* calculate distance to the match */ + distance = (unsigned int)(anchor - ref); + + /* update hash table */ + htab[hval] = (uint32_t) (anchor - ibase); + + if (distance == 0 || (distance >= MAX_FARDISTANCE)) { + LITERAL(ip, op, op_limit, anchor, copy) + continue; + } + + /* is this a match? check the first 4 bytes */ + if (BLOSCLZ_UNLIKELY(BLOSCLZ_READU32(ref) == BLOSCLZ_READU32(ip))) { + ref += 4; + } else { + /* no luck, copy as a literal */ + LITERAL(ip, op, op_limit, anchor, copy) + continue; + } + + /* last matched byte */ + ip = anchor + 4; + + /* distance is biased */ + distance--; + + /* get runs or matches; zero distance means a run */ + ip = get_run_or_match(ip, ip_bound, ref, !distance); + + /* length is biased, '1' means a match of 3 bytes */ + ip -= ipshift; + + unsigned len = (int)(ip - anchor); + + // Encoding short lengths is expensive during decompression + if (len < minlen || (len <= 5 && distance >= MAX_DISTANCE)) { + LITERAL(ip, op, op_limit, anchor, copy) + continue; + } + + /* if we have copied something, adjust the copy count */ + if (copy) + /* copy is biased, '0' means 1 byte copy */ + *(op - copy - 1) = (uint8_t)(copy - 1); + else + /* back, to overwrite the copy count */ + op--; + /* reset literal counter */ + copy = 0; + + /* encode the match */ + if (distance < MAX_DISTANCE) { + if (len < 7) { + MATCH_SHORT(op, op_limit, len, distance) + } else { + MATCH_LONG(op, op_limit, len, distance) + } + } else { + /* far away, but not yet in the another galaxy... */ + distance -= MAX_DISTANCE; + if (len < 7) { + MATCH_SHORT_FAR(op, op_limit, len, distance) + } else { + MATCH_LONG_FAR(op, op_limit, len, distance) + } + } + + /* update the hash at match boundary */ + seq = BLOSCLZ_READU32(ip); + HASH_FUNCTION(hval, seq, hashlog) + htab[hval] = (uint32_t) (ip++ - ibase); + if (ctx->clevel == 9) { + // In some situations, including a second hash proves to be useful, + // but not in others. Activating here in max clevel only. + seq >>= 8U; + HASH_FUNCTION(hval, seq, hashlog) + htab[hval] = (uint32_t) (ip++ - ibase); + } + else { + ip++; + } + + if (BLOSCLZ_UNLIKELY(op + 1 > op_limit)) + goto out; + + /* assuming literal copy */ + *op++ = MAX_COPY - 1; + } + + /* left-over as literal copy */ + while (BLOSCLZ_UNLIKELY(ip <= ip_bound)) { + if (BLOSCLZ_UNLIKELY(op + 2 > op_limit)) goto out; + *op++ = *ip++; + copy++; + if (BLOSCLZ_UNLIKELY(copy == MAX_COPY)) { + copy = 0; + *op++ = MAX_COPY - 1; + } + } + + /* if we have copied something, adjust the copy length */ + if (copy) + *(op - copy - 1) = (uint8_t)(copy - 1); + else + op--; + + /* marker for blosclz */ + *(uint8_t*)output |= (1U << 5U); + + return (int)(op - (uint8_t*)output); + + out: + return 0; +} + +// See https://habr.com/en/company/yandex/blog/457612/ +#if defined(__AVX2__) + +#if defined(_MSC_VER) +#define ALIGNED_(x) __declspec(align(x)) +#else +#if defined(__GNUC__) +#define ALIGNED_(x) __attribute__ ((aligned(x))) +#endif +#endif +#define ALIGNED_TYPE_(t, x) t ALIGNED_(x) + +static unsigned char* copy_match_16(unsigned char *op, const unsigned char *match, int32_t len) +{ + size_t offset = op - match; + while (len >= 16) { + + static const ALIGNED_TYPE_(uint8_t, 16) masks[] = + { + 0, 1, 2, 1, 4, 1, 4, 2, 8, 7, 6, 5, 4, 3, 2, 1, // offset = 0, not used as mask, but for shift + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // offset = 1 + 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, + 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, + 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, + 0, 1, 2, 3, 4, 0, 1, 2, 3, 4, 0, 1, 2, 3, 4, 0, + 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, + 0, 1, 2, 3, 4, 5, 6, 0, 1, 2, 3, 4, 5, 6, 0, 1, + 0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 0, 1, 2, 3, 4, 5, 6, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 0, 1, 2, 3, 4, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 0, 1, 2, 3, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 0, 1, 2, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 0, 1, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 0, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, // offset = 16 + }; + + _mm_storeu_si128((__m128i *)(op), + _mm_shuffle_epi8(_mm_loadu_si128((const __m128i *)(match)), + _mm_load_si128((const __m128i *)(masks) + offset))); + + match += masks[offset]; + + op += 16; + len -= 16; + } + // Deal with remainders + for (; len > 0; len--) { + *op++ = *match++; + } + return op; +} +#endif + +// LZ4 wildCopy which can reach excellent copy bandwidth (even if insecure) +static inline void wild_copy(uint8_t *out, const uint8_t* from, uint8_t* end) { + uint8_t* d = out; + const uint8_t* s = from; + uint8_t* const e = end; + + do { memcpy(d,s,8); d+=8; s+=8; } while (d= 32) { + // match + int32_t len = (int32_t)(ctrl >> 5U) - 1 ; + int32_t ofs = (int32_t)(ctrl & 31U) << 8U; + uint8_t code; + const uint8_t* ref = op - ofs; + + if (len == 7 - 1) { + do { + if (BLOSCLZ_UNLIKELY(ip + 1 >= ip_limit)) { + return 0; + } + code = *ip++; + len += code; + } while (code == 255); + } + else { + if (BLOSCLZ_UNLIKELY(ip + 1 >= ip_limit)) { + return 0; + } + } + code = *ip++; + len += 3; + ref -= code; + + /* match from 16-bit distance */ + if (BLOSCLZ_UNLIKELY(code == 255)) { + if (ofs == (31U << 8U)) { + if (ip + 1 >= ip_limit) { + return 0; + } + ofs = (*ip++) << 8U; + ofs += *ip++; + ref = op - ofs - MAX_DISTANCE; + } + } + + if (BLOSCLZ_UNLIKELY(op + len > op_limit)) { + return 0; + } + + if (BLOSCLZ_UNLIKELY(ref - 1 < (uint8_t*)output)) { + return 0; + } + + if (BLOSCLZ_UNLIKELY(ip >= ip_limit)) break; + ctrl = *ip++; + + ref--; + if (ref == op - 1) { + /* optimized copy for a run */ + memset(op, *ref, len); + op += len; + } + else if ((op - ref >= 8) && (op_limit - op >= len + 8)) { + // copy with an overlap not larger than 8 + wild_copy(op, ref, op + len); + op += len; + } + else { + // general copy with any overlap +#if defined(__AVX2__) + if (op - ref <= 16) { + // This is not faster on a combination of compilers (clang, gcc, icc) or machines, but + // it is not slower either. Let's activate here for experimentation. + op = copy_match_16(op, ref, len); + } + else { +#endif + op = copy_match(op, ref, (unsigned) len); +#if defined(__AVX2__) + } +#endif + } + } + else { + // literal + ctrl++; + if (BLOSCLZ_UNLIKELY(op + ctrl > op_limit)) { + return 0; + } + if (BLOSCLZ_UNLIKELY(ip + ctrl > ip_limit)) { + return 0; + } + + memcpy(op, ip, ctrl); op += ctrl; ip += ctrl; + // On GCC-6, fastcopy this is still faster than plain memcpy + // However, using recent CLANG/LLVM 9.0, there is almost no difference + // in performance. + // And starting on CLANG/LLVM 10 and GCC 9, memcpy is generally faster. + // op = fastcopy(op, ip, (unsigned) ctrl); ip += ctrl; + + if (BLOSCLZ_UNLIKELY(ip >= ip_limit)) break; + ctrl = *ip++; + } + } + + return (int)(op - (uint8_t*)output); +} diff --git a/src/blosc2/blosc/blosclz.h b/src/blosc2/blosc/blosclz.h new file mode 100644 index 0000000..6adee3c --- /dev/null +++ b/src/blosc2/blosc/blosclz.h @@ -0,0 +1,70 @@ +/********************************************************************* + Blosc - Blocked Shuffling and Compression Library + + Copyright (C) 2021 The Blosc Developers + https://blosc.org + License: BSD 3-Clause (see LICENSE.txt) + + See LICENSE.txt for details about copyright and rights to use. +**********************************************************************/ + +/********************************************************************* + The code in this file is heavily based on FastLZ, a lightning-fast + lossless compression library. See LICENSES/FASTLZ.txt for details + about copyright and rights to use. +**********************************************************************/ + + +#ifndef BLOSCLZ_H +#define BLOSCLZ_H +#include "context.h" + +#if defined (__cplusplus) +extern "C" { +#endif + +#define BLOSCLZ_VERSION_STRING "2.5.1" + + +/** + Compress a block of data in the input buffer and returns the size of + compressed block. The size of input buffer is specified by + length. The minimum input buffer size is 16. + + The output buffer must be at least 5% larger than the input buffer + and can not be smaller than 66 bytes. + + If the input is not compressible, or output does not fit in maxout + bytes, the return value will be 0 and you will have to discard the + output buffer. + + The acceleration parameter is related with the frequency for + updating the internal hash. An acceleration of 1 means that the + internal hash is updated at full rate. A value < 1 is not allowed + and will be silently set to 1. + + The input buffer and the output buffer can not overlap. +*/ + +int blosclz_compress(int opt_level, const void* input, int length, + void* output, int maxout, blosc2_context* ctx); + +/** + Decompress a block of compressed data and returns the size of the + decompressed block. If error occurs, e.g. the compressed data is + corrupted or the output buffer is not large enough, then 0 (zero) + will be returned instead. + + The input buffer and the output buffer can not overlap. + + Decompression is memory safe and guaranteed not to write the output buffer + more than what is specified in maxout. + */ + +int blosclz_decompress(const void* input, int length, void* output, int maxout); + +#if defined (__cplusplus) +} +#endif + +#endif /* BLOSCLZ_H */ diff --git a/src/blosc2/blosc/cmake_install.cmake b/src/blosc2/blosc/cmake_install.cmake new file mode 100644 index 0000000..a236772 --- /dev/null +++ b/src/blosc2/blosc/cmake_install.cmake @@ -0,0 +1,54 @@ +# Install script for directory: /home/fangq/space/git/Project/github/zmat/src/blosc2/blosc + +# Set the install prefix +if(NOT DEFINED CMAKE_INSTALL_PREFIX) + set(CMAKE_INSTALL_PREFIX "/usr/local") +endif() +string(REGEX REPLACE "/$" "" CMAKE_INSTALL_PREFIX "${CMAKE_INSTALL_PREFIX}") + +# Set the install configuration name. +if(NOT DEFINED CMAKE_INSTALL_CONFIG_NAME) + if(BUILD_TYPE) + string(REGEX REPLACE "^[^A-Za-z0-9_]+" "" + CMAKE_INSTALL_CONFIG_NAME "${BUILD_TYPE}") + else() + set(CMAKE_INSTALL_CONFIG_NAME "") + endif() + message(STATUS "Install configuration: \"${CMAKE_INSTALL_CONFIG_NAME}\"") +endif() + +# Set the component getting installed. +if(NOT CMAKE_INSTALL_COMPONENT) + if(COMPONENT) + message(STATUS "Install component: \"${COMPONENT}\"") + set(CMAKE_INSTALL_COMPONENT "${COMPONENT}") + else() + set(CMAKE_INSTALL_COMPONENT) + endif() +endif() + +# Install shared libraries without execute permission? +if(NOT DEFINED CMAKE_INSTALL_SO_NO_EXE) + set(CMAKE_INSTALL_SO_NO_EXE "1") +endif() + +# Is this installation the result of a crosscompile? +if(NOT DEFINED CMAKE_CROSSCOMPILING) + set(CMAKE_CROSSCOMPILING "FALSE") +endif() + +# Set default install directory permissions. +if(NOT DEFINED CMAKE_OBJDUMP) + set(CMAKE_OBJDUMP "/usr/bin/objdump") +endif() + +if(CMAKE_INSTALL_COMPONENT) + set(CMAKE_INSTALL_MANIFEST "install_manifest_${CMAKE_INSTALL_COMPONENT}.txt") +else() + set(CMAKE_INSTALL_MANIFEST "install_manifest.txt") +endif() + +string(REPLACE ";" "\n" CMAKE_INSTALL_MANIFEST_CONTENT + "${CMAKE_INSTALL_MANIFEST_FILES}") +file(WRITE "/home/fangq/space/git/Project/github/zmat/src/blosc2/blosc/${CMAKE_INSTALL_MANIFEST}" + "${CMAKE_INSTALL_MANIFEST_CONTENT}") diff --git a/src/blosc2/blosc/config.h b/src/blosc2/blosc/config.h new file mode 100644 index 0000000..d6296da --- /dev/null +++ b/src/blosc2/blosc/config.h @@ -0,0 +1,11 @@ +#ifndef _CONFIGURATION_HEADER_GUARD_H_ +#define _CONFIGURATION_HEADER_GUARD_H_ + +#define HAVE_ZLIB TRUE +#define HAVE_ZLIB_NG TRUE +#define HAVE_ZSTD TRUE +/* #undef HAVE_IPP */ +/* #undef BLOSC_DLL_EXPORT */ +#define HAVE_PLUGINS TRUE + +#endif diff --git a/src/blosc2/blosc/config.h.in b/src/blosc2/blosc/config.h.in new file mode 100644 index 0000000..66cfd90 --- /dev/null +++ b/src/blosc2/blosc/config.h.in @@ -0,0 +1,11 @@ +#ifndef _CONFIGURATION_HEADER_GUARD_H_ +#define _CONFIGURATION_HEADER_GUARD_H_ + +#cmakedefine HAVE_ZLIB @HAVE_ZLIB@ +#cmakedefine HAVE_ZLIB_NG @HAVE_ZLIB_NG@ +#cmakedefine HAVE_ZSTD @HAVE_ZSTD@ +#cmakedefine HAVE_IPP @HAVE_IPP@ +#cmakedefine BLOSC_DLL_EXPORT @DLL_EXPORT@ +#cmakedefine HAVE_PLUGINS @HAVE_PLUGINS@ + +#endif diff --git a/src/blosc2/blosc/context.h b/src/blosc2/blosc/context.h new file mode 100644 index 0000000..ee16613 --- /dev/null +++ b/src/blosc2/blosc/context.h @@ -0,0 +1,167 @@ +/********************************************************************* + Blosc - Blocked Shuffling and Compression Library + + Copyright (C) 2021 The Blosc Developers + https://blosc.org + License: BSD 3-Clause (see LICENSE.txt) + + See LICENSE.txt for details about copyright and rights to use. +**********************************************************************/ + +#ifndef CONTEXT_H +#define CONTEXT_H + +#if defined(_WIN32) && !defined(__GNUC__) + #include "win32/pthread.h" +#else + #include +#endif + +/* Have problems using posix barriers when symbol value is 200112L */ +/* Requires more investigation, but this will work for the moment */ +#if defined(_POSIX_BARRIERS) && ( (_POSIX_BARRIERS - 20012L) >= 0 && _POSIX_BARRIERS != 200112L) +#define BLOSC_POSIX_BARRIERS +#endif + +#include "blosc2.h" + +#if defined(HAVE_ZSTD) + #include "zstd.h" +#endif /* HAVE_ZSTD */ + +#ifdef HAVE_IPP + #include +#endif /* HAVE_IPP */ + +struct blosc2_context_s { + const uint8_t* src; + /* The source buffer */ + uint8_t* dest; + /* The destination buffer */ + uint8_t header_flags; + /* Flags for header */ + uint8_t blosc2_flags; + /* Flags specific for blosc2 */ + int32_t sourcesize; + /* Number of bytes in source buffer */ + int32_t header_overhead; + /* The number of bytes in chunk header */ + int32_t nblocks; + /* Number of total blocks in buffer */ + int32_t leftover; + /* Extra bytes at end of buffer */ + int32_t blocksize; + /* Length of the block in bytes */ + int32_t splitmode; + /* Whether the blocks should be split or not */ + int32_t output_bytes; + /* Counter for the number of input bytes */ + int32_t srcsize; + /* Counter for the number of output bytes */ + int32_t destsize; + /* Maximum size for destination buffer */ + int32_t typesize; + /* Type size */ + int32_t* bstarts; + /* Starts for every block inside the compressed buffer */ + int32_t special_type; + /* Special type for chunk. 0 if not special. */ + int compcode; + /* Compressor code to use */ + uint8_t compcode_meta; + /* The metainfo for the compressor code */ + int clevel; + /* Compression level (1-9) */ + int use_dict; + /* Whether to use dicts or not */ + void* dict_buffer; + /* The buffer to keep the trained dictionary */ + int32_t dict_size; + /* The size of the trained dictionary */ + void* dict_cdict; + /* The dictionary in digested form for compression */ + void* dict_ddict; + /* The dictionary in digested form for decompression */ + uint8_t filter_flags; + /* The filter flags in the filter pipeline */ + uint8_t filters[BLOSC2_MAX_FILTERS]; + /* the (sequence of) filters */ + uint8_t filters_meta[BLOSC2_MAX_FILTERS]; + /* the metainfo for filters */ + blosc2_filter urfilters[BLOSC2_MAX_UDFILTERS]; + /* The user-defined filters */ + blosc2_prefilter_fn prefilter; + /* prefilter function */ + blosc2_postfilter_fn postfilter; + /* postfilter function */ + blosc2_prefilter_params *preparams; + /* prefilter params */ + blosc2_postfilter_params *postparams; + /* postfilter params */ + bool* block_maskout; + /* The blocks that are not meant to be decompressed. + * If NULL (default), all blocks in a chunk should be read. */ + int block_maskout_nitems; + /* The number of items in block_maskout array (must match + * the number of blocks in chunk) */ + blosc2_schunk* schunk; + /* Associated super-chunk (if available) */ + struct thread_context* serial_context; + /* Cache for temporaries for serial operation */ + int do_compress; + /* 1 if we are compressing, 0 if decompressing */ + void *btune; + /* Entry point for BTune persistence between runs */ + blosc2_btune *udbtune; + /* User-defined BTune parameters */ + /* Threading */ + int16_t nthreads; + int16_t new_nthreads; + int16_t threads_started; + int16_t end_threads; + pthread_t *threads; + struct thread_context *thread_contexts; /* only for user-managed threads */ + pthread_mutex_t count_mutex; + pthread_mutex_t nchunk_mutex; +#ifdef BLOSC_POSIX_BARRIERS + pthread_barrier_t barr_init; + pthread_barrier_t barr_finish; +#else + int count_threads; + pthread_mutex_t count_threads_mutex; + pthread_cond_t count_threads_cv; +#endif +#if !defined(_WIN32) + pthread_attr_t ct_attr; /* creation time attrs for threads */ +#endif + int thread_giveup_code; + /* error code when give up */ + int thread_nblock; /* block counter */ + int dref_not_init; /* data ref in delta not initialized */ + pthread_mutex_t delta_mutex; + pthread_cond_t delta_cv; +}; + +struct thread_context { + blosc2_context* parent_context; + int tid; + uint8_t* tmp; + uint8_t* tmp2; + uint8_t* tmp3; + uint8_t* tmp4; + int32_t tmp_blocksize; /* the blocksize for different temporaries */ + size_t tmp_nbytes; /* keep track of how big the temporary buffers are */ + int32_t zfp_cell_start; /* cell starter index for ZFP fixed-rate mode */ + int32_t zfp_cell_nitems; /* number of items to get for ZFP fixed-rate mode */ +#if defined(HAVE_ZSTD) + /* The contexts for ZSTD */ + ZSTD_CCtx* zstd_cctx; + ZSTD_DCtx* zstd_dctx; +#endif /* HAVE_ZSTD */ +#ifdef HAVE_IPP + Ipp8u* lz4_hash_table; +#endif +}; + + +#endif /* CONTEXT_H */ diff --git a/src/blosc2/blosc/delta.c b/src/blosc2/blosc/delta.c new file mode 100644 index 0000000..1559075 --- /dev/null +++ b/src/blosc2/blosc/delta.c @@ -0,0 +1,159 @@ +/********************************************************************* + Blosc - Blocked Shuffling and Compression Library + + Copyright (C) 2021 The Blosc Developers + https://blosc.org + License: BSD 3-Clause (see LICENSE.txt) + + See LICENSE.txt for details about copyright and rights to use. +**********************************************************************/ + +#include +#include "delta.h" + + +/* Apply the delta filters to src. This can never fail. */ +void delta_encoder(const uint8_t* dref, int32_t offset, int32_t nbytes, int32_t typesize, + const uint8_t* src, uint8_t* dest) { + int32_t i; + if (offset == 0) { + /* This is the reference block, use delta coding in elements */ + switch (typesize) { + case 1: + dest[0] = dref[0]; + for (i = 1; i < nbytes; i++) { + dest[i] = src[i] ^ dref[i-1]; + } + break; + case 2: + ((uint16_t *)dest)[0] = ((uint16_t *)dref)[0]; + for (i = 1; i < nbytes / 2; i++) { + ((uint16_t *)dest)[i] = + ((uint16_t *)src)[i] ^ ((uint16_t *)dref)[i-1]; + } + break; + case 4: + ((uint32_t *)dest)[0] = ((uint32_t *)dref)[0]; + for (i = 1; i < nbytes / 4; i++) { + ((uint32_t *)dest)[i] = + ((uint32_t *)src)[i] ^ ((uint32_t *)dref)[i-1]; + } + break; + case 8: + ((uint64_t *)dest)[0] = ((uint64_t *)dref)[0]; + for (i = 1; i < nbytes / 8; i++) { + ((uint64_t *)dest)[i] = + ((uint64_t *)src)[i] ^ ((uint64_t *)dref)[i-1]; + } + break; + default: + if ((typesize % 8) == 0) { + delta_encoder(dref, offset, nbytes, 8, src, dest); + } else { + delta_encoder(dref, offset, nbytes, 1, src, dest); + } + } + } else { + /* Use delta coding wrt reference block */ + switch (typesize) { + case 1: + for (i = 0; i < nbytes; i++) { + dest[i] = src[i] ^ dref[i]; + } + break; + case 2: + for (i = 0; i < nbytes / 2; i++) { + ((uint16_t *) dest)[i] = + ((uint16_t *) src)[i] ^ ((uint16_t *) dref)[i]; + } + break; + case 4: + for (i = 0; i < nbytes / 4; i++) { + ((uint32_t *) dest)[i] = + ((uint32_t *) src)[i] ^ ((uint32_t *) dref)[i]; + } + break; + case 8: + for (i = 0; i < nbytes / 8; i++) { + ((uint64_t *) dest)[i] = + ((uint64_t *) src)[i] ^ ((uint64_t *) dref)[i]; + } + break; + default: + if ((typesize % 8) == 0) { + delta_encoder(dref, offset, nbytes, 8, src, dest); + } else { + delta_encoder(dref, offset, nbytes, 1, src, dest); + } + } + } +} + + +/* Undo the delta filter in dest. This can never fail. */ +void delta_decoder(const uint8_t* dref, int32_t offset, int32_t nbytes, + int32_t typesize, uint8_t* dest) { + int32_t i; + + if (offset == 0) { + /* Decode delta for the reference block */ + switch (typesize) { + case 1: + for (i = 1; i < nbytes; i++) { + dest[i] ^= dref[i-1]; + } + break; + case 2: + for (i = 1; i < nbytes / 2; i++) { + ((uint16_t *)dest)[i] ^= ((uint16_t *)dref)[i-1]; + } + break; + case 4: + for (i = 1; i < nbytes / 4; i++) { + ((uint32_t *)dest)[i] ^= ((uint32_t *)dref)[i-1]; + } + break; + case 8: + for (i = 1; i < nbytes / 8; i++) { + ((uint64_t *)dest)[i] ^= ((uint64_t *)dref)[i-1]; + } + break; + default: + if ((typesize % 8) == 0) { + delta_decoder(dref, offset, nbytes, 8, dest); + } else { + delta_decoder(dref, offset, nbytes, 1, dest); + } + } + } else { + /* Decode delta for the non-reference blocks */ + switch (typesize) { + case 1: + for (i = 0; i < nbytes; i++) { + dest[i] ^= dref[i]; + } + break; + case 2: + for (i = 0; i < nbytes / 2; i++) { + ((uint16_t *)dest)[i] ^= ((uint16_t *)dref)[i]; + } + break; + case 4: + for (i = 0; i < nbytes / 4; i++) { + ((uint32_t *)dest)[i] ^= ((uint32_t *)dref)[i]; + } + break; + case 8: + for (i = 0; i < nbytes / 8; i++) { + ((uint64_t *)dest)[i] ^= ((uint64_t *)dref)[i]; + } + break; + default: + if ((typesize % 8) == 0) { + delta_decoder(dref, offset, nbytes, 8, dest); + } else { + delta_decoder(dref, offset, nbytes, 1, dest); + } + } + } +} diff --git a/src/blosc2/blosc/delta.h b/src/blosc2/blosc/delta.h new file mode 100644 index 0000000..8ca26de --- /dev/null +++ b/src/blosc2/blosc/delta.h @@ -0,0 +1,23 @@ +/********************************************************************* + Blosc - Blocked Shuffling and Compression Library + + Copyright (C) 2021 The Blosc Developers + https://blosc.org + License: BSD 3-Clause (see LICENSE.txt) + + See LICENSE.txt for details about copyright and rights to use. +**********************************************************************/ + +#ifndef BLOSC_DELTA_H +#define BLOSC_DELTA_H + +#include +#include + +void delta_encoder(const uint8_t* dref, int32_t offset, int32_t nbytes, + int32_t typesize, const uint8_t* src, uint8_t* dest); + +void delta_decoder(const uint8_t* dref, int32_t offset, int32_t nbytes, + int32_t typesize, uint8_t* dest); + +#endif //BLOSC_DELTA_H diff --git a/src/blosc2/blosc/directories.c b/src/blosc2/blosc/directories.c new file mode 100644 index 0000000..be852c8 --- /dev/null +++ b/src/blosc2/blosc/directories.c @@ -0,0 +1,158 @@ +/********************************************************************* + Blosc - Blocked Shuffling and Compression Library + + Copyright (C) 2021 The Blosc Developers + https://blosc.org + License: BSD 3-Clause (see LICENSE.txt) + + See LICENSE.txt for details about copyright and rights to use. +**********************************************************************/ + +#include +#include "blosc2.h" +#include + +#if defined(_WIN32) + #include + #include + #include + + int blosc2_remove_dir(const char* dir_path) { + char* path; + char last_char = dir_path[strlen(dir_path) - 1]; + if (last_char != '\\' || last_char != '/') { + path = malloc(strlen(dir_path) + 2 + 1); + sprintf(path, "%s\\*", dir_path); + } + else { + path = malloc(strlen(dir_path) + 1 + 1); + strcpy(path, dir_path); + strcat(path, "*"); + } + char* fname; + struct _finddata_t cfile; + + intptr_t file = _findfirst(path, &cfile); + free(path); + + if (file == -1) { + BLOSC_TRACE_ERROR("Could not open the file."); + return BLOSC2_ERROR_FILE_OPEN; + } + int ret; + + while ( _findnext(file, &cfile) == 0) { + if (strcmp(".", cfile.name) == 0 || strcmp("..", cfile.name) == 0) { + continue; + } + fname = malloc(strlen(dir_path) + 1 + strlen(cfile.name) + 1); + sprintf(fname, "%s\\%s", dir_path, cfile.name); + + ret = remove(fname); + free(fname); + if (ret < 0) { + BLOSC_TRACE_ERROR("Could not remove file %s", fname); + _findclose(file); + return BLOSC2_ERROR_FAILURE; + } + } + + rmdir(dir_path); + _findclose(file); + return BLOSC2_ERROR_SUCCESS; + } + +#else + #include + #include + +/* Return the directory path with the '/' at the end */ +char* blosc2_normalize_dirpath(const char* dir_path) { + char last_char = dir_path[strlen(dir_path) - 1]; + char* path; + if (last_char != '\\' || last_char != '/'){ + path = malloc(strlen(dir_path) + 1 + 1); + sprintf(path, "%s/", dir_path); + } + else { + path = malloc(strlen(dir_path) + 1); + strcpy(path, dir_path); + } + return path; +} + +/* Function needed for removing each time the directory */ +int blosc2_remove_dir(const char* dir_path) { + char* path = blosc2_normalize_dirpath(dir_path); + + DIR* dr = opendir(path); + struct stat statbuf; + if (dr == NULL) { + BLOSC_TRACE_ERROR("No file or directory found."); + free(path); + return BLOSC2_ERROR_NOT_FOUND; + } + struct dirent *de; + int ret; + char* fname; + while ((de = readdir(dr)) != NULL) { + fname = malloc(strlen(path) + strlen(de->d_name) + 1); + sprintf(fname, "%s%s", path, de->d_name); + if (!strcmp(de->d_name, ".") || !strcmp(de->d_name, "..")) { + free(fname); + continue; + } + if (!stat(fname, &statbuf)) { + ret = unlink(fname); + if (ret < 0) { + BLOSC_TRACE_ERROR("Could not remove file %s", fname); + free(fname); + closedir(dr); + free(path); + return BLOSC2_ERROR_FAILURE; + } + } + free(fname); + } + closedir(dr); + rmdir(path); + free(path); + return BLOSC2_ERROR_SUCCESS; +} + +#endif /* _WIN32 */ + +int blosc2_remove_urlpath(const char* urlpath){ + if (urlpath != NULL) { + struct stat statbuf; + if (stat(urlpath, &statbuf) != 0){ + BLOSC_TRACE_ERROR("Could not access %s", urlpath); + return BLOSC2_ERROR_FAILURE; + } + if ((statbuf.st_mode & S_IFDIR) != 0) { + return blosc2_remove_dir(urlpath); + } + if (remove(urlpath) < 0) { + BLOSC_TRACE_ERROR("Could not remove %s", urlpath); + return BLOSC2_ERROR_FILE_REMOVE; + } + } + return BLOSC2_ERROR_SUCCESS; +} + +int blosc2_rename_urlpath(char* old_urlpath, char* new_urlpath){ + if (old_urlpath != NULL && new_urlpath != NULL) { + struct stat statbuf; + if (stat(old_urlpath, &statbuf) != 0) { + BLOSC_TRACE_ERROR("Could not access %s", old_urlpath); + return BLOSC2_ERROR_FAILURE; + } + int ret = rename(old_urlpath, new_urlpath); + if (ret < 0) { + BLOSC_TRACE_ERROR("Could not rename %s to %s", old_urlpath, new_urlpath); + return BLOSC2_ERROR_FAILURE; + } + } + return BLOSC2_ERROR_SUCCESS; +} + diff --git a/src/blosc2/blosc/fastcopy.c b/src/blosc2/blosc/fastcopy.c new file mode 100644 index 0000000..43f946c --- /dev/null +++ b/src/blosc2/blosc/fastcopy.c @@ -0,0 +1,639 @@ +/********************************************************************* + Blosc - Blocked Shuffling and Compression Library + + Copyright (C) 2021 The Blosc Developers + https://blosc.org + License: BSD 3-Clause (see LICENSE.txt) + + See LICENSE.txt for details about copyright and rights to use. +**********************************************************************/ + +/********************************************************************* + The code in this file is heavily based on memcopy.h, from the + zlib-ng compression library. See LICENSES/ZLIB.txt for details. + See also: https://github.com/Dead2/zlib-ng/blob/develop/zlib.h + + New implementations by Francesc Alted: + * fast_copy() and copy_run() functions + * Support for SSE2/AVX2 copy instructions for these routines +**********************************************************************/ + +#include +#include "blosc2/blosc2-common.h" + +/* + * Use inlined functions for supported systems. + */ +#if defined(_MSC_VER) && !defined(__cplusplus) /* Visual Studio */ +#define inline __inline /* Visual C is not C99, but supports some kind of inline */ +#endif + + +static inline unsigned char *copy_1_bytes(unsigned char *out, const unsigned char *from) { + *out++ = *from; + return out; +} + +static inline unsigned char *copy_2_bytes(unsigned char *out, const unsigned char *from) { +#if defined(BLOSC_STRICT_ALIGN) + uint16_t chunk; + memcpy(&chunk, from, 2); + memcpy(out, &chunk, 2); +#else + *(uint16_t *) out = *(uint16_t *) from; +#endif + return out + 2; +} + +static inline unsigned char *copy_3_bytes(unsigned char *out, const unsigned char *from) { + out = copy_1_bytes(out, from); + return copy_2_bytes(out, from + 1); +} + +static inline unsigned char *copy_4_bytes(unsigned char *out, const unsigned char *from) { +#if defined(BLOSC_STRICT_ALIGN) + uint32_t chunk; + memcpy(&chunk, from, 4); + memcpy(out, &chunk, 4); +#else + *(uint32_t *) out = *(uint32_t *) from; +#endif + return out + 4; +} + +static inline unsigned char *copy_5_bytes(unsigned char *out, const unsigned char *from) { + out = copy_1_bytes(out, from); + return copy_4_bytes(out, from + 1); +} + +static inline unsigned char *copy_6_bytes(unsigned char *out, const unsigned char *from) { + out = copy_2_bytes(out, from); + return copy_4_bytes(out, from + 2); +} + +static inline unsigned char *copy_7_bytes(unsigned char *out, const unsigned char *from) { + out = copy_3_bytes(out, from); + return copy_4_bytes(out, from + 3); +} + +static inline unsigned char *copy_8_bytes(unsigned char *out, const unsigned char *from) { +#if defined(BLOSC_STRICT_ALIGN) + uint64_t chunk; + memcpy(&chunk, from, 8); + memcpy(out, &chunk, 8); +#else + *(uint64_t *) out = *(uint64_t *) from; +#endif + return out + 8; +} + + +static inline unsigned char *copy_16_bytes(unsigned char *out, const unsigned char *from) { +#if defined(__SSE2__) + __m128i chunk; + chunk = _mm_loadu_si128((__m128i*)from); + _mm_storeu_si128((__m128i*)out, chunk); + out += 16; +#elif !defined(BLOSC_STRICT_ALIGN) + *(uint64_t*)out = *(uint64_t*)from; + from += 8; out += 8; + *(uint64_t*)out = *(uint64_t*)from; + from += 8; out += 8; +#else + int i; + for (i = 0; i < 16; i++) { + *out++ = *from++; + } +#endif + return out; +} + +static inline unsigned char *copy_32_bytes(unsigned char *out, const unsigned char *from) { +#if defined(__AVX2__) + __m256i chunk; + chunk = _mm256_loadu_si256((__m256i*)from); + _mm256_storeu_si256((__m256i*)out, chunk); + out += 32; +#elif defined(__SSE2__) + __m128i chunk; + chunk = _mm_loadu_si128((__m128i*)from); + _mm_storeu_si128((__m128i*)out, chunk); + from += 16; out += 16; + chunk = _mm_loadu_si128((__m128i*)from); + _mm_storeu_si128((__m128i*)out, chunk); + out += 16; +#elif !defined(BLOSC_STRICT_ALIGN) + *(uint64_t*)out = *(uint64_t*)from; + from += 8; out += 8; + *(uint64_t*)out = *(uint64_t*)from; + from += 8; out += 8; + *(uint64_t*)out = *(uint64_t*)from; + from += 8; out += 8; + *(uint64_t*)out = *(uint64_t*)from; + from += 8; out += 8; +#else + int i; + for (i = 0; i < 32; i++) { + *out++ = *from++; + } +#endif + return out; +} + +// This is never used, so comment it out +//#if defined(__AVX2__) +//static inline unsigned char *copy_32_bytes_aligned(unsigned char *out, const unsigned char *from) { +// __m256i chunk; +// chunk = _mm256_load_si256((__m256i*)from); +// _mm256_storeu_si256((__m256i*)out, chunk); +// return out + 32; +//} +//#endif // __AVX2__ + +/* Copy LEN bytes (7 or fewer) from FROM into OUT. Return OUT + LEN. */ +static inline unsigned char *copy_bytes(unsigned char *out, const unsigned char *from, unsigned len) { + assert(len < 8); + +#ifdef BLOSC_STRICT_ALIGN + while (len--) { + *out++ = *from++; + } +#else + switch (len) { + case 7: + return copy_7_bytes(out, from); + case 6: + return copy_6_bytes(out, from); + case 5: + return copy_5_bytes(out, from); + case 4: + return copy_4_bytes(out, from); + case 3: + return copy_3_bytes(out, from); + case 2: + return copy_2_bytes(out, from); + case 1: + return copy_1_bytes(out, from); + case 0: + return out; + default: + assert(0); + } +#endif /* BLOSC_STRICT_ALIGN */ + return out; +} + +// Define a symbol for avoiding fall-through warnings emitted by gcc >= 7.0 +#if ((defined(__GNUC__) && BLOSC_GCC_VERSION >= 700) && !defined(__clang__) && \ + !defined(__ICC) && !defined(__ICL)) +#define AVOID_FALLTHROUGH_WARNING +#endif + +/* Byte by byte semantics: copy LEN bytes from FROM and write them to OUT. Return OUT + LEN. */ +static inline unsigned char *chunk_memcpy(unsigned char *out, const unsigned char *from, unsigned len) { + unsigned sz = sizeof(uint64_t); + unsigned rem = len % sz; + unsigned by8; + + assert(len >= sz); + + /* Copy a few bytes to make sure the loop below has a multiple of SZ bytes to be copied. */ + copy_8_bytes(out, from); + + len /= sz; + out += rem; + from += rem; + + by8 = len % 8; + len -= by8; + switch (by8) { + case 7: + out = copy_8_bytes(out, from); + from += sz; + #ifdef AVOID_FALLTHROUGH_WARNING + __attribute__ ((fallthrough)); // Shut-up -Wimplicit-fallthrough warning in GCC + #endif + case 6: + out = copy_8_bytes(out, from); + from += sz; + #ifdef AVOID_FALLTHROUGH_WARNING + __attribute__ ((fallthrough)); + #endif + case 5: + out = copy_8_bytes(out, from); + from += sz; + #ifdef AVOID_FALLTHROUGH_WARNING + __attribute__ ((fallthrough)); + #endif + case 4: + out = copy_8_bytes(out, from); + from += sz; + #ifdef AVOID_FALLTHROUGH_WARNING + __attribute__ ((fallthrough)); + #endif + case 3: + out = copy_8_bytes(out, from); + from += sz; + #ifdef AVOID_FALLTHROUGH_WARNING + __attribute__ ((fallthrough)); + #endif + case 2: + out = copy_8_bytes(out, from); + from += sz; + #ifdef AVOID_FALLTHROUGH_WARNING + __attribute__ ((fallthrough)); + #endif + case 1: + out = copy_8_bytes(out, from); + from += sz; + #ifdef AVOID_FALLTHROUGH_WARNING + __attribute__ ((fallthrough)); + #endif + default: + break; + } + + while (len) { + out = copy_8_bytes(out, from); + from += sz; + out = copy_8_bytes(out, from); + from += sz; + out = copy_8_bytes(out, from); + from += sz; + out = copy_8_bytes(out, from); + from += sz; + out = copy_8_bytes(out, from); + from += sz; + out = copy_8_bytes(out, from); + from += sz; + out = copy_8_bytes(out, from); + from += sz; + out = copy_8_bytes(out, from); + from += sz; + + len -= 8; + } + + return out; +} + +#if (defined(__SSE2__) && defined(__AVX2__)) +/* 16-byte version of chunk_memcpy() */ +static inline unsigned char *chunk_memcpy_16(unsigned char *out, const unsigned char *from, unsigned len) { + unsigned sz = 16; + unsigned rem = len % sz; + unsigned ilen; + + assert(len >= sz); + + /* Copy a few bytes to make sure the loop below has a multiple of SZ bytes to be copied. */ + copy_16_bytes(out, from); + + len /= sz; + out += rem; + from += rem; + + for (ilen = 0; ilen < len; ilen++) { + copy_16_bytes(out, from); + out += sz; + from += sz; + } + + return out; +} +#endif + + +// NOTE: chunk_memcpy_32() and chunk_memcpy_32_unrolled() are not used, so commenting them + +///* 32-byte version of chunk_memcpy() */ +//static inline unsigned char *chunk_memcpy_32(unsigned char *out, const unsigned char *from, unsigned len) { +// unsigned sz = 32; +// unsigned rem = len % sz; +// unsigned ilen; +// +// assert(len >= sz); +// +// /* Copy a few bytes to make sure the loop below has a multiple of SZ bytes to be copied. */ +// copy_32_bytes(out, from); +// +// len /= sz; +// out += rem; +// from += rem; +// +// for (ilen = 0; ilen < len; ilen++) { +// copy_32_bytes(out, from); +// out += sz; +// from += sz; +// } +// +// return out; +//} +// +///* 32-byte *unrolled* version of chunk_memcpy() */ +//static inline unsigned char *chunk_memcpy_32_unrolled(unsigned char *out, const unsigned char *from, unsigned len) { +// unsigned sz = 32; +// unsigned rem = len % sz; +// unsigned by8; +// +// assert(len >= sz); +// +// /* Copy a few bytes to make sure the loop below has a multiple of SZ bytes to be copied. */ +// copy_32_bytes(out, from); +// +// len /= sz; +// out += rem; +// from += rem; +// +// by8 = len % 8; +// len -= by8; +// switch (by8) { +// case 7: +// out = copy_32_bytes(out, from); +// from += sz; +// case 6: +// out = copy_32_bytes(out, from); +// from += sz; +// case 5: +// out = copy_32_bytes(out, from); +// from += sz; +// case 4: +// out = copy_32_bytes(out, from); +// from += sz; +// case 3: +// out = copy_32_bytes(out, from); +// from += sz; +// case 2: +// out = copy_32_bytes(out, from); +// from += sz; +// case 1: +// out = copy_32_bytes(out, from); +// from += sz; +// default: +// break; +// } +// +// while (len) { +// out = copy_32_bytes(out, from); +// from += sz; +// out = copy_32_bytes(out, from); +// from += sz; +// out = copy_32_bytes(out, from); +// from += sz; +// out = copy_32_bytes(out, from); +// from += sz; +// out = copy_32_bytes(out, from); +// from += sz; +// out = copy_32_bytes(out, from); +// from += sz; +// out = copy_32_bytes(out, from); +// from += sz; +// out = copy_32_bytes(out, from); +// from += sz; +// +// len -= 8; +// } +// +// return out; +//} + + +/* SSE2/AVX2 *unaligned* version of chunk_memcpy() */ +#if defined(__SSE2__) || defined(__AVX2__) +static inline unsigned char *chunk_memcpy_unaligned(unsigned char *out, const unsigned char *from, unsigned len) { +#if defined(__AVX2__) + unsigned sz = sizeof(__m256i); +#elif defined(__SSE2__) + unsigned sz = sizeof(__m128i); +#endif + unsigned rem = len % sz; + unsigned ilen; + + assert(len >= sz); + + /* Copy a few bytes to make sure the loop below has a multiple of SZ bytes to be copied. */ +#if defined(__AVX2__) + copy_32_bytes(out, from); +#elif defined(__SSE2__) + copy_16_bytes(out, from); +#endif + + len /= sz; + out += rem; + from += rem; + + for (ilen = 0; ilen < len; ilen++) { +#if defined(__AVX2__) + copy_32_bytes(out, from); +#elif defined(__SSE2__) + copy_16_bytes(out, from); +#endif + out += sz; + from += sz; + } + + return out; +} +#endif // __SSE2__ || __AVX2__ + + +// NOTE: chunk_memcpy_aligned() is not used, so commenting it + +//#if defined(__SSE2__) || defined(__AVX2__) +///* SSE2/AVX2 *aligned* version of chunk_memcpy() */ +//static inline unsigned char *chunk_memcpy_aligned(unsigned char *out, const unsigned char *from, unsigned len) { +//#if defined(__AVX2__) +// unsigned sz = sizeof(__m256i); +// __m256i chunk; +//#elif defined(__SSE2__) +// unsigned sz = sizeof(__m128i); +// __m128i chunk; +//#endif +// unsigned bytes_to_align = sz - (unsigned)(((uintptr_t)(const void *)(from)) % sz); +// unsigned corrected_len = len - bytes_to_align; +// unsigned rem = corrected_len % sz; +// unsigned ilen; +// +// assert(len >= sz); +// +// /* Copy a few bytes to make sure the loop below has aligned access. */ +//#if defined(__AVX2__) +// chunk = _mm256_loadu_si256((__m256i *) from); +// _mm256_storeu_si256((__m256i *) out, chunk); +//#elif defined(__SSE2__) +// chunk = _mm_loadu_si128((__m128i *) from); +// _mm_storeu_si128((__m128i *) out, chunk); +//#endif +// out += bytes_to_align; +// from += bytes_to_align; +// +// len = corrected_len / sz; +// for (ilen = 0; ilen < len; ilen++) { +//#if defined(__AVX2__) +// chunk = _mm256_load_si256((__m256i *) from); /* *aligned* load */ +// _mm256_storeu_si256((__m256i *) out, chunk); +//#elif defined(__SSE2__) +// chunk = _mm_load_si128((__m128i *) from); /* *aligned* load */ +// _mm_storeu_si128((__m128i *) out, chunk); +//#endif +// out += sz; +// from += sz; +// } +// +// /* Copy remaining bytes */ +// if (rem < 8) { +// out = copy_bytes(out, from, rem); +// } +// else { +// out = chunk_memcpy(out, from, rem); +// } +// +// return out; +//} +//#endif // __AVX2__ || __SSE2__ + + +/* Byte by byte semantics: copy LEN bytes from FROM and write them to OUT. Return OUT + LEN. */ +unsigned char *fastcopy(unsigned char *out, const unsigned char *from, unsigned len) { + switch (len) { + case 32: + return copy_32_bytes(out, from); + case 16: + return copy_16_bytes(out, from); + case 8: + return copy_8_bytes(out, from); + default: { + } + } + if (len < 8) { + return copy_bytes(out, from, len); + } +#if defined(__SSE2__) + if (len < 16) { + return chunk_memcpy(out, from, len); + } +#if !defined(__AVX2__) + return chunk_memcpy_unaligned(out, from, len); +#else + if (len < 32) { + return chunk_memcpy_16(out, from, len); + } + return chunk_memcpy_unaligned(out, from, len); +#endif // !__AVX2__ +#else + return chunk_memcpy(out, from, len); +#endif // __SSE2__ +} + + +/* Copy a run */ +unsigned char* copy_match(unsigned char *out, const unsigned char *from, unsigned len) { +#if defined(__AVX2__) + unsigned sz = sizeof(__m256i); +#elif defined(__SSE2__) + unsigned sz = sizeof(__m128i); +#else + unsigned sz = sizeof(uint64_t); +#endif + +#if ((defined(__GNUC__) && BLOSC_GCC_VERSION < 800) && !defined(__clang__) && !defined(__ICC) && !defined(__ICL)) + // GCC < 8 in fully optimization mode seems to have problems with the code further below so stop here + for (; len > 0; len--) { + *out++ = *from++; + } + return out; +#endif + + // If out and from are away more than the size of the copy, then a fastcopy is safe + unsigned overlap_dist = (unsigned) (out - from); + if (overlap_dist > sz) { + return fastcopy(out, from, len); + } + + // Otherwise we need to be more careful so as not to overwrite destination + switch (overlap_dist) { + case 32: + for (; len >= 32; len -= 32) { + out = copy_32_bytes(out, from); + } + break; + case 30: + for (; len >= 30; len -= 30) { + out = copy_16_bytes(out, from); + out = copy_8_bytes(out, from + 16); + out = copy_4_bytes(out, from + 24); + out = copy_2_bytes(out, from + 28); + } + break; + case 28: + for (; len >= 28; len -= 28) { + out = copy_16_bytes(out, from); + out = copy_8_bytes(out, from + 16); + out = copy_4_bytes(out, from + 24); + } + break; + case 26: + for (; len >= 26; len -= 26) { + out = copy_16_bytes(out, from); + out = copy_8_bytes(out, from + 16); + out = copy_2_bytes(out, from + 24); + } + break; + case 24: + for (; len >= 24; len -= 24) { + out = copy_16_bytes(out, from); + out = copy_8_bytes(out, from + 16); + } + break; + case 22: + for (; len >= 22; len -= 22) { + out = copy_16_bytes(out, from); + out = copy_4_bytes(out, from + 16); + out = copy_2_bytes(out, from + 20); + } + break; + case 20: + for (; len >= 20; len -= 20) { + out = copy_16_bytes(out, from); + out = copy_4_bytes(out, from + 16); + } + break; + case 18: + for (; len >= 18; len -= 18) { + out = copy_16_bytes(out, from); + out = copy_2_bytes(out, from + 16); + } + break; + case 16: + for (; len >= 16; len -= 16) { + out = copy_16_bytes(out, from); + } + break; + case 8: + for (; len >= 8; len -= 8) { + out = copy_8_bytes(out, from); + } + break; + case 4: + for (; len >= 4; len -= 4) { + out = copy_4_bytes(out, from); + } + break; + case 2: + for (; len >= 2; len -= 2) { + out = copy_2_bytes(out, from); + } + break; + default: + for (; len > 0; len--) { + *out++ = *from++; + } + } + + // Copy the leftovers + for (; len > 0; len--) { + *out++ = *from++; + } + + return out; +} diff --git a/src/blosc2/blosc/fastcopy.h b/src/blosc2/blosc/fastcopy.h new file mode 100644 index 0000000..0993486 --- /dev/null +++ b/src/blosc2/blosc/fastcopy.h @@ -0,0 +1,20 @@ +/********************************************************************* + Blosc - Blocked Shuffling and Compression Library + + Copyright (C) 2021 The Blosc Developers + https://blosc.org + License: BSD 3-Clause (see LICENSE.txt) + + See LICENSE.txt for details about copyright and rights to use. +**********************************************************************/ + +#ifndef BLOSC_FASTCOPY_H +#define BLOSC_FASTCOPY_H + +/* Same semantics than memcpy() */ +unsigned char *fastcopy(unsigned char *out, const unsigned char *from, unsigned len); + +/* Same as fastcopy() but without overwriting origin or destination when they overlap */ +unsigned char* copy_match(unsigned char *out, const unsigned char *from, unsigned len); + +#endif //BLOSC_FASTCOPY_H diff --git a/src/blosc2/blosc/frame.c b/src/blosc2/blosc/frame.c new file mode 100644 index 0000000..98adcbd --- /dev/null +++ b/src/blosc2/blosc/frame.c @@ -0,0 +1,3477 @@ +/********************************************************************* + Blosc - Blocked Shuffling and Compression Library + + Copyright (C) 2021 The Blosc Developers + https://blosc.org + License: BSD 3-Clause (see LICENSE.txt) + + See LICENSE.txt for details about copyright and rights to use. +**********************************************************************/ + +#include +#include +#include +#include +#include +#include "blosc2.h" +#include "blosc-private.h" +#include "context.h" +#include "frame.h" +#include "sframe.h" +#include + +#if defined(_WIN32) +#include + #include + +/* stdint.h only available in VS2010 (VC++ 16.0) and newer */ + #if defined(_MSC_VER) && _MSC_VER < 1600 + #include "win32/stdint-windows.h" + #else + #include + #endif + +#endif /* _WIN32 */ + +/* If C11 is supported, use it's built-in aligned allocation. */ +#if __STDC_VERSION__ >= 201112L +#include +#endif + + +/* Create a new (empty) frame */ +blosc2_frame_s* frame_new(const char* urlpath) { + blosc2_frame_s* new_frame = calloc(1, sizeof(blosc2_frame_s)); + if (urlpath != NULL) { + char* new_urlpath = malloc(strlen(urlpath) + 1); // + 1 for the trailing NULL + new_frame->urlpath = strcpy(new_urlpath, urlpath); + new_frame->file_offset = 0; + } + return new_frame; +} + + +/* Free memory from a frame. */ +int frame_free(blosc2_frame_s* frame) { + + if (frame->cframe != NULL && !frame->avoid_cframe_free) { + free(frame->cframe); + } + + if (frame->coffsets != NULL) { + free(frame->coffsets); + } + + if (frame->urlpath != NULL) { + free(frame->urlpath); + } + + free(frame); + + return 0; +} + + +void *new_header_frame(blosc2_schunk *schunk, blosc2_frame_s *frame) { + if (frame == NULL) { + return NULL; + } + uint8_t* h2 = calloc(FRAME_HEADER_MINLEN, 1); + uint8_t* h2p = h2; + + // The msgpack header starts here + *h2p = 0x90; // fixarray... + *h2p += 14; // ...with 13 elements + h2p += 1; + + // Magic number + *h2p = 0xa0 + 8; // str with 8 elements + h2p += 1; + if (h2p - h2 >= FRAME_HEADER_MINLEN) { + return NULL; + } + strcpy((char*)h2p, "b2frame"); + h2p += 8; + + // Header size + *h2p = 0xd2; // int32 + h2p += 1 + 4; + if (h2p - h2 >= FRAME_HEADER_MINLEN) { + return NULL; + } + + // Total frame size + *h2p = 0xcf; // uint64 + // Fill it with frame->len which is known *after* the creation of the frame (e.g. when updating the header) + int64_t flen = frame->len; + to_big(h2 + FRAME_LEN, &flen, sizeof(flen)); + h2p += 1 + 8; + if (h2p - h2 >= FRAME_HEADER_MINLEN) { + return NULL; + } + + // Flags + *h2p = 0xa0 + 4; // str with 4 elements + h2p += 1; + if (h2p - h2 >= FRAME_HEADER_MINLEN) { + return NULL; + } + // General flags + *h2p = BLOSC2_VERSION_FRAME_FORMAT; // version + *h2p += 0x10; // 64-bit offsets. We only support this for now. + h2p += 1; + if (h2p - h2 >= FRAME_HEADER_MINLEN) { + return NULL; + } + + // Frame type + // We only support contiguous and sparse directories frames currently + *h2p = frame->sframe ? 1 : 0; + h2p += 1; + if (h2p - h2 >= FRAME_HEADER_MINLEN) { + return NULL; + } + + // Codec flags + *h2p = schunk->compcode; + if (schunk->compcode >= BLOSC_LAST_CODEC) { + *h2p = BLOSC_UDCODEC_FORMAT; + } + *h2p += (schunk->clevel) << 4u; // clevel + h2p += 1; + if (h2p - h2 >= FRAME_HEADER_MINLEN) { + return NULL; + } + + // Reserved flags + *h2p = 0; + h2p += 1; + if (h2p - h2 >= FRAME_HEADER_MINLEN) { + return NULL; + } + + // Uncompressed size + *h2p = 0xd3; // int64 + h2p += 1; + int64_t nbytes = schunk->nbytes; + to_big(h2p, &nbytes, sizeof(nbytes)); + h2p += 8; + if (h2p - h2 >= FRAME_HEADER_MINLEN) { + return NULL; + } + + // Compressed size + *h2p = 0xd3; // int64 + h2p += 1; + int64_t cbytes = schunk->cbytes; + to_big(h2p, &cbytes, sizeof(cbytes)); + h2p += 8; + if (h2p - h2 >= FRAME_HEADER_MINLEN) { + return NULL; + } + + // Type size + *h2p = 0xd2; // int32 + h2p += 1; + int32_t typesize = schunk->typesize; + to_big(h2p, &typesize, sizeof(typesize)); + h2p += 4; + if (h2p - h2 >= FRAME_HEADER_MINLEN) { + return NULL; + } + + // Block size + *h2p = 0xd2; // int32 + h2p += 1; + int32_t blocksize = schunk->blocksize; + to_big(h2p, &blocksize, sizeof(blocksize)); + h2p += 4; + if (h2p - h2 >= FRAME_HEADER_MINLEN) { + return NULL; + } + + // Chunk size + *h2p = 0xd2; // int32 + h2p += 1; + int32_t chunksize = schunk->chunksize; + to_big(h2p, &chunksize, sizeof(chunksize)); + h2p += 4; + if (h2p - h2 >= FRAME_HEADER_MINLEN) { + return NULL; + } + + // Number of threads for compression + *h2p = 0xd1; // int16 + h2p += 1; + int16_t nthreads = (int16_t)schunk->cctx->nthreads; + to_big(h2p, &nthreads, sizeof(nthreads)); + h2p += 2; + if (h2p - h2 >= FRAME_HEADER_MINLEN) { + return NULL; + } + + // Number of threads for decompression + *h2p = 0xd1; // int16 + h2p += 1; + nthreads = (int16_t)schunk->dctx->nthreads; + to_big(h2p, &nthreads, sizeof(nthreads)); + h2p += 2; + if (h2p - h2 >= FRAME_HEADER_MINLEN) { + return NULL; + } + + // The boolean for variable-length metalayers + *h2p = (schunk->nvlmetalayers > 0) ? (uint8_t)0xc3 : (uint8_t)0xc2; + h2p += 1; + if (h2p - h2 >= FRAME_HEADER_MINLEN) { + return NULL; + } + + // The space for FRAME_FILTER_PIPELINE + *h2p = 0xd8; // fixext 16 + h2p += 1; + if (BLOSC2_MAX_FILTERS > FRAME_FILTER_PIPELINE_MAX) { + return NULL; + } + // Store the filter pipeline in header + uint8_t* mp_filters = h2 + FRAME_FILTER_PIPELINE + 1; + uint8_t* mp_meta = h2 + FRAME_FILTER_PIPELINE + 1 + FRAME_FILTER_PIPELINE_MAX; + for (int i = 0; i < BLOSC2_MAX_FILTERS; i++) { + mp_filters[i] = schunk->filters[i]; + mp_meta[i] = schunk->filters_meta[i]; + } + *h2p = (uint8_t) BLOSC2_MAX_FILTERS; + h2p += 1; + h2p += 16; + + // User-defined codec and codec metadata + uint8_t* udcodec = h2 + FRAME_UDCODEC; + *udcodec = schunk->compcode; + uint8_t* codec_meta = h2 + FRAME_CODEC_META; + *codec_meta = schunk->compcode_meta; + + if (h2p - h2 != FRAME_HEADER_MINLEN) { + return NULL; + } + + int32_t hsize = FRAME_HEADER_MINLEN; + + // Now, deal with metalayers + uint16_t nmetalayers = schunk->nmetalayers; + if (nmetalayers > BLOSC2_MAX_METALAYERS) { + return NULL; + } + + // Make space for the header of metalayers (array marker, size, map of offsets) + h2 = realloc(h2, (size_t)hsize + 1 + 1 + 2 + 1 + 2); + h2p = h2 + hsize; + + // The msgpack header for the metalayers (array_marker, size, map of offsets, list of metalayers) + *h2p = 0x90 + 3; // array with 3 elements + h2p += 1; + + // Size for the map (index) of offsets, including this uint16 size (to be filled out later on) + *h2p = 0xcd; // uint16 + h2p += 1 + 2; + + // Map (index) of offsets for optional metalayers + *h2p = 0xde; // map 16 with N keys + h2p += 1; + to_big(h2p, &nmetalayers, sizeof(nmetalayers)); + h2p += sizeof(nmetalayers); + int32_t current_header_len = (int32_t)(h2p - h2); + int32_t *offtooff = malloc(nmetalayers * sizeof(int32_t)); + for (int nmetalayer = 0; nmetalayer < nmetalayers; nmetalayer++) { + if (frame == NULL) { + return NULL; + } + blosc2_metalayer *metalayer = schunk->metalayers[nmetalayer]; + uint8_t namelen = (uint8_t) strlen(metalayer->name); + h2 = realloc(h2, (size_t)current_header_len + 1 + namelen + 1 + 4); + h2p = h2 + current_header_len; + // Store the metalayer + if (namelen >= (1U << 5U)) { // metalayer strings cannot be longer than 32 bytes + free(offtooff); + return NULL; + } + *h2p = (uint8_t)0xa0 + namelen; // str + h2p += 1; + memcpy(h2p, metalayer->name, namelen); + h2p += namelen; + // Space for storing the offset for the value of this metalayer + *h2p = 0xd2; // int32 + h2p += 1; + offtooff[nmetalayer] = (int32_t)(h2p - h2); + h2p += 4; + current_header_len += 1 + namelen + 1 + 4; + } + int32_t hsize2 = (int32_t)(h2p - h2); + if (hsize2 != current_header_len) { // sanity check + return NULL; + } + + // Map size + int16 size + if ((uint32_t) (hsize2 - hsize) >= (1U << 16U)) { + return NULL; + } + uint16_t map_size = (uint16_t) (hsize2 - hsize); + to_big(h2 + FRAME_IDX_SIZE, &map_size, sizeof(map_size)); + + // Make space for an (empty) array + hsize = (int32_t)(h2p - h2); + h2 = realloc(h2, (size_t)hsize + 2 + 1 + 2); + h2p = h2 + hsize; + + // Now, store the values in an array + *h2p = 0xdc; // array 16 with N elements + h2p += 1; + to_big(h2p, &nmetalayers, sizeof(nmetalayers)); + h2p += sizeof(nmetalayers); + current_header_len = (int32_t)(h2p - h2); + for (int nmetalayer = 0; nmetalayer < nmetalayers; nmetalayer++) { + if (frame == NULL) { + return NULL; + } + blosc2_metalayer *metalayer = schunk->metalayers[nmetalayer]; + h2 = realloc(h2, (size_t)current_header_len + 1 + 4 + metalayer->content_len); + h2p = h2 + current_header_len; + // Store the serialized contents for this metalayer + *h2p = 0xc6; // bin32 + h2p += 1; + to_big(h2p, &(metalayer->content_len), sizeof(metalayer->content_len)); + h2p += 4; + memcpy(h2p, metalayer->content, metalayer->content_len); // buffer, no need to swap + h2p += metalayer->content_len; + // Update the offset now that we know it + to_big(h2 + offtooff[nmetalayer], ¤t_header_len, sizeof(current_header_len)); + current_header_len += 1 + 4 + metalayer->content_len; + } + free(offtooff); + hsize = (int32_t)(h2p - h2); + if (hsize != current_header_len) { // sanity check + return NULL; + } + + // Set the length of the whole header now that we know it + to_big(h2 + FRAME_HEADER_LEN, &hsize, sizeof(hsize)); + + return h2; +} + + +int get_header_info(blosc2_frame_s *frame, int32_t *header_len, int64_t *frame_len, int64_t *nbytes, int64_t *cbytes, + int32_t *blocksize, int32_t *chunksize, int64_t *nchunks, int32_t *typesize, uint8_t *compcode, + uint8_t *compcode_meta, uint8_t *clevel, uint8_t *filters, uint8_t *filters_meta, const blosc2_io *io) { + uint8_t* framep = frame->cframe; + uint8_t header[FRAME_HEADER_MINLEN]; + + blosc2_io_cb *io_cb = blosc2_get_io_cb(io->id); + if (io_cb == NULL) { + BLOSC_TRACE_ERROR("Error getting the input/output API"); + return BLOSC2_ERROR_PLUGIN_IO; + } + + if (frame->len <= 0) { + return BLOSC2_ERROR_READ_BUFFER; + } + + if (frame->cframe == NULL) { + int64_t rbytes = 0; + void* fp = NULL; + if (frame->sframe) { + fp = sframe_open_index(frame->urlpath, "rb", + io); + } + else { + fp = io_cb->open(frame->urlpath, "rb", io->params); + io_cb->seek(fp, frame->file_offset, SEEK_SET); + } + if (fp != NULL) { + rbytes = io_cb->read(header, 1, FRAME_HEADER_MINLEN, fp); + io_cb->close(fp); + } + (void) rbytes; + if (rbytes != FRAME_HEADER_MINLEN) { + return BLOSC2_ERROR_FILE_READ; + } + framep = header; + } + + // Consistency check for frame type + uint8_t frame_type = framep[FRAME_TYPE]; + if (frame->sframe) { + if (frame_type != FRAME_DIRECTORY_TYPE) { + return BLOSC2_ERROR_FRAME_TYPE; + } + } else { + if (frame_type != FRAME_CONTIGUOUS_TYPE) { + return BLOSC2_ERROR_FRAME_TYPE; + } + } + + // Fetch some internal lengths + from_big(header_len, framep + FRAME_HEADER_LEN, sizeof(*header_len)); + if (*header_len < FRAME_HEADER_MINLEN) { + BLOSC_TRACE_ERROR("Header length is zero or smaller than min allowed."); + return BLOSC2_ERROR_INVALID_HEADER; + } + from_big(frame_len, framep + FRAME_LEN, sizeof(*frame_len)); + if (*header_len > *frame_len) { + BLOSC_TRACE_ERROR("Header length exceeds length of the frame."); + return BLOSC2_ERROR_INVALID_HEADER; + } + from_big(nbytes, framep + FRAME_NBYTES, sizeof(*nbytes)); + from_big(cbytes, framep + FRAME_CBYTES, sizeof(*cbytes)); + from_big(blocksize, framep + FRAME_BLOCKSIZE, sizeof(*blocksize)); + if (chunksize != NULL) { + from_big(chunksize, framep + FRAME_CHUNKSIZE, sizeof(*chunksize)); + } + if (typesize != NULL) { + from_big(typesize, framep + FRAME_TYPESIZE, sizeof(*typesize)); + if (*typesize <= 0 || *typesize > BLOSC_MAX_TYPESIZE) { + BLOSC_TRACE_ERROR("`typesize` is zero or greater than max allowed."); + return BLOSC2_ERROR_INVALID_HEADER; + } + } + + // Codecs + uint8_t frame_codecs = framep[FRAME_CODECS]; + if (clevel != NULL) { + *clevel = frame_codecs >> 4u; + } + if (compcode != NULL) { + *compcode = frame_codecs & 0xFu; + if (*compcode == BLOSC_UDCODEC_FORMAT) { + from_big(compcode, framep + FRAME_UDCODEC, sizeof(*compcode)); + } + } + + + if (compcode_meta != NULL) { + from_big(compcode_meta, framep + FRAME_CODEC_META, sizeof(*compcode_meta)); + } + + // Filters + if (filters != NULL && filters_meta != NULL) { + uint8_t nfilters = framep[FRAME_FILTER_PIPELINE]; + if (nfilters > BLOSC2_MAX_FILTERS) { + BLOSC_TRACE_ERROR("The number of filters in frame header are too large for Blosc2."); + return BLOSC2_ERROR_INVALID_HEADER; + } + uint8_t *filters_ = framep + FRAME_FILTER_PIPELINE + 1; + uint8_t *filters_meta_ = framep + FRAME_FILTER_PIPELINE + 1 + FRAME_FILTER_PIPELINE_MAX; + for (int i = 0; i < nfilters; i++) { + filters[i] = filters_[i]; + filters_meta[i] = filters_meta_[i]; + } + } + + if (*nbytes > 0 && *chunksize > 0) { + // We can compute the number of chunks only when the frame has actual data + *nchunks = *nbytes / *chunksize; + if (*nbytes % *chunksize > 0) { + if (*nchunks == INT32_MAX) { + BLOSC_TRACE_ERROR("Number of chunks exceeds maximum allowed."); + return BLOSC2_ERROR_INVALID_HEADER; + } + *nchunks += 1; + } + + // Sanity check for compressed sizes + if ((*cbytes < 0) || ((int64_t)*nchunks * *chunksize < *nbytes)) { + BLOSC_TRACE_ERROR("Invalid compressed size in frame header."); + return BLOSC2_ERROR_INVALID_HEADER; + } + } else { + *nchunks = 0; + } + + return 0; +} + + +int64_t get_trailer_offset(blosc2_frame_s *frame, int32_t header_len, bool has_coffsets) { + if (!has_coffsets) { + // No data chunks yet + return header_len; + } + return frame->len - frame->trailer_len; +} + + +// Update the length in the header +int update_frame_len(blosc2_frame_s* frame, int64_t len) { + int rc = 1; + blosc2_io_cb *io_cb = blosc2_get_io_cb(frame->schunk->storage->io->id); + if (io_cb == NULL) { + BLOSC_TRACE_ERROR("Error getting the input/output API"); + return BLOSC2_ERROR_PLUGIN_IO; + } + + if (frame->cframe != NULL) { + to_big(frame->cframe + FRAME_LEN, &len, sizeof(int64_t)); + } + else { + void* fp = NULL; + if (frame->sframe) { + fp = sframe_open_index(frame->urlpath, "rb+", + frame->schunk->storage->io); + } + else { + fp = io_cb->open(frame->urlpath, "rb+", frame->schunk->storage->io->params); + } + io_cb->seek(fp, frame->file_offset + FRAME_LEN, SEEK_SET); + int64_t swap_len; + to_big(&swap_len, &len, sizeof(int64_t)); + int64_t wbytes = io_cb->write(&swap_len, 1, sizeof(int64_t), fp); + io_cb->close(fp); + if (wbytes != sizeof(int64_t)) { + BLOSC_TRACE_ERROR("Cannot write the frame length in header."); + return BLOSC2_ERROR_FILE_WRITE; + } + } + return rc; +} + + +int frame_update_trailer(blosc2_frame_s* frame, blosc2_schunk* schunk) { + if (frame != NULL && frame->len == 0) { + BLOSC_TRACE_ERROR("The trailer cannot be updated on empty frames."); + } + + // Create the trailer in msgpack (see the frame format document) + int64_t trailer_len = FRAME_TRAILER_MINLEN; + uint8_t* trailer = (uint8_t*)calloc((size_t)trailer_len, 1); + uint8_t* ptrailer = trailer; + *ptrailer = 0x90 + 4; // fixarray with 4 elements + ptrailer += 1; + // Trailer format version + *ptrailer = FRAME_TRAILER_VERSION; + ptrailer += 1; + + int32_t current_trailer_len = (int32_t)(ptrailer - trailer); + + // Now, deal with variable-length metalayers + int16_t nvlmetalayers = schunk->nvlmetalayers; + if (nvlmetalayers < 0 || nvlmetalayers > BLOSC2_MAX_METALAYERS) { + return -1; + } + + // Make space for the header of metalayers (array marker, size, map of offsets) + trailer = realloc(trailer, (size_t) current_trailer_len + 1 + 1 + 2 + 1 + 2); + ptrailer = trailer + current_trailer_len; + + // The msgpack header for the metalayers (array_marker, size, map of offsets, list of metalayers) + *ptrailer = 0x90 + 3; // array with 3 elements + ptrailer += 1; + + int32_t tsize = (int32_t)(ptrailer - trailer); + + // Size for the map (index) of metalayer offsets, including this uint16 size (to be filled out later on) + *ptrailer = 0xcd; // uint16 + ptrailer += 1 + 2; + + // Map (index) of offsets for optional metalayers + *ptrailer = 0xde; // map 16 with N keys + ptrailer += 1; + to_big(ptrailer, &nvlmetalayers, sizeof(nvlmetalayers)); + ptrailer += sizeof(nvlmetalayers); + current_trailer_len = (int32_t)(ptrailer - trailer); + int32_t *offtodata = malloc(nvlmetalayers * sizeof(int32_t)); + for (int nvlmetalayer = 0; nvlmetalayer < nvlmetalayers; nvlmetalayer++) { + if (frame == NULL) { + return -1; + } + blosc2_metalayer *vlmetalayer = schunk->vlmetalayers[nvlmetalayer]; + uint8_t name_len = (uint8_t) strlen(vlmetalayer->name); + trailer = realloc(trailer, (size_t)current_trailer_len + 1 + name_len + 1 + 4); + ptrailer = trailer + current_trailer_len; + // Store the vlmetalayer + if (name_len >= (1U << 5U)) { // metalayer strings cannot be longer than 32 bytes + free(offtodata); + return -1; + } + *ptrailer = (uint8_t)0xa0 + name_len; // str + ptrailer += 1; + memcpy(ptrailer, vlmetalayer->name, name_len); + ptrailer += name_len; + // Space for storing the offset for the value of this vlmetalayer + *ptrailer = 0xd2; // int32 + ptrailer += 1; + offtodata[nvlmetalayer] = (int32_t)(ptrailer - trailer); + ptrailer += 4; + current_trailer_len += 1 + name_len + 1 + 4; + } + int32_t tsize2 = (int32_t)(ptrailer - trailer); + if (tsize2 != current_trailer_len) { // sanity check + return -1; + } + + // Map size + int16 size + if ((uint32_t) (tsize2 - tsize) >= (1U << 16U)) { + return -1; + } + uint16_t map_size = (uint16_t) (tsize2 - tsize); + to_big(trailer + 4, &map_size, sizeof(map_size)); + + // Make space for an (empty) array + tsize = (int32_t)(ptrailer - trailer); + trailer = realloc(trailer, (size_t) tsize + 2 + 1 + 2); + ptrailer = trailer + tsize; + + // Now, store the values in an array + *ptrailer = 0xdc; // array 16 with N elements + ptrailer += 1; + to_big(ptrailer, &nvlmetalayers, sizeof(nvlmetalayers)); + ptrailer += sizeof(nvlmetalayers); + current_trailer_len = (int32_t)(ptrailer - trailer); + for (int nvlmetalayer = 0; nvlmetalayer < nvlmetalayers; nvlmetalayer++) { + if (frame == NULL) { + return -1; + } + blosc2_metalayer *vlmetalayer = schunk->vlmetalayers[nvlmetalayer]; + trailer = realloc(trailer, (size_t)current_trailer_len + 1 + 4 + vlmetalayer->content_len); + ptrailer = trailer + current_trailer_len; + // Store the serialized contents for this vlmetalayer + *ptrailer = 0xc6; // bin32 + ptrailer += 1; + to_big(ptrailer, &(vlmetalayer->content_len), sizeof(vlmetalayer->content_len)); + ptrailer += 4; + memcpy(ptrailer, vlmetalayer->content, vlmetalayer->content_len); // buffer, no need to swap + ptrailer += vlmetalayer->content_len; + // Update the offset now that we know it + to_big(trailer + offtodata[nvlmetalayer], ¤t_trailer_len, sizeof(current_trailer_len)); + current_trailer_len += 1 + 4 + vlmetalayer->content_len; + } + free(offtodata); + tsize = (int32_t)(ptrailer - trailer); + if (tsize != current_trailer_len) { // sanity check + return -1; + } + + trailer = realloc(trailer, (size_t)current_trailer_len + 23); + ptrailer = trailer + current_trailer_len; + trailer_len = (ptrailer - trailer) + 23; + + // Trailer length + *ptrailer = 0xce; // uint32 + ptrailer += 1; + to_big(ptrailer, &trailer_len, sizeof(uint32_t)); + ptrailer += sizeof(uint32_t); + // Up to 16 bytes for frame fingerprint (using XXH3 included in https://github.com/Cyan4973/xxHash) + // Maybe someone would need 256-bit in the future, but for the time being 128-bit seems like a good tradeoff + *ptrailer = 0xd8; // fixext 16 + ptrailer += 1; + *ptrailer = 0; // fingerprint type: 0 -> no fp; 1 -> 32-bit; 2 -> 64-bit; 3 -> 128-bit + ptrailer += 1; + + // Remove call to memset when we compute an actual fingerprint + memset(ptrailer, 0, 16); + // Uncomment call to memcpy when we compute an actual fingerprint + // memcpy(ptrailer, xxh3_fingerprint, sizeof(xxh3_fingerprint)); + ptrailer += 16; + + // Sanity check + if (ptrailer - trailer != trailer_len) { + return BLOSC2_ERROR_DATA; + } + + int32_t header_len; + int64_t frame_len; + int64_t nbytes; + int64_t cbytes; + int32_t blocksize; + int32_t chunksize; + int64_t nchunks; + int ret = get_header_info(frame, &header_len, &frame_len, &nbytes, &cbytes, + &blocksize, &chunksize, &nchunks, + NULL, NULL, NULL, NULL, NULL, NULL, + frame->schunk->storage->io); + if (ret < 0) { + BLOSC_TRACE_ERROR("Unable to get meta info from frame."); + return ret; + } + + int64_t trailer_offset = get_trailer_offset(frame, header_len, nbytes > 0); + + if (trailer_offset < BLOSC_EXTENDED_HEADER_LENGTH) { + BLOSC_TRACE_ERROR("Unable to get trailer offset in frame."); + return BLOSC2_ERROR_READ_BUFFER; + } + + blosc2_io_cb *io_cb = blosc2_get_io_cb(frame->schunk->storage->io->id); + if (io_cb == NULL) { + BLOSC_TRACE_ERROR("Error getting the input/output API"); + return BLOSC2_ERROR_PLUGIN_IO; + } + // Update the trailer. As there are no internal offsets to the trailer section, + // and it is always at the end of the frame, we can just write (or overwrite) it + // at the end of the frame. + if (frame->cframe != NULL) { + frame->cframe = realloc(frame->cframe, (size_t)(trailer_offset + trailer_len)); + if (frame->cframe == NULL) { + BLOSC_TRACE_ERROR("Cannot realloc space for the frame."); + return BLOSC2_ERROR_MEMORY_ALLOC; + } + memcpy(frame->cframe + trailer_offset, trailer, trailer_len); + } + else { + void* fp = NULL; + if (frame->sframe) { + fp = sframe_open_index(frame->urlpath, "rb+", + frame->schunk->storage->io); + } + else { + fp = io_cb->open(frame->urlpath, "rb+", frame->schunk->storage->io->params); + } + if (fp == NULL) { + BLOSC_TRACE_ERROR("Cannot open the frame for reading and writing."); + return BLOSC2_ERROR_FILE_OPEN; + } + io_cb->seek(fp, frame->file_offset + trailer_offset, SEEK_SET); + int64_t wbytes = io_cb->write(trailer, 1, trailer_len, fp); + if (wbytes != trailer_len) { + BLOSC_TRACE_ERROR("Cannot write the trailer length in trailer."); + return BLOSC2_ERROR_FILE_WRITE; + } + if (io_cb->truncate(fp, trailer_offset + trailer_len) != 0) { + BLOSC_TRACE_ERROR("Cannot truncate the frame."); + return BLOSC2_ERROR_FILE_TRUNCATE; + } + io_cb->close(fp); + + } + free(trailer); + + int rc = update_frame_len(frame, trailer_offset + trailer_len); + if (rc < 0) { + return rc; + } + frame->len = trailer_offset + trailer_len; + frame->trailer_len = trailer_len; + + return 1; +} + + +// Remove a file:/// prefix +// This is a temporary workaround for allowing to use proper URLs for local files/dirs +static char* normalize_urlpath(const char* urlpath) { + char* localpath = strstr(urlpath, "file:///"); + if (localpath == urlpath) { + // There is a file:/// prefix. Get rid of it. + localpath += strlen("file:///"); + } + else { + localpath = (char*)urlpath; + } + return localpath; +} + + +/* Initialize a frame out of a file */ +blosc2_frame_s* frame_from_file_offset(const char* urlpath, const blosc2_io *io, int64_t offset) { + // Get the length of the frame + uint8_t header[FRAME_HEADER_MINLEN]; + uint8_t trailer[FRAME_TRAILER_MINLEN]; + + void* fp = NULL; + bool sframe = false; + struct stat path_stat; + + urlpath = normalize_urlpath(urlpath); + + if(stat(urlpath, &path_stat) < 0) { + BLOSC_TRACE_ERROR("Cannot get information about the path %s.", urlpath); + return NULL; + } + + blosc2_io_cb *io_cb = blosc2_get_io_cb(io->id); + if (io_cb == NULL) { + BLOSC_TRACE_ERROR("Error getting the input/output API"); + return NULL; + } + + char* urlpath_cpy; + if (path_stat.st_mode & S_IFDIR) { + urlpath_cpy = malloc(strlen(urlpath) + 1); + strcpy(urlpath_cpy, urlpath); + char last_char = urlpath[strlen(urlpath) - 1]; + if (last_char == '\\' || last_char == '/') { + urlpath_cpy[strlen(urlpath) - 1] = '\0'; + } + else { + } + fp = sframe_open_index(urlpath_cpy, "rb", io); + sframe = true; + } + else { + urlpath_cpy = malloc(strlen(urlpath) + 1); + strcpy(urlpath_cpy, urlpath); + fp = io_cb->open(urlpath, "rb", io->params); + } + io_cb->seek(fp, offset, SEEK_SET); + int64_t rbytes = io_cb->read(header, 1, FRAME_HEADER_MINLEN, fp); + if (rbytes != FRAME_HEADER_MINLEN) { + BLOSC_TRACE_ERROR("Cannot read from file '%s'.", urlpath); + io_cb->close(fp); + free(urlpath_cpy); + return NULL; + } + int64_t frame_len; + to_big(&frame_len, header + FRAME_LEN, sizeof(frame_len)); + + blosc2_frame_s* frame = calloc(1, sizeof(blosc2_frame_s)); + frame->urlpath = urlpath_cpy; + frame->len = frame_len; + frame->sframe = sframe; + frame->file_offset = offset; + + // Now, the trailer length + io_cb->seek(fp, offset + frame_len - FRAME_TRAILER_MINLEN, SEEK_SET); + rbytes = io_cb->read(trailer, 1, FRAME_TRAILER_MINLEN, fp); + io_cb->close(fp); + if (rbytes != FRAME_TRAILER_MINLEN) { + BLOSC_TRACE_ERROR("Cannot read from file '%s'.", urlpath); + free(urlpath_cpy); + free(frame); + return NULL; + } + int trailer_offset = FRAME_TRAILER_MINLEN - FRAME_TRAILER_LEN_OFFSET; + if (trailer[trailer_offset - 1] != 0xce) { + free(urlpath_cpy); + free(frame); + return NULL; + } + uint32_t trailer_len; + to_big(&trailer_len, trailer + trailer_offset, sizeof(trailer_len)); + frame->trailer_len = trailer_len; + + return frame; +} + + +/* Initialize a frame out of a contiguous frame buffer */ +blosc2_frame_s* frame_from_cframe(uint8_t *cframe, int64_t len, bool copy) { + // Get the length of the frame + const uint8_t* header = cframe; + int64_t frame_len; + if (len < FRAME_HEADER_MINLEN) { + return NULL; + } + + from_big(&frame_len, header + FRAME_LEN, sizeof(frame_len)); + if (frame_len != len) { // sanity check + return NULL; + } + + blosc2_frame_s* frame = calloc(1, sizeof(blosc2_frame_s)); + frame->len = frame_len; + frame->file_offset = 0; + + // Now, the trailer length + const uint8_t* trailer = cframe + frame_len - FRAME_TRAILER_MINLEN; + int trailer_offset = FRAME_TRAILER_MINLEN - FRAME_TRAILER_LEN_OFFSET; + if (trailer[trailer_offset - 1] != 0xce) { + free(frame); + return NULL; + } + uint32_t trailer_len; + from_big(&trailer_len, trailer + trailer_offset, sizeof(trailer_len)); + frame->trailer_len = trailer_len; + + if (copy) { + frame->cframe = malloc((size_t)len); + memcpy(frame->cframe, cframe, (size_t)len); + } + else { + frame->cframe = cframe; + frame->avoid_cframe_free = true; + } + + return frame; +} + + +/* Create a frame out of a super-chunk. */ +int64_t frame_from_schunk(blosc2_schunk *schunk, blosc2_frame_s *frame) { + frame->file_offset = 0; + int64_t nchunks = schunk->nchunks; + int64_t cbytes = schunk->cbytes; + int32_t chunk_cbytes; + int32_t chunk_nbytes; + void* fp = NULL; + int rc; + + uint8_t* h2 = new_header_frame(schunk, frame); + if (h2 == NULL) { + return BLOSC2_ERROR_DATA; + } + uint32_t h2len; + from_big(&h2len, h2 + FRAME_HEADER_LEN, sizeof(h2len)); + // Build the offsets chunk + int32_t chunksize = -1; + int32_t off_cbytes = 0; + uint64_t coffset = 0; + int32_t off_nbytes = (int32_t) (nchunks * sizeof(int64_t)); + uint64_t* data_tmp = malloc(off_nbytes); + bool needs_free = false; + for (int i = 0; i < nchunks; i++) { + uint8_t* data_chunk; + data_chunk = schunk->data[i]; + rc = blosc2_cbuffer_sizes(data_chunk, &chunk_nbytes, &chunk_cbytes, NULL); + if (rc < 0) { + return rc; + } + data_tmp[i] = coffset; + coffset += chunk_cbytes; + int32_t chunksize_ = chunk_nbytes; + if (i == 0) { + chunksize = chunksize_; + } + else if (chunksize != chunksize_) { + // Variable size // TODO: update flags for this (or do not use them at all) + chunksize = 0; + } + if (needs_free) { + free(data_chunk); + } + } + if ((int64_t)coffset != cbytes) { + free(data_tmp); + return BLOSC2_ERROR_DATA; + } + uint8_t *off_chunk = NULL; + if (nchunks > 0) { + // Compress the chunk of offsets + off_chunk = malloc(off_nbytes + BLOSC2_MAX_OVERHEAD); + blosc2_context *cctx = blosc2_create_cctx(BLOSC2_CPARAMS_DEFAULTS); + cctx->typesize = sizeof(int64_t); + off_cbytes = blosc2_compress_ctx(cctx, data_tmp, off_nbytes, off_chunk, + off_nbytes + BLOSC2_MAX_OVERHEAD); + blosc2_free_ctx(cctx); + if (off_cbytes < 0) { + free(off_chunk); + free(h2); + return off_cbytes; + } + } + else { + off_cbytes = 0; + } + free(data_tmp); + + // Now that we know them, fill the chunksize and frame length in header + to_big(h2 + FRAME_CHUNKSIZE, &chunksize, sizeof(chunksize)); + frame->len = h2len + cbytes + off_cbytes + FRAME_TRAILER_MINLEN; + if (frame->sframe) { + frame->len = h2len + off_cbytes + FRAME_TRAILER_MINLEN; + } + int64_t tbytes = frame->len; + to_big(h2 + FRAME_LEN, &tbytes, sizeof(tbytes)); + + blosc2_io_cb *io_cb = blosc2_get_io_cb(frame->schunk->storage->io->id); + if (io_cb == NULL) { + BLOSC_TRACE_ERROR("Error getting the input/output API"); + return BLOSC2_ERROR_PLUGIN_IO; + } + + // Create the frame and put the header at the beginning + if (frame->urlpath == NULL) { + frame->cframe = malloc((size_t)frame->len); + memcpy(frame->cframe, h2, h2len); + } + else { + if (frame->sframe) { + fp = sframe_open_index(frame->urlpath, "wb", + frame->schunk->storage->io); + } + else { + fp = io_cb->open(frame->urlpath, "wb", frame->schunk->storage->io->params); + } + io_cb->write(h2, h2len, 1, fp); + } + free(h2); + + // Fill the frame with the actual data chunks + if (!frame->sframe) { + coffset = 0; + for (int i = 0; i < nchunks; i++) { + uint8_t* data_chunk = schunk->data[i]; + rc = blosc2_cbuffer_sizes(data_chunk, NULL, &chunk_cbytes, NULL); + if (rc < 0) { + return rc; + } + if (frame->urlpath == NULL) { + memcpy(frame->cframe + h2len + coffset, data_chunk, (size_t)chunk_cbytes); + } else { + io_cb->write(data_chunk, chunk_cbytes, 1, fp); + } + coffset += chunk_cbytes; + } + if ((int64_t)coffset != cbytes) { + return BLOSC2_ERROR_FAILURE; + } + } + + // Copy the offsets chunk at the end of the frame + if (frame->urlpath == NULL) { + memcpy(frame->cframe + h2len + cbytes, off_chunk, off_cbytes); + } + else { + io_cb->write(off_chunk, off_cbytes, 1, fp); + io_cb->close(fp); + } + free(off_chunk); + rc = frame_update_trailer(frame, schunk); + if (rc < 0) { + return rc; + } + + return frame->len; +} + + +// Get the compressed data offsets +uint8_t* get_coffsets(blosc2_frame_s *frame, int32_t header_len, int64_t cbytes, + int64_t nchunks, int32_t *off_cbytes) { + int32_t chunk_cbytes; + int rc; + + if (frame->coffsets != NULL) { + if (off_cbytes != NULL) { + rc = blosc2_cbuffer_sizes(frame->coffsets, NULL, &chunk_cbytes, NULL); + if (rc < 0) { + return NULL; + } + *off_cbytes = (int32_t)chunk_cbytes; + } + return frame->coffsets; + } + if (frame->cframe != NULL) { + int64_t off_pos = header_len; + if (cbytes < INT64_MAX - header_len) { + off_pos += cbytes; + } + // Check that there is enough room to read Blosc header + if (off_pos < 0 || off_pos > INT64_MAX - BLOSC_EXTENDED_HEADER_LENGTH || + off_pos + BLOSC_EXTENDED_HEADER_LENGTH > frame->len) { + BLOSC_TRACE_ERROR("Cannot read the offsets outside of frame boundary."); + return NULL; + } + // For in-memory frames, the coffset is just one pointer away + uint8_t* off_start = frame->cframe + off_pos; + if (off_cbytes != NULL) { + int32_t chunk_nbytes; + int32_t chunk_blocksize; + rc = blosc2_cbuffer_sizes(off_start, &chunk_nbytes, &chunk_cbytes, &chunk_blocksize); + if (rc < 0) { + return NULL; + } + *off_cbytes = (int32_t)chunk_cbytes; + if (*off_cbytes < 0 || off_pos + *off_cbytes > frame->len) { + BLOSC_TRACE_ERROR("Cannot read the cbytes outside of frame boundary."); + return NULL; + } + if ((uint64_t)chunk_nbytes != nchunks * sizeof(int64_t)) { + BLOSC_TRACE_ERROR("The number of chunks in offset idx " + "does not match the ones in the header frame."); + return NULL; + } + + } + return off_start; + } + + int64_t trailer_offset = get_trailer_offset(frame, header_len, true); + + if (trailer_offset < BLOSC_EXTENDED_HEADER_LENGTH || trailer_offset + FRAME_TRAILER_MINLEN > frame->len) { + BLOSC_TRACE_ERROR("Cannot read the trailer out of the frame."); + return NULL; + } + + int32_t coffsets_cbytes; + if (frame->sframe) { + coffsets_cbytes = (int32_t)(trailer_offset - (header_len + 0)); + } + else { + coffsets_cbytes = (int32_t)(trailer_offset - (header_len + cbytes)); + } + + if (off_cbytes != NULL) { + *off_cbytes = coffsets_cbytes; + } + + blosc2_io_cb *io_cb = blosc2_get_io_cb(frame->schunk->storage->io->id); + if (io_cb == NULL) { + BLOSC_TRACE_ERROR("Error getting the input/output API"); + return NULL; + } + + void* fp = NULL; + uint8_t* coffsets = malloc((size_t)coffsets_cbytes); + if (frame->sframe) { + fp = sframe_open_index(frame->urlpath, "rb", + frame->schunk->storage->io); + io_cb->seek(fp, header_len + 0, SEEK_SET); + } + else { + fp = io_cb->open(frame->urlpath, "rb", frame->schunk->storage->io->params); + io_cb->seek(fp, frame->file_offset + header_len + cbytes, SEEK_SET); + } + int64_t rbytes = io_cb->read(coffsets, 1, coffsets_cbytes, fp); + io_cb->close(fp); + if (rbytes != coffsets_cbytes) { + BLOSC_TRACE_ERROR("Cannot read the offsets out of the frame."); + free(coffsets); + return NULL; + } + frame->coffsets = coffsets; + return coffsets; +} + + +// Get the data offsets from a frame +int64_t* blosc2_frame_get_offsets(blosc2_schunk *schunk) { + if (schunk->frame == NULL) { + BLOSC_TRACE_ERROR("This function needs a frame."); + return NULL; + } + blosc2_frame_s* frame = (blosc2_frame_s*)schunk->frame; + + // Get header info + int32_t header_len; + int64_t frame_len; + int64_t nbytes; + int64_t cbytes; + int32_t blocksize; + int32_t chunksize; + int64_t nchunks; + int ret = get_header_info(frame, &header_len, &frame_len, &nbytes, &cbytes, + &blocksize, &chunksize, &nchunks, + NULL, NULL, NULL, NULL, NULL, NULL, + frame->schunk->storage->io); + if (ret < 0) { + BLOSC_TRACE_ERROR("Cannot get the header info for the frame."); + return NULL; + } + + int32_t off_nbytes = (int32_t) (nchunks * sizeof(int64_t)); + int64_t* offsets = (int64_t *) malloc((size_t)off_nbytes); + + int32_t coffsets_cbytes = 0; + uint8_t *coffsets = get_coffsets(frame, header_len, cbytes, nchunks, &coffsets_cbytes); + // Decompress offsets + blosc2_dparams off_dparams = BLOSC2_DPARAMS_DEFAULTS; + blosc2_context *dctx = blosc2_create_dctx(off_dparams); + int32_t prev_nbytes = blosc2_decompress_ctx(dctx, coffsets, coffsets_cbytes, + offsets, off_nbytes); + blosc2_free_ctx(dctx); + if (prev_nbytes < 0) { + free(offsets); + BLOSC_TRACE_ERROR("Cannot decompress the offsets chunk."); + return NULL; + } + return offsets; +} + + +int frame_update_header(blosc2_frame_s* frame, blosc2_schunk* schunk, bool new) { + uint8_t* framep = frame->cframe; + uint8_t header[FRAME_HEADER_MINLEN]; + + if (frame->len <= 0) { + return BLOSC2_ERROR_INVALID_PARAM; + } + + if (new && schunk->cbytes > 0) { + BLOSC_TRACE_ERROR("New metalayers cannot be added after actual data " + "has been appended."); + return BLOSC2_ERROR_INVALID_PARAM; + } + + blosc2_io_cb *io_cb = blosc2_get_io_cb(frame->schunk->storage->io->id); + if (io_cb == NULL) { + BLOSC_TRACE_ERROR("Error getting the input/output API"); + return BLOSC2_ERROR_PLUGIN_IO; + } + + if (frame->cframe == NULL) { + int64_t rbytes = 0; + void* fp = NULL; + if (frame->sframe) { + fp = sframe_open_index(frame->urlpath, "rb+", + frame->schunk->storage->io); + } + else { + fp = io_cb->open(frame->urlpath, "rb", frame->schunk->storage->io->params); + io_cb->seek(fp, frame->file_offset, SEEK_SET); + } + if (fp != NULL) { + rbytes = io_cb->read(header, 1, FRAME_HEADER_MINLEN, fp); + io_cb->close(fp); + } + (void) rbytes; + if (rbytes != FRAME_HEADER_MINLEN) { + return BLOSC2_ERROR_FILE_WRITE; + } + framep = header; + } + uint32_t prev_h2len; + from_big(&prev_h2len, framep + FRAME_HEADER_LEN, sizeof(prev_h2len)); + + // Build a new header + uint8_t* h2 = new_header_frame(schunk, frame); + uint32_t h2len; + from_big(&h2len, h2 + FRAME_HEADER_LEN, sizeof(h2len)); + + // The frame length is outdated when adding a new metalayer, so update it + if (new) { + int64_t frame_len = h2len; // at adding time, we only have to worry of the header for now + to_big(h2 + FRAME_LEN, &frame_len, sizeof(frame_len)); + frame->len = frame_len; + } + + if (!new && prev_h2len != h2len) { + BLOSC_TRACE_ERROR("The new metalayer sizes should be equal the existing ones."); + return BLOSC2_ERROR_DATA; + } + + void* fp = NULL; + if (frame->cframe == NULL) { + // Write updated header down to file + if (frame->sframe) { + fp = sframe_open_index(frame->urlpath, "rb+", + frame->schunk->storage->io); + } + else { + fp = io_cb->open(frame->urlpath, "rb+", frame->schunk->storage->io->params); + } + if (fp != NULL) { + io_cb->seek(fp, frame->file_offset, SEEK_SET); + io_cb->write(h2, h2len, 1, fp); + io_cb->close(fp); + } + } + else { + if (new) { + frame->cframe = realloc(frame->cframe, h2len); + } + memcpy(frame->cframe, h2, h2len); + } + free(h2); + + return 1; +} + + +static int get_meta_from_header(blosc2_frame_s* frame, blosc2_schunk* schunk, uint8_t* header, + int32_t header_len) { + BLOSC_UNUSED_PARAM(frame); + int64_t header_pos = FRAME_IDX_SIZE; + + // Get the size for the index of metalayers + uint16_t idx_size; + header_pos += sizeof(idx_size); + if (header_len < header_pos) { + return BLOSC2_ERROR_READ_BUFFER; + } + from_big(&idx_size, header + FRAME_IDX_SIZE, sizeof(idx_size)); + + // Get the actual index of metalayers + uint8_t* metalayers_idx = header + FRAME_IDX_SIZE + 2; + header_pos += 1; + if (header_len < header_pos) { + return BLOSC2_ERROR_READ_BUFFER; + } + if (metalayers_idx[0] != 0xde) { // sanity check + return BLOSC2_ERROR_DATA; + } + uint8_t* idxp = metalayers_idx + 1; + uint16_t nmetalayers; + header_pos += sizeof(nmetalayers); + if (header_len < header_pos) { + return BLOSC2_ERROR_READ_BUFFER; + } + from_big(&nmetalayers, idxp, sizeof(uint16_t)); + idxp += 2; + if (nmetalayers > BLOSC2_MAX_METALAYERS) { + return BLOSC2_ERROR_DATA; + } + schunk->nmetalayers = nmetalayers; + + // Populate the metalayers and its serialized values + for (int nmetalayer = 0; nmetalayer < nmetalayers; nmetalayer++) { + header_pos += 1; + if (header_len < header_pos) { + return BLOSC2_ERROR_READ_BUFFER; + } + if ((*idxp & 0xe0u) != 0xa0u) { // sanity check + return BLOSC2_ERROR_DATA; + } + blosc2_metalayer* metalayer = calloc(sizeof(blosc2_metalayer), 1); + schunk->metalayers[nmetalayer] = metalayer; + + // Populate the metalayer string + int8_t nslen = *idxp & (uint8_t)0x1F; + idxp += 1; + header_pos += nslen; + if (header_len < header_pos) { + return BLOSC2_ERROR_READ_BUFFER; + } + char* ns = malloc((size_t)nslen + 1); + memcpy(ns, idxp, nslen); + ns[nslen] = '\0'; + idxp += nslen; + metalayer->name = ns; + + // Populate the serialized value for this metalayer + // Get the offset + header_pos += 1; + if (header_len < header_pos) { + return BLOSC2_ERROR_READ_BUFFER; + } + if ((*idxp & 0xffu) != 0xd2u) { // sanity check + return BLOSC2_ERROR_DATA; + } + idxp += 1; + int32_t offset; + header_pos += sizeof(offset); + if (header_len < header_pos) { + return BLOSC2_ERROR_READ_BUFFER; + } + from_big(&offset, idxp, sizeof(offset)); + idxp += 4; + if (offset < 0 || offset >= header_len) { + // Offset is less than zero or exceeds header length + return BLOSC2_ERROR_DATA; + } + // Go to offset and see if we have the correct marker + uint8_t* content_marker = header + offset; + if (header_len < offset + 1 + 4) { + return BLOSC2_ERROR_READ_BUFFER; + } + if (*content_marker != 0xc6) { + return BLOSC2_ERROR_DATA; + } + + // Read the size of the content + int32_t content_len; + from_big(&content_len, content_marker + 1, sizeof(content_len)); + if (content_len < 0) { + return BLOSC2_ERROR_DATA; + } + metalayer->content_len = content_len; + + // Finally, read the content + if (header_len < offset + 1 + 4 + content_len) { + return BLOSC2_ERROR_READ_BUFFER; + } + char* content = malloc((size_t)content_len); + memcpy(content, content_marker + 1 + 4, (size_t)content_len); + metalayer->content = (uint8_t*)content; + } + + return 1; +} + +int frame_get_metalayers(blosc2_frame_s* frame, blosc2_schunk* schunk) { + int32_t header_len; + int64_t frame_len; + int64_t nbytes; + int64_t cbytes; + int32_t blocksize; + int32_t chunksize; + int64_t nchunks; + int ret = get_header_info(frame, &header_len, &frame_len, &nbytes, &cbytes, + &blocksize, &chunksize, &nchunks, + NULL, NULL, NULL, NULL, NULL, NULL, + schunk->storage->io); + if (ret < 0) { + BLOSC_TRACE_ERROR("Unable to get the header info from frame."); + return ret; + } + + // Get the header + uint8_t* header = NULL; + if (frame->cframe != NULL) { + header = frame->cframe; + } else { + int64_t rbytes = 0; + header = malloc(header_len); + blosc2_io_cb *io_cb = blosc2_get_io_cb(frame->schunk->storage->io->id); + if (io_cb == NULL) { + BLOSC_TRACE_ERROR("Error getting the input/output API"); + return BLOSC2_ERROR_PLUGIN_IO; + } + + void* fp = NULL; + if (frame->sframe) { + fp = sframe_open_index(frame->urlpath, "rb", + frame->schunk->storage->io); + } + else { + fp = io_cb->open(frame->urlpath, "rb", frame->schunk->storage->io->params); + io_cb->seek(fp, frame->file_offset, SEEK_SET); + } + if (fp != NULL) { + rbytes = io_cb->read(header, 1, header_len, fp); + io_cb->close(fp); + } + if (rbytes != header_len) { + BLOSC_TRACE_ERROR("Cannot access the header out of the frame."); + free(header); + return BLOSC2_ERROR_FILE_READ; + } + } + + ret = get_meta_from_header(frame, schunk, header, header_len); + + if (frame->cframe == NULL) { + free(header); + } + + return ret; +} + +static int get_vlmeta_from_trailer(blosc2_frame_s* frame, blosc2_schunk* schunk, uint8_t* trailer, + int32_t trailer_len) { + + BLOSC_UNUSED_PARAM(frame); + int64_t trailer_pos = FRAME_TRAILER_VLMETALAYERS + 2; + uint8_t* idxp = trailer + trailer_pos; + + // Get the size for the index of metalayers + trailer_pos += 2; + if (trailer_len < trailer_pos) { + return BLOSC2_ERROR_READ_BUFFER; + } + uint16_t idx_size; + from_big(&idx_size, idxp, sizeof(idx_size)); + idxp += 2; + + trailer_pos += 1; + // Get the actual index of metalayers + if (trailer_len < trailer_pos) { + return BLOSC2_ERROR_READ_BUFFER; + } + if (idxp[0] != 0xde) { // sanity check + return BLOSC2_ERROR_DATA; + } + idxp += 1; + + int16_t nmetalayers; + trailer_pos += sizeof(nmetalayers); + if (trailer_len < trailer_pos) { + return BLOSC2_ERROR_READ_BUFFER; + } + from_big(&nmetalayers, idxp, sizeof(uint16_t)); + idxp += 2; + if (nmetalayers > BLOSC2_MAX_VLMETALAYERS) { + return BLOSC2_ERROR_DATA; + } + schunk->nvlmetalayers = nmetalayers; + + // Populate the metalayers and its serialized values + for (int nmetalayer = 0; nmetalayer < nmetalayers; nmetalayer++) { + trailer_pos += 1; + if (trailer_len < trailer_pos) { + return BLOSC2_ERROR_READ_BUFFER; + } + if ((*idxp & 0xe0u) != 0xa0u) { // sanity check + return BLOSC2_ERROR_DATA; + } + blosc2_metalayer* metalayer = calloc(sizeof(blosc2_metalayer), 1); + schunk->vlmetalayers[nmetalayer] = metalayer; + + // Populate the metalayer string + int8_t nslen = *idxp & (uint8_t)0x1F; + idxp += 1; + trailer_pos += nslen; + if (trailer_len < trailer_pos) { + return BLOSC2_ERROR_READ_BUFFER; + } + char* ns = malloc((size_t)nslen + 1); + memcpy(ns, idxp, nslen); + ns[nslen] = '\0'; + idxp += nslen; + metalayer->name = ns; + + // Populate the serialized value for this metalayer + // Get the offset + trailer_pos += 1; + if (trailer_len < trailer_pos) { + return BLOSC2_ERROR_READ_BUFFER; + } + if ((*idxp & 0xffu) != 0xd2u) { // sanity check + return BLOSC2_ERROR_DATA; + } + idxp += 1; + int32_t offset; + trailer_pos += sizeof(offset); + if (trailer_len < trailer_pos) { + return BLOSC2_ERROR_READ_BUFFER; + } + from_big(&offset, idxp, sizeof(offset)); + idxp += 4; + if (offset < 0 || offset >= trailer_len) { + // Offset is less than zero or exceeds trailer length + return BLOSC2_ERROR_DATA; + } + // Go to offset and see if we have the correct marker + uint8_t* content_marker = trailer + offset; + if (trailer_len < offset + 1 + 4) { + return BLOSC2_ERROR_READ_BUFFER; + } + if (*content_marker != 0xc6) { + return BLOSC2_ERROR_DATA; + } + + // Read the size of the content + int32_t content_len; + from_big(&content_len, content_marker + 1, sizeof(content_len)); + if (content_len < 0) { + return BLOSC2_ERROR_DATA; + } + metalayer->content_len = content_len; + + // Finally, read the content + if (trailer_len < offset + 1 + 4 + content_len) { + return BLOSC2_ERROR_READ_BUFFER; + } + char* content = malloc((size_t)content_len); + memcpy(content, content_marker + 1 + 4, (size_t)content_len); + metalayer->content = (uint8_t*)content; + } + return 1; +} + +int frame_get_vlmetalayers(blosc2_frame_s* frame, blosc2_schunk* schunk) { + int32_t header_len; + int64_t frame_len; + int64_t nbytes; + int64_t cbytes; + int32_t blocksize; + int32_t chunksize; + int64_t nchunks; + int ret = get_header_info(frame, &header_len, &frame_len, &nbytes, &cbytes, + &blocksize, &chunksize, &nchunks, + NULL, NULL, NULL, NULL, NULL, NULL, + schunk->storage->io); + if (ret < 0) { + BLOSC_TRACE_ERROR("Unable to get the trailer info from frame."); + return ret; + } + + int64_t trailer_offset = get_trailer_offset(frame, header_len, nbytes > 0); + int32_t trailer_len = (int32_t) frame->trailer_len; + + if (trailer_offset < BLOSC_EXTENDED_HEADER_LENGTH || trailer_offset + trailer_len > frame->len) { + BLOSC_TRACE_ERROR("Cannot access the trailer out of the frame."); + return BLOSC2_ERROR_READ_BUFFER; + } + + // Get the trailer + uint8_t* trailer = NULL; + if (frame->cframe != NULL) { + trailer = frame->cframe + trailer_offset; + } else { + int64_t rbytes = 0; + trailer = malloc(trailer_len); + + blosc2_io_cb *io_cb = blosc2_get_io_cb(frame->schunk->storage->io->id); + if (io_cb == NULL) { + BLOSC_TRACE_ERROR("Error getting the input/output API"); + return BLOSC2_ERROR_PLUGIN_IO; + } + + void* fp = NULL; + if (frame->sframe) { + char* eframe_name = malloc(strlen(frame->urlpath) + strlen("/chunks.b2frame") + 1); + sprintf(eframe_name, "%s/chunks.b2frame", frame->urlpath); + fp = io_cb->open(eframe_name, "rb", frame->schunk->storage->io->params); + free(eframe_name); + io_cb->seek(fp, trailer_offset, SEEK_SET); + } + else { + fp = io_cb->open(frame->urlpath, "rb", frame->schunk->storage->io->params); + io_cb->seek(fp, frame->file_offset + trailer_offset, SEEK_SET); + } + if (fp != NULL) { + rbytes = io_cb->read(trailer, 1, trailer_len, fp); + io_cb->close(fp); + } + if (rbytes != trailer_len) { + BLOSC_TRACE_ERROR("Cannot access the trailer out of the fileframe."); + free(trailer); + return BLOSC2_ERROR_FILE_READ; + } + } + + ret = get_vlmeta_from_trailer(frame, schunk, trailer, trailer_len); + + if (frame->cframe == NULL) { + free(trailer); + } + + return ret; +} + + +blosc2_storage* get_new_storage(const blosc2_storage* storage, + const blosc2_cparams* cdefaults, + const blosc2_dparams* ddefaults, + const blosc2_io* iodefaults) { + + blosc2_storage* new_storage = (blosc2_storage*)calloc(1, sizeof(blosc2_storage)); + memcpy(new_storage, storage, sizeof(blosc2_storage)); + if (storage->urlpath != NULL) { + char* urlpath = normalize_urlpath(storage->urlpath); + new_storage->urlpath = malloc(strlen(urlpath) + 1); + strcpy(new_storage->urlpath, urlpath); + } + + // cparams + blosc2_cparams* cparams = malloc(sizeof(blosc2_cparams)); + if (storage->cparams != NULL) { + memcpy(cparams, storage->cparams, sizeof(blosc2_cparams)); + } else { + memcpy(cparams, cdefaults, sizeof(blosc2_cparams)); + } + new_storage->cparams = cparams; + + // dparams + blosc2_dparams* dparams = malloc(sizeof(blosc2_dparams)); + if (storage->dparams != NULL) { + memcpy(dparams, storage->dparams, sizeof(blosc2_dparams)); + } + else { + memcpy(dparams, ddefaults, sizeof(blosc2_dparams)); + } + new_storage->dparams = dparams; + + // iodefaults + blosc2_io* udio = malloc(sizeof(blosc2_io)); + if (storage->io != NULL) { + memcpy(udio, storage->io, sizeof(blosc2_io)); + } + else { + memcpy(udio, iodefaults, sizeof(blosc2_io)); + } + new_storage->io = udio; + + return new_storage; +} + + +/* Get a super-chunk out of a frame */ +blosc2_schunk* frame_to_schunk(blosc2_frame_s* frame, bool copy, const blosc2_io *udio) { + int32_t header_len; + int64_t frame_len; + int rc; + blosc2_schunk* schunk = calloc(1, sizeof(blosc2_schunk)); + schunk->frame = (blosc2_frame*)frame; + frame->schunk = schunk; + + rc = get_header_info(frame, &header_len, &frame_len, &schunk->nbytes, + &schunk->cbytes, &schunk->blocksize, + &schunk->chunksize, &schunk->nchunks, &schunk->typesize, + &schunk->compcode, &schunk->compcode_meta, &schunk->clevel, schunk->filters, + schunk->filters_meta, udio); + if (rc < 0) { + BLOSC_TRACE_ERROR("Unable to get meta info from frame."); + blosc2_schunk_free(schunk); + return NULL; + } + int64_t nchunks = schunk->nchunks; + int64_t nbytes = schunk->nbytes; + (void) nbytes; + int64_t cbytes = schunk->cbytes; + + // Compression and decompression contexts + blosc2_cparams *cparams; + blosc2_schunk_get_cparams(schunk, &cparams); + schunk->cctx = blosc2_create_cctx(*cparams); + blosc2_dparams *dparams; + blosc2_schunk_get_dparams(schunk, &dparams); + schunk->dctx = blosc2_create_dctx(*dparams); + blosc2_storage storage = {.contiguous = copy ? false : true}; + schunk->storage = get_new_storage(&storage, cparams, dparams, udio); + free(cparams); + free(dparams); + if (!copy) { + goto out; + } + + // We are not attached to a frame anymore + schunk->frame = NULL; + + if (nchunks == 0) { + frame->schunk = NULL; + goto out; + } + + // Get the compressed offsets + int32_t coffsets_cbytes = 0; + uint8_t* coffsets = get_coffsets(frame, header_len, cbytes, nchunks, &coffsets_cbytes); + if (coffsets == NULL) { + blosc2_schunk_free(schunk); + BLOSC_TRACE_ERROR("Cannot get the offsets for the frame."); + return NULL; + } + + // Decompress offsets + blosc2_dparams off_dparams = BLOSC2_DPARAMS_DEFAULTS; + blosc2_context *dctx = blosc2_create_dctx(off_dparams); + int64_t* offsets = (int64_t *) malloc((size_t)nchunks * sizeof(int64_t)); + int32_t off_nbytes = blosc2_decompress_ctx(dctx, coffsets, coffsets_cbytes, + offsets, (int32_t)(nchunks * sizeof(int64_t))); + blosc2_free_ctx(dctx); + if (off_nbytes < 0) { + free(offsets); + blosc2_schunk_free(schunk); + BLOSC_TRACE_ERROR("Cannot decompress the offsets chunk."); + return NULL; + } + + // We want the contiguous schunk, so create the actual data chunks (and, while doing this, + // get a guess at the blocksize used in this frame) + int64_t acc_nbytes = 0; + int64_t acc_cbytes = 0; + int32_t blocksize = 0; + int32_t chunk_nbytes; + int32_t chunk_cbytes; + int32_t chunk_blocksize; + size_t prev_alloc = BLOSC_EXTENDED_HEADER_LENGTH; + uint8_t* data_chunk = NULL; + bool needs_free = false; + const blosc2_io_cb *io_cb = blosc2_get_io_cb(udio->id); + if (io_cb == NULL) { + blosc2_schunk_free(schunk); + BLOSC_TRACE_ERROR("Error getting the input/output API"); + return NULL; + } + + void* fp = NULL; + if (frame->cframe == NULL) { + data_chunk = malloc((size_t)prev_alloc); + needs_free = true; + if (!frame->sframe) { + // If not the chunks won't be in the frame + fp = io_cb->open(frame->urlpath, "rb", udio->params); + if (fp == NULL) { + rc = BLOSC2_ERROR_FILE_OPEN; + goto end; + } + } + } + schunk->data = malloc(nchunks * sizeof(void*)); + for (int i = 0; i < nchunks; i++) { + if (frame->cframe != NULL) { + if (needs_free) { + free(data_chunk); + } + if (offsets[i] < 0) { + int64_t rbytes = frame_get_lazychunk(frame, offsets[i], &data_chunk, &needs_free); + if (rbytes < 0) { + break; + } + } + else { + data_chunk = frame->cframe + header_len + offsets[i]; + } + rc = blosc2_cbuffer_sizes(data_chunk, NULL, &chunk_cbytes, NULL); + if (rc < 0) { + break; + } + } + else { + int64_t rbytes; + if (frame->sframe) { + if (needs_free) { + free(data_chunk); + } + rbytes = frame_get_lazychunk(frame, offsets[i], &data_chunk, &needs_free); + if (rbytes < 0) { + break; + } + } + else { + io_cb->seek(fp, frame->file_offset + header_len + offsets[i], SEEK_SET); + rbytes = io_cb->read(data_chunk, 1, BLOSC_EXTENDED_HEADER_LENGTH, fp); + } + if (rbytes != BLOSC_EXTENDED_HEADER_LENGTH) { + rc = BLOSC2_ERROR_READ_BUFFER; + break; + } + rc = blosc2_cbuffer_sizes(data_chunk, NULL, &chunk_cbytes, NULL); + if (rc < 0) { + break; + } + if (chunk_cbytes > (int32_t)prev_alloc) { + data_chunk = realloc(data_chunk, chunk_cbytes); + prev_alloc = chunk_cbytes; + } + if (!frame->sframe) { + io_cb->seek(fp, frame->file_offset + header_len + offsets[i], SEEK_SET); + rbytes = io_cb->read(data_chunk, 1, chunk_cbytes, fp); + if (rbytes != chunk_cbytes) { + rc = BLOSC2_ERROR_READ_BUFFER; + break; + } + } + } + uint8_t* new_chunk = malloc(chunk_cbytes); + memcpy(new_chunk, data_chunk, chunk_cbytes); + schunk->data[i] = new_chunk; + rc = blosc2_cbuffer_sizes(data_chunk, &chunk_nbytes, NULL, &chunk_blocksize); + if (rc < 0) { + break; + } + acc_nbytes += chunk_nbytes; + acc_cbytes += chunk_cbytes; + if (i == 0) { + blocksize = chunk_blocksize; + } + else if (blocksize != chunk_blocksize) { + // Blocksize varies + blocksize = 0; + } + } + + // We are not attached to a schunk anymore + frame->schunk = NULL; + + end: + if (needs_free) { + free(data_chunk); + } + if (frame->cframe == NULL) { + if (!frame->sframe) { + io_cb->close(fp); + } + } + free(offsets); + + // cframes and sframes have different ways to store chunks with special values: + // 1) cframes represent special chunks as negative offsets + // 2) sframes does not have the concept of offsets, but rather of data pointers (.data) + // so they always have a pointer to a special chunk + // This is why cframes and sframes have different cbytes and hence, we cannot enforce acc_bytes == schunk->cbytes + // In the future, maybe we could provide special meanings for .data[i] > 0x7FFFFFFF, but not there yet + // if (rc < 0 || acc_nbytes != nbytes || acc_cbytes != cbytes) { + if (rc < 0 || acc_nbytes != nbytes) { + blosc2_schunk_free(schunk); + return NULL; + } + // Update counters + schunk->cbytes = acc_cbytes; + schunk->blocksize = blocksize; + + out: + rc = frame_get_metalayers(frame, schunk); + if (rc < 0) { + blosc2_schunk_free(schunk); + BLOSC_TRACE_ERROR("Cannot access the metalayers."); + return NULL; + } + + rc = frame_get_vlmetalayers(frame, schunk); + if (rc < 0) { + blosc2_schunk_free(schunk); + BLOSC_TRACE_ERROR("Cannot access the vlmetalayers."); + return NULL; + } + + return schunk; +} + + +struct csize_idx { + int32_t val; + int32_t idx; +}; + +// Helper function for qsorting block offsets +int sort_offset(const void* a, const void* b) { + int32_t a_ = ((struct csize_idx*)a)->val; + int32_t b_ = ((struct csize_idx*)b)->val; + return a_ - b_; +} + + +int get_coffset(blosc2_frame_s* frame, int32_t header_len, int64_t cbytes, + int64_t nchunk, int64_t nchunks, int64_t *offset) { + int32_t off_cbytes; + // Get the offset to nchunk + uint8_t *coffsets = get_coffsets(frame, header_len, cbytes, nchunks, &off_cbytes); + if (coffsets == NULL) { + BLOSC_TRACE_ERROR("Cannot get the offset for chunk %" PRId64 " for the frame.", nchunk); + return BLOSC2_ERROR_DATA; + } + + // Get the 64-bit offset + int rc = blosc2_getitem(coffsets, off_cbytes, (int32_t)nchunk, 1, offset, (int32_t)sizeof(int64_t)); + if (rc < 0) { + BLOSC_TRACE_ERROR("Problems retrieving a chunk offset."); + } else if (!frame->sframe && *offset > frame->len) { + BLOSC_TRACE_ERROR("Cannot read chunk %" PRId64 " outside of frame boundary.", nchunk); + rc = BLOSC2_ERROR_READ_BUFFER; + } + + return rc; +} + + +// Detect and return a chunk with special values in offsets (only zeros, NaNs and non initialized) +int frame_special_chunk(int64_t special_value, int32_t nbytes, int32_t typesize, int32_t blocksize, + uint8_t** chunk, int32_t cbytes, bool *needs_free) { + int rc = 0; + *chunk = malloc(cbytes); + *needs_free = true; + + // Detect the kind of special value + uint64_t zeros_mask = (uint64_t) BLOSC2_SPECIAL_ZERO << (8 * 7); // chunk of zeros + uint64_t nans_mask = (uint64_t) BLOSC2_SPECIAL_NAN << (8 * 7); // chunk of NaNs + uint64_t uninit_mask = (uint64_t) BLOSC2_SPECIAL_UNINIT << (8 * 7); // chunk of uninit values + + blosc2_cparams cparams = BLOSC2_CPARAMS_DEFAULTS; + cparams.typesize = typesize; + cparams.blocksize = blocksize; + if (special_value & zeros_mask) { + rc = blosc2_chunk_zeros(cparams, nbytes, *chunk, cbytes); + if (rc < 0) { + BLOSC_TRACE_ERROR("Error creating a zero chunk"); + } + } + else if (special_value & uninit_mask) { + rc = blosc2_chunk_uninit(cparams, nbytes, *chunk, cbytes); + if (rc < 0) { + BLOSC_TRACE_ERROR("Error creating a non initialized chunk"); + } + } + else if (special_value & nans_mask) { + rc = blosc2_chunk_nans(cparams, nbytes, *chunk, cbytes); + if (rc < 0) { + BLOSC_TRACE_ERROR("Error creating a nan chunk"); + } + } + else { + BLOSC_TRACE_ERROR("Special value not recognized: %" PRId64 "", special_value); + rc = BLOSC2_ERROR_DATA; + } + + if (rc < 0) { + free(*chunk); + *needs_free = false; + *chunk = NULL; + } + + return rc; +} + + +/* Return a compressed chunk that is part of a frame in the `chunk` parameter. + * If the frame is disk-based, a buffer is allocated for the (compressed) chunk, + * and hence a free is needed. You can check if the chunk requires a free with the `needs_free` + * parameter. + * If the chunk does not need a free, it means that a pointer to the location in frame is returned + * in the `chunk` parameter. + * + * The size of the (compressed) chunk is returned. If some problem is detected, a negative code + * is returned instead. +*/ +int frame_get_chunk(blosc2_frame_s *frame, int64_t nchunk, uint8_t **chunk, bool *needs_free) { + int32_t header_len; + int64_t frame_len; + int64_t nbytes; + int64_t cbytes; + int32_t blocksize; + int32_t chunksize; + int64_t nchunks; + int32_t typesize; + int64_t offset; + int32_t chunk_cbytes; + int rc; + + *chunk = NULL; + *needs_free = false; + rc = get_header_info(frame, &header_len, &frame_len, &nbytes, &cbytes, + &blocksize, &chunksize, &nchunks, + &typesize, NULL, NULL, NULL, NULL, NULL, + frame->schunk->storage->io); + if (rc < 0) { + BLOSC_TRACE_ERROR("Unable to get meta info from frame."); + return rc; + } + + if (nchunk >= nchunks) { + BLOSC_TRACE_ERROR("nchunk ('%" PRId64 "') exceeds the number of chunks " + "('%" PRId64 "') in frame.", nchunk, nchunks); + return BLOSC2_ERROR_INVALID_PARAM; + } + + // Get the offset to nchunk + rc = get_coffset(frame, header_len, cbytes, nchunk, nchunks, &offset); + if (rc < 0) { + BLOSC_TRACE_ERROR("Unable to get offset to chunk %" PRId64 ".", nchunk); + return rc; + } + + if (offset < 0) { + // Special value + chunk_cbytes = BLOSC_EXTENDED_HEADER_LENGTH; + int32_t chunksize_ = chunksize; + if ((nchunk == nchunks - 1) && (nbytes % chunksize)) { + // Last chunk is incomplete. Compute its actual size. + chunksize_ = (int32_t) nbytes % chunksize; + } + rc = frame_special_chunk(offset, chunksize_, typesize, blocksize, chunk, chunk_cbytes, needs_free); + if (rc < 0) { + return rc; + } + goto end; + } + + if (frame->sframe) { + // Sparse on-disk + nchunk = offset; + return sframe_get_chunk(frame, nchunk, chunk, needs_free); + } + + blosc2_io_cb *io_cb = blosc2_get_io_cb(frame->schunk->storage->io->id); + if (io_cb == NULL) { + BLOSC_TRACE_ERROR("Error getting the input/output API"); + return BLOSC2_ERROR_PLUGIN_IO; + } + + if (frame->cframe == NULL) { + uint8_t header[BLOSC_EXTENDED_HEADER_LENGTH]; + void* fp = io_cb->open(frame->urlpath, "rb", frame->schunk->storage->io->params); + io_cb->seek(fp, frame->file_offset + header_len + offset, SEEK_SET); + int64_t rbytes = io_cb->read(header, 1, sizeof(header), fp); + if (rbytes != sizeof(header)) { + BLOSC_TRACE_ERROR("Cannot read the cbytes for chunk in the frame."); + io_cb->close(fp); + return BLOSC2_ERROR_FILE_READ; + } + rc = blosc2_cbuffer_sizes(header, NULL, &chunk_cbytes, NULL); + if (rc < 0) { + BLOSC_TRACE_ERROR("Cannot read the cbytes for chunk in the frame."); + io_cb->close(fp); + return rc; + } + *chunk = malloc(chunk_cbytes); + io_cb->seek(fp, frame->file_offset + header_len + offset, SEEK_SET); + rbytes = io_cb->read(*chunk, 1, chunk_cbytes, fp); + io_cb->close(fp); + if (rbytes != chunk_cbytes) { + BLOSC_TRACE_ERROR("Cannot read the chunk out of the frame."); + return BLOSC2_ERROR_FILE_READ; + } + *needs_free = true; + } else { + // The chunk is in memory and just one pointer away + *chunk = frame->cframe + header_len + offset; + rc = blosc2_cbuffer_sizes(*chunk, NULL, &chunk_cbytes, NULL); + if (rc < 0) { + return rc; + } + } + + end: + return (int32_t)chunk_cbytes; +} + + +/* Return a compressed chunk that is part of a frame in the `chunk` parameter. + * If the frame is disk-based, a buffer is allocated for the (lazy) chunk, + * and hence a free is needed. You can check if the chunk requires a free with the `needs_free` + * parameter. + * If the chunk does not need a free, it means that the frame is in memory and that just a + * pointer to the location of the chunk in memory is returned. + * + * The size of the (compressed, potentially lazy) chunk is returned. If some problem is detected, + * a negative code is returned instead. +*/ +int frame_get_lazychunk(blosc2_frame_s *frame, int64_t nchunk, uint8_t **chunk, bool *needs_free) { + int32_t header_len; + int64_t frame_len; + int64_t nbytes; + int64_t cbytes; + int32_t blocksize; + int32_t chunksize; + int64_t nchunks; + int32_t typesize; + int32_t lazychunk_cbytes; + int64_t offset; + void* fp = NULL; + + *chunk = NULL; + *needs_free = false; + int rc = get_header_info(frame, &header_len, &frame_len, &nbytes, &cbytes, + &blocksize, &chunksize, &nchunks, + &typesize, NULL, NULL, NULL, NULL, NULL, + frame->schunk->storage->io); + if (rc < 0) { + BLOSC_TRACE_ERROR("Unable to get meta info from frame."); + return rc; + } + + if (nchunk >= nchunks) { + BLOSC_TRACE_ERROR("nchunk ('%" PRId64 "') exceeds the number of chunks " + "('%" PRId64 "') in frame.", nchunk, nchunks); + return BLOSC2_ERROR_INVALID_PARAM; + } + + // Get the offset to nchunk + rc = get_coffset(frame, header_len, cbytes, nchunk, nchunks, &offset); + if (rc < 0) { + BLOSC_TRACE_ERROR("Unable to get offset to chunk %" PRId64 ".", nchunk); + return rc; + } + + if (offset < 0) { + // Special value + lazychunk_cbytes = BLOSC_EXTENDED_HEADER_LENGTH; + int32_t chunksize_ = chunksize; + if ((nchunk == nchunks - 1) && (nbytes % chunksize)) { + // Last chunk is incomplete. Compute its actual size. + chunksize_ = (int32_t) nbytes % chunksize; + } + rc = frame_special_chunk(offset, chunksize_, typesize, blocksize, chunk, + (int32_t)lazychunk_cbytes, needs_free); + goto end; + } + + blosc2_io_cb *io_cb = blosc2_get_io_cb(frame->schunk->storage->io->id); + if (io_cb == NULL) { + BLOSC_TRACE_ERROR("Error getting the input/output API"); + rc = BLOSC2_ERROR_PLUGIN_IO; + goto end; + } + + if (frame->cframe == NULL) { + // TODO: make this portable across different endianness + // Get info for building a lazy chunk + int32_t chunk_nbytes; + int32_t chunk_cbytes; + int32_t chunk_blocksize; + uint8_t header[BLOSC_EXTENDED_HEADER_LENGTH]; + if (frame->sframe) { + // The chunk is not in the frame + fp = sframe_open_chunk(frame->urlpath, offset, "rb", + frame->schunk->storage->io); + } + else { + fp = io_cb->open(frame->urlpath, "rb", frame->schunk->storage->io->params); + io_cb->seek(fp, frame->file_offset + header_len + offset, SEEK_SET); + } + int64_t rbytes = io_cb->read(header, 1, BLOSC_EXTENDED_HEADER_LENGTH, fp); + if (rbytes != BLOSC_EXTENDED_HEADER_LENGTH) { + BLOSC_TRACE_ERROR("Cannot read the header for chunk in the frame."); + rc = BLOSC2_ERROR_FILE_READ; + goto end; + } + rc = blosc2_cbuffer_sizes(header, &chunk_nbytes, &chunk_cbytes, &chunk_blocksize); + if (rc < 0) { + goto end; + } + size_t nblocks = chunk_nbytes / chunk_blocksize; + size_t leftover_block = chunk_nbytes % chunk_blocksize; + nblocks = leftover_block ? nblocks + 1 : nblocks; + // Allocate space for the lazy chunk + int32_t trailer_len; + int32_t special_type = (header[BLOSC2_CHUNK_BLOSC2_FLAGS] >> 4) & BLOSC2_SPECIAL_MASK; + int memcpyed = header[BLOSC2_CHUNK_FLAGS] & (uint8_t) BLOSC_MEMCPYED; + + int32_t trailer_offset = BLOSC_EXTENDED_HEADER_LENGTH; + size_t streams_offset = BLOSC_EXTENDED_HEADER_LENGTH; + if (special_type == 0) { + // Regular values have offsets for blocks + trailer_offset += (int32_t) (nblocks * sizeof(int32_t)); + if (memcpyed) { + streams_offset += 0; + } else { + streams_offset += nblocks * sizeof(int32_t); + } + trailer_len = (int32_t) (sizeof(int32_t) + sizeof(int64_t) + nblocks * sizeof(int32_t)); + lazychunk_cbytes = trailer_offset + trailer_len; + } + else if (special_type == BLOSC2_SPECIAL_VALUE) { + trailer_offset += typesize; + streams_offset += typesize; + trailer_len = 0; + lazychunk_cbytes = trailer_offset + trailer_len; + } + else { + rc = BLOSC2_ERROR_INVALID_HEADER; + goto end; + } + *chunk = malloc(lazychunk_cbytes); + *needs_free = true; + + // Read just the full header and bstarts section too (lazy partial length) + if (frame->sframe) { + io_cb->seek(fp, 0, SEEK_SET); + } + else { + io_cb->seek(fp, frame->file_offset + header_len + offset, SEEK_SET); + } + + rbytes = io_cb->read(*chunk, 1, (int64_t)streams_offset, fp); + if (rbytes != (int64_t)streams_offset) { + BLOSC_TRACE_ERROR("Cannot read the (lazy) chunk out of the frame."); + rc = BLOSC2_ERROR_FILE_READ; + goto end; + } + if (special_type == BLOSC2_SPECIAL_VALUE) { + // Value runlen is not returning a lazy chunk. We are done. + goto end; + } + + // Mark chunk as lazy + uint8_t* blosc2_flags = *chunk + BLOSC2_CHUNK_BLOSC2_FLAGS; + *blosc2_flags |= 0x08U; + + // Add the trailer (currently, nchunk + offset + block_csizes) + if (frame->sframe) { + *(int32_t*)(*chunk + trailer_offset) = (int32_t)offset; // offset is nchunk for sframes + *(int64_t*)(*chunk + trailer_offset + sizeof(int32_t)) = offset; + } + else { + *(int32_t*)(*chunk + trailer_offset) = (int32_t)nchunk; + *(int64_t*)(*chunk + trailer_offset + sizeof(int32_t)) = header_len + offset; + } + + int32_t* block_csizes = malloc(nblocks * sizeof(int32_t)); + + if (memcpyed) { + // When memcpyed the blocksizes are trivial to compute + for (int i = 0; i < (int)nblocks - 1; i++) { + block_csizes[i] = (int)chunk_blocksize; + } + // The last block could be incomplete, mainly due to the fact that the block size is not divisible + // by the typesize + block_csizes[nblocks - 1] = (int32_t)leftover_block ? (int32_t)leftover_block : chunk_blocksize; + } + else { + // In regular, compressed chunks, we need to sort the bstarts (they can be out + // of order because of multi-threading), and get a reverse index too. + memcpy(block_csizes, *chunk + BLOSC_EXTENDED_HEADER_LENGTH, nblocks * sizeof(int32_t)); + // Helper structure to keep track of original indexes + struct csize_idx *csize_idx = malloc(nblocks * sizeof(struct csize_idx)); + for (int n = 0; n < (int)nblocks; n++) { + csize_idx[n].val = block_csizes[n]; + csize_idx[n].idx = n; + } + qsort(csize_idx, nblocks, sizeof(struct csize_idx), &sort_offset); + // Compute the actual csizes + int idx; + for (int n = 0; n < (int)nblocks - 1; n++) { + idx = csize_idx[n].idx; + block_csizes[idx] = csize_idx[n + 1].val - csize_idx[n].val; + } + idx = csize_idx[nblocks - 1].idx; + block_csizes[idx] = (int)chunk_cbytes - csize_idx[nblocks - 1].val; + free(csize_idx); + } + // Copy the csizes at the end of the trailer + void *trailer_csizes = *chunk + lazychunk_cbytes - nblocks * sizeof(int32_t); + memcpy(trailer_csizes, block_csizes, nblocks * sizeof(int32_t)); + free(block_csizes); + } else { + // The chunk is in memory and just one pointer away + int64_t chunk_header_offset = header_len + offset; + int64_t chunk_cbytes_offset = chunk_header_offset + BLOSC_MIN_HEADER_LENGTH; + + *chunk = frame->cframe + chunk_header_offset; + + if (chunk_cbytes_offset > frame->len) { + BLOSC_TRACE_ERROR("Cannot read the header for chunk in the (contiguous) frame."); + rc = BLOSC2_ERROR_READ_BUFFER; + } else { + rc = blosc2_cbuffer_sizes(*chunk, NULL, &lazychunk_cbytes, NULL); + if (rc && chunk_cbytes_offset + lazychunk_cbytes > frame_len) { + BLOSC_TRACE_ERROR("Compressed bytes exceed beyond frame length."); + rc = BLOSC2_ERROR_READ_BUFFER; + } + } + } + + end: + if (fp != NULL) { + io_cb->close(fp); + } + if (rc < 0) { + if (*needs_free) { + free(*chunk); + *chunk = NULL; + *needs_free = false; + } + return rc; + } + + return (int)lazychunk_cbytes; +} + + +/* Fill an empty frame with special values (fast path). */ +int64_t frame_fill_special(blosc2_frame_s* frame, int64_t nitems, int special_value, + int32_t chunksize, blosc2_schunk* schunk) { + int32_t header_len; + int64_t frame_len; + int64_t nbytes; + int64_t cbytes; + int32_t blocksize; + int32_t typesize; + int64_t nchunks; + + int rc = get_header_info(frame, &header_len, &frame_len, &nbytes, &cbytes, &blocksize, NULL, + &nchunks, &typesize, NULL, NULL, NULL, NULL, NULL, + schunk->storage->io); + if (rc < 0) { + BLOSC_TRACE_ERROR("Unable to get meta info from frame."); + return BLOSC2_ERROR_DATA; + } + + if (nitems == 0) { + return frame_len; + } + + if ((nitems / chunksize) > INT_MAX) { + BLOSC_TRACE_ERROR("nitems is too large. Try increasing the chunksize."); + return BLOSC2_ERROR_FRAME_SPECIAL; + } + + if ((nbytes > 0) || (cbytes > 0)) { + BLOSC_TRACE_ERROR("Filling with special values only works on empty frames"); + return BLOSC2_ERROR_FRAME_SPECIAL; + } + + // Compute the number of chunks and the length of the offsets chunk + int32_t chunkitems = chunksize / typesize; + nchunks = nitems / chunkitems; + int32_t leftover_items = (int32_t)(nitems % chunkitems); + if (leftover_items) { + nchunks += 1; + } + + blosc2_cparams* cparams; + blosc2_schunk_get_cparams(schunk, &cparams); + + // Build the offsets with a special chunk + int new_off_cbytes = BLOSC_EXTENDED_HEADER_LENGTH + sizeof(int64_t); + uint8_t* off_chunk = malloc(new_off_cbytes); + uint64_t offset_value = ((uint64_t)1 << 63); + uint8_t* sample_chunk = malloc(BLOSC_EXTENDED_HEADER_LENGTH); + int csize; + switch (special_value) { + case BLOSC2_SPECIAL_ZERO: + offset_value += (uint64_t) BLOSC2_SPECIAL_ZERO << (8 * 7); + csize = blosc2_chunk_zeros(*cparams, chunksize, sample_chunk, BLOSC_EXTENDED_HEADER_LENGTH); + break; + case BLOSC2_SPECIAL_UNINIT: + offset_value += (uint64_t) BLOSC2_SPECIAL_UNINIT << (8 * 7); + csize = blosc2_chunk_uninit(*cparams, chunksize, sample_chunk, BLOSC_EXTENDED_HEADER_LENGTH); + break; + case BLOSC2_SPECIAL_NAN: + offset_value += (uint64_t)BLOSC2_SPECIAL_NAN << (8 * 7); + csize = blosc2_chunk_nans(*cparams, chunksize, sample_chunk, BLOSC_EXTENDED_HEADER_LENGTH); + break; + default: + BLOSC_TRACE_ERROR("Only zeros, NaNs or non-initialized values are supported."); + return BLOSC2_ERROR_FRAME_SPECIAL; + } + if (csize < 0) { + BLOSC_TRACE_ERROR("Error creating sample chunk"); + return BLOSC2_ERROR_FRAME_SPECIAL; + } + cparams->typesize = sizeof(int64_t); // change it to offsets typesize + // cparams->blocksize = 0; // automatic blocksize + cparams->blocksize = 8 * 2 * 1024; // based on experiments with create_frame.c bench + cparams->clevel = 5; + cparams->compcode = BLOSC_BLOSCLZ; + int32_t special_nbytes = (int32_t) (nchunks * sizeof(int64_t)); + rc = blosc2_chunk_repeatval(*cparams, special_nbytes, off_chunk, new_off_cbytes, &offset_value); + free(cparams); + if (rc < 0) { + BLOSC_TRACE_ERROR("Error creating a special offsets chunk"); + return BLOSC2_ERROR_DATA; + } + + // Get the blocksize associated to the sample chunk + blosc2_cbuffer_sizes(sample_chunk, NULL, NULL, &blocksize); + free(sample_chunk); + // and use it for the super-chunk + schunk->blocksize = blocksize; + // schunk->blocksize = 0; // for experimenting with automatic blocksize + + // We have the new offsets; update the frame. + blosc2_io_cb *io_cb = blosc2_get_io_cb(frame->schunk->storage->io->id); + if (io_cb == NULL) { + BLOSC_TRACE_ERROR("Error getting the input/output API"); + return BLOSC2_ERROR_PLUGIN_IO; + } + + int64_t new_frame_len = header_len + new_off_cbytes + frame->trailer_len; + void* fp = NULL; + if (frame->cframe != NULL) { + uint8_t* framep = frame->cframe; + /* Make space for the new chunk and copy it */ + frame->cframe = framep = realloc(framep, (size_t)new_frame_len); + if (framep == NULL) { + BLOSC_TRACE_ERROR("Cannot realloc space for the frame."); + return BLOSC2_ERROR_FRAME_SPECIAL; + } + /* Copy the offsets */ + memcpy(framep + header_len, off_chunk, (size_t)new_off_cbytes); + } + else { + size_t wbytes; + if (frame->sframe) { + // Update the offsets chunk in the chunks frame + fp = sframe_open_index(frame->urlpath, "rb+", frame->schunk->storage->io); + io_cb->seek(fp, frame->file_offset + header_len, SEEK_SET); + } + else { + // Regular frame + fp = io_cb->open(frame->urlpath, "rb+", schunk->storage->io->params); + io_cb->seek(fp, frame->file_offset + header_len + cbytes, SEEK_SET); + } + wbytes = io_cb->write(off_chunk, 1, new_off_cbytes, fp); // the new offsets + io_cb->close(fp); + if (wbytes != (size_t)new_off_cbytes) { + BLOSC_TRACE_ERROR("Cannot write the offsets to frame."); + return BLOSC2_ERROR_FRAME_SPECIAL; + } + } + + // Invalidate the cache for chunk offsets + if (frame->coffsets != NULL) { + free(frame->coffsets); + frame->coffsets = NULL; + } + free(off_chunk); + + frame->len = new_frame_len; + rc = frame_update_header(frame, schunk, false); + if (rc < 0) { + return BLOSC2_ERROR_FRAME_SPECIAL; + } + + rc = frame_update_trailer(frame, schunk); + if (rc < 0) { + return BLOSC2_ERROR_FRAME_SPECIAL; + } + + return frame->len; +} + + +/* Append an existing chunk into a frame. */ +void* frame_append_chunk(blosc2_frame_s* frame, void* chunk, blosc2_schunk* schunk) { + int8_t* chunk_ = chunk; + int32_t header_len; + int64_t frame_len; + int64_t nbytes; + int64_t cbytes; + int32_t blocksize; + int32_t chunksize; + int64_t nchunks; + int rc = get_header_info(frame, &header_len, &frame_len, &nbytes, &cbytes, &blocksize, &chunksize, + &nchunks, NULL, NULL, NULL, NULL, NULL, NULL, + frame->schunk->storage->io); + if (rc < 0) { + BLOSC_TRACE_ERROR("Unable to get meta info from frame."); + return NULL; + } + + /* The uncompressed and compressed sizes start at byte 4 and 12 */ + int32_t chunk_nbytes; + int32_t chunk_cbytes; + rc = blosc2_cbuffer_sizes(chunk, &chunk_nbytes, &chunk_cbytes, NULL); + if (rc < 0) { + return NULL; + } + + if ((nchunks > 0) && (chunk_nbytes > chunksize)) { + BLOSC_TRACE_ERROR("Appending chunks with a larger chunksize than frame is " + "not allowed yet %d != %d.", chunk_nbytes, chunksize); + return NULL; + } + + // Check that we are not appending a small chunk after another small chunk + int32_t chunk_nbytes_last; + if (chunksize == 0 && (nchunks > 0) && (chunk_nbytes < chunksize)) { + uint8_t* last_chunk; + bool needs_free; + rc = frame_get_lazychunk(frame, nchunks - 1, &last_chunk, &needs_free); + if (rc < 0) { + BLOSC_TRACE_ERROR("Cannot get the last chunk (in position %" PRId64 ").", nchunks - 1); + } else { + rc = blosc2_cbuffer_sizes(last_chunk, &chunk_nbytes_last, NULL, NULL); + } + if (needs_free) { + free(last_chunk); + } + if (rc < 0) { + return NULL; + } + if ((chunk_nbytes_last < chunksize) && (nbytes < chunksize)) { + BLOSC_TRACE_ERROR("Appending two consecutive chunks with a chunksize smaller " + "than the frame chunksize is not allowed yet: %d != %d.", + chunk_nbytes, chunksize); + return NULL; + } + } + + // Get the current offsets and add one more + int32_t off_nbytes = (int32_t) ((nchunks + 1) * sizeof(int64_t)); + int64_t* offsets = (int64_t *) malloc((size_t)off_nbytes); + if (nchunks > 0) { + int32_t coffsets_cbytes; + uint8_t *coffsets = get_coffsets(frame, header_len, cbytes, nchunks, &coffsets_cbytes); + if (coffsets == NULL) { + BLOSC_TRACE_ERROR("Cannot get the offsets for the frame."); + free(offsets); + return NULL; + } + if (coffsets_cbytes == 0) { + coffsets_cbytes = (int32_t)cbytes; + } + + // Decompress offsets + blosc2_dparams off_dparams = BLOSC2_DPARAMS_DEFAULTS; + blosc2_context *dctx = blosc2_create_dctx(off_dparams); + int32_t prev_nbytes = blosc2_decompress_ctx(dctx, coffsets, coffsets_cbytes, offsets, + off_nbytes); + blosc2_free_ctx(dctx); + if (prev_nbytes < 0) { + free(offsets); + BLOSC_TRACE_ERROR("Cannot decompress the offsets chunk."); + return NULL; + } + } + + // Add the new offset + int64_t sframe_chunk_id = -1; + int special_value = (chunk_[BLOSC2_CHUNK_BLOSC2_FLAGS] >> 4) & BLOSC2_SPECIAL_MASK; + uint64_t offset_value = ((uint64_t)1 << 63); + switch (special_value) { + case BLOSC2_SPECIAL_ZERO: + // Zero chunk. Code it in a special way. + offset_value += (uint64_t) BLOSC2_SPECIAL_ZERO << (8 * 7); // chunk of zeros + to_little(offsets + nchunks, &offset_value, sizeof(uint64_t)); + chunk_cbytes = 0; // we don't need to store the chunk + break; + case BLOSC2_SPECIAL_UNINIT: + // Non initizalized values chunk. Code it in a special way. + offset_value += (uint64_t) BLOSC2_SPECIAL_UNINIT << (8 * 7); // chunk of uninit values + to_little(offsets + nchunks, &offset_value, sizeof(uint64_t)); + chunk_cbytes = 0; // we don't need to store the chunk + break; + case BLOSC2_SPECIAL_NAN: + // NaN chunk. Code it in a special way. + offset_value += (uint64_t)BLOSC2_SPECIAL_NAN << (8 * 7); // chunk of NANs + to_little(offsets + nchunks, &offset_value, sizeof(uint64_t)); + chunk_cbytes = 0; // we don't need to store the chunk + break; + default: + if (frame->sframe) { + // Compute the sframe_chunk_id value + for (int i = 0; i < nchunks; ++i) { + if (offsets[i] > sframe_chunk_id) { + sframe_chunk_id = offsets[i]; + } + } + offsets[nchunks] = ++sframe_chunk_id; + } + else { + offsets[nchunks] = cbytes; + } + } + + // Re-compress the offsets again + blosc2_cparams cparams = BLOSC2_CPARAMS_DEFAULTS; + cparams.splitmode = BLOSC_NEVER_SPLIT; + cparams.typesize = sizeof(int64_t); + cparams.blocksize = 16 * 1024; // based on experiments with create_frame.c bench + cparams.nthreads = 4; // 4 threads seems a decent default for nowadays CPUs + cparams.compcode = BLOSC_BLOSCLZ; + blosc2_context* cctx = blosc2_create_cctx(cparams); + void* off_chunk = malloc((size_t)off_nbytes + BLOSC2_MAX_OVERHEAD); + int32_t new_off_cbytes = blosc2_compress_ctx(cctx, offsets, off_nbytes, + off_chunk, off_nbytes + BLOSC2_MAX_OVERHEAD); + blosc2_free_ctx(cctx); + free(offsets); + if (new_off_cbytes < 0) { + free(off_chunk); + return NULL; + } + // printf("%f\n", (double) off_nbytes / new_off_cbytes); + + int64_t new_cbytes = cbytes + chunk_cbytes; + int64_t new_frame_len; + if (frame->sframe) { + new_frame_len = header_len + 0 + new_off_cbytes + frame->trailer_len; + } + else { + new_frame_len = header_len + new_cbytes + new_off_cbytes + frame->trailer_len; + } + + void* fp = NULL; + if (frame->cframe != NULL) { + uint8_t* framep = frame->cframe; + /* Make space for the new chunk and copy it */ + frame->cframe = framep = realloc(framep, (size_t)new_frame_len); + if (framep == NULL) { + BLOSC_TRACE_ERROR("Cannot realloc space for the frame."); + return NULL; + } + /* Copy the chunk */ + memcpy(framep + header_len + cbytes, chunk, (size_t)chunk_cbytes); + /* Copy the offsets */ + memcpy(framep + header_len + new_cbytes, off_chunk, (size_t)new_off_cbytes); + } + else { + int64_t wbytes; + blosc2_io_cb *io_cb = blosc2_get_io_cb(frame->schunk->storage->io->id); + if (io_cb == NULL) { + BLOSC_TRACE_ERROR("Error getting the input/output API"); + return NULL; + } + + if (frame->sframe) { + // Update the offsets chunk in the chunks frame + if (chunk_cbytes != 0) { + if (sframe_chunk_id < 0) { + BLOSC_TRACE_ERROR("The chunk id (%" PRId64 ") is not correct", sframe_chunk_id); + return NULL; + } + if (sframe_create_chunk(frame, chunk, sframe_chunk_id, chunk_cbytes) == NULL) { + BLOSC_TRACE_ERROR("Cannot write the full chunk."); + return NULL; + } + } + fp = sframe_open_index(frame->urlpath, "rb+", + frame->schunk->storage->io); + io_cb->seek(fp, frame->file_offset + header_len, SEEK_SET); + } + else { + // Regular frame + fp = io_cb->open(frame->urlpath, "rb+", frame->schunk->storage->io->params); + io_cb->seek(fp, frame->file_offset + header_len + cbytes, SEEK_SET); + wbytes = io_cb->write(chunk, 1, chunk_cbytes, fp); // the new chunk + if (wbytes != chunk_cbytes) { + BLOSC_TRACE_ERROR("Cannot write the full chunk to frame."); + io_cb->close(fp); + return NULL; + } + } + wbytes = io_cb->write(off_chunk, 1, new_off_cbytes, fp); // the new offsets + io_cb->close(fp); + if (wbytes != new_off_cbytes) { + BLOSC_TRACE_ERROR("Cannot write the offsets to frame."); + return NULL; + } + } + // Invalidate the cache for chunk offsets + if (frame->coffsets != NULL) { + free(frame->coffsets); + frame->coffsets = NULL; + } + free(chunk); // chunk has always to be a copy when reaching here... + free(off_chunk); + + frame->len = new_frame_len; + rc = frame_update_header(frame, schunk, false); + if (rc < 0) { + return NULL; + } + + rc = frame_update_trailer(frame, schunk); + if (rc < 0) { + return NULL; + } + + return frame; +} + + +void* frame_insert_chunk(blosc2_frame_s* frame, int64_t nchunk, void* chunk, blosc2_schunk* schunk) { + uint8_t* chunk_ = chunk; + int32_t header_len; + int64_t frame_len; + int64_t nbytes; + int64_t cbytes; + int32_t blocksize; + int32_t chunksize; + int64_t nchunks; + int rc = get_header_info(frame, &header_len, &frame_len, &nbytes, &cbytes, + &blocksize, &chunksize, &nchunks, + NULL, NULL, NULL, NULL, NULL, NULL, + frame->schunk->storage->io); + if (rc < 0) { + BLOSC_TRACE_ERROR("Unable to get meta info from frame."); + return NULL; + } + int32_t chunk_cbytes; + rc = blosc2_cbuffer_sizes(chunk_, NULL, &chunk_cbytes, NULL); + if (rc < 0) { + return NULL; + } + + // Get the current offsets + int32_t off_nbytes = (int32_t) ((nchunks + 1) * sizeof(int64_t)); + int64_t* offsets = (int64_t *) malloc((size_t)off_nbytes); + if (nchunks > 0) { + int32_t coffsets_cbytes = 0; + uint8_t *coffsets = get_coffsets(frame, header_len, cbytes, nchunks, &coffsets_cbytes); + if (coffsets == NULL) { + BLOSC_TRACE_ERROR("Cannot get the offsets for the frame."); + return NULL; + } + if (coffsets_cbytes == 0) { + coffsets_cbytes = (int32_t)cbytes; + } + + // Decompress offsets + blosc2_dparams off_dparams = BLOSC2_DPARAMS_DEFAULTS; + blosc2_context *dctx = blosc2_create_dctx(off_dparams); + int32_t prev_nbytes = blosc2_decompress_ctx(dctx, coffsets, coffsets_cbytes, offsets, off_nbytes); + blosc2_free_ctx(dctx); + if (prev_nbytes < 0) { + free(offsets); + BLOSC_TRACE_ERROR("Cannot decompress the offsets chunk."); + return NULL; + } + } + + // TODO: Improvement: Check if new chunk is smaller than previous one + + // Move offsets + for (int64_t i = nchunks; i > nchunk; i--) { + offsets[i] = offsets[i - 1]; + } + // Add the new offset + int64_t sframe_chunk_id = -1; + int special_value = (chunk_[BLOSC2_CHUNK_BLOSC2_FLAGS] >> 4) & BLOSC2_SPECIAL_MASK; + uint64_t offset_value = ((uint64_t)1 << 63); + switch (special_value) { + case BLOSC2_SPECIAL_ZERO: + // Zero chunk. Code it in a special way. + offset_value += (uint64_t)BLOSC2_SPECIAL_ZERO << (8 * 7); // indicate a chunk of zeros + to_little(offsets + nchunk, &offset_value, sizeof(uint64_t)); + chunk_cbytes = 0; // we don't need to store the chunk + break; + case BLOSC2_SPECIAL_UNINIT: + // Non initizalized values chunk. Code it in a special way. + offset_value += (uint64_t) BLOSC2_SPECIAL_UNINIT << (8 * 7); // chunk of uninit values + to_little(offsets + nchunk, &offset_value, sizeof(uint64_t)); + chunk_cbytes = 0; // we don't need to store the chunk + break; + case BLOSC2_SPECIAL_NAN: + // NaN chunk. Code it in a special way. + offset_value += (uint64_t)BLOSC2_SPECIAL_NAN << (8 * 7); // indicate a chunk of NANs + to_little(offsets + nchunk, &offset_value, sizeof(uint64_t)); + chunk_cbytes = 0; // we don't need to store the chunk + break; + default: + if (frame->sframe) { + for (int i = 0; i <= nchunks; ++i) { + // offsets[nchunk] is still uninitialized here + if (i != nchunk && offsets[i] > sframe_chunk_id) { + sframe_chunk_id = offsets[i]; + } + } + offsets[nchunk] = ++sframe_chunk_id; + } + else { + offsets[nchunk] = cbytes; + } + } + + // Re-compress the offsets again + blosc2_cparams cparams = BLOSC2_CPARAMS_DEFAULTS; + cparams.splitmode = BLOSC_NEVER_SPLIT; + cparams.typesize = sizeof(int64_t); + cparams.blocksize = 16 * 1024; // based on experiments with create_frame.c bench + cparams.nthreads = 4; // 4 threads seems a decent default for nowadays CPUs + cparams.compcode = BLOSC_BLOSCLZ; + blosc2_context* cctx = blosc2_create_cctx(cparams); + void* off_chunk = malloc((size_t)off_nbytes + BLOSC2_MAX_OVERHEAD); + int32_t new_off_cbytes = blosc2_compress_ctx(cctx, offsets, off_nbytes, + off_chunk, off_nbytes + BLOSC2_MAX_OVERHEAD); + blosc2_free_ctx(cctx); + + free(offsets); + if (new_off_cbytes < 0) { + free(off_chunk); + return NULL; + } + + int64_t new_cbytes = cbytes + chunk_cbytes; + + int64_t new_frame_len; + if (frame->sframe) { + new_frame_len = header_len + 0 + new_off_cbytes + frame->trailer_len; + } + else { + new_frame_len = header_len + new_cbytes + new_off_cbytes + frame->trailer_len; + } + + // Add the chunk and update meta + void* fp = NULL; + if (frame->cframe != NULL) { + uint8_t* framep = frame->cframe; + /* Make space for the new chunk and copy it */ + frame->cframe = framep = realloc(framep, (size_t)new_frame_len); + if (framep == NULL) { + BLOSC_TRACE_ERROR("Cannot realloc space for the frame."); + return NULL; + } + /* Copy the chunk */ + memcpy(framep + header_len + cbytes, chunk, (size_t)chunk_cbytes); + /* Copy the offsets */ + memcpy(framep + header_len + new_cbytes, off_chunk, (size_t)new_off_cbytes); + } else { + int64_t wbytes; + + blosc2_io_cb *io_cb = blosc2_get_io_cb(frame->schunk->storage->io->id); + if (io_cb == NULL) { + BLOSC_TRACE_ERROR("Error getting the input/output API"); + return NULL; + } + + if (frame->sframe) { + if (chunk_cbytes != 0) { + if (sframe_chunk_id < 0) { + BLOSC_TRACE_ERROR("The chunk id (%" PRId64 ") is not correct", sframe_chunk_id); + return NULL; + } + if (sframe_create_chunk(frame, chunk, sframe_chunk_id, chunk_cbytes) == NULL) { + BLOSC_TRACE_ERROR("Cannot write the full chunk."); + return NULL; + } + } + // Update the offsets chunk in the chunks frame + fp = sframe_open_index(frame->urlpath, "rb+", + frame->schunk->storage->io); + io_cb->seek(fp, frame->file_offset + header_len + 0, SEEK_SET); + } + else { + // Regular frame + fp = io_cb->open(frame->urlpath, "rb+", frame->schunk->storage->io->params); + io_cb->seek(fp, frame->file_offset + header_len + cbytes, SEEK_SET); + wbytes = io_cb->write(chunk, 1, chunk_cbytes, fp); // the new chunk + if (wbytes != chunk_cbytes) { + BLOSC_TRACE_ERROR("Cannot write the full chunk to frame."); + io_cb->close(fp); + return NULL; + } + } + wbytes = io_cb->write(off_chunk, 1, new_off_cbytes, fp); // the new offsets + io_cb->close(fp); + if (wbytes != new_off_cbytes) { + BLOSC_TRACE_ERROR("Cannot write the offsets to frame."); + return NULL; + } + // Invalidate the cache for chunk offsets + if (frame->coffsets != NULL) { + free(frame->coffsets); + frame->coffsets = NULL; + } + } + free(chunk); // chunk has always to be a copy when reaching here... + free(off_chunk); + + frame->len = new_frame_len; + rc = frame_update_header(frame, schunk, false); + if (rc < 0) { + return NULL; + } + + rc = frame_update_trailer(frame, schunk); + if (rc < 0) { + return NULL; + } + + return frame; +} + + +void* frame_update_chunk(blosc2_frame_s* frame, int64_t nchunk, void* chunk, blosc2_schunk* schunk) { + uint8_t *chunk_ = (uint8_t *) chunk; + int32_t header_len; + int64_t frame_len; + int64_t nbytes; + int64_t cbytes; + int32_t blocksize; + int32_t chunksize; + int64_t nchunks; + int rc = get_header_info(frame, &header_len, &frame_len, &nbytes, &cbytes, + &blocksize, &chunksize, &nchunks, + NULL, NULL, NULL, NULL, NULL, NULL, + frame->schunk->storage->io); + if (rc < 0) { + BLOSC_TRACE_ERROR("Unable to get meta info from frame."); + return NULL; + } + if (nchunk >= nchunks) { + BLOSC_TRACE_ERROR("The chunk must already exist."); + return NULL; + } + + int32_t chunk_cbytes; + rc = blosc2_cbuffer_sizes(chunk, NULL, &chunk_cbytes, NULL); + if (rc < 0) { + return NULL; + } + + // Get the current offsets + int32_t off_nbytes = (int32_t) (nchunks * sizeof(int64_t)); + int64_t* offsets = (int64_t *) malloc((size_t)off_nbytes); + if (nchunks > 0) { + int32_t coffsets_cbytes = 0; + uint8_t *coffsets = get_coffsets(frame, header_len, cbytes, nchunks, &coffsets_cbytes); + if (coffsets == NULL) { + BLOSC_TRACE_ERROR("Cannot get the offsets for the frame."); + return NULL; + } + if (coffsets_cbytes == 0) { + coffsets_cbytes = (int32_t)cbytes; + } + + // Decompress offsets + blosc2_dparams off_dparams = BLOSC2_DPARAMS_DEFAULTS; + blosc2_context *dctx = blosc2_create_dctx(off_dparams); + int32_t prev_nbytes = blosc2_decompress_ctx(dctx, coffsets, coffsets_cbytes, offsets, off_nbytes); + blosc2_free_ctx(dctx); + if (prev_nbytes < 0) { + free(offsets); + BLOSC_TRACE_ERROR("Cannot decompress the offsets chunk."); + return NULL; + } + } + int32_t cbytes_old; + int64_t old_offset; + if (!frame->sframe) { + // See how big would be the space + old_offset = offsets[nchunk]; + bool needs_free; + uint8_t *chunk_old; + int err = blosc2_schunk_get_chunk(schunk, nchunk, &chunk_old, &needs_free); + if (err < 0) { + BLOSC_TRACE_ERROR("%" PRId64 " chunk can not be obtained from schunk.", nchunk); + return NULL; + } + + if (chunk_old == NULL) { + cbytes_old = 0; + } + else { + cbytes_old = sw32_(chunk_old + BLOSC2_CHUNK_CBYTES); + if (cbytes_old == BLOSC2_MAX_OVERHEAD) { + cbytes_old = 0; + } + } + if (needs_free) { + free(chunk_old); + } + } + + // Add the new offset + int64_t sframe_chunk_id; + if (frame->sframe) { + if ((int64_t)offsets[nchunk] < 0) { + sframe_chunk_id = -1; + } + else { + // In case there was a reorder in a sframe + sframe_chunk_id = offsets[nchunk]; + } + } + int special_value = (chunk_[BLOSC2_CHUNK_BLOSC2_FLAGS] >> 4) & BLOSC2_SPECIAL_MASK; + uint64_t offset_value = ((uint64_t)1 << 63); + switch (special_value) { + case BLOSC2_SPECIAL_ZERO: + // Zero chunk. Code it in a special way. + offset_value += (uint64_t)BLOSC2_SPECIAL_ZERO << (8 * 7); // indicate a chunk of zeros + to_little(offsets + nchunk, &offset_value, sizeof(uint64_t)); + chunk_cbytes = 0; // we don't need to store the chunk + break; + case BLOSC2_SPECIAL_UNINIT: + // Non initizalized values chunk. Code it in a special way. + offset_value += (uint64_t)BLOSC2_SPECIAL_UNINIT << (8 * 7); // indicate a chunk of uninit values + to_little(offsets + nchunk, &offset_value, sizeof(uint64_t)); + chunk_cbytes = 0; // we don't need to store the chunk + break; + case BLOSC2_SPECIAL_NAN: + // NaN chunk. Code it in a special way. + offset_value += (uint64_t)BLOSC2_SPECIAL_NAN << (8 * 7); // indicate a chunk of NANs + to_little(offsets + nchunk, &offset_value, sizeof(uint64_t)); + chunk_cbytes = 0; // we don't need to store the chunk + break; + default: + if (frame->sframe) { + if (sframe_chunk_id < 0) { + for (int i = 0; i < nchunks; ++i) { + if (offsets[i] > sframe_chunk_id) { + sframe_chunk_id = offsets[i]; + } + } + offsets[nchunk] = ++sframe_chunk_id; + } + } + else { + // Add the new offset + offsets[nchunk] = cbytes; + } + } + + if (!frame->sframe && chunk_cbytes != 0 && cbytes_old >= chunk_cbytes) { + offsets[nchunk] = old_offset; + cbytes = old_offset; + } + // Re-compress the offsets again + blosc2_cparams cparams = BLOSC2_CPARAMS_DEFAULTS; + cparams.splitmode = BLOSC_NEVER_SPLIT; + cparams.typesize = sizeof(int64_t); + cparams.blocksize = 16 * 1024; // based on experiments with create_frame.c bench + cparams.nthreads = 4; // 4 threads seems a decent default for nowadays CPUs + cparams.compcode = BLOSC_BLOSCLZ; + blosc2_context* cctx = blosc2_create_cctx(cparams); + void* off_chunk = malloc((size_t)off_nbytes + BLOSC2_MAX_OVERHEAD); + int32_t new_off_cbytes = blosc2_compress_ctx(cctx, offsets, off_nbytes, + off_chunk, off_nbytes + BLOSC2_MAX_OVERHEAD); + blosc2_free_ctx(cctx); + + free(offsets); + if (new_off_cbytes < 0) { + free(off_chunk); + return NULL; + } + + int64_t new_cbytes = schunk->cbytes; + int64_t new_frame_len; + if (frame->sframe) { + // The chunk is not stored in the frame + new_frame_len = header_len + 0 + new_off_cbytes + frame->trailer_len; + } + else { + new_frame_len = header_len + new_cbytes + new_off_cbytes + frame->trailer_len; + } + + void* fp = NULL; + if (frame->cframe != NULL) { + uint8_t* framep = frame->cframe; + /* Make space for the new chunk and copy it */ + frame->cframe = framep = realloc(framep, (size_t)new_frame_len); + if (framep == NULL) { + BLOSC_TRACE_ERROR("Cannot realloc space for the frame."); + return NULL; + } + /* Copy the chunk */ + memcpy(framep + header_len + cbytes, chunk, (size_t)chunk_cbytes); + /* Copy the offsets */ + memcpy(framep + header_len + new_cbytes, off_chunk, (size_t)new_off_cbytes); + } else { + int64_t wbytes; + + blosc2_io_cb *io_cb = blosc2_get_io_cb(frame->schunk->storage->io->id); + if (io_cb == NULL) { + BLOSC_TRACE_ERROR("Error getting the input/output API"); + return NULL; + } + + if (frame->sframe) { + // Create the chunks file, if it's a special value this will delete its old content + if (sframe_chunk_id >= 0) { + if (sframe_create_chunk(frame, chunk, sframe_chunk_id, chunk_cbytes) == NULL) { + BLOSC_TRACE_ERROR("Cannot write the full chunk."); + return NULL; + } + } + // Update the offsets chunk in the chunks frame + fp = sframe_open_index(frame->urlpath, "rb+", + frame->schunk->storage->io); + io_cb->seek(fp, frame->file_offset + header_len + 0, SEEK_SET); + } + else { + // Regular frame + fp = io_cb->open(frame->urlpath, "rb+", frame->schunk->storage->io->params); + io_cb->seek(fp, frame->file_offset + header_len + cbytes, SEEK_SET); + wbytes = io_cb->write(chunk, 1, chunk_cbytes, fp); // the new chunk + if (wbytes != chunk_cbytes) { + BLOSC_TRACE_ERROR("Cannot write the full chunk to frame."); + io_cb->close(fp); + return NULL; + } + io_cb->seek(fp, frame->file_offset + header_len + new_cbytes, SEEK_SET); + } + wbytes = io_cb->write(off_chunk, 1, new_off_cbytes, fp); // the new offsets + io_cb->close(fp); + if (wbytes != new_off_cbytes) { + BLOSC_TRACE_ERROR("Cannot write the offsets to frame."); + return NULL; + } + // Invalidate the cache for chunk offsets + if (frame->coffsets != NULL) { + free(frame->coffsets); + frame->coffsets = NULL; + } + } + free(chunk); // chunk has always to be a copy when reaching here... + free(off_chunk); + + frame->len = new_frame_len; + rc = frame_update_header(frame, schunk, false); + if (rc < 0) { + return NULL; + } + + rc = frame_update_trailer(frame, schunk); + if (rc < 0) { + return NULL; + } + + return frame; +} + + +void* frame_delete_chunk(blosc2_frame_s* frame, int64_t nchunk, blosc2_schunk* schunk) { + int32_t header_len; + int64_t frame_len; + int64_t nbytes; + int64_t cbytes; + int32_t blocksize; + int32_t chunksize; + int64_t nchunks; + int rc = get_header_info(frame, &header_len, &frame_len, &nbytes, &cbytes, + &blocksize, &chunksize, &nchunks, + NULL, NULL, NULL, NULL, NULL, NULL, frame->schunk->storage->io); + if (rc < 0) { + BLOSC_TRACE_ERROR("Unable to get meta info from frame."); + return NULL; + } + + // Get the current offsets + int32_t off_nbytes = (int32_t) (nchunks * sizeof(int64_t)); + int64_t* offsets = (int64_t *) malloc((size_t)off_nbytes); + if (nchunks > 0) { + int32_t coffsets_cbytes = 0; + uint8_t *coffsets = get_coffsets(frame, header_len, cbytes, nchunks, &coffsets_cbytes); + if (coffsets == NULL) { + BLOSC_TRACE_ERROR("Cannot get the offsets for the frame."); + return NULL; + } + if (coffsets_cbytes == 0) { + coffsets_cbytes = (int32_t)cbytes; + } + + // Decompress offsets + blosc2_dparams off_dparams = BLOSC2_DPARAMS_DEFAULTS; + blosc2_context *dctx = blosc2_create_dctx(off_dparams); + int32_t prev_nbytes = blosc2_decompress_ctx(dctx, coffsets, coffsets_cbytes, offsets, off_nbytes); + blosc2_free_ctx(dctx); + if (prev_nbytes < 0) { + free(offsets); + BLOSC_TRACE_ERROR("Cannot decompress the offsets chunk."); + return NULL; + } + } + + // Delete the new offset + for (int64_t i = nchunk; i < nchunks - 1; i++) { + offsets[i] = offsets[i + 1]; + } + offsets[nchunks - 1] = 0; + + // Re-compress the offsets again + blosc2_cparams cparams = BLOSC2_CPARAMS_DEFAULTS; + cparams.splitmode = BLOSC_NEVER_SPLIT; + cparams.typesize = sizeof(int64_t); + cparams.blocksize = 16 * 1024; // based on experiments with create_frame.c bench + cparams.nthreads = 4; // 4 threads seems a decent default for nowadays CPUs + cparams.compcode = BLOSC_BLOSCLZ; + blosc2_context* cctx = blosc2_create_cctx(cparams); + void* off_chunk = malloc((size_t)off_nbytes + BLOSC2_MAX_OVERHEAD); + int32_t new_off_cbytes = blosc2_compress_ctx(cctx, offsets, off_nbytes - (int32_t)sizeof(int64_t), + off_chunk, off_nbytes + BLOSC2_MAX_OVERHEAD); + blosc2_free_ctx(cctx); + + free(offsets); + if (new_off_cbytes < 0) { + free(off_chunk); + return NULL; + } + + int64_t new_cbytes = cbytes; + + int64_t new_frame_len; + if (frame->sframe) { + new_frame_len = header_len + 0 + new_off_cbytes + frame->trailer_len; + } + else { + new_frame_len = header_len + new_cbytes + new_off_cbytes + frame->trailer_len; + } + + // Add the chunk and update meta + FILE* fp = NULL; + if (frame->cframe != NULL) { + uint8_t* framep = frame->cframe; + /* Make space for the new chunk and copy it */ + frame->cframe = framep = realloc(framep, (size_t)new_frame_len); + if (framep == NULL) { + BLOSC_TRACE_ERROR("Cannot realloc space for the frame."); + return NULL; + } + /* Copy the offsets */ + memcpy(framep + header_len + new_cbytes, off_chunk, (size_t)new_off_cbytes); + } else { + blosc2_io_cb *io_cb = blosc2_get_io_cb(frame->schunk->storage->io->id); + if (io_cb == NULL) { + BLOSC_TRACE_ERROR("Error getting the input/output API"); + return NULL; + } + + size_t wbytes; + if (frame->sframe) { + int64_t offset; + rc = get_coffset(frame, header_len, cbytes, nchunk, nchunks, &offset); + if (rc < 0) { + BLOSC_TRACE_ERROR("Unable to get offset to chunk %" PRId64 ".", nchunk); + return NULL; + } + if (offset >= 0){ + // Remove the chunk file only if it is not a special value chunk + int err = sframe_delete_chunk(frame->urlpath, offset); + if (err != 0) { + BLOSC_TRACE_ERROR("Unable to delete chunk!"); + return NULL; + } + } + // Update the offsets chunk in the chunks frame + fp = sframe_open_index(frame->urlpath, "rb+", frame->schunk->storage->io); + io_cb->seek(fp, frame->file_offset + header_len + 0, SEEK_SET); + } + else { + // Regular frame + fp = io_cb->open(frame->urlpath, "rb+", frame->schunk->storage->io); + io_cb->seek(fp, frame->file_offset + header_len + cbytes, SEEK_SET); + } + wbytes = io_cb->write(off_chunk, 1, new_off_cbytes, fp); // the new offsets + io_cb->close(fp); + if (wbytes != (size_t)new_off_cbytes) { + BLOSC_TRACE_ERROR("Cannot write the offsets to frame."); + return NULL; + } + // Invalidate the cache for chunk offsets + if (frame->coffsets != NULL) { + free(frame->coffsets); + frame->coffsets = NULL; + } + } + free(off_chunk); + + frame->len = new_frame_len; + rc = frame_update_header(frame, schunk, false); + if (rc < 0) { + return NULL; + } + + rc = frame_update_trailer(frame, schunk); + if (rc < 0) { + return NULL; + } + + return frame; +} + + +int frame_reorder_offsets(blosc2_frame_s* frame, const int64_t* offsets_order, blosc2_schunk* schunk) { + // Get header info + int32_t header_len; + int64_t frame_len; + int64_t nbytes; + int64_t cbytes; + int32_t blocksize; + int32_t chunksize; + int64_t nchunks; + int ret = get_header_info(frame, &header_len, &frame_len, &nbytes, &cbytes, + &blocksize, &chunksize, &nchunks, + NULL, NULL, NULL, NULL, NULL, NULL, + frame->schunk->storage->io); + if (ret < 0) { + BLOSC_TRACE_ERROR("Cannot get the header info for the frame."); + return ret; + } + + // Get the current offsets and add one more + int32_t off_nbytes = (int32_t) (nchunks * sizeof(int64_t)); + int64_t* offsets = (int64_t *) malloc((size_t)off_nbytes); + + int32_t coffsets_cbytes = 0; + uint8_t *coffsets = get_coffsets(frame, header_len, cbytes, nchunks, &coffsets_cbytes); + if (coffsets == NULL) { + BLOSC_TRACE_ERROR("Cannot get the offsets for the frame."); + free(offsets); + return BLOSC2_ERROR_DATA; + } + + // Decompress offsets + blosc2_dparams off_dparams = BLOSC2_DPARAMS_DEFAULTS; + blosc2_context *dctx = blosc2_create_dctx(off_dparams); + int32_t prev_nbytes = blosc2_decompress_ctx(dctx, coffsets, coffsets_cbytes, + offsets, off_nbytes); + blosc2_free_ctx(dctx); + if (prev_nbytes < 0) { + free(offsets); + BLOSC_TRACE_ERROR("Cannot decompress the offsets chunk."); + return prev_nbytes; + } + + // Make a copy of the chunk offsets and reorder it + int64_t *offsets_copy = malloc(prev_nbytes); + memcpy(offsets_copy, offsets, prev_nbytes); + + for (int i = 0; i < nchunks; ++i) { + offsets[i] = offsets_copy[offsets_order[i]]; + } + free(offsets_copy); + + // Re-compress the offsets again + blosc2_cparams cparams = BLOSC2_CPARAMS_DEFAULTS; + cparams.splitmode = BLOSC_NEVER_SPLIT; + cparams.typesize = sizeof(int64_t); + cparams.blocksize = 16 * 1024; // based on experiments with create_frame.c bench + cparams.nthreads = 4; // 4 threads seems a decent default for nowadays CPUs + cparams.compcode = BLOSC_BLOSCLZ; + blosc2_context* cctx = blosc2_create_cctx(cparams); + void* off_chunk = malloc((size_t)off_nbytes + BLOSC2_MAX_OVERHEAD); + int32_t new_off_cbytes = blosc2_compress_ctx(cctx, offsets, off_nbytes, + off_chunk, off_nbytes + BLOSC2_MAX_OVERHEAD); + blosc2_free_ctx(cctx); + + if (new_off_cbytes < 0) { + free(offsets); + free(off_chunk); + return new_off_cbytes; + } + free(offsets); + int64_t new_frame_len; + if (frame->sframe) { + // The chunks are not in the frame + new_frame_len = header_len + 0 + new_off_cbytes + frame->trailer_len; + } + else { + new_frame_len = header_len + cbytes + new_off_cbytes + frame->trailer_len; + } + + if (frame->cframe != NULL) { + uint8_t* framep = frame->cframe; + /* Make space for the new chunk and copy it */ + frame->cframe = framep = realloc(framep, (size_t)new_frame_len); + if (framep == NULL) { + BLOSC_TRACE_ERROR("Cannot realloc space for the frame."); + return BLOSC2_ERROR_MEMORY_ALLOC; + } + /* Copy the offsets */ + memcpy(framep + header_len + cbytes, off_chunk, (size_t)new_off_cbytes); + } + else { + void* fp = NULL; + + blosc2_io_cb *io_cb = blosc2_get_io_cb(frame->schunk->storage->io->id); + if (io_cb == NULL) { + BLOSC_TRACE_ERROR("Error getting the input/output API"); + return BLOSC2_ERROR_PLUGIN_IO; + } + + if (frame->sframe) { + // Update the offsets chunk in the chunks frame + fp = sframe_open_index(frame->urlpath, "rb+", + frame->schunk->storage->io); + io_cb->seek(fp, frame->file_offset + header_len + 0, SEEK_SET); + } + else { + // Regular frame + fp = io_cb->open(frame->urlpath, "rb+", frame->schunk->storage->io->params); + io_cb->seek(fp, frame->file_offset + header_len + cbytes, SEEK_SET); + } + int64_t wbytes = io_cb->write(off_chunk, 1, new_off_cbytes, fp); // the new offsets + io_cb->close(fp); + if (wbytes != new_off_cbytes) { + BLOSC_TRACE_ERROR("Cannot write the offsets to frame."); + return BLOSC2_ERROR_FILE_WRITE; + } + } + + // Invalidate the cache for chunk offsets + if (frame->coffsets != NULL) { + free(frame->coffsets); + frame->coffsets = NULL; + } + free(off_chunk); + + frame->len = new_frame_len; + int rc = frame_update_header(frame, schunk, false); + if (rc < 0) { + return rc; + } + + rc = frame_update_trailer(frame, schunk); + if (rc < 0) { + return rc; + } + + return 0; +} + + +/* Decompress and return a chunk that is part of a frame. */ +int frame_decompress_chunk(blosc2_context *dctx, blosc2_frame_s* frame, int64_t nchunk, void *dest, int32_t nbytes) { + uint8_t* src; + bool needs_free; + int32_t chunk_nbytes; + int32_t chunk_cbytes; + int rc; + + // Use a lazychunk here in order to do a potential parallel read. + rc = frame_get_lazychunk(frame, nchunk, &src, &needs_free); + if (rc < 0) { + BLOSC_TRACE_ERROR("Cannot get the chunk in position %" PRId64 ".", nchunk); + goto end; + } + chunk_cbytes = rc; + if (chunk_cbytes < (signed)sizeof(int32_t)) { + /* Not enough input to read `nbytes` */ + rc = BLOSC2_ERROR_READ_BUFFER; + } + + rc = blosc2_cbuffer_sizes(src, &chunk_nbytes, &chunk_cbytes, NULL); + if (rc < 0) { + goto end; + } + + /* Create a buffer for destination */ + if (chunk_nbytes > nbytes) { + BLOSC_TRACE_ERROR("Not enough space for decompressing in dest."); + rc = BLOSC2_ERROR_WRITE_BUFFER; + goto end; + } + /* And decompress it */ + dctx->header_overhead = BLOSC_EXTENDED_HEADER_LENGTH; + int chunksize = rc = blosc2_decompress_ctx(dctx, src, chunk_cbytes, dest, nbytes); + if (chunksize < 0 || chunksize != chunk_nbytes) { + BLOSC_TRACE_ERROR("Error in decompressing chunk."); + if (chunksize >= 0) + rc = BLOSC2_ERROR_FAILURE; + } + end: + if (needs_free) { + free(src); + } + return rc; +} diff --git a/src/blosc2/blosc/frame.h b/src/blosc2/blosc/frame.h new file mode 100644 index 0000000..942ddc5 --- /dev/null +++ b/src/blosc2/blosc/frame.h @@ -0,0 +1,158 @@ +/********************************************************************* + Blosc - Blocked Shuffling and Compression Library + + Copyright (C) 2021 The Blosc Developers + https://blosc.org + License: BSD 3-Clause (see LICENSE.txt) + + See LICENSE.txt for details about copyright and rights to use. +**********************************************************************/ + +#ifndef BLOSC_FRAME_H +#define BLOSC_FRAME_H + +#include +#include + +// Different types of frames +#define FRAME_CONTIGUOUS_TYPE 0 +#define FRAME_DIRECTORY_TYPE 1 + + +// Constants for metadata placement in header +#define FRAME_HEADER_MAGIC 2 +#define FRAME_HEADER_LEN (FRAME_HEADER_MAGIC + 8 + 1) // 11 +#define FRAME_LEN (FRAME_HEADER_LEN + 4 + 1) // 16 +#define FRAME_FLAGS (FRAME_LEN + 8 + 1) // 25 +#define FRAME_TYPE (FRAME_FLAGS + 1) // 26 +#define FRAME_CODECS (FRAME_FLAGS + 2) // 27 +#define FRAME_NBYTES (FRAME_FLAGS + 4 + 1) // 30 +#define FRAME_CBYTES (FRAME_NBYTES + 8 + 1) // 39 +#define FRAME_TYPESIZE (FRAME_CBYTES + 8 + 1) // 48 +#define FRAME_BLOCKSIZE (FRAME_TYPESIZE + 4 + 1) // 53 +#define FRAME_CHUNKSIZE (FRAME_BLOCKSIZE + 4 + 1) // 58 +#define FRAME_NTHREADS_C (FRAME_CHUNKSIZE + 4 + 1) // 63 +#define FRAME_NTHREADS_D (FRAME_NTHREADS_C + 2 + 1) // 66 +#define FRAME_HAS_VLMETALAYERS (FRAME_NTHREADS_D + 2) // 68 +#define FRAME_FILTER_PIPELINE (FRAME_HAS_VLMETALAYERS + 1 + 1) // 70 +#define FRAME_UDCODEC (FRAME_FILTER_PIPELINE + 1 + 6) // 77 +#define FRAME_CODEC_META (FRAME_FILTER_PIPELINE + 1 + 7) // 78 +#define FRAME_HEADER_MINLEN (FRAME_FILTER_PIPELINE + 1 + 16) // 87 <- minimum length +#define FRAME_METALAYERS (FRAME_HEADER_MINLEN) // 87 +#define FRAME_IDX_SIZE (FRAME_METALAYERS + 1 + 1) // 89 + +#define FRAME_FILTER_PIPELINE_MAX (8) // the maximum number of filters that can be stored in header + +#define FRAME_TRAILER_VERSION_BETA2 (0U) // for beta.2 and former +#define FRAME_TRAILER_VERSION (1U) // can be up to 127 + +#define FRAME_TRAILER_MINLEN (25) // minimum length for the trailer (msgpack overhead) +#define FRAME_TRAILER_LEN_OFFSET (22) // offset to trailer length (counting from the end) +#define FRAME_TRAILER_VLMETALAYERS (2) + + +typedef struct { + char* urlpath; //!< The name of the file or directory if it's an sframe; if NULL, this is in-memory + uint8_t* cframe; //!< The in-memory, contiguous frame buffer + bool avoid_cframe_free; //!< Whether the cframe can be freed (false) or not (true). + uint8_t* coffsets; //!< Pointers to the (compressed, on-disk) chunk offsets + int64_t len; //!< The current length of the frame in (compressed) bytes + int64_t maxlen; //!< The maximum length of the frame; if 0, there is no maximum + uint32_t trailer_len; //!< The current length of the trailer in (compressed) bytes + bool sframe; //!< Whether the frame is sparse (true) or not + blosc2_schunk *schunk; //!< The schunk associated + int64_t file_offset; //!< The offset where the frame starts inside the file +} blosc2_frame_s; + + +/********************************************************************* + Frame struct related functions. + These are rather low-level and the blosc2_schunk interface is + recommended instead. +*********************************************************************/ + +/** + * @brief Create a new frame. + * + * @param urlpath The filename of the frame. If not persistent, pass NULL. + * + * @return The new frame. + */ +blosc2_frame_s* frame_new(const char* urlpath); + +/** + * @brief Create a frame from a super-chunk. + * + * @param schunk The super-chunk from where the frame will be created. + * @param frame The pointer for the frame that will be populated. + * + * @note If frame->urlpath is NULL, a frame is created in-memory; else it is created + * on-disk. + * + * @return The size in bytes of the frame. If an error occurs it returns a negative value. + */ +int64_t frame_from_schunk(blosc2_schunk* schunk, blosc2_frame_s* frame); + +/** + * @brief Free all memory from a frame. + * + * @param frame The frame to be freed. + * + * @return 0 if succeeds. + */ +int frame_free(blosc2_frame_s *frame); + +/** + * @brief Initialize a frame out of a file. + * + * @param urlpath The file name. + * + * @return The frame created from the file. + */ +blosc2_frame_s* frame_from_file_offset(const char *urlpath, const blosc2_io *io_cb, int64_t offset); + +/** + * @brief Initialize a frame out of a frame buffer. + * + * @param buffer The buffer for the frame. + * @param len The length of buffer for the frame. + * @param copy Whether the frame buffer should be copied internally or not. + * + * @return The frame created from the frame buffer. + */ +blosc2_frame_s* frame_from_cframe(uint8_t *cframe, int64_t len, bool copy); + +/** + * @brief Create a super-chunk from a frame. + * + * @param frame The frame from which the super-chunk will be created. + * @param copy If true, a new frame buffer is created + * internally to serve as storage for the super-chunk. Else, the + * super-chunk will be backed by @p frame (i.e. no copies are made). + * + * @return The super-chunk corresponding to the frame. + */ +blosc2_schunk* frame_to_schunk(blosc2_frame_s* frame, bool copy, const blosc2_io *udio); + +blosc2_storage * +get_new_storage(const blosc2_storage *storage, const blosc2_cparams *cdefaults, const blosc2_dparams *ddefaults, + const blosc2_io *iodefaults); + +void* frame_append_chunk(blosc2_frame_s* frame, void* chunk, blosc2_schunk* schunk); +void* frame_insert_chunk(blosc2_frame_s* frame, int64_t nchunk, void* chunk, blosc2_schunk* schunk); +void* frame_update_chunk(blosc2_frame_s* frame, int64_t nchunk, void* chunk, blosc2_schunk* schunk); +void* frame_delete_chunk(blosc2_frame_s* frame, int64_t nchunk, blosc2_schunk* schunk); +int frame_reorder_offsets(blosc2_frame_s *frame, const int64_t *offsets_order, blosc2_schunk* schunk); + +int frame_get_chunk(blosc2_frame_s* frame, int64_t nchunk, uint8_t **chunk, bool *needs_free); +int frame_get_lazychunk(blosc2_frame_s* frame, int64_t nchunk, uint8_t **chunk, bool *needs_free); +int frame_decompress_chunk(blosc2_context* dctx, blosc2_frame_s* frame, int64_t nchunk, + void *dest, int32_t nbytes); + +int frame_update_header(blosc2_frame_s* frame, blosc2_schunk* schunk, bool new); +int frame_update_trailer(blosc2_frame_s* frame, blosc2_schunk* schunk); + +int64_t frame_fill_special(blosc2_frame_s* frame, int64_t nitems, int special_value, + int32_t chunksize, blosc2_schunk* schunk); + +#endif //BLOSC_FRAME_H diff --git a/src/blosc2/blosc/schunk.c b/src/blosc2/blosc/schunk.c new file mode 100644 index 0000000..8ef51dc --- /dev/null +++ b/src/blosc2/blosc/schunk.c @@ -0,0 +1,1563 @@ +/********************************************************************* + Blosc - Blocked Shuffling and Compression Library + + Copyright (C) 2021 The Blosc Developers + https://blosc.org + License: BSD 3-Clause (see LICENSE.txt) + + See LICENSE.txt for details about copyright and rights to use. +**********************************************************************/ + + +#include +#include +#include +#include +#include "blosc2.h" +#include "frame.h" +#include "stune.h" +#include + +#if defined(_WIN32) + #include + #include + #include + + #define mkdir(D, M) _mkdir(D) + +/* stdint.h only available in VS2010 (VC++ 16.0) and newer */ + #if defined(_MSC_VER) && _MSC_VER < 1600 + #include "win32/stdint-windows.h" + #else + #include + #endif + +#endif /* _WIN32 */ + + +/* If C11 is supported, use it's built-in aligned allocation. */ +#if __STDC_VERSION__ >= 201112L + #include +#endif + + +/* Get the cparams associated with a super-chunk */ +int blosc2_schunk_get_cparams(blosc2_schunk *schunk, blosc2_cparams **cparams) { + *cparams = calloc(sizeof(blosc2_cparams), 1); + (*cparams)->schunk = schunk; + for (int i = 0; i < BLOSC2_MAX_FILTERS; i++) { + (*cparams)->filters[i] = schunk->filters[i]; + (*cparams)->filters_meta[i] = schunk->filters_meta[i]; + } + (*cparams)->compcode = schunk->compcode; + (*cparams)->compcode_meta = schunk->compcode_meta; + (*cparams)->clevel = schunk->clevel; + (*cparams)->typesize = schunk->typesize; + (*cparams)->blocksize = schunk->blocksize; + if (schunk->cctx == NULL) { + (*cparams)->nthreads = BLOSC2_CPARAMS_DEFAULTS.nthreads; + } + else { + (*cparams)->nthreads = (int16_t)schunk->cctx->nthreads; + } + return 0; +} + + +/* Get the dparams associated with a super-chunk */ +int blosc2_schunk_get_dparams(blosc2_schunk *schunk, blosc2_dparams **dparams) { + *dparams = calloc(sizeof(blosc2_dparams), 1); + (*dparams)->schunk = schunk; + if (schunk->dctx == NULL) { + (*dparams)->nthreads = BLOSC2_DPARAMS_DEFAULTS.nthreads; + } + else { + (*dparams)->nthreads = schunk->dctx->nthreads; + } + return 0; +} + + +void update_schunk_properties(struct blosc2_schunk* schunk) { + blosc2_cparams* cparams = schunk->storage->cparams; + blosc2_dparams* dparams = schunk->storage->dparams; + + for (int i = 0; i < BLOSC2_MAX_FILTERS; i++) { + schunk->filters[i] = cparams->filters[i]; + schunk->filters_meta[i] = cparams->filters_meta[i]; + } + schunk->compcode = cparams->compcode; + schunk->compcode_meta = cparams->compcode_meta; + schunk->clevel = cparams->clevel; + schunk->typesize = cparams->typesize; + schunk->blocksize = cparams->blocksize; + schunk->chunksize = -1; + + /* The compression context */ + if (schunk->cctx != NULL) { + blosc2_free_ctx(schunk->cctx); + } + cparams->schunk = schunk; + schunk->cctx = blosc2_create_cctx(*cparams); + + /* The decompression context */ + if (schunk->dctx != NULL) { + blosc2_free_ctx(schunk->dctx); + } + dparams->schunk = schunk; + schunk->dctx = blosc2_create_dctx(*dparams); +} + + +static bool file_exists (char *filename) { + struct stat buffer; + return (stat (filename, &buffer) == 0); +} + + +/* Create a new super-chunk */ +blosc2_schunk* blosc2_schunk_new(blosc2_storage *storage) { + blosc2_schunk* schunk = calloc(1, sizeof(blosc2_schunk)); + schunk->version = 0; /* pre-first version */ + + // Get the storage with proper defaults + schunk->storage = get_new_storage(storage, &BLOSC2_CPARAMS_DEFAULTS, &BLOSC2_DPARAMS_DEFAULTS, &BLOSC2_IO_DEFAULTS); + // Update the (local variable) storage + storage = schunk->storage; + + schunk->udbtune = malloc(sizeof(blosc2_btune)); + if (schunk->storage->cparams->udbtune == NULL) { + memcpy(schunk->udbtune, &BTUNE_DEFAULTS, sizeof(blosc2_btune)); + } else { + memcpy(schunk->udbtune, schunk->storage->cparams->udbtune, sizeof(blosc2_btune)); + } + schunk->storage->cparams->udbtune = schunk->udbtune; + + // ...and update internal properties + update_schunk_properties(schunk); + + schunk->cctx->udbtune->btune_init(schunk->udbtune->btune_config, schunk->cctx, schunk->dctx); + + if (!storage->contiguous && storage->urlpath != NULL){ + char* urlpath; + char last_char = storage->urlpath[strlen(storage->urlpath) - 1]; + urlpath = malloc(strlen(storage->urlpath) + 1); + strcpy(urlpath, storage->urlpath); + if (last_char == '\\' || last_char == '/') { + urlpath[strlen(storage->urlpath) - 1] = '\0'; + } + // Create directory + if (mkdir(urlpath, 0777) == -1) { + BLOSC_TRACE_ERROR("Error during the creation of the directory, maybe it already exists."); + return NULL; + } + // We want a sparse (directory) frame as storage + blosc2_frame_s* frame = frame_new(urlpath); + free(urlpath); + frame->sframe = true; + // Initialize frame (basically, encode the header) + frame->schunk = schunk; + int64_t frame_len = frame_from_schunk(schunk, frame); + if (frame_len < 0) { + BLOSC_TRACE_ERROR("Error during the conversion of schunk to frame."); + return NULL; + } + schunk->frame = (blosc2_frame*)frame; + } + if (storage->contiguous){ + // We want a contiguous frame as storage + if (storage->urlpath != NULL) { + if (file_exists(storage->urlpath)) { + BLOSC_TRACE_ERROR("You are trying to overwrite an existing frame. Remove it first!"); + return NULL; + } + } + blosc2_frame_s* frame = frame_new(storage->urlpath); + frame->sframe = false; + // Initialize frame (basically, encode the header) + frame->schunk = schunk; + int64_t frame_len = frame_from_schunk(schunk, frame); + if (frame_len < 0) { + BLOSC_TRACE_ERROR("Error during the conversion of schunk to frame."); + return NULL; + } + schunk->frame = (blosc2_frame*)frame; + } + + return schunk; +} + + +/* Create a copy of a super-chunk */ +blosc2_schunk* blosc2_schunk_copy(blosc2_schunk *schunk, blosc2_storage *storage) { + if (schunk == NULL) { + BLOSC_TRACE_ERROR("Can not copy a NULL `schunk`."); + return NULL; + } + + // Check if cparams are equals + bool cparams_equal = true; + blosc2_cparams cparams = {0}; + if (storage->cparams == NULL) { + // When cparams are not specified, just use the same of schunk + cparams.typesize = schunk->cctx->typesize; + cparams.clevel = schunk->cctx->clevel; + cparams.compcode = schunk->cctx->compcode; + cparams.compcode_meta = schunk->cctx->compcode_meta; + cparams.use_dict = schunk->cctx->use_dict; + cparams.blocksize = schunk->cctx->blocksize; + memcpy(cparams.filters, schunk->cctx->filters, BLOSC2_MAX_FILTERS); + memcpy(cparams.filters_meta, schunk->cctx->filters_meta, BLOSC2_MAX_FILTERS); + storage->cparams = &cparams; + } + else { + cparams = *storage->cparams; + } + if (cparams.blocksize == 0) { + // TODO: blocksize should be read from schunk->blocksize + // For this, it should be updated during the first append + // (or change API to make this a property during schunk creation). + cparams.blocksize = schunk->cctx->blocksize; + } + + if (cparams.typesize != schunk->cctx->typesize || + cparams.clevel != schunk->cctx->clevel || + cparams.compcode != schunk->cctx->compcode || + cparams.use_dict != schunk->cctx->use_dict || + cparams.blocksize != schunk->cctx->blocksize || + // In case of prefilters or postfilters, force their execution. + schunk->cctx->prefilter != NULL || + schunk->dctx->postfilter != NULL) { + cparams_equal = false; + } + for (int i = 0; i < BLOSC2_MAX_FILTERS; ++i) { + if (cparams.filters[i] != schunk->cctx->filters[i] || + cparams.filters_meta[i] != schunk->cctx->filters_meta[i]) { + cparams_equal = false; + } + } + + // Create new schunk + blosc2_schunk *new_schunk = blosc2_schunk_new(storage); + if (new_schunk == NULL) { + BLOSC_TRACE_ERROR("Can not create a new schunk"); + return NULL; + } + + // Copy metalayers + for (int nmeta = 0; nmeta < schunk->nmetalayers; ++nmeta) { + blosc2_metalayer *meta = schunk->metalayers[nmeta]; + if (blosc2_meta_add(new_schunk, meta->name, meta->content, meta->content_len) < 0) { + BLOSC_TRACE_ERROR("Can not add %s `metalayer`.", meta->name); + return NULL; + } + } + + // Copy chunks + if (cparams_equal) { + for (int nchunk = 0; nchunk < schunk->nchunks; ++nchunk) { + uint8_t *chunk; + bool needs_free; + if (blosc2_schunk_get_chunk(schunk, nchunk, &chunk, &needs_free) < 0) { + BLOSC_TRACE_ERROR("Can not get the `chunk` %d.", nchunk); + return NULL; + } + if (blosc2_schunk_append_chunk(new_schunk, chunk, !needs_free) < 0) { + BLOSC_TRACE_ERROR("Can not append the `chunk` into super-chunk."); + return NULL; + } + } + } else { + int32_t chunksize = schunk->chunksize == -1 ? 0 : schunk->chunksize; + uint8_t *buffer = malloc(chunksize); + for (int nchunk = 0; nchunk < schunk->nchunks; ++nchunk) { + if (blosc2_schunk_decompress_chunk(schunk, nchunk, buffer, schunk->chunksize) < 0) { + BLOSC_TRACE_ERROR("Can not decompress the `chunk` %d.", nchunk); + return NULL; + } + if (blosc2_schunk_append_buffer(new_schunk, buffer, schunk->chunksize) < 0) { + BLOSC_TRACE_ERROR("Can not append the `buffer` into super-chunk."); + return NULL; + } + } + free(buffer); + } + + // Copy vlmetalayers + for (int nmeta = 0; nmeta < schunk->nvlmetalayers; ++nmeta) { + uint8_t *content; + int32_t content_len; + char* name = schunk->vlmetalayers[nmeta]->name; + if (blosc2_vlmeta_get(schunk, name, &content, &content_len) < 0) { + BLOSC_TRACE_ERROR("Can not get %s `vlmetalayer`.", name); + } + if (blosc2_vlmeta_add(new_schunk, name, content, content_len, NULL) < 0) { + BLOSC_TRACE_ERROR("Can not add %s `vlmetalayer`.", name); + return NULL; + } + free(content); + } + return new_schunk; +} + + +/* Open an existing super-chunk that is on-disk (no copy is made). */ +blosc2_schunk* blosc2_schunk_open_udio(const char* urlpath, const blosc2_io *udio) { + if (urlpath == NULL) { + BLOSC_TRACE_ERROR("You need to supply a urlpath."); + return NULL; + } + + blosc2_frame_s* frame = frame_from_file_offset(urlpath, udio, 0); + if (frame == NULL) { + return NULL; + } + blosc2_schunk* schunk = frame_to_schunk(frame, false, udio); + + // Set the storage with proper defaults + size_t pathlen = strlen(urlpath); + schunk->storage->urlpath = malloc(pathlen + 1); + strcpy(schunk->storage->urlpath, urlpath); + schunk->storage->contiguous = !frame->sframe; + + return schunk; +} + +blosc2_schunk* blosc2_schunk_open(const char* urlpath) { + return blosc2_schunk_open_udio(urlpath, &BLOSC2_IO_DEFAULTS); +} + +BLOSC_EXPORT blosc2_schunk* blosc2_schunk_open_offset(const char* urlpath, int64_t offset) { + if (urlpath == NULL) { + BLOSC_TRACE_ERROR("You need to supply a urlpath."); + return NULL; + } + + blosc2_frame_s* frame = frame_from_file_offset(urlpath, &BLOSC2_IO_DEFAULTS, offset); + if (frame == NULL) { + return NULL; + } + blosc2_schunk* schunk = frame_to_schunk(frame, false, &BLOSC2_IO_DEFAULTS); + + // Set the storage with proper defaults + size_t pathlen = strlen(urlpath); + schunk->storage->urlpath = malloc(pathlen + 1); + strcpy(schunk->storage->urlpath, urlpath); + schunk->storage->contiguous = !frame->sframe; + + return schunk; +} + +int64_t blosc2_schunk_to_buffer(blosc2_schunk* schunk, uint8_t** dest, bool* needs_free) { + blosc2_frame_s* frame; + int64_t cframe_len; + + // Initialize defaults in case of errors + *dest = NULL; + *needs_free = false; + + if ((schunk->storage->contiguous == true) && (schunk->storage->urlpath == NULL)) { + frame = (blosc2_frame_s*)(schunk->frame); + *dest = frame->cframe; + cframe_len = frame->len; + *needs_free = false; + } + else { + // Copy to a contiguous storage + blosc2_storage frame_storage = {.contiguous=true}; + blosc2_schunk* schunk_copy = blosc2_schunk_copy(schunk, &frame_storage); + if (schunk_copy == NULL) { + BLOSC_TRACE_ERROR("Error during the conversion of schunk to buffer."); + return BLOSC2_ERROR_SCHUNK_COPY; + } + frame = (blosc2_frame_s*)(schunk_copy->frame); + *dest = frame->cframe; + cframe_len = frame->len; + *needs_free = true; + frame->avoid_cframe_free = true; + blosc2_schunk_free(schunk_copy); + } + + return cframe_len; + +} + + +/* Write an in-memory frame out to a file. */ +int64_t frame_to_file(blosc2_frame_s* frame, const char* urlpath) { + blosc2_io_cb *io_cb = blosc2_get_io_cb(frame->schunk->storage->io->id); + if (io_cb == NULL) { + BLOSC_TRACE_ERROR("Error getting the input/output API"); + return BLOSC2_ERROR_PLUGIN_IO; + } + void* fp = io_cb->open(urlpath, "wb", frame->schunk->storage->io); + int64_t nitems = io_cb->write(frame->cframe, frame->len, 1, fp); + io_cb->close(fp); + return nitems * frame->len; +} + + +/* Append an in-memory frame to a file. */ +int64_t append_frame_to_file(blosc2_frame_s* frame, const char* urlpath) { + blosc2_io_cb *io_cb = blosc2_get_io_cb(frame->schunk->storage->io->id); + if (io_cb == NULL) { + BLOSC_TRACE_ERROR("Error getting the input/output API"); + return BLOSC2_ERROR_PLUGIN_IO; + } + void* fp = io_cb->open(urlpath, "ab", frame->schunk->storage->io); + int64_t offset; + +# if (UNIX) + offset = io_cb->tell(fp); +# else + io_cb->seek(fp, 0, SEEK_END); + offset = io_cb->tell(fp); +# endif + + io_cb->write(frame->cframe, frame->len, 1, fp); + io_cb->close(fp); + return offset; +} + + +/* Write super-chunk out to a file. */ +int64_t blosc2_schunk_to_file(blosc2_schunk* schunk, const char* urlpath) { + if (urlpath == NULL) { + BLOSC_TRACE_ERROR("urlpath cannot be NULL"); + return BLOSC2_ERROR_INVALID_PARAM; + } + + // Accelerated path for in-memory frames + if (schunk->storage->contiguous && schunk->storage->urlpath == NULL) { + int64_t len = frame_to_file((blosc2_frame_s*)(schunk->frame), urlpath); + if (len <= 0) { + BLOSC_TRACE_ERROR("Error writing to file"); + return len; + } + return len; + } + + // Copy to a contiguous file + blosc2_storage frame_storage = {.contiguous=true, .urlpath=(char*)urlpath}; + blosc2_schunk* schunk_copy = blosc2_schunk_copy(schunk, &frame_storage); + if (schunk_copy == NULL) { + BLOSC_TRACE_ERROR("Error during the conversion of schunk to buffer."); + return BLOSC2_ERROR_SCHUNK_COPY; + } + blosc2_frame_s* frame = (blosc2_frame_s*)(schunk_copy->frame); + int64_t frame_len = frame->len; + blosc2_schunk_free(schunk_copy); + return frame_len; +} + + +/* Append a super-chunk to a file. */ +int64_t blosc2_schunk_append_file(blosc2_schunk* schunk, const char* urlpath) { + if (urlpath == NULL) { + BLOSC_TRACE_ERROR("urlpath cannot be NULL"); + return BLOSC2_ERROR_INVALID_PARAM; + } + + // Accelerated path for in-memory frames + if (schunk->storage->contiguous && schunk->storage->urlpath == NULL) { + int64_t offset = append_frame_to_file((blosc2_frame_s*)(schunk->frame), urlpath); + if (offset <= 0) { + BLOSC_TRACE_ERROR("Error writing to file"); + return offset; + } + return offset; + } + + // Copy to a contiguous file + blosc2_storage frame_storage = {.contiguous=true, .urlpath=NULL}; + blosc2_schunk* schunk_copy = blosc2_schunk_copy(schunk, &frame_storage); + if (schunk_copy == NULL) { + BLOSC_TRACE_ERROR("Error during the conversion of schunk to buffer."); + return BLOSC2_ERROR_SCHUNK_COPY; + } + blosc2_frame_s* frame = (blosc2_frame_s*)(schunk_copy->frame); + int64_t offset = append_frame_to_file(frame, urlpath); + blosc2_schunk_free(schunk_copy); + return offset; +} + + +/* Free all memory from a super-chunk. */ +int blosc2_schunk_free(blosc2_schunk *schunk) { + if (schunk->data != NULL) { + for (int i = 0; i < schunk->nchunks; i++) { + free(schunk->data[i]); + } + free(schunk->data); + } + if (schunk->cctx != NULL) + blosc2_free_ctx(schunk->cctx); + if (schunk->dctx != NULL) + blosc2_free_ctx(schunk->dctx); + if (schunk->blockshape != NULL) + free(schunk->blockshape); + + if (schunk->nmetalayers > 0) { + for (int i = 0; i < schunk->nmetalayers; i++) { + if (schunk->metalayers[i] != NULL) { + if (schunk->metalayers[i]->name != NULL) + free(schunk->metalayers[i]->name); + if (schunk->metalayers[i]->content != NULL) + free(schunk->metalayers[i]->content); + free(schunk->metalayers[i]); + } + } + schunk->nmetalayers = 0; + } + + if (schunk->storage != NULL) { + if (schunk->storage->urlpath != NULL) { + free(schunk->storage->urlpath); + } + free(schunk->storage->cparams); + free(schunk->storage->dparams); + free(schunk->storage->io); + free(schunk->storage); + } + + if (schunk->frame != NULL) { + frame_free((blosc2_frame_s *) schunk->frame); + } + + if (schunk->nvlmetalayers > 0) { + for (int i = 0; i < schunk->nvlmetalayers; ++i) { + if (schunk->vlmetalayers[i] != NULL) { + if (schunk->vlmetalayers[i]->name != NULL) + free(schunk->vlmetalayers[i]->name); + if (schunk->vlmetalayers[i]->content != NULL) + free(schunk->vlmetalayers[i]->content); + free(schunk->vlmetalayers[i]); + } + } + } + + if (schunk->udbtune != NULL) { + free(schunk->udbtune); + } + free(schunk); + + return 0; +} + + +/* Create a super-chunk out of a contiguous frame buffer */ +blosc2_schunk* blosc2_schunk_from_buffer(uint8_t *cframe, int64_t len, bool copy) { + blosc2_frame_s* frame = frame_from_cframe(cframe, len, false); + if (frame == NULL) { + return NULL; + } + // Check that the buffer actually comes from a cframe + char *magic_number = (char *)cframe; + magic_number += FRAME_HEADER_MAGIC; + if (strcmp(magic_number, "b2frame\0") != 0) { + return NULL; + } + blosc2_schunk* schunk = frame_to_schunk(frame, copy, &BLOSC2_IO_DEFAULTS); + if (schunk && copy) { + // Super-chunk has its own copy of frame + frame_free(frame); + } + return schunk; +} + +/* Fill an empty frame with special values (fast path). */ +int64_t blosc2_schunk_fill_special(blosc2_schunk* schunk, int64_t nitems, int special_value, + int32_t chunksize) { + if (nitems == 0) { + return 0; + } + + int32_t typesize = schunk->typesize; + + if ((nitems * typesize / chunksize) > INT_MAX) { + BLOSC_TRACE_ERROR("nitems is too large. Try increasing the chunksize."); + return BLOSC2_ERROR_SCHUNK_SPECIAL; + } + + if ((schunk->nbytes > 0) || (schunk->cbytes > 0)) { + BLOSC_TRACE_ERROR("Filling with special values only works on empty super-chunks"); + return BLOSC2_ERROR_FRAME_SPECIAL; + } + + // Compute the number of chunks and the length of the offsets chunk + int32_t chunkitems = chunksize / typesize; + int64_t nchunks = nitems / chunkitems; + int32_t leftover_items = (int32_t)(nitems % chunkitems); + + if (schunk->frame == NULL) { + // Build the special chunks + int32_t leftover_size = leftover_items * typesize; + void* chunk = malloc(BLOSC_EXTENDED_HEADER_LENGTH); + void* chunk2 = malloc(BLOSC_EXTENDED_HEADER_LENGTH); + blosc2_cparams* cparams; + blosc2_schunk_get_cparams(schunk, &cparams); + int csize, csize2; + switch (special_value) { + case BLOSC2_SPECIAL_ZERO: + csize = blosc2_chunk_zeros(*cparams, chunksize, chunk, BLOSC_EXTENDED_HEADER_LENGTH); + csize2 = blosc2_chunk_zeros(*cparams, leftover_size, chunk2, BLOSC_EXTENDED_HEADER_LENGTH); + break; + case BLOSC2_SPECIAL_UNINIT: + csize = blosc2_chunk_uninit(*cparams, chunksize, chunk, BLOSC_EXTENDED_HEADER_LENGTH); + csize2 = blosc2_chunk_uninit(*cparams, leftover_size, chunk2, BLOSC_EXTENDED_HEADER_LENGTH); + break; + case BLOSC2_SPECIAL_NAN: + csize = blosc2_chunk_nans(*cparams, chunksize, chunk, BLOSC_EXTENDED_HEADER_LENGTH); + csize2 = blosc2_chunk_nans(*cparams, leftover_size, chunk2, BLOSC_EXTENDED_HEADER_LENGTH); + break; + default: + BLOSC_TRACE_ERROR("Only zeros, NaNs or non-initialized values are supported."); + return BLOSC2_ERROR_SCHUNK_SPECIAL; + } + free(cparams); + if (csize < 0 || csize2 < 0) { + BLOSC_TRACE_ERROR("Error creating special chunks."); + return BLOSC2_ERROR_SCHUNK_SPECIAL; + } + + for (int nchunk = 0; nchunk < nchunks; nchunk++) { + int64_t nchunk_ = blosc2_schunk_append_chunk(schunk, chunk, true); + if (nchunk_ != nchunk + 1) { + BLOSC_TRACE_ERROR("Error appending special chunks."); + return BLOSC2_ERROR_SCHUNK_SPECIAL; + } + } + + if (leftover_items) { + int64_t nchunk_ = blosc2_schunk_append_chunk(schunk, chunk2, true); + if (nchunk_ != nchunks + 1) { + BLOSC_TRACE_ERROR("Error appending last special chunk."); + return BLOSC2_ERROR_SCHUNK_SPECIAL; + } + } + free(chunk); + free(chunk2); + } + else { + /* Fill an empty frame with special values (fast path). */ + blosc2_frame_s *frame = (blosc2_frame_s *) schunk->frame; + /* Update counters (necessary for the frame_fill_special() logic) */ + if (leftover_items) { + nchunks += 1; + } + schunk->chunksize = chunksize; + schunk->nchunks = nchunks; + schunk->nbytes = nitems * typesize; + int64_t frame_len = frame_fill_special(frame, nitems, special_value, chunksize, schunk); + if (frame_len < 0) { + BLOSC_TRACE_ERROR("Error creating special frame."); + return frame_len; + } + } + + return schunk->nchunks; +} + +/* Append an existing chunk into a super-chunk. */ +int64_t blosc2_schunk_append_chunk(blosc2_schunk *schunk, uint8_t *chunk, bool copy) { + int32_t chunk_nbytes; + int32_t chunk_cbytes; + int64_t nchunks = schunk->nchunks; + + int rc = blosc2_cbuffer_sizes(chunk, &chunk_nbytes, &chunk_cbytes, NULL); + if (rc < 0) { + return rc; + } + + if (schunk->chunksize == -1) { + schunk->chunksize = chunk_nbytes; // The super-chunk is initialized now + } + if (chunk_nbytes > schunk->chunksize) { + BLOSC_TRACE_ERROR("Appending chunks that have different lengths in the same schunk " + "is not supported yet: %d > %d.", chunk_nbytes, schunk->chunksize); + return BLOSC2_ERROR_CHUNK_APPEND; + } + + /* Update counters */ + schunk->current_nchunk = nchunks; + schunk->nchunks = nchunks + 1; + schunk->nbytes += chunk_nbytes; + if (schunk->frame == NULL) { + schunk->cbytes += chunk_cbytes; + } else { + // A frame + int special_value = (chunk[BLOSC2_CHUNK_BLOSC2_FLAGS] >> 4) & BLOSC2_SPECIAL_MASK; + switch (special_value) { + case BLOSC2_SPECIAL_ZERO: + case BLOSC2_SPECIAL_NAN: + case BLOSC2_SPECIAL_UNINIT: + schunk->cbytes += 0; + break; + default: + schunk->cbytes += chunk_cbytes; + } + } + + if (copy) { + // Make a copy of the chunk + uint8_t *chunk_copy = malloc(chunk_cbytes); + memcpy(chunk_copy, chunk, chunk_cbytes); + chunk = chunk_copy; + } + + // Update super-chunk or frame + blosc2_frame_s* frame = (blosc2_frame_s*)schunk->frame; + if (frame == NULL) { + // Check that we are not appending a small chunk after another small chunk + if ((schunk->nchunks > 0) && (chunk_nbytes < schunk->chunksize)) { + uint8_t* last_chunk = schunk->data[nchunks - 1]; + int32_t last_nbytes; + rc = blosc2_cbuffer_sizes(last_chunk, &last_nbytes, NULL, NULL); + if (rc < 0) { + return rc; + } + if ((last_nbytes < schunk->chunksize) && (chunk_nbytes < schunk->chunksize)) { + BLOSC_TRACE_ERROR( + "Appending two consecutive chunks with a chunksize smaller than the schunk chunksize " + "is not allowed yet: %d != %d.", chunk_nbytes, schunk->chunksize); + return BLOSC2_ERROR_CHUNK_APPEND; + } + } + + if (!copy && (chunk_cbytes < chunk_nbytes)) { + // We still want to do a shrink of the chunk + chunk = realloc(chunk, chunk_cbytes); + } + + /* Make space for appending the copy of the chunk and do it */ + if ((nchunks + 1) * sizeof(void *) > schunk->data_len) { + // Extend the data pointer by one memory page (4k) + schunk->data_len += 4096; // must be a multiple of sizeof(void*) + schunk->data = realloc(schunk->data, schunk->data_len); + } + schunk->data[nchunks] = chunk; + } + else { + if (frame_append_chunk(frame, chunk, schunk) == NULL) { + BLOSC_TRACE_ERROR("Problems appending a chunk."); + return BLOSC2_ERROR_CHUNK_APPEND; + } + } + return schunk->nchunks; +} + + +/* Insert an existing @p chunk in a specified position on a super-chunk */ +int64_t blosc2_schunk_insert_chunk(blosc2_schunk *schunk, int64_t nchunk, uint8_t *chunk, bool copy) { + int32_t chunk_nbytes; + int32_t chunk_cbytes; + int64_t nchunks = schunk->nchunks; + + int rc = blosc2_cbuffer_sizes(chunk, &chunk_nbytes, &chunk_cbytes, NULL); + if (rc < 0) { + return rc; + } + + if (schunk->chunksize == -1) { + schunk->chunksize = chunk_nbytes; // The super-chunk is initialized now + } + + if (chunk_nbytes > schunk->chunksize) { + BLOSC_TRACE_ERROR("Inserting chunks that have different lengths in the same schunk " + "is not supported yet: %d > %d.", chunk_nbytes, schunk->chunksize); + return BLOSC2_ERROR_CHUNK_INSERT; + } + + /* Update counters */ + schunk->current_nchunk = nchunk; + schunk->nchunks = nchunks + 1; + schunk->nbytes += chunk_nbytes; + if (schunk->frame == NULL) { + schunk->cbytes += chunk_cbytes; + } else { + // A frame + int special_value = (chunk[BLOSC2_CHUNK_BLOSC2_FLAGS] >> 4) & BLOSC2_SPECIAL_MASK; + switch (special_value) { + case BLOSC2_SPECIAL_ZERO: + case BLOSC2_SPECIAL_NAN: + case BLOSC2_SPECIAL_UNINIT: + schunk->cbytes += 0; + break; + default: + schunk->cbytes += chunk_cbytes; + } + } + + if (copy) { + // Make a copy of the chunk + uint8_t *chunk_copy = malloc(chunk_cbytes); + memcpy(chunk_copy, chunk, chunk_cbytes); + chunk = chunk_copy; + } + + // Update super-chunk or frame + blosc2_frame_s* frame = (blosc2_frame_s*)schunk->frame; + if (frame == NULL) { + // Check that we are not appending a small chunk after another small chunk + if ((schunk->nchunks > 0) && (chunk_nbytes < schunk->chunksize)) { + uint8_t* last_chunk = schunk->data[nchunks - 1]; + int32_t last_nbytes; + rc = blosc2_cbuffer_sizes(last_chunk, &last_nbytes, NULL, NULL); + if (rc < 0) { + return rc; + } + if ((last_nbytes < schunk->chunksize) && (chunk_nbytes < schunk->chunksize)) { + BLOSC_TRACE_ERROR("Appending two consecutive chunks with a chunksize smaller " + "than the schunk chunksize is not allowed yet: %d != %d", + chunk_nbytes, schunk->chunksize); + return BLOSC2_ERROR_CHUNK_APPEND; + } + } + + if (!copy && (chunk_cbytes < chunk_nbytes)) { + // We still want to do a shrink of the chunk + chunk = realloc(chunk, chunk_cbytes); + } + + // Make space for appending the copy of the chunk and do it + if ((nchunks + 1) * sizeof(void *) > schunk->data_len) { + // Extend the data pointer by one memory page (4k) + schunk->data_len += 4096; // must be a multiple of sizeof(void*) + schunk->data = realloc(schunk->data, schunk->data_len); + } + + // Reorder the offsets and insert the new chunk + for (int64_t i = nchunks; i > nchunk; --i) { + schunk->data[i] = schunk->data[i-1]; + } + schunk->data[nchunk] = chunk; + } + + else { + if (frame_insert_chunk(frame, nchunk, chunk, schunk) == NULL) { + BLOSC_TRACE_ERROR("Problems inserting a chunk in a frame."); + return BLOSC2_ERROR_CHUNK_INSERT; + } + } + return schunk->nchunks; +} + + +int64_t blosc2_schunk_update_chunk(blosc2_schunk *schunk, int64_t nchunk, uint8_t *chunk, bool copy) { + int32_t chunk_nbytes; + int32_t chunk_cbytes; + + int rc = blosc2_cbuffer_sizes(chunk, &chunk_nbytes, &chunk_cbytes, NULL); + if (rc < 0) { + return rc; + } + + if (schunk->chunksize == -1) { + schunk->chunksize = chunk_nbytes; // The super-chunk is initialized now + } + + if ((schunk->chunksize != 0) && (chunk_nbytes != schunk->chunksize)) { + BLOSC_TRACE_ERROR("Inserting chunks that have different lengths in the same schunk " + "is not supported yet: %d > %d.", chunk_nbytes, schunk->chunksize); + return BLOSC2_ERROR_CHUNK_INSERT; + } + + bool needs_free; + uint8_t *chunk_old; + int err = blosc2_schunk_get_chunk(schunk, nchunk, &chunk_old, &needs_free); + if (err < 0) { + BLOSC_TRACE_ERROR("%" PRId64 " chunk can not be obtained from schunk.", nchunk); + return -1; + } + int32_t chunk_nbytes_old = 0; + int32_t chunk_cbytes_old = 0; + schunk->current_nchunk = nchunk; + + if (chunk_old != 0) { + rc = blosc2_cbuffer_sizes(chunk_old, &chunk_nbytes_old, &chunk_cbytes_old, NULL); + if (rc < 0) { + return rc; + } + if (chunk_cbytes_old == BLOSC2_MAX_OVERHEAD) { + chunk_cbytes_old = 0; + } + } + if (needs_free) { + free(chunk_old); + } + + if (copy) { + // Make a copy of the chunk + uint8_t *chunk_copy = malloc(chunk_cbytes); + memcpy(chunk_copy, chunk, chunk_cbytes); + chunk = chunk_copy; + } + + blosc2_frame_s* frame = (blosc2_frame_s*)(schunk->frame); + if (schunk->frame == NULL) { + /* Update counters */ + schunk->nbytes += chunk_nbytes; + schunk->nbytes -= chunk_nbytes_old; + schunk->cbytes += chunk_cbytes; + schunk->cbytes -= chunk_cbytes_old; + } else { + // A frame + int special_value = (chunk[BLOSC2_CHUNK_BLOSC2_FLAGS] >> 4) & BLOSC2_SPECIAL_MASK; + switch (special_value) { + case BLOSC2_SPECIAL_ZERO: + case BLOSC2_SPECIAL_NAN: + case BLOSC2_SPECIAL_UNINIT: + schunk->nbytes += chunk_nbytes; + schunk->nbytes -= chunk_nbytes_old; + if (frame->sframe) { + schunk->cbytes -= chunk_cbytes_old; + } + break; + default: + /* Update counters */ + schunk->nbytes += chunk_nbytes; + schunk->nbytes -= chunk_nbytes_old; + schunk->cbytes += chunk_cbytes; + if (frame->sframe) { + schunk->cbytes -= chunk_cbytes_old; + } + else { + if (chunk_cbytes_old >= chunk_cbytes) { + schunk->cbytes -= chunk_cbytes; + } + } + } + } + + // Update super-chunk or frame + if (schunk->frame == NULL) { + if (!copy && (chunk_cbytes < chunk_nbytes)) { + // We still want to do a shrink of the chunk + chunk = realloc(chunk, chunk_cbytes); + } + + // Free old chunk and add reference to new chunk + if (schunk->data[nchunk] != 0) { + free(schunk->data[nchunk]); + } + schunk->data[nchunk] = chunk; + } + else { + if (frame_update_chunk(frame, nchunk, chunk, schunk) == NULL) { + BLOSC_TRACE_ERROR("Problems updating a chunk in a frame."); + return BLOSC2_ERROR_CHUNK_UPDATE; + } + } + + return schunk->nchunks; +} + +int64_t blosc2_schunk_delete_chunk(blosc2_schunk *schunk, int64_t nchunk) { + int rc; + if (schunk->nchunks < nchunk) { + BLOSC_TRACE_ERROR("The schunk has not enough chunks (%" PRId64 ")!", schunk->nchunks); + } + + bool needs_free; + uint8_t *chunk_old; + int err = blosc2_schunk_get_chunk(schunk, nchunk, &chunk_old, &needs_free); + if (err < 0) { + BLOSC_TRACE_ERROR("%" PRId64 "chunk can not be obtained from schunk.", nchunk); + return -1; + } + int32_t chunk_nbytes_old = 0; + int32_t chunk_cbytes_old = 0; + schunk->current_nchunk = nchunk; + + if (chunk_old != 0) { + rc = blosc2_cbuffer_sizes(chunk_old, &chunk_nbytes_old, &chunk_cbytes_old, NULL); + if (rc < 0) { + return rc; + } + if (chunk_cbytes_old == BLOSC2_MAX_OVERHEAD) { + chunk_cbytes_old = 0; + } + } + if (needs_free) { + free(chunk_old); + } + + blosc2_frame_s* frame = (blosc2_frame_s*)(schunk->frame); + schunk->nchunks -= 1; + if (schunk->frame == NULL) { + /* Update counters */ + schunk->nbytes -= chunk_nbytes_old; + schunk->cbytes -= chunk_cbytes_old; + } else { + // A frame + schunk->nbytes -= chunk_nbytes_old; + if (frame->sframe) { + schunk->cbytes -= chunk_cbytes_old; + } + } + + // Update super-chunk or frame + if (schunk->frame == NULL) { + // Free old chunk + if (schunk->data[nchunk] != 0) { + free(schunk->data[nchunk]); + } + // Reorder the offsets and insert the new chunk + for (int64_t i = nchunk; i < schunk->nchunks; i++) { + schunk->data[i] = schunk->data[i + 1]; + } + schunk->data[schunk->nchunks] = NULL; + + } + else { + if (frame_delete_chunk(frame, nchunk, schunk) == NULL) { + BLOSC_TRACE_ERROR("Problems deleting a chunk in a frame."); + return BLOSC2_ERROR_CHUNK_UPDATE; + } + } + return schunk->nchunks; +} + + +/* Append a data buffer to a super-chunk. */ +int64_t blosc2_schunk_append_buffer(blosc2_schunk *schunk, void *src, int32_t nbytes) { + uint8_t* chunk = malloc(nbytes + BLOSC2_MAX_OVERHEAD); + schunk->current_nchunk = schunk->nchunks; + /* Compress the src buffer using super-chunk context */ + int cbytes = blosc2_compress_ctx(schunk->cctx, src, nbytes, chunk, + nbytes + BLOSC2_MAX_OVERHEAD); + if (cbytes < 0) { + free(chunk); + return cbytes; + } + // We don't need a copy of the chunk, as it will be shrunk if necessary + int64_t nchunks = blosc2_schunk_append_chunk(schunk, chunk, false); + if (nchunks < 0) { + BLOSC_TRACE_ERROR("Error appending a buffer in super-chunk"); + return nchunks; + } + + return nchunks; +} + + +/* Decompress and return a chunk that is part of a super-chunk. */ +int blosc2_schunk_decompress_chunk(blosc2_schunk *schunk, int64_t nchunk, + void *dest, int32_t nbytes) { + int32_t chunk_nbytes; + int32_t chunk_cbytes; + int chunksize; + int rc; + blosc2_frame_s* frame = (blosc2_frame_s*)schunk->frame; + + schunk->current_nchunk = nchunk; + if (frame == NULL) { + if (nchunk >= schunk->nchunks) { + BLOSC_TRACE_ERROR("nchunk ('%" PRId64 "') exceeds the number of chunks " + "('%" PRId64 "') in super-chunk.", nchunk, schunk->nchunks); + return BLOSC2_ERROR_INVALID_PARAM; + } + uint8_t* src = schunk->data[nchunk]; + if (src == 0) { + return 0; + } + + rc = blosc2_cbuffer_sizes(src, &chunk_nbytes, &chunk_cbytes, NULL); + if (rc < 0) { + return rc; + } + + if (nbytes < chunk_nbytes) { + BLOSC_TRACE_ERROR("Buffer size is too small for the decompressed buffer " + "('%d' bytes, but '%d' are needed).", nbytes, chunk_nbytes); + return BLOSC2_ERROR_INVALID_PARAM; + } + + chunksize = blosc2_decompress_ctx(schunk->dctx, src, chunk_cbytes, dest, nbytes); + if (chunksize < 0 || chunksize != chunk_nbytes) { + BLOSC_TRACE_ERROR("Error in decompressing chunk."); + if (chunksize < 0) + return chunksize; + return BLOSC2_ERROR_FAILURE; + } + } else { + chunksize = frame_decompress_chunk(schunk->dctx, frame, nchunk, dest, nbytes); + if (chunksize < 0) { + return chunksize; + } + } + return chunksize; +} + + +/* Return a compressed chunk that is part of a super-chunk in the `chunk` parameter. + * If the super-chunk is backed by a frame that is disk-based, a buffer is allocated for the + * (compressed) chunk, and hence a free is needed. You can check if the chunk requires a free + * with the `needs_free` parameter. + * If the chunk does not need a free, it means that a pointer to the location in the super-chunk + * (or the backing in-memory frame) is returned in the `chunk` parameter. + * + * The size of the (compressed) chunk is returned. If some problem is detected, a negative code + * is returned instead. +*/ +int blosc2_schunk_get_chunk(blosc2_schunk *schunk, int64_t nchunk, uint8_t **chunk, bool *needs_free) { + if (schunk->dctx->threads_started > 1) { + pthread_mutex_lock(&schunk->dctx->nchunk_mutex); + schunk->current_nchunk = nchunk; + pthread_mutex_unlock(&schunk->dctx->nchunk_mutex); + } + else { + schunk->current_nchunk = nchunk; + } + blosc2_frame_s* frame = (blosc2_frame_s*)schunk->frame; + if (frame != NULL) { + return frame_get_chunk(frame, nchunk, chunk, needs_free); + } + + if (nchunk >= schunk->nchunks) { + BLOSC_TRACE_ERROR("nchunk ('%" PRId64 "') exceeds the number of chunks " + "('%" PRId64 "') in schunk.", nchunk, schunk->nchunks); + return BLOSC2_ERROR_INVALID_PARAM; + } + + *chunk = schunk->data[nchunk]; + if (*chunk == 0) { + *needs_free = 0; + return 0; + } + + *needs_free = false; + int32_t chunk_cbytes; + int rc = blosc2_cbuffer_sizes(*chunk, NULL, &chunk_cbytes, NULL); + if (rc < 0) { + return rc; + } + return (int)chunk_cbytes; +} + + +/* Return a compressed chunk that is part of a super-chunk in the `chunk` parameter. + * If the super-chunk is backed by a frame that is disk-based, a buffer is allocated for the + * (compressed) chunk, and hence a free is needed. You can check if the chunk requires a free + * with the `needs_free` parameter. + * If the chunk does not need a free, it means that a pointer to the location in the super-chunk + * (or the backing in-memory frame) is returned in the `chunk` parameter. + * + * The size of the (compressed) chunk is returned. If some problem is detected, a negative code + * is returned instead. +*/ +int blosc2_schunk_get_lazychunk(blosc2_schunk *schunk, int64_t nchunk, uint8_t **chunk, bool *needs_free) { + if (schunk->dctx->threads_started > 1) { + pthread_mutex_lock(&schunk->dctx->nchunk_mutex); + schunk->current_nchunk = nchunk; + pthread_mutex_unlock(&schunk->dctx->nchunk_mutex); + } + else { + schunk->current_nchunk = nchunk; + } + blosc2_frame_s* frame = (blosc2_frame_s*)schunk->frame; + if (schunk->frame != NULL) { + return frame_get_lazychunk(frame, nchunk, chunk, needs_free); + } + + if (nchunk >= schunk->nchunks) { + BLOSC_TRACE_ERROR("nchunk ('%" PRId64 "') exceeds the number of chunks " + "('%" PRId64 "') in schunk.", nchunk, schunk->nchunks); + return BLOSC2_ERROR_INVALID_PARAM; + } + + *chunk = schunk->data[nchunk]; + if (*chunk == 0) { + *needs_free = 0; + return 0; + } + + *needs_free = false; + int32_t chunk_cbytes; + int rc = blosc2_cbuffer_sizes(*chunk, NULL, &chunk_cbytes, NULL); + if (rc < 0) { + return rc; + } + return (int)chunk_cbytes; +} + + +/* Find whether the schunk has a metalayer or not. + * + * If successful, return the index of the metalayer. Else, return a negative value. + */ +int blosc2_meta_exists(blosc2_schunk *schunk, const char *name) { + if (strlen(name) > BLOSC2_METALAYER_NAME_MAXLEN) { + BLOSC_TRACE_ERROR("Metalayers cannot be larger than %d chars.", BLOSC2_METALAYER_NAME_MAXLEN); + return BLOSC2_ERROR_INVALID_PARAM; + } + + if (schunk == NULL) { + BLOSC_TRACE_ERROR("Schunk must not be NUll."); + return BLOSC2_ERROR_INVALID_PARAM; + } + + for (int nmetalayer = 0; nmetalayer < schunk->nmetalayers; nmetalayer++) { + if (strcmp(name, schunk->metalayers[nmetalayer]->name) == 0) { + return nmetalayer; + } + } + return BLOSC2_ERROR_NOT_FOUND; +} + +/* Reorder the chunk offsets of an existing super-chunk. */ +int blosc2_schunk_reorder_offsets(blosc2_schunk *schunk, int64_t *offsets_order) { + // Check that the offsets order are correct + bool *index_check = (bool *) calloc(schunk->nchunks, sizeof(bool)); + for (int i = 0; i < schunk->nchunks; ++i) { + int64_t index = offsets_order[i]; + if (index >= schunk->nchunks) { + BLOSC_TRACE_ERROR("Index is bigger than the number of chunks."); + free(index_check); + return BLOSC2_ERROR_DATA; + } + if (index_check[index] == false) { + index_check[index] = true; + } else { + BLOSC_TRACE_ERROR("Index is yet used."); + free(index_check); + return BLOSC2_ERROR_DATA; + } + } + free(index_check); + + blosc2_frame_s* frame = (blosc2_frame_s*)schunk->frame; + if (frame != NULL) { + return frame_reorder_offsets(frame, offsets_order, schunk); + } + uint8_t **offsets = schunk->data; + + // Make a copy of the chunk offsets and reorder it + uint8_t **offsets_copy = malloc(schunk->data_len); + memcpy(offsets_copy, offsets, schunk->data_len); + + for (int i = 0; i < schunk->nchunks; ++i) { + offsets[i] = offsets_copy[offsets_order[i]]; + } + free(offsets_copy); + + return 0; +} + + +// Get the length (in bytes) of the internal frame of the super-chunk +int64_t blosc2_schunk_frame_len(blosc2_schunk* schunk) { + int64_t len; + blosc2_frame_s* frame_s = (blosc2_frame_s*)(schunk->frame); + if (frame_s != NULL) { + len = frame_s->len; + } + else { + // No attached frame, but we can still come with an estimate + len = (int64_t) (schunk->cbytes + schunk->nchunks * sizeof(int64_t)); + } + + return len; +} + + +/** + * @brief Flush metalayers content into a possible attached frame. + * + * @param schunk The super-chunk to which the flush should be applied. + * + * @return If successful, a 0 is returned. Else, return a negative value. + */ +// Initially, this was a public function, but as it is really meant to be used only +// in the schunk_add_metalayer(), I decided to convert it into private and call it +// implicitly instead of requiring the user to do so. The only drawback is that +// each add operation requires a complete frame re-build, but as users should need +// very few metalayers, this overhead should be negligible in practice. +int metalayer_flush(blosc2_schunk* schunk) { + int rc = BLOSC2_ERROR_SUCCESS; + blosc2_frame_s* frame = (blosc2_frame_s*)schunk->frame; + if (frame == NULL) { + return rc; + } + rc = frame_update_header(frame, schunk, true); + if (rc < 0) { + BLOSC_TRACE_ERROR("Unable to update metalayers into frame."); + return rc; + } + rc = frame_update_trailer(frame, schunk); + if (rc < 0) { + BLOSC_TRACE_ERROR("Unable to update trailer into frame."); + return rc; + } + return rc; +} + + +/* Add content into a new metalayer. + * + * If successful, return the index of the new metalayer. Else, return a negative value. + */ +int blosc2_meta_add(blosc2_schunk *schunk, const char *name, uint8_t *content, int32_t content_len) { + int nmetalayer = blosc2_meta_exists(schunk, name); + if (nmetalayer >= 0) { + BLOSC_TRACE_ERROR("Metalayer \"%s\" already exists.", name); + return BLOSC2_ERROR_INVALID_PARAM; + } + + // Add the metalayer + blosc2_metalayer *metalayer = malloc(sizeof(blosc2_metalayer)); + char* name_ = malloc(strlen(name) + 1); + strcpy(name_, name); + metalayer->name = name_; + uint8_t* content_buf = malloc((size_t)content_len); + memcpy(content_buf, content, content_len); + metalayer->content = content_buf; + metalayer->content_len = content_len; + schunk->metalayers[schunk->nmetalayers] = metalayer; + schunk->nmetalayers += 1; + + int rc = metalayer_flush(schunk); + if (rc < 0) { + return rc; + } + + return schunk->nmetalayers - 1; +} + + +/* Update the content of an existing metalayer. + * + * If successful, return the index of the new metalayer. Else, return a negative value. + */ +int blosc2_meta_update(blosc2_schunk *schunk, const char *name, uint8_t *content, int32_t content_len) { + int nmetalayer = blosc2_meta_exists(schunk, name); + if (nmetalayer < 0) { + BLOSC_TRACE_ERROR("Metalayer \"%s\" not found.", name); + return nmetalayer; + } + + blosc2_metalayer *metalayer = schunk->metalayers[nmetalayer]; + if (content_len > metalayer->content_len) { + BLOSC_TRACE_ERROR("`content_len` cannot exceed the existing size of %d bytes.", metalayer->content_len); + return nmetalayer; + } + + // Update the contents of the metalayer + memcpy(metalayer->content, content, content_len); + + // Update the metalayers in frame (as size has not changed, we don't need to update the trailer) + blosc2_frame_s* frame = (blosc2_frame_s*)schunk->frame; + if (frame != NULL) { + int rc = frame_update_header(frame, schunk, false); + if (rc < 0) { + BLOSC_TRACE_ERROR("Unable to update meta info from frame."); + return rc; + } + } + + return nmetalayer; +} + + +/* Get the content out of a metalayer. + * + * The `**content` receives a malloc'ed copy of the content. The user is responsible of freeing it. + * + * If successful, return the index of the new metalayer. Else, return a negative value. + */ +int blosc2_meta_get(blosc2_schunk *schunk, const char *name, uint8_t **content, + int32_t *content_len) { + int nmetalayer = blosc2_meta_exists(schunk, name); + if (nmetalayer < 0) { + BLOSC_TRACE_ERROR("Metalayer \"%s\" not found.", name); + return nmetalayer; + } + *content_len = schunk->metalayers[nmetalayer]->content_len; + *content = malloc((size_t)*content_len); + memcpy(*content, schunk->metalayers[nmetalayer]->content, (size_t)*content_len); + return nmetalayer; +} + +/* Find whether the schunk has a variable-length metalayer or not. + * + * If successful, return the index of the variable-length metalayer. Else, return a negative value. + */ +int blosc2_vlmeta_exists(blosc2_schunk *schunk, const char *name) { + if (strlen(name) > BLOSC2_METALAYER_NAME_MAXLEN) { + BLOSC_TRACE_ERROR("Variable-length metalayer names cannot be larger than %d chars.", BLOSC2_METALAYER_NAME_MAXLEN); + return BLOSC2_ERROR_INVALID_PARAM; + } + + for (int nvlmetalayer = 0; nvlmetalayer < schunk->nvlmetalayers; nvlmetalayer++) { + if (strcmp(name, schunk->vlmetalayers[nvlmetalayer]->name) == 0) { + return nvlmetalayer; + } + } + return BLOSC2_ERROR_NOT_FOUND; +} + +int vlmetalayer_flush(blosc2_schunk* schunk) { + int rc = BLOSC2_ERROR_SUCCESS; + blosc2_frame_s* frame = (blosc2_frame_s*)schunk->frame; + if (frame == NULL) { + return rc; + } + rc = frame_update_header(frame, schunk, false); + if (rc < 0) { + BLOSC_TRACE_ERROR("Unable to update metalayers into frame."); + return rc; + } + rc = frame_update_trailer(frame, schunk); + if (rc < 0) { + BLOSC_TRACE_ERROR("Unable to update trailer into frame."); + return rc; + } + return rc; +} + +/* Add content into a new variable-length metalayer. + * + * If successful, return the index of the new variable-length metalayer. Else, return a negative value. + */ +int blosc2_vlmeta_add(blosc2_schunk *schunk, const char *name, uint8_t *content, int32_t content_len, + blosc2_cparams *cparams) { + int nvlmetalayer = blosc2_vlmeta_exists(schunk, name); + if (nvlmetalayer >= 0) { + BLOSC_TRACE_ERROR("Variable-length metalayer \"%s\" already exists.", name); + return BLOSC2_ERROR_INVALID_PARAM; + } + + // Add the vlmetalayer + blosc2_metalayer *vlmetalayer = malloc(sizeof(blosc2_metalayer)); + vlmetalayer->name = strdup(name); + uint8_t* content_buf = malloc((size_t) content_len + BLOSC2_MAX_OVERHEAD); + + blosc2_context *cctx; + if (cparams != NULL) { + cctx = blosc2_create_cctx(*cparams); + } else { + cctx = blosc2_create_cctx(BLOSC2_CPARAMS_DEFAULTS); + } + + int csize = blosc2_compress_ctx(cctx, content, content_len, content_buf, content_len + BLOSC2_MAX_OVERHEAD); + if (csize < 0) { + BLOSC_TRACE_ERROR("Can not compress the `%s` variable-length metalayer.", name); + return csize; + } + blosc2_free_ctx(cctx); + + vlmetalayer->content = realloc(content_buf, csize); + vlmetalayer->content_len = csize; + schunk->vlmetalayers[schunk->nvlmetalayers] = vlmetalayer; + schunk->nvlmetalayers += 1; + + // Propagate to frames + int rc = vlmetalayer_flush(schunk); + if (rc < 0) { + BLOSC_TRACE_ERROR("Can not propagate de `%s` variable-length metalayer to a frame.", name); + return rc; + } + + return schunk->nvlmetalayers - 1; +} + + +int blosc2_vlmeta_get(blosc2_schunk *schunk, const char *name, uint8_t **content, + int32_t *content_len) { + int nvlmetalayer = blosc2_vlmeta_exists(schunk, name); + if (nvlmetalayer < 0) { + BLOSC_TRACE_ERROR("User metalayer \"%s\" not found.", name); + return nvlmetalayer; + } + blosc2_metalayer *meta = schunk->vlmetalayers[nvlmetalayer]; + int32_t nbytes, cbytes; + blosc2_cbuffer_sizes(meta->content, &nbytes, &cbytes, NULL); + if (cbytes != meta->content_len) { + BLOSC_TRACE_ERROR("User metalayer \"%s\" is corrupted.", meta->name); + return BLOSC2_ERROR_DATA; + } + *content_len = nbytes; + *content = malloc((size_t) nbytes); + blosc2_context *dctx = blosc2_create_dctx(*schunk->storage->dparams); + int nbytes_ = blosc2_decompress_ctx(dctx, meta->content, meta->content_len, *content, nbytes); + blosc2_free_ctx(dctx); + if (nbytes_ != nbytes) { + BLOSC_TRACE_ERROR("User metalayer \"%s\" is corrupted.", meta->name); + return BLOSC2_ERROR_READ_BUFFER; + } + return nvlmetalayer; +} + +int blosc2_vlmeta_update(blosc2_schunk *schunk, const char *name, uint8_t *content, int32_t content_len, + blosc2_cparams *cparams) { + int nvlmetalayer = blosc2_vlmeta_exists(schunk, name); + if (nvlmetalayer < 0) { + BLOSC_TRACE_ERROR("User vlmetalayer \"%s\" not found.", name); + return nvlmetalayer; + } + + blosc2_metalayer *vlmetalayer = schunk->vlmetalayers[nvlmetalayer]; + free(vlmetalayer->content); + uint8_t* content_buf = malloc((size_t) content_len + BLOSC2_MAX_OVERHEAD); + + blosc2_context *cctx; + if (cparams != NULL) { + cctx = blosc2_create_cctx(*cparams); + } else { + cctx = blosc2_create_cctx(BLOSC2_CPARAMS_DEFAULTS); + } + + int csize = blosc2_compress_ctx(cctx, content, content_len, content_buf, content_len + BLOSC2_MAX_OVERHEAD); + if (csize < 0) { + BLOSC_TRACE_ERROR("Can not compress the `%s` variable-length metalayer.", name); + return csize; + } + blosc2_free_ctx(cctx); + + vlmetalayer->content = realloc(content_buf, csize); + vlmetalayer->content_len = csize; + + // Propagate to frames + int rc = vlmetalayer_flush(schunk); + if (rc < 0) { + BLOSC_TRACE_ERROR("Can not propagate de `%s` variable-length metalayer to a frame.", name); + return rc; + } + + return nvlmetalayer; +} + +int blosc2_vlmeta_delete(blosc2_schunk *schunk, const char *name) { + int nvlmetalayer = blosc2_vlmeta_exists(schunk, name); + if (nvlmetalayer < 0) { + BLOSC_TRACE_ERROR("User vlmetalayer \"%s\" not found.", name); + return nvlmetalayer; + } + + blosc2_metalayer *vlmetalayer = schunk->vlmetalayers[nvlmetalayer]; + for (int i = nvlmetalayer; i < (schunk->nvlmetalayers - 1); i++) { + schunk->vlmetalayers[i] = schunk->vlmetalayers[i + 1]; + } + free(vlmetalayer->content); + schunk->nvlmetalayers--; + + // Propagate to frames + int rc = vlmetalayer_flush(schunk); + if (rc < 0) { + BLOSC_TRACE_ERROR("Can not propagate de `%s` variable-length metalayer to a frame.", name); + return rc; + } + + return schunk->nvlmetalayers; +} + + +int blosc2_vlmeta_get_names(blosc2_schunk *schunk, char **names) { + int16_t nvlmetalayers = schunk->nvlmetalayers; + + for (int i = 0; i < nvlmetalayers; ++i) { + names[i] = schunk->vlmetalayers[i]->name; + } + + return nvlmetalayers; +} diff --git a/src/blosc2/blosc/sframe.c b/src/blosc2/blosc/sframe.c new file mode 100644 index 0000000..6ff0f4c --- /dev/null +++ b/src/blosc2/blosc/sframe.c @@ -0,0 +1,120 @@ +/********************************************************************* + Blosc - Blocked Shuffling and Compression Library + + Copyright (C) 2021 The Blosc Developers + https://blosc.org + License: BSD 3-Clause (see LICENSE.txt) + + See LICENSE.txt for details about copyright and rights to use. +**********************************************************************/ + +#include +#include +#include +#include "blosc2.h" +#include "frame.h" + + +/* If C11 is supported, use it's built-in aligned allocation. */ +#if __STDC_VERSION__ >= 201112L +#include +#endif + + +/* Open sparse frame index chunk */ +void* sframe_open_index(const char* urlpath, const char* mode, const blosc2_io *io) { + void* fp = NULL; + char* index_path = malloc(strlen(urlpath) + strlen("/chunks.b2frame") + 1); + if (index_path) { + sprintf(index_path, "%s/chunks.b2frame", urlpath); + blosc2_io_cb *io_cb = blosc2_get_io_cb(io->id); + if (io_cb == NULL) { + BLOSC_TRACE_ERROR("Error getting the input/output API"); + return NULL; + } + fp = io_cb->open(index_path, mode, io->params); + free(index_path); + } + return fp; +} + +/* Open directory/nchunk.chunk with 8 zeros of padding */ +void* sframe_open_chunk(const char* urlpath, int64_t nchunk, const char* mode, const blosc2_io *io) { + void* fp = NULL; + char* chunk_path = malloc(strlen(urlpath) + 1 + 8 + strlen(".chunk") + 1); + if (chunk_path) { + sprintf(chunk_path, "%s/%08X.chunk", urlpath, (unsigned int)nchunk); + blosc2_io_cb *io_cb = blosc2_get_io_cb(io->id); + if (io_cb == NULL) { + BLOSC_TRACE_ERROR("Error getting the input/output API"); + return NULL; + } + fp = io_cb->open(chunk_path, mode, io->params); + free(chunk_path); + } + return fp; +} + +/* Append an existing chunk into a sparse frame. */ +void* sframe_create_chunk(blosc2_frame_s* frame, uint8_t* chunk, int64_t nchunk, int64_t cbytes) { + void* fpc = sframe_open_chunk(frame->urlpath, nchunk, "wb", frame->schunk->storage->io); + if (fpc == NULL) { + BLOSC_TRACE_ERROR("Cannot open the chunkfile."); + return NULL; + } + blosc2_io_cb *io_cb = blosc2_get_io_cb(frame->schunk->storage->io->id); + if (io_cb == NULL) { + BLOSC_TRACE_ERROR("Error getting the input/output API"); + return NULL; + } + int64_t wbytes = io_cb->write(chunk, 1, cbytes, fpc); + io_cb->close(fpc); + if (wbytes != cbytes) { + BLOSC_TRACE_ERROR("Cannot write the full chunk."); + return NULL; + } + + return frame; +} + +/* Append an existing chunk into a sparse frame. */ +int sframe_delete_chunk(const char *urlpath, int64_t nchunk) { + char* chunk_path = malloc(strlen(urlpath) + 1 + 8 + strlen(".chunk") + 1); + if (chunk_path) { + sprintf(chunk_path, "%s/%08X.chunk", urlpath, (unsigned int)nchunk); + int rc = remove(chunk_path); + free(chunk_path); + return rc; + } + return BLOSC2_ERROR_FILE_REMOVE; +} + +/* Get chunk from sparse frame. */ +int32_t sframe_get_chunk(blosc2_frame_s* frame, int64_t nchunk, uint8_t** chunk, bool* needs_free){ + void *fpc = sframe_open_chunk(frame->urlpath, nchunk, "rb", frame->schunk->storage->io); + if(fpc == NULL){ + BLOSC_TRACE_ERROR("Cannot open the chunkfile."); + return BLOSC2_ERROR_FILE_OPEN; + } + + blosc2_io_cb *io_cb = blosc2_get_io_cb(frame->schunk->storage->io->id); + if (io_cb == NULL) { + BLOSC_TRACE_ERROR("Error getting the input/output API"); + return BLOSC2_ERROR_PLUGIN_IO; + } + + io_cb->seek(fpc, 0L, SEEK_END); + int64_t chunk_cbytes = io_cb->tell(fpc); + *chunk = malloc((size_t)chunk_cbytes); + + io_cb->seek(fpc, 0L, SEEK_SET); + int64_t rbytes = io_cb->read(*chunk, 1, chunk_cbytes, fpc); + io_cb->close(fpc); + if (rbytes != chunk_cbytes) { + BLOSC_TRACE_ERROR("Cannot read the chunk out of the chunkfile."); + return BLOSC2_ERROR_FILE_READ; + } + *needs_free = true; + + return (int32_t)chunk_cbytes; +} diff --git a/src/blosc2/blosc/sframe.h b/src/blosc2/blosc/sframe.h new file mode 100644 index 0000000..75ea239 --- /dev/null +++ b/src/blosc2/blosc/sframe.h @@ -0,0 +1,20 @@ +/********************************************************************* + Blosc - Blocked Shuffling and Compression Library + + Copyright (C) 2021 The Blosc Developers + https://blosc.org + License: BSD 3-Clause (see LICENSE.txt) + + See LICENSE.txt for details about copyright and rights to use. +**********************************************************************/ + +#ifndef BLOSC_SFRAME_H +#define BLOSC_SFRAME_H + +void* sframe_open_index(const char* urlpath, const char* mode, const blosc2_io *io); +void* sframe_open_chunk(const char* urlpath, int64_t nchunk, const char* mode, const blosc2_io *io); +int sframe_delete_chunk(const char* urlpath, int64_t nchunk); +void* sframe_create_chunk(blosc2_frame_s* frame, uint8_t* chunk, int64_t nchunk, int64_t cbytes); +int32_t sframe_get_chunk(blosc2_frame_s* frame, int64_t nchunk, uint8_t** chunk, bool* needs_free); + +#endif //BLOSC_SFRAME_H diff --git a/src/blosc2/blosc/shuffle-altivec.c b/src/blosc2/blosc/shuffle-altivec.c new file mode 100644 index 0000000..98ec909 --- /dev/null +++ b/src/blosc2/blosc/shuffle-altivec.c @@ -0,0 +1,423 @@ +/********************************************************************* + Blosc - Blocked Shuffling and Compression Library + + Copyright (C) 2021 The Blosc developers and Jerome Kieffer + https://blosc.org + License: BSD 3-Clause (see LICENSE.txt) + + See LICENSE.txt for details about copyright and rights to use. +**********************************************************************/ + +#include "shuffle-generic.h" +#include "shuffle-altivec.h" + +/* Make sure ALTIVEC is available for the compilation target and compiler. */ +#if !defined(__ALTIVEC__) + #error ALTIVEC is not supported by the target architecture/platform and/or this compiler. +#endif + +#include +#include "transpose-altivec.h" + +/* Routine optimized for shuffling a buffer for a type size of 2 bytes. */ +static void +shuffle2_altivec(uint8_t* const dest, const uint8_t* const src, + const int32_t vectorizable_elements, const int32_t total_elements){ + static const int32_t bytesoftype = 2; + uint32_t i, j; + __vector uint8_t xmm0[2]; + + for (j = 0; j < vectorizable_elements; j += 16){ + /* Fetch 16 elements (32 bytes) */ + for (i = 0; i < bytesoftype; i++) + xmm0[i] = vec_xl(bytesoftype * j + 16 * i, src); + + /* Transpose vectors */ + transpose2x16(xmm0); + + /* Store the result vectors */ + for (i = 0; i < bytesoftype; i++) + vec_xst(xmm0[i], j + i * total_elements, dest); + } +} + +/* Routine optimized for shuffling a buffer for a type size of 4 bytes. */ +static void +shuffle4_altivec(uint8_t* const dest, const uint8_t* const src, + const int32_t vectorizable_elements, const int32_t total_elements){ + static const int32_t bytesoftype = 4; + int32_t i, j; + __vector uint8_t xmm0[4]; + + for (j = 0; j < vectorizable_elements; j += 16) + { + /* Fetch 16 elements (64 bytes, 4 vectors) */ + for (i = 0; i < bytesoftype; i++) + xmm0[i] = vec_xl(bytesoftype * j + 16 * i, src); + + + /* Transpose vectors */ + transpose4x16(xmm0); + + /* Store the result vectors */ + for (i = 0; i < bytesoftype; i ++){ + vec_xst(xmm0[i], j + i*total_elements, dest); + } + } +} + + +/* Routine optimized for shuffling a buffer for a type size of 8 bytes. */ +static void +shuffle8_altivec(uint8_t* const dest, const uint8_t* const src, + const int32_t vectorizable_elements, const int32_t total_elements) { + static const uint8_t bytesoftype = 8; + int32_t i, j; + __vector uint8_t xmm0[8]; + + for (j = 0; j < vectorizable_elements; j += 16) + { + /* Fetch 16 elements (128 bytes, 8 vectors) */ + for (i = 0; i < bytesoftype; i++) + xmm0[i] = vec_xl(bytesoftype * j + 16 * i, src); + + /* Transpose vectors */ + transpose8x16(xmm0); + + /* Store the result vectors */ + for (i = 0; i < bytesoftype; i++) + vec_xst(xmm0[i], j + i*total_elements, dest); + } +} + +/* Routine optimized for shuffling a buffer for a type size of 16 bytes. */ +static void +shuffle16_altivec(uint8_t* const dest, const uint8_t* const src, + const int32_t vectorizable_elements, const int32_t total_elements) { + static const int32_t bytesoftype = 16; + int32_t i, j; + __vector uint8_t xmm0[16]; + + for (j = 0; j < vectorizable_elements; j += 16) + { + /* Fetch 16 elements (256 bytes, 16 vectors) */ + for (i = 0; i < bytesoftype; i++) + xmm0[i] = vec_xl(bytesoftype * j + 16 * i, src); + + // Do the job ! + transpose16x16(xmm0); + + /* Store the result vectors */ + for (i = 0; i < bytesoftype; i ++) + vec_xst(xmm0[i], j + i * total_elements, dest); + } +} + + +/* Routine optimized for shuffling a buffer for a type size larger than 16 bytes. */ +static void +shuffle16_tiled_altivec(uint8_t* const dest, const uint8_t* const src, + const int32_t vectorizable_elements, const int32_t total_elements, + const int32_t bytesoftype) { + int32_t j, k; + const int32_t vecs_per_el_rem = bytesoftype & 0xF; + __vector uint8_t xmm[16]; + + for (j = 0; j < vectorizable_elements; j += 16) { + /* Advance the offset into the type by the vector size (in bytes), unless this is + the initial iteration and the type size is not a multiple of the vector size. + In that case, only advance by the number of bytes necessary so that the number + of remaining bytes in the type will be a multiple of the vector size. */ + int32_t offset_into_type; + for (offset_into_type = 0; offset_into_type < bytesoftype; + offset_into_type += (offset_into_type == 0 && + vecs_per_el_rem > 0 ? vecs_per_el_rem : 16)) { + + /* Fetch elements in groups of 256 bytes */ + const uint8_t* const src_with_offset = src + offset_into_type; + for (k = 0; k < 16; k++) + xmm[k] = vec_xl((j + k) * bytesoftype, src_with_offset); + // Do the Job! + transpose16x16(xmm); + /* Store the result vectors */ + for (k = 0; k < 16; k++) { + vec_xst(xmm[k], j + total_elements * (offset_into_type + k), dest); + } + } + } +} +/* Routine optimized for unshuffling a buffer for a type size of 2 bytes. */ +static void +unshuffle2_altivec(uint8_t* const dest, const uint8_t* const src, + const int32_t vectorizable_elements, const int32_t total_elements) { + static const int32_t bytesoftype = 2; + uint32_t i, j; + __vector uint8_t xmm0[2], xmm1[2]; + + for (j = 0; j < vectorizable_elements; j += 16) { + /* Load 16 elements (32 bytes) into 2 vectors registers. */ + for (i = 0; i < bytesoftype; i++) + xmm0[i] = vec_xl(j + i * total_elements, src); + + /* Shuffle bytes */ + /* Note the shuffling is different from intel's SSE2 */ + xmm1[0] = vec_vmrghb(xmm0[0], xmm0[1]); + xmm1[1] = vec_vmrglb(xmm0[0], xmm0[1]); + + /* Store the result vectors*/ + for (i = 0; i < bytesoftype; i++) + vec_xst(xmm1[i], bytesoftype * j + 16 * i, dest); + } +} + +/* Routine optimized for unshuffling a buffer for a type size of 4 bytes. */ +static void +unshuffle4_altivec(uint8_t* const dest, const uint8_t* const src, + const int32_t vectorizable_elements, const int32_t total_elements) { + static const int32_t bytesoftype = 4; + uint32_t i, j; + __vector uint8_t xmm0[4], xmm1[4]; + + for (j = 0; j < vectorizable_elements; j += 16) { + /* Load 16 elements (64 bytes) into 4 vectors registers. */ + for (i = 0; i < bytesoftype; i++) + xmm0[i] = vec_xl(j + i * total_elements, src); + + /* Shuffle bytes */ + for (i = 0; i < 2; i++) { + xmm1[i ] = vec_vmrghb(xmm0[i * 2], xmm0[i * 2 + 1]); + xmm1[i+2] = vec_vmrglb(xmm0[i * 2], xmm0[i * 2 + 1]); + } + /* Shuffle 2-byte words */ + for (i = 0; i < 2; i++) { + /* Compute the low 32 bytes */ + xmm0[i] = (__vector uint8_t) vec_vmrghh((__vector uint16_t)xmm1[i * 2], + (__vector uint16_t) xmm1[i * 2 + 1]); + /* Compute the hi 32 bytes */ + xmm0[i+2] = (__vector uint8_t) vec_vmrglh((__vector uint16_t)xmm1[i * 2], + (__vector uint16_t)xmm1[i * 2 + 1]); + } + /* Store the result vectors in proper order */ + vec_xst(xmm0[0], bytesoftype * j, dest); + vec_xst(xmm0[2], bytesoftype * j + 16, dest); + vec_xst(xmm0[1], bytesoftype * j + 32, dest); + vec_xst(xmm0[3], bytesoftype * j + 48, dest); + } +} + +/* Routine optimized for unshuffling a buffer for a type size of 8 bytes. */ +static void +unshuffle8_altivec(uint8_t* const dest, const uint8_t* const src, + const int32_t vectorizable_elements, const int32_t total_elements) { + static const uint8_t bytesoftype = 8; + uint32_t i, j; + __vector uint8_t xmm0[8], xmm1[8]; + + // Initialize permutations for writing + for (j = 0; j < vectorizable_elements; j += 16) { + /* Load 16 elements (64 bytes) into 4 vectors registers. */ + for (i = 0; i < bytesoftype; i++) + xmm0[i] = vec_xl(j + i * total_elements, src); + /* Shuffle bytes */ + for (i = 0; i < 4; i++) { + xmm1[i] = vec_vmrghb(xmm0[i * 2], xmm0[i * 2 + 1]); + xmm1[4 + i] = vec_vmrglb(xmm0[i * 2], xmm0[i * 2 + 1]); + } + /* Shuffle 2-byte words */ + for (i = 0; i < 4; i++) { + xmm0[i] = (__vector uint8_t)vec_vmrghh((__vector uint16_t)xmm1[i * 2], + (__vector uint16_t)xmm1[i * 2 + 1]); + xmm0[4 + i] = (__vector uint8_t)vec_vmrglh((__vector uint16_t)xmm1[i * 2], + (__vector uint16_t)xmm1[i * 2 + 1]); + } + /* Shuffle 4-byte dwords */ + for (i = 0; i < 4; i++) { + xmm1[i] = (__vector uint8_t)vec_vmrghw((__vector uint32_t)xmm0[i * 2], + (__vector uint32_t)xmm0[i * 2 + 1]); + xmm1[4 + i] = (__vector uint8_t)vec_vmrglw((__vector uint32_t)xmm0[i * 2], + (__vector uint32_t)xmm0[i * 2 + 1]); + } + /* Store the result vectors in proper order */ + vec_xst(xmm1[0], bytesoftype * j, dest); + vec_xst(xmm1[4], bytesoftype * j + 16, dest); + vec_xst(xmm1[2], bytesoftype * j + 32, dest); + vec_xst(xmm1[6], bytesoftype * j + 48, dest); + vec_xst(xmm1[1], bytesoftype * j + 64, dest); + vec_xst(xmm1[5], bytesoftype * j + 80, dest); + vec_xst(xmm1[3], bytesoftype * j + 96, dest); + vec_xst(xmm1[7], bytesoftype * j + 112, dest); + } +} + + +/* Routine optimized for unshuffling a buffer for a type size of 16 bytes. */ +static void +unshuffle16_altivec(uint8_t* const dest, const uint8_t* const src, + const int32_t vectorizable_elements, const int32_t total_elements) { + static const int32_t bytesoftype = 16; + uint32_t i, j; + __vector uint8_t xmm0[16]; + + for (j = 0; j < vectorizable_elements; j += 16) { + /* Load 16 elements (64 bytes) into 4 vectors registers. */ + for (i = 0; i < bytesoftype; i++) + xmm0[i] = vec_xl(j + i * total_elements, src); + + // Do the Job! + transpose16x16(xmm0); + + /* Store the result vectors*/ + for (i = 0; i < 16; i++) + vec_st(xmm0[i], bytesoftype * (i+j), dest); + } +} + + +/* Routine optimized for unshuffling a buffer for a type size larger than 16 bytes. */ +static void +unshuffle16_tiled_altivec(uint8_t* const dest, const uint8_t* const orig, + const int32_t vectorizable_elements, const int32_t total_elements, + const int32_t bytesoftype) { + int32_t i, j, offset_into_type; + const int32_t vecs_per_el_rem = bytesoftype & 0xF; + __vector uint8_t xmm[16]; + + + /* Advance the offset into the type by the vector size (in bytes), unless this is + the initial iteration and the type size is not a multiple of the vector size. + In that case, only advance by the number of bytes necessary so that the number + of remaining bytes in the type will be a multiple of the vector size. */ + + for (offset_into_type = 0; offset_into_type < bytesoftype; + offset_into_type += (offset_into_type == 0 && + vecs_per_el_rem > 0 ? vecs_per_el_rem : 16)) { + for (i = 0; i < vectorizable_elements; i += 16) { + /* Load the first 128 bytes in 16 XMM registers */ + for (j = 0; j < 16; j++) + xmm[j] = vec_xl(total_elements * (offset_into_type + j) + i, orig); + + // Do the Job ! + transpose16x16(xmm); + + /* Store the result vectors in proper order */ + for (j = 0; j < 16; j++) + vec_xst(xmm[j], (i + j) * bytesoftype + offset_into_type, dest); + } + } +} + +/* Shuffle a block. This can never fail. */ +void +shuffle_altivec(const int32_t bytesoftype, const int32_t blocksize, + const uint8_t *_src, uint8_t *_dest) +{ + int32_t vectorized_chunk_size; + vectorized_chunk_size = bytesoftype * 16; + + + /* If the blocksize is not a multiple of both the typesize and + the vector size, round the blocksize down to the next value + which is a multiple of both. The vectorized shuffle can be + used for that portion of the data, and the naive implementation + can be used for the remaining portion. */ + const int32_t vectorizable_bytes = blocksize - (blocksize % vectorized_chunk_size); + const int32_t vectorizable_elements = vectorizable_bytes / bytesoftype; + const int32_t total_elements = blocksize / bytesoftype; + + /* If the block size is too small to be vectorized, + use the generic implementation. */ + if (blocksize < vectorized_chunk_size) { + shuffle_generic(bytesoftype, blocksize, _src, _dest); + return; + } + + /* Optimized shuffle implementations */ + switch (bytesoftype) { + case 2: + shuffle2_altivec(_dest, _src, vectorizable_elements, total_elements); + break; + case 4: + shuffle4_altivec(_dest, _src, vectorizable_elements, total_elements); + break; + case 8: + shuffle8_altivec(_dest, _src, vectorizable_elements, total_elements); + break; + case 16: + shuffle16_altivec(_dest, _src, vectorizable_elements, total_elements); + break; + default: + if (bytesoftype > 16) { + shuffle16_tiled_altivec(_dest, _src, vectorizable_elements, total_elements, bytesoftype); + } + else { + /* Non-optimized shuffle */ + shuffle_generic(bytesoftype, blocksize, _src, _dest); + /* The non-optimized function covers the whole buffer, + so we're done processing here. */ + return; + } + } + + /* If the buffer had any bytes at the end which couldn't be handled + by the vectorized implementations, use the non-optimized version + to finish them up. */ + if (vectorizable_bytes < blocksize) { + shuffle_generic_inline(bytesoftype, vectorizable_bytes, blocksize, _src, _dest); + } +} + +/* Unshuffle a block. This can never fail. */ +void +unshuffle_altivec(const int32_t bytesoftype, const int32_t blocksize, + const uint8_t *_src, uint8_t *_dest) { + const int32_t vectorized_chunk_size = bytesoftype * 16; + /* If the blocksize is not a multiple of both the typesize and + the vector size, round the blocksize down to the next value + which is a multiple of both. The vectorized unshuffle can be + used for that portion of the data, and the naive implementation + can be used for the remaining portion. */ + const int32_t vectorizable_bytes = blocksize - (blocksize % vectorized_chunk_size); + const int32_t vectorizable_elements = vectorizable_bytes / bytesoftype; + const int32_t total_elements = blocksize / bytesoftype; + + /* If the block size is too small to be vectorized, + use the generic implementation. */ + if (blocksize < vectorized_chunk_size) { + unshuffle_generic(bytesoftype, blocksize, _src, _dest); + return; + } + + /* Optimized unshuffle implementations */ + switch (bytesoftype) { + case 2: + unshuffle2_altivec(_dest, _src, vectorizable_elements, total_elements); + break; + case 4: + unshuffle4_altivec(_dest, _src, vectorizable_elements, total_elements); + break; + case 8: + unshuffle8_altivec(_dest, _src, vectorizable_elements, total_elements); + break; + case 16: + unshuffle16_altivec(_dest, _src, vectorizable_elements, total_elements); + break; + default: + if (bytesoftype > 16) { + unshuffle16_tiled_altivec(_dest, _src, vectorizable_elements, total_elements, bytesoftype); + } + else { + /* Non-optimized unshuffle */ + unshuffle_generic(bytesoftype, blocksize, _src, _dest); + /* The non-optimized function covers the whole buffer, + so we're done processing here. */ + return; + } + } + + /* If the buffer had any bytes at the end which couldn't be handled + by the vectorized implementations, use the non-optimized version + to finish them up. */ + if (vectorizable_bytes < blocksize) { + unshuffle_generic_inline(bytesoftype, vectorizable_bytes, blocksize, _src, _dest); + } +} diff --git a/src/blosc2/blosc/shuffle-altivec.h b/src/blosc2/blosc/shuffle-altivec.h new file mode 100644 index 0000000..3825557 --- /dev/null +++ b/src/blosc2/blosc/shuffle-altivec.h @@ -0,0 +1,38 @@ +/********************************************************************* + Blosc - Blocked Shuffling and Compression Library + + Copyright (C) 2021 The Blosc Developers + https://blosc.org + License: BSD 3-Clause (see LICENSE.txt) + + See LICENSE.txt for details about copyright and rights to use. +**********************************************************************/ + +/* ALTIVEC-accelerated shuffle/unshuffle routines. */ + +#ifndef SHUFFLE_ALTIVEC_H +#define SHUFFLE_ALTIVEC_H + +#include "blosc2/blosc2-common.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/** + ALTIVEC-accelerated shuffle routine. +*/ +BLOSC_NO_EXPORT void shuffle_altivec(const int32_t bytesoftype, const int32_t blocksize, + const uint8_t *_src, uint8_t *_dest); + +/** + ALTIVEC-accelerated unshuffle routine. +*/ +BLOSC_NO_EXPORT void unshuffle_altivec(const int32_t bytesoftype, const int32_t blocksize, + const uint8_t *_src, uint8_t *_dest); + +#ifdef __cplusplus +} +#endif + +#endif /* SHUFFLE_ALTIVEC_H */ diff --git a/src/blosc2/blosc/shuffle-avx2.c b/src/blosc2/blosc/shuffle-avx2.c new file mode 100644 index 0000000..716b9c6 --- /dev/null +++ b/src/blosc2/blosc/shuffle-avx2.c @@ -0,0 +1,747 @@ +/********************************************************************* + Blosc - Blocked Shuffling and Compression Library + + Copyright (C) 2021 The Blosc Developers + https://blosc.org + License: BSD 3-Clause (see LICENSE.txt) + + See LICENSE.txt for details about copyright and rights to use. +**********************************************************************/ + +#include "shuffle-generic.h" +#include "shuffle-avx2.h" + +/* Make sure AVX2 is available for the compilation target and compiler. */ +#if !defined(__AVX2__) + #error AVX2 is not supported by the target architecture/platform and/or this compiler. +#endif + +#include + + +/* The next is useful for debugging purposes */ +#if 0 +#include +#include + +static void printymm(__m256i ymm0) +{ + uint8_t buf[32]; + + ((__m256i *)buf)[0] = ymm0; + printf("%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x\n", + buf[0], buf[1], buf[2], buf[3], + buf[4], buf[5], buf[6], buf[7], + buf[8], buf[9], buf[10], buf[11], + buf[12], buf[13], buf[14], buf[15], + buf[16], buf[17], buf[18], buf[19], + buf[20], buf[21], buf[22], buf[23], + buf[24], buf[25], buf[26], buf[27], + buf[28], buf[29], buf[30], buf[31]); +} +#endif + +/* GCC doesn't include the split load/store intrinsics + needed for the tiled shuffle, so define them here. */ +#if defined(__GNUC__) && !defined(__clang__) && !defined(__ICC) +static inline __m256i +__attribute__((__always_inline__)) +_mm256_loadu2_m128i(const __m128i* const hiaddr, const __m128i* const loaddr) { + return _mm256_inserti128_si256( + _mm256_castsi128_si256(_mm_loadu_si128(loaddr)), _mm_loadu_si128(hiaddr), 1); +} + +static inline void +__attribute__((__always_inline__)) +_mm256_storeu2_m128i(__m128i* const hiaddr, __m128i* const loaddr, const __m256i a) { + _mm_storeu_si128(loaddr, _mm256_castsi256_si128(a)); + _mm_storeu_si128(hiaddr, _mm256_extracti128_si256(a, 1)); +} +#endif /* defined(__GNUC__) */ + +/* Routine optimized for shuffling a buffer for a type size of 2 bytes. */ +static void +shuffle2_avx2(uint8_t* const dest, const uint8_t* const src, + const int32_t vectorizable_elements, const int32_t total_elements) { + static const int32_t bytesoftype = 2; + int32_t j; + int k; + __m256i ymm0[2], ymm1[2]; + + /* Create the shuffle mask. + NOTE: The XMM/YMM 'set' intrinsics require the arguments to be ordered from + most to least significant (i.e., their order is reversed when compared to + loading the mask from an array). */ + const __m256i shmask = _mm256_set_epi8( + 0x0f, 0x0d, 0x0b, 0x09, 0x07, 0x05, 0x03, 0x01, + 0x0e, 0x0c, 0x0a, 0x08, 0x06, 0x04, 0x02, 0x00, + 0x0f, 0x0d, 0x0b, 0x09, 0x07, 0x05, 0x03, 0x01, + 0x0e, 0x0c, 0x0a, 0x08, 0x06, 0x04, 0x02, 0x00); + + for (j = 0; j < vectorizable_elements; j += sizeof(__m256i)) { + /* Fetch 32 elements (64 bytes) then transpose bytes, words and double words. */ + for (k = 0; k < 2; k++) { + ymm0[k] = _mm256_loadu_si256((__m256i*)(src + (j * bytesoftype) + (k * sizeof(__m256i)))); + ymm1[k] = _mm256_shuffle_epi8(ymm0[k], shmask); + } + + ymm0[0] = _mm256_permute4x64_epi64(ymm1[0], 0xd8); + ymm0[1] = _mm256_permute4x64_epi64(ymm1[1], 0x8d); + + ymm1[0] = _mm256_blend_epi32(ymm0[0], ymm0[1], 0xf0); + ymm0[1] = _mm256_blend_epi32(ymm0[0], ymm0[1], 0x0f); + ymm1[1] = _mm256_permute4x64_epi64(ymm0[1], 0x4e); + + /* Store the result vectors */ + uint8_t* const dest_for_jth_element = dest + j; + for (k = 0; k < 2; k++) { + _mm256_storeu_si256((__m256i*)(dest_for_jth_element + (k * total_elements)), ymm1[k]); + } + } +} + +/* Routine optimized for shuffling a buffer for a type size of 4 bytes. */ +static void +shuffle4_avx2(uint8_t* const dest, const uint8_t* const src, + const int32_t vectorizable_elements, const int32_t total_elements) { + static const int32_t bytesoftype = 4; + int32_t i; + int j; + __m256i ymm0[4], ymm1[4]; + + /* Create the shuffle mask. + NOTE: The XMM/YMM 'set' intrinsics require the arguments to be ordered from + most to least significant (i.e., their order is reversed when compared to + loading the mask from an array). */ + const __m256i mask = _mm256_set_epi32( + 0x07, 0x03, 0x06, 0x02, 0x05, 0x01, 0x04, 0x00); + + for (i = 0; i < vectorizable_elements; i += sizeof(__m256i)) { + /* Fetch 32 elements (128 bytes) then transpose bytes and words. */ + for (j = 0; j < 4; j++) { + ymm0[j] = _mm256_loadu_si256((__m256i*)(src + (i * bytesoftype) + (j * sizeof(__m256i)))); + ymm1[j] = _mm256_shuffle_epi32(ymm0[j], 0xd8); + ymm0[j] = _mm256_shuffle_epi32(ymm0[j], 0x8d); + ymm0[j] = _mm256_unpacklo_epi8(ymm1[j], ymm0[j]); + ymm1[j] = _mm256_shuffle_epi32(ymm0[j], 0x04e); + ymm0[j] = _mm256_unpacklo_epi16(ymm0[j], ymm1[j]); + } + /* Transpose double words */ + for (j = 0; j < 2; j++) { + ymm1[j * 2] = _mm256_unpacklo_epi32(ymm0[j * 2], ymm0[j * 2 + 1]); + ymm1[j * 2 + 1] = _mm256_unpackhi_epi32(ymm0[j * 2], ymm0[j * 2 + 1]); + } + /* Transpose quad words */ + for (j = 0; j < 2; j++) { + ymm0[j * 2] = _mm256_unpacklo_epi64(ymm1[j], ymm1[j + 2]); + ymm0[j * 2 + 1] = _mm256_unpackhi_epi64(ymm1[j], ymm1[j + 2]); + } + for (j = 0; j < 4; j++) { + ymm0[j] = _mm256_permutevar8x32_epi32(ymm0[j], mask); + } + /* Store the result vectors */ + uint8_t* const dest_for_ith_element = dest + i; + for (j = 0; j < 4; j++) { + _mm256_storeu_si256((__m256i*)(dest_for_ith_element + (j * total_elements)), ymm0[j]); + } + } +} + +/* Routine optimized for shuffling a buffer for a type size of 8 bytes. */ +static void +shuffle8_avx2(uint8_t* const dest, const uint8_t* const src, + const int32_t vectorizable_elements, const int32_t total_elements) { + static const int32_t bytesoftype = 8; + int32_t j; + int k, l; + __m256i ymm0[8], ymm1[8]; + + for (j = 0; j < vectorizable_elements; j += sizeof(__m256i)) { + /* Fetch 32 elements (256 bytes) then transpose bytes. */ + for (k = 0; k < 8; k++) { + ymm0[k] = _mm256_loadu_si256((__m256i*)(src + (j * bytesoftype) + (k * sizeof(__m256i)))); + ymm1[k] = _mm256_shuffle_epi32(ymm0[k], 0x4e); + ymm1[k] = _mm256_unpacklo_epi8(ymm0[k], ymm1[k]); + } + /* Transpose words */ + for (k = 0, l = 0; k < 4; k++, l += 2) { + ymm0[k * 2] = _mm256_unpacklo_epi16(ymm1[l], ymm1[l + 1]); + ymm0[k * 2 + 1] = _mm256_unpackhi_epi16(ymm1[l], ymm1[l + 1]); + } + /* Transpose double words */ + for (k = 0, l = 0; k < 4; k++, l++) { + if (k == 2) l += 2; + ymm1[k * 2] = _mm256_unpacklo_epi32(ymm0[l], ymm0[l + 2]); + ymm1[k * 2 + 1] = _mm256_unpackhi_epi32(ymm0[l], ymm0[l + 2]); + } + /* Transpose quad words */ + for (k = 0; k < 4; k++) { + ymm0[k * 2] = _mm256_unpacklo_epi64(ymm1[k], ymm1[k + 4]); + ymm0[k * 2 + 1] = _mm256_unpackhi_epi64(ymm1[k], ymm1[k + 4]); + } + for (k = 0; k < 8; k++) { + ymm1[k] = _mm256_permute4x64_epi64(ymm0[k], 0x72); + ymm0[k] = _mm256_permute4x64_epi64(ymm0[k], 0xD8); + ymm0[k] = _mm256_unpacklo_epi16(ymm0[k], ymm1[k]); + } + /* Store the result vectors */ + uint8_t* const dest_for_jth_element = dest + j; + for (k = 0; k < 8; k++) { + _mm256_storeu_si256((__m256i*)(dest_for_jth_element + (k * total_elements)), ymm0[k]); + } + } +} + +/* Routine optimized for shuffling a buffer for a type size of 16 bytes. */ +static void +shuffle16_avx2(uint8_t* const dest, const uint8_t* const src, + const int32_t vectorizable_elements, const int32_t total_elements) { + static const int32_t bytesoftype = 16; + int32_t j; + int k, l; + __m256i ymm0[16], ymm1[16]; + + /* Create the shuffle mask. + NOTE: The XMM/YMM 'set' intrinsics require the arguments to be ordered from + most to least significant (i.e., their order is reversed when compared to + loading the mask from an array). */ + const __m256i shmask = _mm256_set_epi8( + 0x0f, 0x07, 0x0e, 0x06, 0x0d, 0x05, 0x0c, 0x04, + 0x0b, 0x03, 0x0a, 0x02, 0x09, 0x01, 0x08, 0x00, + 0x0f, 0x07, 0x0e, 0x06, 0x0d, 0x05, 0x0c, 0x04, + 0x0b, 0x03, 0x0a, 0x02, 0x09, 0x01, 0x08, 0x00); + + for (j = 0; j < vectorizable_elements; j += sizeof(__m256i)) { + /* Fetch 32 elements (512 bytes) into 16 YMM registers. */ + for (k = 0; k < 16; k++) { + ymm0[k] = _mm256_loadu_si256((__m256i*)(src + (j * bytesoftype) + (k * sizeof(__m256i)))); + } + /* Transpose bytes */ + for (k = 0, l = 0; k < 8; k++, l += 2) { + ymm1[k * 2] = _mm256_unpacklo_epi8(ymm0[l], ymm0[l + 1]); + ymm1[k * 2 + 1] = _mm256_unpackhi_epi8(ymm0[l], ymm0[l + 1]); + } + /* Transpose words */ + for (k = 0, l = -2; k < 8; k++, l++) { + if ((k % 2) == 0) l += 2; + ymm0[k * 2] = _mm256_unpacklo_epi16(ymm1[l], ymm1[l + 2]); + ymm0[k * 2 + 1] = _mm256_unpackhi_epi16(ymm1[l], ymm1[l + 2]); + } + /* Transpose double words */ + for (k = 0, l = -4; k < 8; k++, l++) { + if ((k % 4) == 0) l += 4; + ymm1[k * 2] = _mm256_unpacklo_epi32(ymm0[l], ymm0[l + 4]); + ymm1[k * 2 + 1] = _mm256_unpackhi_epi32(ymm0[l], ymm0[l + 4]); + } + /* Transpose quad words */ + for (k = 0; k < 8; k++) { + ymm0[k * 2] = _mm256_unpacklo_epi64(ymm1[k], ymm1[k + 8]); + ymm0[k * 2 + 1] = _mm256_unpackhi_epi64(ymm1[k], ymm1[k + 8]); + } + for (k = 0; k < 16; k++) { + ymm0[k] = _mm256_permute4x64_epi64(ymm0[k], 0xd8); + ymm0[k] = _mm256_shuffle_epi8(ymm0[k], shmask); + } + /* Store the result vectors */ + uint8_t* const dest_for_jth_element = dest + j; + for (k = 0; k < 16; k++) { + _mm256_storeu_si256((__m256i*)(dest_for_jth_element + (k * total_elements)), ymm0[k]); + } + } +} + +/* Routine optimized for shuffling a buffer for a type size larger than 16 bytes. */ +static void +shuffle16_tiled_avx2(uint8_t* const dest, const uint8_t* const src, + const int32_t vectorizable_elements, const int32_t total_elements, const int32_t bytesoftype) { + int32_t j; + int k, l; + __m256i ymm0[16], ymm1[16]; + + const lldiv_t vecs_per_el = lldiv(bytesoftype, sizeof(__m128i)); + const int32_t vecs_rem = (int32_t)vecs_per_el.rem; + + /* Create the shuffle mask. + NOTE: The XMM/YMM 'set' intrinsics require the arguments to be ordered from + most to least significant (i.e., their order is reversed when compared to + loading the mask from an array). */ + const __m256i shmask = _mm256_set_epi8( + 0x0f, 0x07, 0x0e, 0x06, 0x0d, 0x05, 0x0c, 0x04, + 0x0b, 0x03, 0x0a, 0x02, 0x09, 0x01, 0x08, 0x00, + 0x0f, 0x07, 0x0e, 0x06, 0x0d, 0x05, 0x0c, 0x04, + 0x0b, 0x03, 0x0a, 0x02, 0x09, 0x01, 0x08, 0x00); + + for (j = 0; j < vectorizable_elements; j += sizeof(__m256i)) { + /* Advance the offset into the type by the vector size (in bytes), unless this is + the initial iteration and the type size is not a multiple of the vector size. + In that case, only advance by the number of bytes necessary so that the number + of remaining bytes in the type will be a multiple of the vector size. */ + int32_t offset_into_type; + for (offset_into_type = 0; offset_into_type < bytesoftype; + offset_into_type += (offset_into_type == 0 && vecs_rem > 0 ? vecs_rem : (int32_t)sizeof(__m128i))) { + + /* Fetch elements in groups of 512 bytes */ + const uint8_t* const src_with_offset = src + offset_into_type; + for (k = 0; k < 16; k++) { + ymm0[k] = _mm256_loadu2_m128i( + (__m128i*)(src_with_offset + (j + (2 * k) + 1) * bytesoftype), + (__m128i*)(src_with_offset + (j + (2 * k)) * bytesoftype)); + } + /* Transpose bytes */ + for (k = 0, l = 0; k < 8; k++, l += 2) { + ymm1[k * 2] = _mm256_unpacklo_epi8(ymm0[l], ymm0[l + 1]); + ymm1[k * 2 + 1] = _mm256_unpackhi_epi8(ymm0[l], ymm0[l + 1]); + } + /* Transpose words */ + for (k = 0, l = -2; k < 8; k++, l++) { + if ((k % 2) == 0) l += 2; + ymm0[k * 2] = _mm256_unpacklo_epi16(ymm1[l], ymm1[l + 2]); + ymm0[k * 2 + 1] = _mm256_unpackhi_epi16(ymm1[l], ymm1[l + 2]); + } + /* Transpose double words */ + for (k = 0, l = -4; k < 8; k++, l++) { + if ((k % 4) == 0) l += 4; + ymm1[k * 2] = _mm256_unpacklo_epi32(ymm0[l], ymm0[l + 4]); + ymm1[k * 2 + 1] = _mm256_unpackhi_epi32(ymm0[l], ymm0[l + 4]); + } + /* Transpose quad words */ + for (k = 0; k < 8; k++) { + ymm0[k * 2] = _mm256_unpacklo_epi64(ymm1[k], ymm1[k + 8]); + ymm0[k * 2 + 1] = _mm256_unpackhi_epi64(ymm1[k], ymm1[k + 8]); + } + for (k = 0; k < 16; k++) { + ymm0[k] = _mm256_permute4x64_epi64(ymm0[k], 0xd8); + ymm0[k] = _mm256_shuffle_epi8(ymm0[k], shmask); + } + /* Store the result vectors */ + uint8_t* const dest_for_jth_element = dest + j; + for (k = 0; k < 16; k++) { + _mm256_storeu_si256((__m256i*)(dest_for_jth_element + (total_elements * (offset_into_type + k))), ymm0[k]); + } + } + } +} + +/* Routine optimized for unshuffling a buffer for a type size of 2 bytes. */ +static void +unshuffle2_avx2(uint8_t* const dest, const uint8_t* const src, + const int32_t vectorizable_elements, const int32_t total_elements) { + static const int32_t bytesoftype = 2; + int32_t i; + int j; + __m256i ymm0[2], ymm1[2]; + + for (i = 0; i < vectorizable_elements; i += sizeof(__m256i)) { + /* Load 32 elements (64 bytes) into 2 YMM registers. */ + const uint8_t* const src_for_ith_element = src + i; + for (j = 0; j < 2; j++) { + ymm0[j] = _mm256_loadu_si256((__m256i*)(src_for_ith_element + (j * total_elements))); + } + /* Shuffle bytes */ + for (j = 0; j < 2; j++) { + ymm0[j] = _mm256_permute4x64_epi64(ymm0[j], 0xd8); + } + /* Compute the low 64 bytes */ + ymm1[0] = _mm256_unpacklo_epi8(ymm0[0], ymm0[1]); + /* Compute the hi 64 bytes */ + ymm1[1] = _mm256_unpackhi_epi8(ymm0[0], ymm0[1]); + /* Store the result vectors in proper order */ + _mm256_storeu_si256((__m256i*)(dest + (i * bytesoftype) + (0 * sizeof(__m256i))), ymm1[0]); + _mm256_storeu_si256((__m256i*)(dest + (i * bytesoftype) + (1 * sizeof(__m256i))), ymm1[1]); + } +} + +/* Routine optimized for unshuffling a buffer for a type size of 4 bytes. */ +static void +unshuffle4_avx2(uint8_t* const dest, const uint8_t* const src, + const int32_t vectorizable_elements, const int32_t total_elements) { + static const int32_t bytesoftype = 4; + int32_t i; + int j; + __m256i ymm0[4], ymm1[4]; + + for (i = 0; i < vectorizable_elements; i += sizeof(__m256i)) { + /* Load 32 elements (128 bytes) into 4 YMM registers. */ + const uint8_t* const src_for_ith_element = src + i; + for (j = 0; j < 4; j++) { + ymm0[j] = _mm256_loadu_si256((__m256i*)(src_for_ith_element + (j * total_elements))); + } + /* Shuffle bytes */ + for (j = 0; j < 2; j++) { + /* Compute the low 64 bytes */ + ymm1[j] = _mm256_unpacklo_epi8(ymm0[j * 2], ymm0[j * 2 + 1]); + /* Compute the hi 64 bytes */ + ymm1[2 + j] = _mm256_unpackhi_epi8(ymm0[j * 2], ymm0[j * 2 + 1]); + } + /* Shuffle 2-byte words */ + for (j = 0; j < 2; j++) { + /* Compute the low 64 bytes */ + ymm0[j] = _mm256_unpacklo_epi16(ymm1[j * 2], ymm1[j * 2 + 1]); + /* Compute the hi 64 bytes */ + ymm0[2 + j] = _mm256_unpackhi_epi16(ymm1[j * 2], ymm1[j * 2 + 1]); + } + ymm1[0] = _mm256_permute2x128_si256(ymm0[0], ymm0[2], 0x20); + ymm1[1] = _mm256_permute2x128_si256(ymm0[1], ymm0[3], 0x20); + ymm1[2] = _mm256_permute2x128_si256(ymm0[0], ymm0[2], 0x31); + ymm1[3] = _mm256_permute2x128_si256(ymm0[1], ymm0[3], 0x31); + + /* Store the result vectors in proper order */ + for (j = 0; j < 4; j++) { + _mm256_storeu_si256((__m256i*)(dest + (i * bytesoftype) + (j * sizeof(__m256i))), ymm1[j]); + } + } +} + +/* Routine optimized for unshuffling a buffer for a type size of 8 bytes. */ +static void +unshuffle8_avx2(uint8_t* const dest, const uint8_t* const src, + const int32_t vectorizable_elements, const int32_t total_elements) { + static const int32_t bytesoftype = 8; + int32_t i; + int j; + __m256i ymm0[8], ymm1[8]; + + for (i = 0; i < vectorizable_elements; i += sizeof(__m256i)) { + /* Fetch 32 elements (256 bytes) into 8 YMM registers. */ + const uint8_t* const src_for_ith_element = src + i; + for (j = 0; j < 8; j++) { + ymm0[j] = _mm256_loadu_si256((__m256i*)(src_for_ith_element + (j * total_elements))); + } + /* Shuffle bytes */ + for (j = 0; j < 4; j++) { + /* Compute the low 32 bytes */ + ymm1[j] = _mm256_unpacklo_epi8(ymm0[j * 2], ymm0[j * 2 + 1]); + /* Compute the hi 32 bytes */ + ymm1[4 + j] = _mm256_unpackhi_epi8(ymm0[j * 2], ymm0[j * 2 + 1]); + } + /* Shuffle words */ + for (j = 0; j < 4; j++) { + /* Compute the low 32 bytes */ + ymm0[j] = _mm256_unpacklo_epi16(ymm1[j * 2], ymm1[j * 2 + 1]); + /* Compute the hi 32 bytes */ + ymm0[4 + j] = _mm256_unpackhi_epi16(ymm1[j * 2], ymm1[j * 2 + 1]); + } + for (j = 0; j < 8; j++) { + ymm0[j] = _mm256_permute4x64_epi64(ymm0[j], 0xd8); + } + + /* Shuffle 4-byte dwords */ + for (j = 0; j < 4; j++) { + /* Compute the low 32 bytes */ + ymm1[j] = _mm256_unpacklo_epi32(ymm0[j * 2], ymm0[j * 2 + 1]); + /* Compute the hi 32 bytes */ + ymm1[4 + j] = _mm256_unpackhi_epi32(ymm0[j * 2], ymm0[j * 2 + 1]); + } + + /* Store the result vectors in proper order */ + _mm256_storeu_si256((__m256i*)(dest + (i * bytesoftype) + (0 * sizeof(__m256i))), ymm1[0]); + _mm256_storeu_si256((__m256i*)(dest + (i * bytesoftype) + (1 * sizeof(__m256i))), ymm1[2]); + _mm256_storeu_si256((__m256i*)(dest + (i * bytesoftype) + (2 * sizeof(__m256i))), ymm1[1]); + _mm256_storeu_si256((__m256i*)(dest + (i * bytesoftype) + (3 * sizeof(__m256i))), ymm1[3]); + _mm256_storeu_si256((__m256i*)(dest + (i * bytesoftype) + (4 * sizeof(__m256i))), ymm1[4]); + _mm256_storeu_si256((__m256i*)(dest + (i * bytesoftype) + (5 * sizeof(__m256i))), ymm1[6]); + _mm256_storeu_si256((__m256i*)(dest + (i * bytesoftype) + (6 * sizeof(__m256i))), ymm1[5]); + _mm256_storeu_si256((__m256i*)(dest + (i * bytesoftype) + (7 * sizeof(__m256i))), ymm1[7]); + } +} + +/* Routine optimized for unshuffling a buffer for a type size of 16 bytes. */ +static void +unshuffle16_avx2(uint8_t* const dest, const uint8_t* const src, + const int32_t vectorizable_elements, const int32_t total_elements) { + static const int32_t bytesoftype = 16; + int32_t i; + int j; + __m256i ymm0[16], ymm1[16]; + + for (i = 0; i < vectorizable_elements; i += sizeof(__m256i)) { + /* Fetch 32 elements (512 bytes) into 16 YMM registers. */ + const uint8_t* const src_for_ith_element = src + i; + for (j = 0; j < 16; j++) { + ymm0[j] = _mm256_loadu_si256((__m256i*)(src_for_ith_element + (j * total_elements))); + } + + /* Shuffle bytes */ + for (j = 0; j < 8; j++) { + /* Compute the low 32 bytes */ + ymm1[j] = _mm256_unpacklo_epi8(ymm0[j * 2], ymm0[j * 2 + 1]); + /* Compute the hi 32 bytes */ + ymm1[8 + j] = _mm256_unpackhi_epi8(ymm0[j * 2], ymm0[j * 2 + 1]); + } + /* Shuffle 2-byte words */ + for (j = 0; j < 8; j++) { + /* Compute the low 32 bytes */ + ymm0[j] = _mm256_unpacklo_epi16(ymm1[j * 2], ymm1[j * 2 + 1]); + /* Compute the hi 32 bytes */ + ymm0[8 + j] = _mm256_unpackhi_epi16(ymm1[j * 2], ymm1[j * 2 + 1]); + } + /* Shuffle 4-byte dwords */ + for (j = 0; j < 8; j++) { + /* Compute the low 32 bytes */ + ymm1[j] = _mm256_unpacklo_epi32(ymm0[j * 2], ymm0[j * 2 + 1]); + /* Compute the hi 32 bytes */ + ymm1[8 + j] = _mm256_unpackhi_epi32(ymm0[j * 2], ymm0[j * 2 + 1]); + } + + /* Shuffle 8-byte qwords */ + for (j = 0; j < 8; j++) { + /* Compute the low 32 bytes */ + ymm0[j] = _mm256_unpacklo_epi64(ymm1[j * 2], ymm1[j * 2 + 1]); + /* Compute the hi 32 bytes */ + ymm0[8 + j] = _mm256_unpackhi_epi64(ymm1[j * 2], ymm1[j * 2 + 1]); + } + + for (j = 0; j < 8; j++) { + ymm1[j] = _mm256_permute2x128_si256(ymm0[j], ymm0[j + 8], 0x20); + ymm1[j + 8] = _mm256_permute2x128_si256(ymm0[j], ymm0[j + 8], 0x31); + } + + /* Store the result vectors in proper order */ + _mm256_storeu_si256((__m256i*)(dest + (i * bytesoftype) + (0 * sizeof(__m256i))), ymm1[0]); + _mm256_storeu_si256((__m256i*)(dest + (i * bytesoftype) + (1 * sizeof(__m256i))), ymm1[4]); + _mm256_storeu_si256((__m256i*)(dest + (i * bytesoftype) + (2 * sizeof(__m256i))), ymm1[2]); + _mm256_storeu_si256((__m256i*)(dest + (i * bytesoftype) + (3 * sizeof(__m256i))), ymm1[6]); + _mm256_storeu_si256((__m256i*)(dest + (i * bytesoftype) + (4 * sizeof(__m256i))), ymm1[1]); + _mm256_storeu_si256((__m256i*)(dest + (i * bytesoftype) + (5 * sizeof(__m256i))), ymm1[5]); + _mm256_storeu_si256((__m256i*)(dest + (i * bytesoftype) + (6 * sizeof(__m256i))), ymm1[3]); + _mm256_storeu_si256((__m256i*)(dest + (i * bytesoftype) + (7 * sizeof(__m256i))), ymm1[7]); + _mm256_storeu_si256((__m256i*)(dest + (i * bytesoftype) + (8 * sizeof(__m256i))), ymm1[8]); + _mm256_storeu_si256((__m256i*)(dest + (i * bytesoftype) + (9 * sizeof(__m256i))), ymm1[12]); + _mm256_storeu_si256((__m256i*)(dest + (i * bytesoftype) + (10 * sizeof(__m256i))), ymm1[10]); + _mm256_storeu_si256((__m256i*)(dest + (i * bytesoftype) + (11 * sizeof(__m256i))), ymm1[14]); + _mm256_storeu_si256((__m256i*)(dest + (i * bytesoftype) + (12 * sizeof(__m256i))), ymm1[9]); + _mm256_storeu_si256((__m256i*)(dest + (i * bytesoftype) + (13 * sizeof(__m256i))), ymm1[13]); + _mm256_storeu_si256((__m256i*)(dest + (i * bytesoftype) + (14 * sizeof(__m256i))), ymm1[11]); + _mm256_storeu_si256((__m256i*)(dest + (i * bytesoftype) + (15 * sizeof(__m256i))), ymm1[15]); + } +} + +/* Routine optimized for unshuffling a buffer for a type size larger than 16 bytes. */ +static void +unshuffle16_tiled_avx2(const uint8_t *dest, const uint8_t* const src, + const int32_t vectorizable_elements, const int32_t total_elements, const int32_t bytesoftype) { + int32_t i; + int j; + __m256i ymm0[16], ymm1[16]; + + const lldiv_t vecs_per_el = lldiv(bytesoftype, sizeof(__m128i)); + const int32_t vecs_rem = (int32_t)vecs_per_el.rem; + + /* The unshuffle loops are inverted (compared to shuffle_tiled16_avx2) + to optimize cache utilization. */ + int32_t offset_into_type; + for (offset_into_type = 0; offset_into_type < bytesoftype; + offset_into_type += (offset_into_type == 0 && vecs_rem > 0 ? vecs_rem : (int32_t)sizeof(__m128i))) { + for (i = 0; i < vectorizable_elements; i += sizeof(__m256i)) { + /* Load the first 16 bytes of 32 adjacent elements (512 bytes) into 16 YMM registers */ + const uint8_t* const src_for_ith_element = src + i; + for (j = 0; j < 16; j++) { + ymm0[j] = _mm256_loadu_si256((__m256i*)(src_for_ith_element + (total_elements * (offset_into_type + j)))); + } + + /* Shuffle bytes */ + for (j = 0; j < 8; j++) { + /* Compute the low 32 bytes */ + ymm1[j] = _mm256_unpacklo_epi8(ymm0[j * 2], ymm0[j * 2 + 1]); + /* Compute the hi 32 bytes */ + ymm1[8 + j] = _mm256_unpackhi_epi8(ymm0[j * 2], ymm0[j * 2 + 1]); + } + /* Shuffle 2-byte words */ + for (j = 0; j < 8; j++) { + /* Compute the low 32 bytes */ + ymm0[j] = _mm256_unpacklo_epi16(ymm1[j * 2], ymm1[j * 2 + 1]); + /* Compute the hi 32 bytes */ + ymm0[8 + j] = _mm256_unpackhi_epi16(ymm1[j * 2], ymm1[j * 2 + 1]); + } + /* Shuffle 4-byte dwords */ + for (j = 0; j < 8; j++) { + /* Compute the low 32 bytes */ + ymm1[j] = _mm256_unpacklo_epi32(ymm0[j * 2], ymm0[j * 2 + 1]); + /* Compute the hi 32 bytes */ + ymm1[8 + j] = _mm256_unpackhi_epi32(ymm0[j * 2], ymm0[j * 2 + 1]); + } + + /* Shuffle 8-byte qwords */ + for (j = 0; j < 8; j++) { + /* Compute the low 32 bytes */ + ymm0[j] = _mm256_unpacklo_epi64(ymm1[j * 2], ymm1[j * 2 + 1]); + /* Compute the hi 32 bytes */ + ymm0[8 + j] = _mm256_unpackhi_epi64(ymm1[j * 2], ymm1[j * 2 + 1]); + } + + for (j = 0; j < 8; j++) { + ymm1[j] = _mm256_permute2x128_si256(ymm0[j], ymm0[j + 8], 0x20); + ymm1[j + 8] = _mm256_permute2x128_si256(ymm0[j], ymm0[j + 8], 0x31); + } + + /* Store the result vectors in proper order */ + const uint8_t* const dest_with_offset = dest + offset_into_type; + _mm256_storeu2_m128i( + (__m128i*)(dest_with_offset + (i + 0x01) * bytesoftype), + (__m128i*)(dest_with_offset + (i + 0x00) * bytesoftype), ymm1[0]); + _mm256_storeu2_m128i( + (__m128i*)(dest_with_offset + (i + 0x03) * bytesoftype), + (__m128i*)(dest_with_offset + (i + 0x02) * bytesoftype), ymm1[4]); + _mm256_storeu2_m128i( + (__m128i*)(dest_with_offset + (i + 0x05) * bytesoftype), + (__m128i*)(dest_with_offset + (i + 0x04) * bytesoftype), ymm1[2]); + _mm256_storeu2_m128i( + (__m128i*)(dest_with_offset + (i + 0x07) * bytesoftype), + (__m128i*)(dest_with_offset + (i + 0x06) * bytesoftype), ymm1[6]); + _mm256_storeu2_m128i( + (__m128i*)(dest_with_offset + (i + 0x09) * bytesoftype), + (__m128i*)(dest_with_offset + (i + 0x08) * bytesoftype), ymm1[1]); + _mm256_storeu2_m128i( + (__m128i*)(dest_with_offset + (i + 0x0b) * bytesoftype), + (__m128i*)(dest_with_offset + (i + 0x0a) * bytesoftype), ymm1[5]); + _mm256_storeu2_m128i( + (__m128i*)(dest_with_offset + (i + 0x0d) * bytesoftype), + (__m128i*)(dest_with_offset + (i + 0x0c) * bytesoftype), ymm1[3]); + _mm256_storeu2_m128i( + (__m128i*)(dest_with_offset + (i + 0x0f) * bytesoftype), + (__m128i*)(dest_with_offset + (i + 0x0e) * bytesoftype), ymm1[7]); + _mm256_storeu2_m128i( + (__m128i*)(dest_with_offset + (i + 0x11) * bytesoftype), + (__m128i*)(dest_with_offset + (i + 0x10) * bytesoftype), ymm1[8]); + _mm256_storeu2_m128i( + (__m128i*)(dest_with_offset + (i + 0x13) * bytesoftype), + (__m128i*)(dest_with_offset + (i + 0x12) * bytesoftype), ymm1[12]); + _mm256_storeu2_m128i( + (__m128i*)(dest_with_offset + (i + 0x15) * bytesoftype), + (__m128i*)(dest_with_offset + (i + 0x14) * bytesoftype), ymm1[10]); + _mm256_storeu2_m128i( + (__m128i*)(dest_with_offset + (i + 0x17) * bytesoftype), + (__m128i*)(dest_with_offset + (i + 0x16) * bytesoftype), ymm1[14]); + _mm256_storeu2_m128i( + (__m128i*)(dest_with_offset + (i + 0x19) * bytesoftype), + (__m128i*)(dest_with_offset + (i + 0x18) * bytesoftype), ymm1[9]); + _mm256_storeu2_m128i( + (__m128i*)(dest_with_offset + (i + 0x1b) * bytesoftype), + (__m128i*)(dest_with_offset + (i + 0x1a) * bytesoftype), ymm1[13]); + _mm256_storeu2_m128i( + (__m128i*)(dest_with_offset + (i + 0x1d) * bytesoftype), + (__m128i*)(dest_with_offset + (i + 0x1c) * bytesoftype), ymm1[11]); + _mm256_storeu2_m128i( + (__m128i*)(dest_with_offset + (i + 0x1f) * bytesoftype), + (__m128i*)(dest_with_offset + (i + 0x1e) * bytesoftype), ymm1[15]); + } + } +} + +/* Shuffle a block. This can never fail. */ +void +shuffle_avx2(const int32_t bytesoftype, const int32_t blocksize, + const uint8_t *_src, uint8_t *_dest) { + const int32_t vectorized_chunk_size = bytesoftype * (int32_t)sizeof(__m256i); + + /* If the block size is too small to be vectorized, + use the generic implementation. */ + if (blocksize < vectorized_chunk_size) { + shuffle_generic(bytesoftype, blocksize, _src, _dest); + return; + } + + /* If the blocksize is not a multiple of both the typesize and + the vector size, round the blocksize down to the next value + which is a multiple of both. The vectorized shuffle can be + used for that portion of the data, and the naive implementation + can be used for the remaining portion. */ + const int32_t vectorizable_bytes = blocksize - (blocksize % vectorized_chunk_size); + + const int32_t vectorizable_elements = vectorizable_bytes / bytesoftype; + const int32_t total_elements = blocksize / bytesoftype; + + /* Optimized shuffle implementations */ + switch (bytesoftype) { + case 2: + shuffle2_avx2(_dest, _src, vectorizable_elements, total_elements); + break; + case 4: + shuffle4_avx2(_dest, _src, vectorizable_elements, total_elements); + break; + case 8: + shuffle8_avx2(_dest, _src, vectorizable_elements, total_elements); + break; + case 16: + shuffle16_avx2(_dest, _src, vectorizable_elements, total_elements); + break; + default: + /* For types larger than 16 bytes, use the AVX2 tiled shuffle. */ + if (bytesoftype > (int32_t)sizeof(__m128i)) { + shuffle16_tiled_avx2(_dest, _src, vectorizable_elements, total_elements, bytesoftype); + } + else { + /* Non-optimized shuffle */ + shuffle_generic(bytesoftype, blocksize, _src, _dest); + /* The non-optimized function covers the whole buffer, + so we're done processing here. */ + return; + } + } + + /* If the buffer had any bytes at the end which couldn't be handled + by the vectorized implementations, use the non-optimized version + to finish them up. */ + if (vectorizable_bytes < blocksize) { + shuffle_generic_inline(bytesoftype, vectorizable_bytes, blocksize, _src, _dest); + } +} + +/* Unshuffle a block. This can never fail. */ +void +unshuffle_avx2(const int32_t bytesoftype, const int32_t blocksize, + const uint8_t *_src, uint8_t *_dest) { + const int32_t vectorized_chunk_size = bytesoftype * (int32_t)sizeof(__m256i); + + /* If the block size is too small to be vectorized, + use the generic implementation. */ + if (blocksize < vectorized_chunk_size) { + unshuffle_generic(bytesoftype, blocksize, _src, _dest); + return; + } + + /* If the blocksize is not a multiple of both the typesize and + the vector size, round the blocksize down to the next value + which is a multiple of both. The vectorized unshuffle can be + used for that portion of the data, and the naive implementation + can be used for the remaining portion. */ + const int32_t vectorizable_bytes = blocksize - (blocksize % vectorized_chunk_size); + + const int32_t vectorizable_elements = vectorizable_bytes / bytesoftype; + const int32_t total_elements = blocksize / bytesoftype; + + /* Optimized unshuffle implementations */ + switch (bytesoftype) { + case 2: + unshuffle2_avx2(_dest, _src, vectorizable_elements, total_elements); + break; + case 4: + unshuffle4_avx2(_dest, _src, vectorizable_elements, total_elements); + break; + case 8: + unshuffle8_avx2(_dest, _src, vectorizable_elements, total_elements); + break; + case 16: + unshuffle16_avx2(_dest, _src, vectorizable_elements, total_elements); + break; + default: + /* For types larger than 16 bytes, use the AVX2 tiled unshuffle. */ + if (bytesoftype > (int32_t)sizeof(__m128i)) { + unshuffle16_tiled_avx2(_dest, _src, vectorizable_elements, total_elements, bytesoftype); + } + else { + /* Non-optimized unshuffle */ + unshuffle_generic(bytesoftype, blocksize, _src, _dest); + /* The non-optimized function covers the whole buffer, + so we're done processing here. */ + return; + } + } + + /* If the buffer had any bytes at the end which couldn't be handled + by the vectorized implementations, use the non-optimized version + to finish them up. */ + if (vectorizable_bytes < blocksize) { + unshuffle_generic_inline(bytesoftype, vectorizable_bytes, blocksize, _src, _dest); + } +} diff --git a/src/blosc2/blosc/shuffle-avx2.c.orig b/src/blosc2/blosc/shuffle-avx2.c.orig new file mode 100644 index 0000000..82f9864 --- /dev/null +++ b/src/blosc2/blosc/shuffle-avx2.c.orig @@ -0,0 +1,747 @@ +/********************************************************************* + Blosc - Blocked Shuffling and Compression Library + + Copyright (C) 2021 The Blosc Developers + https://blosc.org + License: BSD 3-Clause (see LICENSE.txt) + + See LICENSE.txt for details about copyright and rights to use. +**********************************************************************/ + +#include "shuffle-generic.h" +#include "shuffle-avx2.h" + +/* Make sure AVX2 is available for the compilation target and compiler. */ +#if !defined(__AVX2__) + #error AVX2 is not supported by the target architecture/platform and/or this compiler. +#endif + +#include + + +/* The next is useful for debugging purposes */ +#if 0 +#include +#include + +static void printymm(__m256i ymm0) +{ + uint8_t buf[32]; + + ((__m256i *)buf)[0] = ymm0; + printf("%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x\n", + buf[0], buf[1], buf[2], buf[3], + buf[4], buf[5], buf[6], buf[7], + buf[8], buf[9], buf[10], buf[11], + buf[12], buf[13], buf[14], buf[15], + buf[16], buf[17], buf[18], buf[19], + buf[20], buf[21], buf[22], buf[23], + buf[24], buf[25], buf[26], buf[27], + buf[28], buf[29], buf[30], buf[31]); +} +#endif + +/* GCC doesn't include the split load/store intrinsics + needed for the tiled shuffle, so define them here. */ +#if defined(__GNUC__) && !defined(__clang__) +static inline __m256i +__attribute__((__always_inline__)) +_mm256_loadu2_m128i(const __m128i* const hiaddr, const __m128i* const loaddr) +{ + return _mm256_inserti128_si256( + _mm256_castsi128_si256(_mm_loadu_si128(loaddr)), _mm_loadu_si128(hiaddr), 1); +} + +static inline void +__attribute__((__always_inline__)) +_mm256_storeu2_m128i(__m128i* const hiaddr, __m128i* const loaddr, const __m256i a) +{ + _mm_storeu_si128(loaddr, _mm256_castsi256_si128(a)); + _mm_storeu_si128(hiaddr, _mm256_extracti128_si256(a, 1)); +} +#endif /* defined(__GNUC__) */ + +/* Routine optimized for shuffling a buffer for a type size of 2 bytes. */ +static void +shuffle2_avx2(uint8_t* const dest, const uint8_t* const src, + const size_t vectorizable_elements, const size_t total_elements) { + static const size_t bytesoftype = 2; + size_t j; + int k; + __m256i ymm0[2], ymm1[2]; + + /* Create the shuffle mask. + NOTE: The XMM/YMM 'set' intrinsics require the arguments to be ordered from + most to least significant (i.e., their order is reversed when compared to + loading the mask from an array). */ + const __m256i shmask = _mm256_set_epi8( + 0x0f, 0x0d, 0x0b, 0x09, 0x07, 0x05, 0x03, 0x01, + 0x0e, 0x0c, 0x0a, 0x08, 0x06, 0x04, 0x02, 0x00, + 0x0f, 0x0d, 0x0b, 0x09, 0x07, 0x05, 0x03, 0x01, + 0x0e, 0x0c, 0x0a, 0x08, 0x06, 0x04, 0x02, 0x00); + + for (j = 0; j < vectorizable_elements; j += sizeof(__m256i)) { + /* Fetch 32 elements (64 bytes) then transpose bytes, words and double words. */ + for (k = 0; k < 2; k++) { + ymm0[k] = _mm256_loadu_si256((__m256i * )(src + (j * bytesoftype) + (k * sizeof(__m256i)))); + ymm1[k] = _mm256_shuffle_epi8(ymm0[k], shmask); + } + + ymm0[0] = _mm256_permute4x64_epi64(ymm1[0], 0xd8); + ymm0[1] = _mm256_permute4x64_epi64(ymm1[1], 0x8d); + + ymm1[0] = _mm256_blend_epi32(ymm0[0], ymm0[1], 0xf0); + ymm0[1] = _mm256_blend_epi32(ymm0[0], ymm0[1], 0x0f); + ymm1[1] = _mm256_permute4x64_epi64(ymm0[1], 0x4e); + + /* Store the result vectors */ + uint8_t* const dest_for_jth_element = dest + j; + for (k = 0; k < 2; k++) { + _mm256_storeu_si256((__m256i * )(dest_for_jth_element + (k * total_elements)), ymm1[k]); + } + } +} + +/* Routine optimized for shuffling a buffer for a type size of 4 bytes. */ +static void +shuffle4_avx2(uint8_t* const dest, const uint8_t* const src, + const size_t vectorizable_elements, const size_t total_elements) { + static const size_t bytesoftype = 4; + size_t i; + int j; + __m256i ymm0[4], ymm1[4]; + + /* Create the shuffle mask. + NOTE: The XMM/YMM 'set' intrinsics require the arguments to be ordered from + most to least significant (i.e., their order is reversed when compared to + loading the mask from an array). */ + const __m256i mask = _mm256_set_epi32( + 0x07, 0x03, 0x06, 0x02, 0x05, 0x01, 0x04, 0x00); + + for (i = 0; i < vectorizable_elements; i += sizeof(__m256i)) { + /* Fetch 32 elements (128 bytes) then transpose bytes and words. */ + for (j = 0; j < 4; j++) { + ymm0[j] = _mm256_loadu_si256((__m256i * )(src + (i * bytesoftype) + (j * sizeof(__m256i)))); + ymm1[j] = _mm256_shuffle_epi32(ymm0[j], 0xd8); + ymm0[j] = _mm256_shuffle_epi32(ymm0[j], 0x8d); + ymm0[j] = _mm256_unpacklo_epi8(ymm1[j], ymm0[j]); + ymm1[j] = _mm256_shuffle_epi32(ymm0[j], 0x04e); + ymm0[j] = _mm256_unpacklo_epi16(ymm0[j], ymm1[j]); + } + /* Transpose double words */ + for (j = 0; j < 2; j++) { + ymm1[j * 2] = _mm256_unpacklo_epi32(ymm0[j * 2], ymm0[j * 2 + 1]); + ymm1[j * 2 + 1] = _mm256_unpackhi_epi32(ymm0[j * 2], ymm0[j * 2 + 1]); + } + /* Transpose quad words */ + for (j = 0; j < 2; j++) { + ymm0[j * 2] = _mm256_unpacklo_epi64(ymm1[j], ymm1[j + 2]); + ymm0[j * 2 + 1] = _mm256_unpackhi_epi64(ymm1[j], ymm1[j + 2]); + } + for (j = 0; j < 4; j++) { + ymm0[j] = _mm256_permutevar8x32_epi32(ymm0[j], mask); + } + /* Store the result vectors */ + uint8_t* const dest_for_ith_element = dest + i; + for (j = 0; j < 4; j++) { + _mm256_storeu_si256((__m256i * )(dest_for_ith_element + (j * total_elements)), ymm0[j]); + } + } +} + +/* Routine optimized for shuffling a buffer for a type size of 8 bytes. */ +static void +shuffle8_avx2(uint8_t* const dest, const uint8_t* const src, + const size_t vectorizable_elements, const size_t total_elements) { + static const size_t bytesoftype = 8; + size_t j; + int k, l; + __m256i ymm0[8], ymm1[8]; + + for (j = 0; j < vectorizable_elements; j += sizeof(__m256i)) { + /* Fetch 32 elements (256 bytes) then transpose bytes. */ + for (k = 0; k < 8; k++) { + ymm0[k] = _mm256_loadu_si256((__m256i * )(src + (j * bytesoftype) + (k * sizeof(__m256i)))); + ymm1[k] = _mm256_shuffle_epi32(ymm0[k], 0x4e); + ymm1[k] = _mm256_unpacklo_epi8(ymm0[k], ymm1[k]); + } + /* Transpose words */ + for (k = 0, l = 0; k < 4; k++, l += 2) { + ymm0[k * 2] = _mm256_unpacklo_epi16(ymm1[l], ymm1[l + 1]); + ymm0[k * 2 + 1] = _mm256_unpackhi_epi16(ymm1[l], ymm1[l + 1]); + } + /* Transpose double words */ + for (k = 0, l = 0; k < 4; k++, l++) { + if (k == 2) l += 2; + ymm1[k * 2] = _mm256_unpacklo_epi32(ymm0[l], ymm0[l + 2]); + ymm1[k * 2 + 1] = _mm256_unpackhi_epi32(ymm0[l], ymm0[l + 2]); + } + /* Transpose quad words */ + for (k = 0; k < 4; k++) { + ymm0[k * 2] = _mm256_unpacklo_epi64(ymm1[k], ymm1[k + 4]); + ymm0[k * 2 + 1] = _mm256_unpackhi_epi64(ymm1[k], ymm1[k + 4]); + } + for (k = 0; k < 8; k++) { + ymm1[k] = _mm256_permute4x64_epi64(ymm0[k], 0x72); + ymm0[k] = _mm256_permute4x64_epi64(ymm0[k], 0xD8); + ymm0[k] = _mm256_unpacklo_epi16(ymm0[k], ymm1[k]); + } + /* Store the result vectors */ + uint8_t* const dest_for_jth_element = dest + j; + for (k = 0; k < 8; k++) { + _mm256_storeu_si256((__m256i * )(dest_for_jth_element + (k * total_elements)), ymm0[k]); + } + } +} + +/* Routine optimized for shuffling a buffer for a type size of 16 bytes. */ +static void +shuffle16_avx2(uint8_t* const dest, const uint8_t* const src, + const size_t vectorizable_elements, const size_t total_elements) { + static const size_t bytesoftype = 16; + size_t j; + int k, l; + __m256i ymm0[16], ymm1[16]; + + /* Create the shuffle mask. + NOTE: The XMM/YMM 'set' intrinsics require the arguments to be ordered from + most to least significant (i.e., their order is reversed when compared to + loading the mask from an array). */ + const __m256i shmask = _mm256_set_epi8( + 0x0f, 0x07, 0x0e, 0x06, 0x0d, 0x05, 0x0c, 0x04, + 0x0b, 0x03, 0x0a, 0x02, 0x09, 0x01, 0x08, 0x00, + 0x0f, 0x07, 0x0e, 0x06, 0x0d, 0x05, 0x0c, 0x04, + 0x0b, 0x03, 0x0a, 0x02, 0x09, 0x01, 0x08, 0x00); + + for (j = 0; j < vectorizable_elements; j += sizeof(__m256i)) { + /* Fetch 32 elements (512 bytes) into 16 YMM registers. */ + for (k = 0; k < 16; k++) { + ymm0[k] = _mm256_loadu_si256((__m256i * )(src + (j * bytesoftype) + (k * sizeof(__m256i)))); + } + /* Transpose bytes */ + for (k = 0, l = 0; k < 8; k++, l += 2) { + ymm1[k * 2] = _mm256_unpacklo_epi8(ymm0[l], ymm0[l + 1]); + ymm1[k * 2 + 1] = _mm256_unpackhi_epi8(ymm0[l], ymm0[l + 1]); + } + /* Transpose words */ + for (k = 0, l = -2; k < 8; k++, l++) { + if ((k % 2) == 0) l += 2; + ymm0[k * 2] = _mm256_unpacklo_epi16(ymm1[l], ymm1[l + 2]); + ymm0[k * 2 + 1] = _mm256_unpackhi_epi16(ymm1[l], ymm1[l + 2]); + } + /* Transpose double words */ + for (k = 0, l = -4; k < 8; k++, l++) { + if ((k % 4) == 0) l += 4; + ymm1[k * 2] = _mm256_unpacklo_epi32(ymm0[l], ymm0[l + 4]); + ymm1[k * 2 + 1] = _mm256_unpackhi_epi32(ymm0[l], ymm0[l + 4]); + } + /* Transpose quad words */ + for (k = 0; k < 8; k++) { + ymm0[k * 2] = _mm256_unpacklo_epi64(ymm1[k], ymm1[k + 8]); + ymm0[k * 2 + 1] = _mm256_unpackhi_epi64(ymm1[k], ymm1[k + 8]); + } + for (k = 0; k < 16; k++) { + ymm0[k] = _mm256_permute4x64_epi64(ymm0[k], 0xd8); + ymm0[k] = _mm256_shuffle_epi8(ymm0[k], shmask); + } + /* Store the result vectors */ + uint8_t* const dest_for_jth_element = dest + j; + for (k = 0; k < 16; k++) { + _mm256_storeu_si256((__m256i * )(dest_for_jth_element + (k * total_elements)), ymm0[k]); + } + } +} + +/* Routine optimized for shuffling a buffer for a type size larger than 16 bytes. */ +static void +shuffle16_tiled_avx2(uint8_t* const dest, const uint8_t* const src, + const size_t vectorizable_elements, const size_t total_elements, const size_t bytesoftype) { + size_t j; + int k, l; + __m256i ymm0[16], ymm1[16]; + + const lldiv_t vecs_per_el = lldiv(bytesoftype, sizeof(__m128i)); + + /* Create the shuffle mask. + NOTE: The XMM/YMM 'set' intrinsics require the arguments to be ordered from + most to least significant (i.e., their order is reversed when compared to + loading the mask from an array). */ + const __m256i shmask = _mm256_set_epi8( + 0x0f, 0x07, 0x0e, 0x06, 0x0d, 0x05, 0x0c, 0x04, + 0x0b, 0x03, 0x0a, 0x02, 0x09, 0x01, 0x08, 0x00, + 0x0f, 0x07, 0x0e, 0x06, 0x0d, 0x05, 0x0c, 0x04, + 0x0b, 0x03, 0x0a, 0x02, 0x09, 0x01, 0x08, 0x00); + + for (j = 0; j < vectorizable_elements; j += sizeof(__m256i)) { + /* Advance the offset into the type by the vector size (in bytes), unless this is + the initial iteration and the type size is not a multiple of the vector size. + In that case, only advance by the number of bytes necessary so that the number + of remaining bytes in the type will be a multiple of the vector size. */ + size_t offset_into_type; + for (offset_into_type = 0; offset_into_type < bytesoftype; + offset_into_type += (offset_into_type == 0 && vecs_per_el.rem > 0 ? vecs_per_el.rem : sizeof(__m128i))) { + + /* Fetch elements in groups of 512 bytes */ + const uint8_t* const src_with_offset = src + offset_into_type; + for (k = 0; k < 16; k++) { + ymm0[k] = _mm256_loadu2_m128i( + (__m128i * )(src_with_offset + (j + (2 * k) + 1) * bytesoftype), + (__m128i * )(src_with_offset + (j + (2 * k)) * bytesoftype)); + } + /* Transpose bytes */ + for (k = 0, l = 0; k < 8; k++, l += 2) { + ymm1[k * 2] = _mm256_unpacklo_epi8(ymm0[l], ymm0[l + 1]); + ymm1[k * 2 + 1] = _mm256_unpackhi_epi8(ymm0[l], ymm0[l + 1]); + } + /* Transpose words */ + for (k = 0, l = -2; k < 8; k++, l++) { + if ((k % 2) == 0) l += 2; + ymm0[k * 2] = _mm256_unpacklo_epi16(ymm1[l], ymm1[l + 2]); + ymm0[k * 2 + 1] = _mm256_unpackhi_epi16(ymm1[l], ymm1[l + 2]); + } + /* Transpose double words */ + for (k = 0, l = -4; k < 8; k++, l++) { + if ((k % 4) == 0) l += 4; + ymm1[k * 2] = _mm256_unpacklo_epi32(ymm0[l], ymm0[l + 4]); + ymm1[k * 2 + 1] = _mm256_unpackhi_epi32(ymm0[l], ymm0[l + 4]); + } + /* Transpose quad words */ + for (k = 0; k < 8; k++) { + ymm0[k * 2] = _mm256_unpacklo_epi64(ymm1[k], ymm1[k + 8]); + ymm0[k * 2 + 1] = _mm256_unpackhi_epi64(ymm1[k], ymm1[k + 8]); + } + for (k = 0; k < 16; k++) { + ymm0[k] = _mm256_permute4x64_epi64(ymm0[k], 0xd8); + ymm0[k] = _mm256_shuffle_epi8(ymm0[k], shmask); + } + /* Store the result vectors */ + uint8_t* const dest_for_jth_element = dest + j; + for (k = 0; k < 16; k++) { + _mm256_storeu_si256((__m256i * )(dest_for_jth_element + (total_elements * (offset_into_type + k))), ymm0[k]); + } + } + } +} + +/* Routine optimized for unshuffling a buffer for a type size of 2 bytes. */ +static void +unshuffle2_avx2(uint8_t* const dest, const uint8_t* const src, + const size_t vectorizable_elements, const size_t total_elements) { + static const size_t bytesoftype = 2; + size_t i; + int j; + __m256i ymm0[2], ymm1[2]; + + for (i = 0; i < vectorizable_elements; i += sizeof(__m256i)) { + /* Load 32 elements (64 bytes) into 2 YMM registers. */ + const uint8_t* const src_for_ith_element = src + i; + for (j = 0; j < 2; j++) { + ymm0[j] = _mm256_loadu_si256((__m256i * )(src_for_ith_element + (j * total_elements))); + } + /* Shuffle bytes */ + for (j = 0; j < 2; j++) { + ymm0[j] = _mm256_permute4x64_epi64(ymm0[j], 0xd8); + } + /* Compute the low 64 bytes */ + ymm1[0] = _mm256_unpacklo_epi8(ymm0[0], ymm0[1]); + /* Compute the hi 64 bytes */ + ymm1[1] = _mm256_unpackhi_epi8(ymm0[0], ymm0[1]); + /* Store the result vectors in proper order */ + _mm256_storeu_si256((__m256i * )(dest + (i * bytesoftype) + (0 * sizeof(__m256i))), ymm1[0]); + _mm256_storeu_si256((__m256i * )(dest + (i * bytesoftype) + (1 * sizeof(__m256i))), ymm1[1]); + } +} + +/* Routine optimized for unshuffling a buffer for a type size of 4 bytes. */ +static void +unshuffle4_avx2(uint8_t* const dest, const uint8_t* const src, + const size_t vectorizable_elements, const size_t total_elements) { + static const size_t bytesoftype = 4; + size_t i; + int j; + __m256i ymm0[4], ymm1[4]; + + for (i = 0; i < vectorizable_elements; i += sizeof(__m256i)) { + /* Load 32 elements (128 bytes) into 4 YMM registers. */ + const uint8_t* const src_for_ith_element = src + i; + for (j = 0; j < 4; j++) { + ymm0[j] = _mm256_loadu_si256((__m256i * )(src_for_ith_element + (j * total_elements))); + } + /* Shuffle bytes */ + for (j = 0; j < 2; j++) { + /* Compute the low 64 bytes */ + ymm1[j] = _mm256_unpacklo_epi8(ymm0[j * 2], ymm0[j * 2 + 1]); + /* Compute the hi 64 bytes */ + ymm1[2 + j] = _mm256_unpackhi_epi8(ymm0[j * 2], ymm0[j * 2 + 1]); + } + /* Shuffle 2-byte words */ + for (j = 0; j < 2; j++) { + /* Compute the low 64 bytes */ + ymm0[j] = _mm256_unpacklo_epi16(ymm1[j * 2], ymm1[j * 2 + 1]); + /* Compute the hi 64 bytes */ + ymm0[2 + j] = _mm256_unpackhi_epi16(ymm1[j * 2], ymm1[j * 2 + 1]); + } + ymm1[0] = _mm256_permute2x128_si256(ymm0[0], ymm0[2], 0x20); + ymm1[1] = _mm256_permute2x128_si256(ymm0[1], ymm0[3], 0x20); + ymm1[2] = _mm256_permute2x128_si256(ymm0[0], ymm0[2], 0x31); + ymm1[3] = _mm256_permute2x128_si256(ymm0[1], ymm0[3], 0x31); + + /* Store the result vectors in proper order */ + for (j = 0; j < 4; j++) { + _mm256_storeu_si256((__m256i * )(dest + (i * bytesoftype) + (j * sizeof(__m256i))), ymm1[j]); + } + } +} + +/* Routine optimized for unshuffling a buffer for a type size of 8 bytes. */ +static void +unshuffle8_avx2(uint8_t* const dest, const uint8_t* const src, + const size_t vectorizable_elements, const size_t total_elements) { + static const size_t bytesoftype = 8; + size_t i; + int j; + __m256i ymm0[8], ymm1[8]; + + for (i = 0; i < vectorizable_elements; i += sizeof(__m256i)) { + /* Fetch 32 elements (256 bytes) into 8 YMM registers. */ + const uint8_t* const src_for_ith_element = src + i; + for (j = 0; j < 8; j++) { + ymm0[j] = _mm256_loadu_si256((__m256i * )(src_for_ith_element + (j * total_elements))); + } + /* Shuffle bytes */ + for (j = 0; j < 4; j++) { + /* Compute the low 32 bytes */ + ymm1[j] = _mm256_unpacklo_epi8(ymm0[j * 2], ymm0[j * 2 + 1]); + /* Compute the hi 32 bytes */ + ymm1[4 + j] = _mm256_unpackhi_epi8(ymm0[j * 2], ymm0[j * 2 + 1]); + } + /* Shuffle words */ + for (j = 0; j < 4; j++) { + /* Compute the low 32 bytes */ + ymm0[j] = _mm256_unpacklo_epi16(ymm1[j * 2], ymm1[j * 2 + 1]); + /* Compute the hi 32 bytes */ + ymm0[4 + j] = _mm256_unpackhi_epi16(ymm1[j * 2], ymm1[j * 2 + 1]); + } + for (j = 0; j < 8; j++) { + ymm0[j] = _mm256_permute4x64_epi64(ymm0[j], 0xd8); + } + + /* Shuffle 4-byte dwords */ + for (j = 0; j < 4; j++) { + /* Compute the low 32 bytes */ + ymm1[j] = _mm256_unpacklo_epi32(ymm0[j * 2], ymm0[j * 2 + 1]); + /* Compute the hi 32 bytes */ + ymm1[4 + j] = _mm256_unpackhi_epi32(ymm0[j * 2], ymm0[j * 2 + 1]); + } + + /* Store the result vectors in proper order */ + _mm256_storeu_si256((__m256i * )(dest + (i * bytesoftype) + (0 * sizeof(__m256i))), ymm1[0]); + _mm256_storeu_si256((__m256i * )(dest + (i * bytesoftype) + (1 * sizeof(__m256i))), ymm1[2]); + _mm256_storeu_si256((__m256i * )(dest + (i * bytesoftype) + (2 * sizeof(__m256i))), ymm1[1]); + _mm256_storeu_si256((__m256i * )(dest + (i * bytesoftype) + (3 * sizeof(__m256i))), ymm1[3]); + _mm256_storeu_si256((__m256i * )(dest + (i * bytesoftype) + (4 * sizeof(__m256i))), ymm1[4]); + _mm256_storeu_si256((__m256i * )(dest + (i * bytesoftype) + (5 * sizeof(__m256i))), ymm1[6]); + _mm256_storeu_si256((__m256i * )(dest + (i * bytesoftype) + (6 * sizeof(__m256i))), ymm1[5]); + _mm256_storeu_si256((__m256i * )(dest + (i * bytesoftype) + (7 * sizeof(__m256i))), ymm1[7]); + } +} + +/* Routine optimized for unshuffling a buffer for a type size of 16 bytes. */ +static void +unshuffle16_avx2(uint8_t* const dest, const uint8_t* const src, + const size_t vectorizable_elements, const size_t total_elements) { + static const size_t bytesoftype = 16; + size_t i; + int j; + __m256i ymm0[16], ymm1[16]; + + for (i = 0; i < vectorizable_elements; i += sizeof(__m256i)) { + /* Fetch 32 elements (512 bytes) into 16 YMM registers. */ + const uint8_t* const src_for_ith_element = src + i; + for (j = 0; j < 16; j++) { + ymm0[j] = _mm256_loadu_si256((__m256i * )(src_for_ith_element + (j * total_elements))); + } + + /* Shuffle bytes */ + for (j = 0; j < 8; j++) { + /* Compute the low 32 bytes */ + ymm1[j] = _mm256_unpacklo_epi8(ymm0[j * 2], ymm0[j * 2 + 1]); + /* Compute the hi 32 bytes */ + ymm1[8 + j] = _mm256_unpackhi_epi8(ymm0[j * 2], ymm0[j * 2 + 1]); + } + /* Shuffle 2-byte words */ + for (j = 0; j < 8; j++) { + /* Compute the low 32 bytes */ + ymm0[j] = _mm256_unpacklo_epi16(ymm1[j * 2], ymm1[j * 2 + 1]); + /* Compute the hi 32 bytes */ + ymm0[8 + j] = _mm256_unpackhi_epi16(ymm1[j * 2], ymm1[j * 2 + 1]); + } + /* Shuffle 4-byte dwords */ + for (j = 0; j < 8; j++) { + /* Compute the low 32 bytes */ + ymm1[j] = _mm256_unpacklo_epi32(ymm0[j * 2], ymm0[j * 2 + 1]); + /* Compute the hi 32 bytes */ + ymm1[8 + j] = _mm256_unpackhi_epi32(ymm0[j * 2], ymm0[j * 2 + 1]); + } + + /* Shuffle 8-byte qwords */ + for (j = 0; j < 8; j++) { + /* Compute the low 32 bytes */ + ymm0[j] = _mm256_unpacklo_epi64(ymm1[j * 2], ymm1[j * 2 + 1]); + /* Compute the hi 32 bytes */ + ymm0[8 + j] = _mm256_unpackhi_epi64(ymm1[j * 2], ymm1[j * 2 + 1]); + } + + for (j = 0; j < 8; j++) { + ymm1[j] = _mm256_permute2x128_si256(ymm0[j], ymm0[j + 8], 0x20); + ymm1[j + 8] = _mm256_permute2x128_si256(ymm0[j], ymm0[j + 8], 0x31); + } + + /* Store the result vectors in proper order */ + _mm256_storeu_si256((__m256i * )(dest + (i * bytesoftype) + (0 * sizeof(__m256i))), ymm1[0]); + _mm256_storeu_si256((__m256i * )(dest + (i * bytesoftype) + (1 * sizeof(__m256i))), ymm1[4]); + _mm256_storeu_si256((__m256i * )(dest + (i * bytesoftype) + (2 * sizeof(__m256i))), ymm1[2]); + _mm256_storeu_si256((__m256i * )(dest + (i * bytesoftype) + (3 * sizeof(__m256i))), ymm1[6]); + _mm256_storeu_si256((__m256i * )(dest + (i * bytesoftype) + (4 * sizeof(__m256i))), ymm1[1]); + _mm256_storeu_si256((__m256i * )(dest + (i * bytesoftype) + (5 * sizeof(__m256i))), ymm1[5]); + _mm256_storeu_si256((__m256i * )(dest + (i * bytesoftype) + (6 * sizeof(__m256i))), ymm1[3]); + _mm256_storeu_si256((__m256i * )(dest + (i * bytesoftype) + (7 * sizeof(__m256i))), ymm1[7]); + _mm256_storeu_si256((__m256i * )(dest + (i * bytesoftype) + (8 * sizeof(__m256i))), ymm1[8]); + _mm256_storeu_si256((__m256i * )(dest + (i * bytesoftype) + (9 * sizeof(__m256i))), ymm1[12]); + _mm256_storeu_si256((__m256i * )(dest + (i * bytesoftype) + (10 * sizeof(__m256i))), ymm1[10]); + _mm256_storeu_si256((__m256i * )(dest + (i * bytesoftype) + (11 * sizeof(__m256i))), ymm1[14]); + _mm256_storeu_si256((__m256i * )(dest + (i * bytesoftype) + (12 * sizeof(__m256i))), ymm1[9]); + _mm256_storeu_si256((__m256i * )(dest + (i * bytesoftype) + (13 * sizeof(__m256i))), ymm1[13]); + _mm256_storeu_si256((__m256i * )(dest + (i * bytesoftype) + (14 * sizeof(__m256i))), ymm1[11]); + _mm256_storeu_si256((__m256i * )(dest + (i * bytesoftype) + (15 * sizeof(__m256i))), ymm1[15]); + } +} + +/* Routine optimized for unshuffling a buffer for a type size larger than 16 bytes. */ +static void +unshuffle16_tiled_avx2(uint8_t* const dest, const uint8_t* const src, + const size_t vectorizable_elements, const size_t total_elements, const size_t bytesoftype) { + size_t i; + int j; + __m256i ymm0[16], ymm1[16]; + + const lldiv_t vecs_per_el = lldiv(bytesoftype, sizeof(__m128i)); + + /* The unshuffle loops are inverted (compared to shuffle_tiled16_avx2) + to optimize cache utilization. */ + size_t offset_into_type; + for (offset_into_type = 0; offset_into_type < bytesoftype; + offset_into_type += (offset_into_type == 0 && vecs_per_el.rem > 0 ? vecs_per_el.rem : sizeof(__m128i))) { + for (i = 0; i < vectorizable_elements; i += sizeof(__m256i)) { + /* Load the first 16 bytes of 32 adjacent elements (512 bytes) into 16 YMM registers */ + const uint8_t* const src_for_ith_element = src + i; + for (j = 0; j < 16; j++) { + ymm0[j] = _mm256_loadu_si256((__m256i * )(src_for_ith_element + (total_elements * (offset_into_type + j)))); + } + + /* Shuffle bytes */ + for (j = 0; j < 8; j++) { + /* Compute the low 32 bytes */ + ymm1[j] = _mm256_unpacklo_epi8(ymm0[j * 2], ymm0[j * 2 + 1]); + /* Compute the hi 32 bytes */ + ymm1[8 + j] = _mm256_unpackhi_epi8(ymm0[j * 2], ymm0[j * 2 + 1]); + } + /* Shuffle 2-byte words */ + for (j = 0; j < 8; j++) { + /* Compute the low 32 bytes */ + ymm0[j] = _mm256_unpacklo_epi16(ymm1[j * 2], ymm1[j * 2 + 1]); + /* Compute the hi 32 bytes */ + ymm0[8 + j] = _mm256_unpackhi_epi16(ymm1[j * 2], ymm1[j * 2 + 1]); + } + /* Shuffle 4-byte dwords */ + for (j = 0; j < 8; j++) { + /* Compute the low 32 bytes */ + ymm1[j] = _mm256_unpacklo_epi32(ymm0[j * 2], ymm0[j * 2 + 1]); + /* Compute the hi 32 bytes */ + ymm1[8 + j] = _mm256_unpackhi_epi32(ymm0[j * 2], ymm0[j * 2 + 1]); + } + + /* Shuffle 8-byte qwords */ + for (j = 0; j < 8; j++) { + /* Compute the low 32 bytes */ + ymm0[j] = _mm256_unpacklo_epi64(ymm1[j * 2], ymm1[j * 2 + 1]); + /* Compute the hi 32 bytes */ + ymm0[8 + j] = _mm256_unpackhi_epi64(ymm1[j * 2], ymm1[j * 2 + 1]); + } + + for (j = 0; j < 8; j++) { + ymm1[j] = _mm256_permute2x128_si256(ymm0[j], ymm0[j + 8], 0x20); + ymm1[j + 8] = _mm256_permute2x128_si256(ymm0[j], ymm0[j + 8], 0x31); + } + + /* Store the result vectors in proper order */ + const uint8_t* const dest_with_offset = dest + offset_into_type; + _mm256_storeu2_m128i( + (__m128i * )(dest_with_offset + (i + 0x01) * bytesoftype), + (__m128i * )(dest_with_offset + (i + 0x00) * bytesoftype), ymm1[0]); + _mm256_storeu2_m128i( + (__m128i * )(dest_with_offset + (i + 0x03) * bytesoftype), + (__m128i * )(dest_with_offset + (i + 0x02) * bytesoftype), ymm1[4]); + _mm256_storeu2_m128i( + (__m128i * )(dest_with_offset + (i + 0x05) * bytesoftype), + (__m128i * )(dest_with_offset + (i + 0x04) * bytesoftype), ymm1[2]); + _mm256_storeu2_m128i( + (__m128i * )(dest_with_offset + (i + 0x07) * bytesoftype), + (__m128i * )(dest_with_offset + (i + 0x06) * bytesoftype), ymm1[6]); + _mm256_storeu2_m128i( + (__m128i * )(dest_with_offset + (i + 0x09) * bytesoftype), + (__m128i * )(dest_with_offset + (i + 0x08) * bytesoftype), ymm1[1]); + _mm256_storeu2_m128i( + (__m128i * )(dest_with_offset + (i + 0x0b) * bytesoftype), + (__m128i * )(dest_with_offset + (i + 0x0a) * bytesoftype), ymm1[5]); + _mm256_storeu2_m128i( + (__m128i * )(dest_with_offset + (i + 0x0d) * bytesoftype), + (__m128i * )(dest_with_offset + (i + 0x0c) * bytesoftype), ymm1[3]); + _mm256_storeu2_m128i( + (__m128i * )(dest_with_offset + (i + 0x0f) * bytesoftype), + (__m128i * )(dest_with_offset + (i + 0x0e) * bytesoftype), ymm1[7]); + _mm256_storeu2_m128i( + (__m128i * )(dest_with_offset + (i + 0x11) * bytesoftype), + (__m128i * )(dest_with_offset + (i + 0x10) * bytesoftype), ymm1[8]); + _mm256_storeu2_m128i( + (__m128i * )(dest_with_offset + (i + 0x13) * bytesoftype), + (__m128i * )(dest_with_offset + (i + 0x12) * bytesoftype), ymm1[12]); + _mm256_storeu2_m128i( + (__m128i * )(dest_with_offset + (i + 0x15) * bytesoftype), + (__m128i * )(dest_with_offset + (i + 0x14) * bytesoftype), ymm1[10]); + _mm256_storeu2_m128i( + (__m128i * )(dest_with_offset + (i + 0x17) * bytesoftype), + (__m128i * )(dest_with_offset + (i + 0x16) * bytesoftype), ymm1[14]); + _mm256_storeu2_m128i( + (__m128i * )(dest_with_offset + (i + 0x19) * bytesoftype), + (__m128i * )(dest_with_offset + (i + 0x18) * bytesoftype), ymm1[9]); + _mm256_storeu2_m128i( + (__m128i * )(dest_with_offset + (i + 0x1b) * bytesoftype), + (__m128i * )(dest_with_offset + (i + 0x1a) * bytesoftype), ymm1[13]); + _mm256_storeu2_m128i( + (__m128i * )(dest_with_offset + (i + 0x1d) * bytesoftype), + (__m128i * )(dest_with_offset + (i + 0x1c) * bytesoftype), ymm1[11]); + _mm256_storeu2_m128i( + (__m128i * )(dest_with_offset + (i + 0x1f) * bytesoftype), + (__m128i * )(dest_with_offset + (i + 0x1e) * bytesoftype), ymm1[15]); + } + } +} + +/* Shuffle a block. This can never fail. */ +void +shuffle_avx2(const size_t bytesoftype, const size_t blocksize, + const uint8_t* const _src, uint8_t* const _dest) { + const size_t vectorized_chunk_size = bytesoftype * sizeof(__m256i); + + /* If the block size is too small to be vectorized, + use the generic implementation. */ + if (blocksize < vectorized_chunk_size) { + shuffle_generic(bytesoftype, blocksize, _src, _dest); + return; + } + + /* If the blocksize is not a multiple of both the typesize and + the vector size, round the blocksize down to the next value + which is a multiple of both. The vectorized shuffle can be + used for that portion of the data, and the naive implementation + can be used for the remaining portion. */ + const size_t vectorizable_bytes = blocksize - (blocksize % vectorized_chunk_size); + + const size_t vectorizable_elements = vectorizable_bytes / bytesoftype; + const size_t total_elements = blocksize / bytesoftype; + + /* Optimized shuffle implementations */ + switch (bytesoftype) { + case 2: + shuffle2_avx2(_dest, _src, vectorizable_elements, total_elements); + break; + case 4: + shuffle4_avx2(_dest, _src, vectorizable_elements, total_elements); + break; + case 8: + shuffle8_avx2(_dest, _src, vectorizable_elements, total_elements); + break; + case 16: + shuffle16_avx2(_dest, _src, vectorizable_elements, total_elements); + break; + default: + /* For types larger than 16 bytes, use the AVX2 tiled shuffle. */ + if (bytesoftype > sizeof(__m128i)) { + shuffle16_tiled_avx2(_dest, _src, vectorizable_elements, total_elements, bytesoftype); + } + else { + /* Non-optimized shuffle */ + shuffle_generic(bytesoftype, blocksize, _src, _dest); + /* The non-optimized function covers the whole buffer, + so we're done processing here. */ + return; + } + } + + /* If the buffer had any bytes at the end which couldn't be handled + by the vectorized implementations, use the non-optimized version + to finish them up. */ + if (vectorizable_bytes < blocksize) { + shuffle_generic_inline(bytesoftype, vectorizable_bytes, blocksize, _src, _dest); + } +} + +/* Unshuffle a block. This can never fail. */ +void +unshuffle_avx2(const size_t bytesoftype, const size_t blocksize, + const uint8_t* const _src, uint8_t* const _dest) { + const size_t vectorized_chunk_size = bytesoftype * sizeof(__m256i); + + /* If the block size is too small to be vectorized, + use the generic implementation. */ + if (blocksize < vectorized_chunk_size) { + unshuffle_generic(bytesoftype, blocksize, _src, _dest); + return; + } + + /* If the blocksize is not a multiple of both the typesize and + the vector size, round the blocksize down to the next value + which is a multiple of both. The vectorized unshuffle can be + used for that portion of the data, and the naive implementation + can be used for the remaining portion. */ + const size_t vectorizable_bytes = blocksize - (blocksize % vectorized_chunk_size); + + const size_t vectorizable_elements = vectorizable_bytes / bytesoftype; + const size_t total_elements = blocksize / bytesoftype; + + /* Optimized unshuffle implementations */ + switch (bytesoftype) { + case 2: + unshuffle2_avx2(_dest, _src, vectorizable_elements, total_elements); + break; + case 4: + unshuffle4_avx2(_dest, _src, vectorizable_elements, total_elements); + break; + case 8: + unshuffle8_avx2(_dest, _src, vectorizable_elements, total_elements); + break; + case 16: + unshuffle16_avx2(_dest, _src, vectorizable_elements, total_elements); + break; + default: + /* For types larger than 16 bytes, use the AVX2 tiled unshuffle. */ + if (bytesoftype > sizeof(__m128i)) { + unshuffle16_tiled_avx2(_dest, _src, vectorizable_elements, total_elements, bytesoftype); + } + else { + /* Non-optimized unshuffle */ + unshuffle_generic(bytesoftype, blocksize, _src, _dest); + /* The non-optimized function covers the whole buffer, + so we're done processing here. */ + return; + } + } + + /* If the buffer had any bytes at the end which couldn't be handled + by the vectorized implementations, use the non-optimized version + to finish them up. */ + if (vectorizable_bytes < blocksize) { + unshuffle_generic_inline(bytesoftype, vectorizable_bytes, blocksize, _src, _dest); + } +} diff --git a/src/blosc2/blosc/shuffle-avx2.h b/src/blosc2/blosc/shuffle-avx2.h new file mode 100644 index 0000000..98c053f --- /dev/null +++ b/src/blosc2/blosc/shuffle-avx2.h @@ -0,0 +1,38 @@ +/********************************************************************* + Blosc - Blocked Shuffling and Compression Library + + Copyright (C) 2021 The Blosc Developers + https://blosc.org + License: BSD 3-Clause (see LICENSE.txt) + + See LICENSE.txt for details about copyright and rights to use. +**********************************************************************/ + +/* AVX2-accelerated shuffle/unshuffle routines. */ + +#ifndef SHUFFLE_AVX2_H +#define SHUFFLE_AVX2_H + +#include "blosc2/blosc2-common.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/** + AVX2-accelerated shuffle routine. +*/ +BLOSC_NO_EXPORT void shuffle_avx2(const int32_t bytesoftype, const int32_t blocksize, + const uint8_t *_src, uint8_t *_dest); + +/** + AVX2-accelerated unshuffle routine. +*/ +BLOSC_NO_EXPORT void unshuffle_avx2(const int32_t bytesoftype, const int32_t blocksize, + const uint8_t *_src, uint8_t *_dest); + +#ifdef __cplusplus +} +#endif + +#endif /* SHUFFLE_AVX2_H */ diff --git a/src/blosc2/blosc/shuffle-generic.c b/src/blosc2/blosc/shuffle-generic.c new file mode 100644 index 0000000..ebd0720 --- /dev/null +++ b/src/blosc2/blosc/shuffle-generic.c @@ -0,0 +1,25 @@ +/********************************************************************* + Blosc - Blocked Shuffling and Compression Library + + Copyright (C) 2021 The Blosc Developers + https://blosc.org + License: BSD 3-Clause (see LICENSE.txt) + + See LICENSE.txt for details about copyright and rights to use. +**********************************************************************/ + +#include "shuffle-generic.h" + +/* Shuffle a block. This can never fail. */ +void shuffle_generic(const int32_t bytesoftype, const int32_t blocksize, + const uint8_t *_src, uint8_t *_dest) { + /* Non-optimized shuffle */ + shuffle_generic_inline(bytesoftype, 0, blocksize, _src, _dest); +} + +/* Unshuffle a block. This can never fail. */ +void unshuffle_generic(const int32_t bytesoftype, const int32_t blocksize, + const uint8_t *_src, uint8_t *_dest) { + /* Non-optimized unshuffle */ + unshuffle_generic_inline(bytesoftype, 0, blocksize, _src, _dest); +} diff --git a/src/blosc2/blosc/shuffle-generic.h b/src/blosc2/blosc/shuffle-generic.h new file mode 100644 index 0000000..6ed95fa --- /dev/null +++ b/src/blosc2/blosc/shuffle-generic.h @@ -0,0 +1,102 @@ +/********************************************************************* + Blosc - Blocked Shuffling and Compression Library + + Copyright (C) 2021 The Blosc Developers + https://blosc.org + License: BSD 3-Clause (see LICENSE.txt) + + See LICENSE.txt for details about copyright and rights to use. +**********************************************************************/ + +/********************************************************************* + Generic (non-hardware-accelerated) shuffle/unshuffle routines. + These are used when hardware-accelerated functions aren't available + for a particular platform; they are also used by the hardware- + accelerated functions to handle any remaining elements in a block + which isn't a multiple of the hardware's vector size. +**********************************************************************/ + + +#ifndef SHUFFLE_GENERIC_H +#define SHUFFLE_GENERIC_H + +#include "blosc2/blosc2-common.h" +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/** + Generic (non-hardware-accelerated) shuffle routine. + This is the pure element-copying nested loop. It is used by the + generic shuffle implementation and also by the vectorized shuffle + implementations to process any remaining elements in a block which + is not a multiple of (type_size * vector_size). +*/ +inline static void shuffle_generic_inline(const int32_t type_size, + const int32_t vectorizable_blocksize, const int32_t blocksize, + const uint8_t *_src, uint8_t *_dest) { + int32_t i, j; + /* Calculate the number of elements in the block. */ + const int32_t neblock_quot = blocksize / type_size; + const int32_t neblock_rem = blocksize % type_size; + const int32_t vectorizable_elements = vectorizable_blocksize / type_size; + + + /* Non-optimized shuffle */ + for (j = 0; j < type_size; j++) { + for (i = vectorizable_elements; i < (int32_t)neblock_quot; i++) { + _dest[j * neblock_quot + i] = _src[i * type_size + j]; + } + } + + /* Copy any leftover bytes in the block without shuffling them. */ + memcpy(_dest + (blocksize - neblock_rem), _src + (blocksize - neblock_rem), neblock_rem); +} + +/** + Generic (non-hardware-accelerated) unshuffle routine. + This is the pure element-copying nested loop. It is used by the + generic unshuffle implementation and also by the vectorized unshuffle + implementations to process any remaining elements in a block which + is not a multiple of (type_size * vector_size). +*/ +inline static void unshuffle_generic_inline(const int32_t type_size, + const int32_t vectorizable_blocksize, const int32_t blocksize, + const uint8_t *_src, uint8_t *_dest) { + int32_t i, j; + + /* Calculate the number of elements in the block. */ + const int32_t neblock_quot = blocksize / type_size; + const int32_t neblock_rem = blocksize % type_size; + const int32_t vectorizable_elements = vectorizable_blocksize / type_size; + + /* Non-optimized unshuffle */ + for (i = vectorizable_elements; i < (int32_t)neblock_quot; i++) { + for (j = 0; j < type_size; j++) { + _dest[i * type_size + j] = _src[j * neblock_quot + i]; + } + } + + /* Copy any leftover bytes in the block without unshuffling them. */ + memcpy(_dest + (blocksize - neblock_rem), _src + (blocksize - neblock_rem), neblock_rem); +} + +/** + Generic (non-hardware-accelerated) shuffle routine. +*/ +BLOSC_NO_EXPORT void shuffle_generic(const int32_t bytesoftype, const int32_t blocksize, + const uint8_t *_src, uint8_t *_dest); + +/** + Generic (non-hardware-accelerated) unshuffle routine. +*/ +BLOSC_NO_EXPORT void unshuffle_generic(const int32_t bytesoftype, const int32_t blocksize, + const uint8_t *_src, uint8_t *_dest); + +#ifdef __cplusplus +} +#endif + +#endif /* SHUFFLE_GENERIC_H */ diff --git a/src/blosc2/blosc/shuffle-neon.c b/src/blosc2/blosc/shuffle-neon.c new file mode 100644 index 0000000..97c84da --- /dev/null +++ b/src/blosc2/blosc/shuffle-neon.c @@ -0,0 +1,416 @@ +/********************************************************************* + Blosc - Blocked Shuffling and Compression Library + + Copyright (C) 2021 Lucian Marc + https://blosc.org + License: BSD 3-Clause (see LICENSE.txt) + + See LICENSE.txt for details about copyright and rights to use. +**********************************************************************/ + + +#include "shuffle-generic.h" +#include "shuffle-neon.h" + +/* Make sure NEON is available for the compilation target and compiler. */ +#if !defined(__ARM_NEON) +#error NEON is not supported by the target architecture/platform and/or this compiler. +#endif + +#include + + +/* The next is useful for debugging purposes */ +#if 0 +#include +#include + +static void printmem(uint8_t* buf) +{ + printf("%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,\n", + buf[0], buf[1], buf[2], buf[3], + buf[4], buf[5], buf[6], buf[7], + buf[8], buf[9], buf[10], buf[11], + buf[12], buf[13], buf[14], buf[15]); +} +#endif + + +/* Routine optimized for shuffling a buffer for a type size of 2 bytes. */ +static void +shuffle2_neon(uint8_t *const dest, const uint8_t *const src, + const size_t vectorizable_elements, const size_t total_elements) { + size_t i, k; + static const size_t bytesoftype = 2; + uint8x16x2_t r0; + + for (i = 0, k = 0; i < vectorizable_elements * bytesoftype; i += 32, k++) { + /* Load (and permute) 32 bytes to the structure r0 */ + r0 = vld2q_u8(src + i); + /* Store the results in the destination vector */ + vst1q_u8(dest + total_elements * 0 + k * 16, r0.val[0]); + vst1q_u8(dest + total_elements * 1 + k * 16, r0.val[1]); + } +} + +/* Routine optimized for shuffling a buffer for a type size of 4 bytes. */ +static void +shuffle4_neon(uint8_t *const dest, const uint8_t *const src, + const size_t vectorizable_elements, const size_t total_elements) { + size_t i, k; + static const size_t bytesoftype = 4; + uint8x16x4_t r0; + + for (i = 0, k = 0; i < vectorizable_elements * bytesoftype; i += 64, k++) { + /* Load (and permute) 64 bytes to the structure r0 */ + r0 = vld4q_u8(src + i); + /* Store the results in the destination vector */ + vst1q_u8(dest + total_elements * 0 + k * 16, r0.val[0]); + vst1q_u8(dest + total_elements * 1 + k * 16, r0.val[1]); + vst1q_u8(dest + total_elements * 2 + k * 16, r0.val[2]); + vst1q_u8(dest + total_elements * 3 + k * 16, r0.val[3]); + } +} + +/* Routine optimized for shuffling a buffer for a type size of 8 bytes. */ +static void +shuffle8_neon(uint8_t *const dest, const uint8_t *const src, + const size_t vectorizable_elements, const size_t total_elements) { + size_t i, k; + static const size_t bytesoftype = 8; + uint8x8x2_t r0[4]; + uint16x4x2_t r1[4]; + uint32x2x2_t r2[4]; + + for (i = 0, k = 0; i < vectorizable_elements * bytesoftype; i += 64, k++) { + /* Load and interleave groups of 8 bytes (64 bytes) to the structure r0 */ + r0[0] = vzip_u8(vld1_u8(src + i + 0 * 8), vld1_u8(src + i + 1 * 8)); + r0[1] = vzip_u8(vld1_u8(src + i + 2 * 8), vld1_u8(src + i + 3 * 8)); + r0[2] = vzip_u8(vld1_u8(src + i + 4 * 8), vld1_u8(src + i + 5 * 8)); + r0[3] = vzip_u8(vld1_u8(src + i + 6 * 8), vld1_u8(src + i + 7 * 8)); + /* Interleave 16 bytes */ + r1[0] = vzip_u16(vreinterpret_u16_u8(r0[0].val[0]), vreinterpret_u16_u8(r0[1].val[0])); + r1[1] = vzip_u16(vreinterpret_u16_u8(r0[0].val[1]), vreinterpret_u16_u8(r0[1].val[1])); + r1[2] = vzip_u16(vreinterpret_u16_u8(r0[2].val[0]), vreinterpret_u16_u8(r0[3].val[0])); + r1[3] = vzip_u16(vreinterpret_u16_u8(r0[2].val[1]), vreinterpret_u16_u8(r0[3].val[1])); + /* Interleave 32 bytes */ + r2[0] = vzip_u32(vreinterpret_u32_u16(r1[0].val[0]), vreinterpret_u32_u16(r1[2].val[0])); + r2[1] = vzip_u32(vreinterpret_u32_u16(r1[0].val[1]), vreinterpret_u32_u16(r1[2].val[1])); + r2[2] = vzip_u32(vreinterpret_u32_u16(r1[1].val[0]), vreinterpret_u32_u16(r1[3].val[0])); + r2[3] = vzip_u32(vreinterpret_u32_u16(r1[1].val[1]), vreinterpret_u32_u16(r1[3].val[1])); + /* Store the results in the destination vector */ + vst1_u8(dest + k * 8 + 0 * total_elements, vreinterpret_u8_u32(r2[0].val[0])); + vst1_u8(dest + k * 8 + 1 * total_elements, vreinterpret_u8_u32(r2[0].val[1])); + vst1_u8(dest + k * 8 + 2 * total_elements, vreinterpret_u8_u32(r2[1].val[0])); + vst1_u8(dest + k * 8 + 3 * total_elements, vreinterpret_u8_u32(r2[1].val[1])); + vst1_u8(dest + k * 8 + 4 * total_elements, vreinterpret_u8_u32(r2[2].val[0])); + vst1_u8(dest + k * 8 + 5 * total_elements, vreinterpret_u8_u32(r2[2].val[1])); + vst1_u8(dest + k * 8 + 6 * total_elements, vreinterpret_u8_u32(r2[3].val[0])); + vst1_u8(dest + k * 8 + 7 * total_elements, vreinterpret_u8_u32(r2[3].val[1])); + } +} + +/* Routine optimized for shuffling a buffer for a type size of 16 bytes. */ +static void +shuffle16_neon(uint8_t *const dest, const uint8_t *const src, + const size_t vectorizable_elements, const size_t total_elements) { + size_t i, k; + static const size_t bytesoftype = 16; + uint8x8x2_t r0[8]; + uint16x4x2_t r1[8]; + uint32x2x2_t r2[8]; + + for (i = 0, k = 0; i < vectorizable_elements * bytesoftype; i += 128, k++) { + /* Load and interleave groups of 16 bytes (128 bytes) to the structure r0 */ + r0[0] = vzip_u8(vld1_u8(src + i + 0 * 8), vld1_u8(src + i + 2 * 8)); + r0[1] = vzip_u8(vld1_u8(src + i + 1 * 8), vld1_u8(src + i + 3 * 8)); + r0[2] = vzip_u8(vld1_u8(src + i + 4 * 8), vld1_u8(src + i + 6 * 8)); + r0[3] = vzip_u8(vld1_u8(src + i + 5 * 8), vld1_u8(src + i + 7 * 8)); + r0[4] = vzip_u8(vld1_u8(src + i + 8 * 8), vld1_u8(src + i + 10 * 8)); + r0[5] = vzip_u8(vld1_u8(src + i + 9 * 8), vld1_u8(src + i + 11 * 8)); + r0[6] = vzip_u8(vld1_u8(src + i + 12 * 8), vld1_u8(src + i + 14 * 8)); + r0[7] = vzip_u8(vld1_u8(src + i + 13 * 8), vld1_u8(src + i + 15 * 8)); + /* Interleave 16 bytes */ + r1[0] = vzip_u16(vreinterpret_u16_u8(r0[0].val[0]), vreinterpret_u16_u8(r0[2].val[0])); + r1[1] = vzip_u16(vreinterpret_u16_u8(r0[0].val[1]), vreinterpret_u16_u8(r0[2].val[1])); + r1[2] = vzip_u16(vreinterpret_u16_u8(r0[1].val[0]), vreinterpret_u16_u8(r0[3].val[0])); + r1[3] = vzip_u16(vreinterpret_u16_u8(r0[1].val[1]), vreinterpret_u16_u8(r0[3].val[1])); + r1[4] = vzip_u16(vreinterpret_u16_u8(r0[4].val[0]), vreinterpret_u16_u8(r0[6].val[0])); + r1[5] = vzip_u16(vreinterpret_u16_u8(r0[4].val[1]), vreinterpret_u16_u8(r0[6].val[1])); + r1[6] = vzip_u16(vreinterpret_u16_u8(r0[5].val[0]), vreinterpret_u16_u8(r0[7].val[0])); + r1[7] = vzip_u16(vreinterpret_u16_u8(r0[5].val[1]), vreinterpret_u16_u8(r0[7].val[1])); + /* Interleave 32 bytes */ + r2[0] = vzip_u32(vreinterpret_u32_u16(r1[0].val[0]), vreinterpret_u32_u16(r1[4].val[0])); + r2[1] = vzip_u32(vreinterpret_u32_u16(r1[0].val[1]), vreinterpret_u32_u16(r1[4].val[1])); + r2[2] = vzip_u32(vreinterpret_u32_u16(r1[1].val[0]), vreinterpret_u32_u16(r1[5].val[0])); + r2[3] = vzip_u32(vreinterpret_u32_u16(r1[1].val[1]), vreinterpret_u32_u16(r1[5].val[1])); + r2[4] = vzip_u32(vreinterpret_u32_u16(r1[2].val[0]), vreinterpret_u32_u16(r1[6].val[0])); + r2[5] = vzip_u32(vreinterpret_u32_u16(r1[2].val[1]), vreinterpret_u32_u16(r1[6].val[1])); + r2[6] = vzip_u32(vreinterpret_u32_u16(r1[3].val[0]), vreinterpret_u32_u16(r1[7].val[0])); + r2[7] = vzip_u32(vreinterpret_u32_u16(r1[3].val[1]), vreinterpret_u32_u16(r1[7].val[1])); + /* Store the results to the destination vector */ + vst1_u8(dest + k * 8 + 0 * total_elements, vreinterpret_u8_u32(r2[0].val[0])); + vst1_u8(dest + k * 8 + 1 * total_elements, vreinterpret_u8_u32(r2[0].val[1])); + vst1_u8(dest + k * 8 + 2 * total_elements, vreinterpret_u8_u32(r2[1].val[0])); + vst1_u8(dest + k * 8 + 3 * total_elements, vreinterpret_u8_u32(r2[1].val[1])); + vst1_u8(dest + k * 8 + 4 * total_elements, vreinterpret_u8_u32(r2[2].val[0])); + vst1_u8(dest + k * 8 + 5 * total_elements, vreinterpret_u8_u32(r2[2].val[1])); + vst1_u8(dest + k * 8 + 6 * total_elements, vreinterpret_u8_u32(r2[3].val[0])); + vst1_u8(dest + k * 8 + 7 * total_elements, vreinterpret_u8_u32(r2[3].val[1])); + vst1_u8(dest + k * 8 + 8 * total_elements, vreinterpret_u8_u32(r2[4].val[0])); + vst1_u8(dest + k * 8 + 9 * total_elements, vreinterpret_u8_u32(r2[4].val[1])); + vst1_u8(dest + k * 8 + 10 * total_elements, vreinterpret_u8_u32(r2[5].val[0])); + vst1_u8(dest + k * 8 + 11 * total_elements, vreinterpret_u8_u32(r2[5].val[1])); + vst1_u8(dest + k * 8 + 12 * total_elements, vreinterpret_u8_u32(r2[6].val[0])); + vst1_u8(dest + k * 8 + 13 * total_elements, vreinterpret_u8_u32(r2[6].val[1])); + vst1_u8(dest + k * 8 + 14 * total_elements, vreinterpret_u8_u32(r2[7].val[0])); + vst1_u8(dest + k * 8 + 15 * total_elements, vreinterpret_u8_u32(r2[7].val[1])); + } +} + +/* Routine optimized for unshuffling a buffer for a type size of 2 bytes. */ +static void +unshuffle2_neon(uint8_t *const dest, const uint8_t *const src, + const size_t vectorizable_elements, const size_t total_elements) { + size_t i, k; + static const size_t bytesoftype = 2; + uint8x16x2_t r0; + + for (i = 0, k = 0; i < vectorizable_elements * bytesoftype; i += 32, k++) { + /* Load 32 bytes to the structure r0 */ + r0.val[0] = vld1q_u8(src + total_elements * 0 + k * 16); + r0.val[1] = vld1q_u8(src + total_elements * 1 + k * 16); + /* Store (with permutation) the results in the destination vector */ + vst2q_u8(dest + k * 32, r0); + } +} + +/* Routine optimized for unshuffling a buffer for a type size of 4 bytes. */ +static void +unshuffle4_neon(uint8_t *const dest, const uint8_t *const src, + const size_t vectorizable_elements, const size_t total_elements) { + size_t i, k; + static const size_t bytesoftype = 4; + uint8x16x4_t r0; + + for (i = 0, k = 0; i < vectorizable_elements * bytesoftype; i += 64, k++) { + /* load 64 bytes to the structure r0 */ + r0.val[0] = vld1q_u8(src + total_elements * 0 + k * 16); + r0.val[1] = vld1q_u8(src + total_elements * 1 + k * 16); + r0.val[2] = vld1q_u8(src + total_elements * 2 + k * 16); + r0.val[3] = vld1q_u8(src + total_elements * 3 + k * 16); + /* Store (with permutation) the results in the destination vector */ + vst4q_u8(dest + k * 64, r0); + } +} + +/* Routine optimized for unshuffling a buffer for a type size of 8 bytes. */ +static void +unshuffle8_neon(uint8_t *const dest, const uint8_t *const src, + const size_t vectorizable_elements, const size_t total_elements) { + size_t i, k; + static const size_t bytesoftype = 8; + uint8x8x2_t r0[4]; + uint16x4x2_t r1[4]; + uint32x2x2_t r2[4]; + + for (i = 0, k = 0; i < vectorizable_elements * bytesoftype; i += 64, k++) { + /* Load and interleave groups of 8 bytes (64 bytes) to the structure r0 */ + r0[0] = vzip_u8(vld1_u8(src + 0 * total_elements + k * 8), vld1_u8(src + 1 * total_elements + k * 8)); + r0[1] = vzip_u8(vld1_u8(src + 2 * total_elements + k * 8), vld1_u8(src + 3 * total_elements + k * 8)); + r0[2] = vzip_u8(vld1_u8(src + 4 * total_elements + k * 8), vld1_u8(src + 5 * total_elements + k * 8)); + r0[3] = vzip_u8(vld1_u8(src + 6 * total_elements + k * 8), vld1_u8(src + 7 * total_elements + k * 8)); + /* Interleave 16 bytes */ + r1[0] = vzip_u16(vreinterpret_u16_u8(r0[0].val[0]), vreinterpret_u16_u8(r0[1].val[0])); + r1[1] = vzip_u16(vreinterpret_u16_u8(r0[0].val[1]), vreinterpret_u16_u8(r0[1].val[1])); + r1[2] = vzip_u16(vreinterpret_u16_u8(r0[2].val[0]), vreinterpret_u16_u8(r0[3].val[0])); + r1[3] = vzip_u16(vreinterpret_u16_u8(r0[2].val[1]), vreinterpret_u16_u8(r0[3].val[1])); + /* Interleave 32 bytes */ + r2[0] = vzip_u32(vreinterpret_u32_u16(r1[0].val[0]), vreinterpret_u32_u16(r1[2].val[0])); + r2[1] = vzip_u32(vreinterpret_u32_u16(r1[0].val[1]), vreinterpret_u32_u16(r1[2].val[1])); + r2[2] = vzip_u32(vreinterpret_u32_u16(r1[1].val[0]), vreinterpret_u32_u16(r1[3].val[0])); + r2[3] = vzip_u32(vreinterpret_u32_u16(r1[1].val[1]), vreinterpret_u32_u16(r1[3].val[1])); + /* Store the results in the destination vector */ + vst1_u8(dest + i + 0 * 8, vreinterpret_u8_u32(r2[0].val[0])); + vst1_u8(dest + i + 1 * 8, vreinterpret_u8_u32(r2[0].val[1])); + vst1_u8(dest + i + 2 * 8, vreinterpret_u8_u32(r2[1].val[0])); + vst1_u8(dest + i + 3 * 8, vreinterpret_u8_u32(r2[1].val[1])); + vst1_u8(dest + i + 4 * 8, vreinterpret_u8_u32(r2[2].val[0])); + vst1_u8(dest + i + 5 * 8, vreinterpret_u8_u32(r2[2].val[1])); + vst1_u8(dest + i + 6 * 8, vreinterpret_u8_u32(r2[3].val[0])); + vst1_u8(dest + i + 7 * 8, vreinterpret_u8_u32(r2[3].val[1])); + + } +} + +/* Routine optimized for unshuffling a buffer for a type size of 16 bytes. */ +static void +unshuffle16_neon(uint8_t *const dest, const uint8_t *const src, + const size_t vectorizable_elements, const size_t total_elements) { + size_t i, k; + static const size_t bytesoftype = 16; + uint8x8x2_t r0[8]; + uint16x4x2_t r1[8]; + uint32x2x2_t r2[8]; + + for (i = 0, k = 0; i < vectorizable_elements * bytesoftype; i += 128, k++) { + /* Load and interleave groups of 16 bytes (128 bytes) to the structure r0*/ + r0[0] = vzip_u8(vld1_u8(src + k * 8 + 0 * total_elements), vld1_u8(src + k * 8 + 1 * total_elements)); + r0[1] = vzip_u8(vld1_u8(src + k * 8 + 2 * total_elements), vld1_u8(src + k * 8 + 3 * total_elements)); + r0[2] = vzip_u8(vld1_u8(src + k * 8 + 4 * total_elements), vld1_u8(src + k * 8 + 5 * total_elements)); + r0[3] = vzip_u8(vld1_u8(src + k * 8 + 6 * total_elements), vld1_u8(src + k * 8 + 7 * total_elements)); + r0[4] = vzip_u8(vld1_u8(src + k * 8 + 8 * total_elements), vld1_u8(src + k * 8 + 9 * total_elements)); + r0[5] = vzip_u8(vld1_u8(src + k * 8 + 10 * total_elements), vld1_u8(src + k * 8 + 11 * total_elements)); + r0[6] = vzip_u8(vld1_u8(src + k * 8 + 12 * total_elements), vld1_u8(src + k * 8 + 13 * total_elements)); + r0[7] = vzip_u8(vld1_u8(src + k * 8 + 14 * total_elements), vld1_u8(src + k * 8 + 15 * total_elements)); + /* Interleave 16 bytes */ + r1[0] = vzip_u16(vreinterpret_u16_u8(r0[0].val[0]), vreinterpret_u16_u8(r0[1].val[0])); + r1[1] = vzip_u16(vreinterpret_u16_u8(r0[0].val[1]), vreinterpret_u16_u8(r0[1].val[1])); + r1[2] = vzip_u16(vreinterpret_u16_u8(r0[2].val[0]), vreinterpret_u16_u8(r0[3].val[0])); + r1[3] = vzip_u16(vreinterpret_u16_u8(r0[2].val[1]), vreinterpret_u16_u8(r0[3].val[1])); + r1[4] = vzip_u16(vreinterpret_u16_u8(r0[4].val[0]), vreinterpret_u16_u8(r0[5].val[0])); + r1[5] = vzip_u16(vreinterpret_u16_u8(r0[4].val[1]), vreinterpret_u16_u8(r0[5].val[1])); + r1[6] = vzip_u16(vreinterpret_u16_u8(r0[6].val[0]), vreinterpret_u16_u8(r0[7].val[0])); + r1[7] = vzip_u16(vreinterpret_u16_u8(r0[6].val[1]), vreinterpret_u16_u8(r0[7].val[1])); + /* Interleave 32 bytes */ + r2[0] = vzip_u32(vreinterpret_u32_u16(r1[0].val[0]), vreinterpret_u32_u16(r1[2].val[0])); + r2[1] = vzip_u32(vreinterpret_u32_u16(r1[0].val[1]), vreinterpret_u32_u16(r1[2].val[1])); + r2[2] = vzip_u32(vreinterpret_u32_u16(r1[1].val[0]), vreinterpret_u32_u16(r1[3].val[0])); + r2[3] = vzip_u32(vreinterpret_u32_u16(r1[1].val[1]), vreinterpret_u32_u16(r1[3].val[1])); + r2[4] = vzip_u32(vreinterpret_u32_u16(r1[4].val[0]), vreinterpret_u32_u16(r1[6].val[0])); + r2[5] = vzip_u32(vreinterpret_u32_u16(r1[4].val[1]), vreinterpret_u32_u16(r1[6].val[1])); + r2[6] = vzip_u32(vreinterpret_u32_u16(r1[5].val[0]), vreinterpret_u32_u16(r1[7].val[0])); + r2[7] = vzip_u32(vreinterpret_u32_u16(r1[5].val[1]), vreinterpret_u32_u16(r1[7].val[1])); + /* Store the results in the destination vector */ + vst1_u8(dest + i + 0 * 8, vreinterpret_u8_u32(r2[0].val[0])); + vst1_u8(dest + i + 1 * 8, vreinterpret_u8_u32(r2[4].val[0])); + vst1_u8(dest + i + 2 * 8, vreinterpret_u8_u32(r2[0].val[1])); + vst1_u8(dest + i + 3 * 8, vreinterpret_u8_u32(r2[4].val[1])); + vst1_u8(dest + i + 4 * 8, vreinterpret_u8_u32(r2[1].val[0])); + vst1_u8(dest + i + 5 * 8, vreinterpret_u8_u32(r2[5].val[0])); + vst1_u8(dest + i + 6 * 8, vreinterpret_u8_u32(r2[1].val[1])); + vst1_u8(dest + i + 7 * 8, vreinterpret_u8_u32(r2[5].val[1])); + vst1_u8(dest + i + 8 * 8, vreinterpret_u8_u32(r2[2].val[0])); + vst1_u8(dest + i + 9 * 8, vreinterpret_u8_u32(r2[6].val[0])); + vst1_u8(dest + i + 10 * 8, vreinterpret_u8_u32(r2[2].val[1])); + vst1_u8(dest + i + 11 * 8, vreinterpret_u8_u32(r2[6].val[1])); + vst1_u8(dest + i + 12 * 8, vreinterpret_u8_u32(r2[3].val[0])); + vst1_u8(dest + i + 13 * 8, vreinterpret_u8_u32(r2[7].val[0])); + vst1_u8(dest + i + 14 * 8, vreinterpret_u8_u32(r2[3].val[1])); + vst1_u8(dest + i + 15 * 8, vreinterpret_u8_u32(r2[7].val[1])); + } +} + + +/* Shuffle a block. This can never fail. */ +void +shuffle_neon(const int32_t bytesoftype, const int32_t blocksize, + const uint8_t *const _src, uint8_t *_dest) { + int32_t vectorized_chunk_size = 1; + if (bytesoftype == 2 || bytesoftype == 4) { + vectorized_chunk_size = bytesoftype * 16; + } else if (bytesoftype == 8 || bytesoftype == 16) { + vectorized_chunk_size = bytesoftype * 8; + } + /* If the blocksize is not a multiple of both the typesize and + the vector size, round the blocksize down to the next value + which is a multiple of both. The vectorized shuffle can be + used for that portion of the data, and the naive implementation + can be used for the remaining portion. */ + const int32_t vectorizable_bytes = blocksize - (blocksize % vectorized_chunk_size); + const int32_t vectorizable_elements = vectorizable_bytes / bytesoftype; + const int32_t total_elements = blocksize / bytesoftype; + + /* If the block size is too small to be vectorized, + use the generic implementation. */ + if (blocksize < vectorized_chunk_size) { + shuffle_generic(bytesoftype, blocksize, _src, _dest); + return; + } + + /* Optimized shuffle implementations */ + switch (bytesoftype) { + case 2: + shuffle2_neon(_dest, _src, vectorizable_elements, total_elements); + break; + case 4: + shuffle4_neon(_dest, _src, vectorizable_elements, total_elements); + break; + case 8: + shuffle8_neon(_dest, _src, vectorizable_elements, total_elements); + break; + case 16: + shuffle16_neon(_dest, _src, vectorizable_elements, total_elements); + break; + default: + /* Non-optimized shuffle */ + shuffle_generic(bytesoftype, blocksize, _src, _dest); + /* The non-optimized function covers the whole buffer, + so we're done processing here. */ + return; + } + + /* If the buffer had any bytes at the end which couldn't be handled + by the vectorized implementations, use the non-optimized version + to finish them up. */ + if (vectorizable_bytes < blocksize) { + shuffle_generic_inline(bytesoftype, vectorizable_bytes, blocksize, _src, _dest); + } +} + +/* Unshuffle a block. This can never fail. */ +void +unshuffle_neon(const int32_t bytesoftype, const int32_t blocksize, + const uint8_t *const _src, uint8_t *_dest) { + int32_t vectorized_chunk_size = 1; + if (bytesoftype == 2 || bytesoftype == 4) { + vectorized_chunk_size = bytesoftype * 16; + } else if (bytesoftype == 8 || bytesoftype == 16) { + vectorized_chunk_size = bytesoftype * 8; + } + /* If the blocksize is not a multiple of both the typesize and + the vector size, round the blocksize down to the next value + which is a multiple of both. The vectorized unshuffle can be + used for that portion of the data, and the naive implementation + can be used for the remaining portion. */ + const int32_t vectorizable_bytes = blocksize - (blocksize % vectorized_chunk_size); + const int32_t vectorizable_elements = vectorizable_bytes / bytesoftype; + const int32_t total_elements = blocksize / bytesoftype; + + + /* If the block size is too small to be vectorized, + use the generic implementation. */ + if (blocksize < vectorized_chunk_size) { + unshuffle_generic(bytesoftype, blocksize, _src, _dest); + return; + } + + /* Optimized unshuffle implementations */ + switch (bytesoftype) { + case 2: + unshuffle2_neon(_dest, _src, vectorizable_elements, total_elements); + break; + case 4: + unshuffle4_neon(_dest, _src, vectorizable_elements, total_elements); + break; + case 8: + unshuffle8_neon(_dest, _src, vectorizable_elements, total_elements); + break; + case 16: + unshuffle16_neon(_dest, _src, vectorizable_elements, total_elements); + break; + default: + /* Non-optimized unshuffle */ + unshuffle_generic(bytesoftype, blocksize, _src, _dest); + /* The non-optimized function covers the whole buffer, + so we're done processing here. */ + return; + } + + /* If the buffer had any bytes at the end which couldn't be handled + by the vectorized implementations, use the non-optimized version + to finish them up. */ + if (vectorizable_bytes < blocksize) { + unshuffle_generic_inline(bytesoftype, vectorizable_bytes, blocksize, _src, _dest); + } +} diff --git a/src/blosc2/blosc/shuffle-neon.h b/src/blosc2/blosc/shuffle-neon.h new file mode 100644 index 0000000..3b54655 --- /dev/null +++ b/src/blosc2/blosc/shuffle-neon.h @@ -0,0 +1,40 @@ +/********************************************************************* + Blosc - Blocked Shuffling and Compression Library + + Copyright (C) 2021 The Blosc Developers + https://blosc.org + License: BSD 3-Clause (see LICENSE.txt) + + Note: Adapted for NEON by Lucian Marc. + + See LICENSE.txt for details about copyright and rights to use. +**********************************************************************/ + +/* NEON-accelerated shuffle/unshuffle routines. */ + +#ifndef SHUFFLE_NEON_H +#define SHUFFLE_NEON_H + +#include "blosc2/blosc2-common.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/** + NEON-accelerated shuffle routine. +*/ +BLOSC_NO_EXPORT void shuffle_neon(const int32_t bytesoftype, const int32_t blocksize, + const uint8_t* const _src, uint8_t* const _dest); + +/** + NEON-accelerated unshuffle routine. +*/ +BLOSC_NO_EXPORT void unshuffle_neon(const int32_t bytesoftype, const int32_t blocksize, + const uint8_t *_src, uint8_t *_dest); + +#ifdef __cplusplus +} +#endif + +#endif /* SHUFFLE_NEON_H */ diff --git a/src/blosc2/blosc/shuffle-sse2.c b/src/blosc2/blosc/shuffle-sse2.c new file mode 100644 index 0000000..7230856 --- /dev/null +++ b/src/blosc2/blosc/shuffle-sse2.c @@ -0,0 +1,617 @@ +/********************************************************************* + Blosc - Blocked Shuffling and Compression Library + + Copyright (C) 2021 The Blosc Developers + https://blosc.org + License: BSD 3-Clause (see LICENSE.txt) + + See LICENSE.txt for details about copyright and rights to use. +**********************************************************************/ + +#include "shuffle-generic.h" +#include "shuffle-sse2.h" + +/* Make sure SSE2 is available for the compilation target and compiler. */ +#if !defined(__SSE2__) + #error SSE2 is not supported by the target architecture/platform and/or this compiler. +#endif + +#include + + +/* The next is useful for debugging purposes */ +#if 0 +#include +#include + +static void printxmm(__m128i xmm0) +{ + uint8_t buf[16]; + + ((__m128i *)buf)[0] = xmm0; + printf("%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x\n", + buf[0], buf[1], buf[2], buf[3], + buf[4], buf[5], buf[6], buf[7], + buf[8], buf[9], buf[10], buf[11], + buf[12], buf[13], buf[14], buf[15]); +} +#endif + + +/* Routine optimized for shuffling a buffer for a type size of 2 bytes. */ +static void +shuffle2_sse2(uint8_t* const dest, const uint8_t* const src, + const int32_t vectorizable_elements, const int32_t total_elements) { + static const int32_t bytesoftype = 2; + int32_t j; + int k; + uint8_t* dest_for_jth_element; + __m128i xmm0[2], xmm1[2]; + + for (j = 0; j < vectorizable_elements; j += sizeof(__m128i)) { + /* Fetch 16 elements (32 bytes) then transpose bytes, words and double words. */ + for (k = 0; k < 2; k++) { + xmm0[k] = _mm_loadu_si128((__m128i*)(src + (j * bytesoftype) + (k * sizeof(__m128i)))); + xmm0[k] = _mm_shufflelo_epi16(xmm0[k], 0xd8); + xmm0[k] = _mm_shufflehi_epi16(xmm0[k], 0xd8); + xmm0[k] = _mm_shuffle_epi32(xmm0[k], 0xd8); + xmm1[k] = _mm_shuffle_epi32(xmm0[k], 0x4e); + xmm0[k] = _mm_unpacklo_epi8(xmm0[k], xmm1[k]); + xmm0[k] = _mm_shuffle_epi32(xmm0[k], 0xd8); + xmm1[k] = _mm_shuffle_epi32(xmm0[k], 0x4e); + xmm0[k] = _mm_unpacklo_epi16(xmm0[k], xmm1[k]); + xmm0[k] = _mm_shuffle_epi32(xmm0[k], 0xd8); + } + /* Transpose quad words */ + for (k = 0; k < 1; k++) { + xmm1[k * 2] = _mm_unpacklo_epi64(xmm0[k], xmm0[k + 1]); + xmm1[k * 2 + 1] = _mm_unpackhi_epi64(xmm0[k], xmm0[k + 1]); + } + /* Store the result vectors */ + dest_for_jth_element = dest + j; + for (k = 0; k < 2; k++) { + _mm_storeu_si128((__m128i*)(dest_for_jth_element + (k * total_elements)), xmm1[k]); + } + } +} + +/* Routine optimized for shuffling a buffer for a type size of 4 bytes. */ +static void +shuffle4_sse2(uint8_t* const dest, const uint8_t* const src, + const int32_t vectorizable_elements, const int32_t total_elements) { + static const int32_t bytesoftype = 4; + int32_t i; + int j; + uint8_t* dest_for_ith_element; + __m128i xmm0[4], xmm1[4]; + + for (i = 0; i < vectorizable_elements; i += sizeof(__m128i)) { + /* Fetch 16 elements (64 bytes) then transpose bytes and words. */ + for (j = 0; j < 4; j++) { + xmm0[j] = _mm_loadu_si128((__m128i*)(src + (i * bytesoftype) + (j * sizeof(__m128i)))); + xmm1[j] = _mm_shuffle_epi32(xmm0[j], 0xd8); + xmm0[j] = _mm_shuffle_epi32(xmm0[j], 0x8d); + xmm0[j] = _mm_unpacklo_epi8(xmm1[j], xmm0[j]); + xmm1[j] = _mm_shuffle_epi32(xmm0[j], 0x04e); + xmm0[j] = _mm_unpacklo_epi16(xmm0[j], xmm1[j]); + } + /* Transpose double words */ + for (j = 0; j < 2; j++) { + xmm1[j * 2] = _mm_unpacklo_epi32(xmm0[j * 2], xmm0[j * 2 + 1]); + xmm1[j * 2 + 1] = _mm_unpackhi_epi32(xmm0[j * 2], xmm0[j * 2 + 1]); + } + /* Transpose quad words */ + for (j = 0; j < 2; j++) { + xmm0[j * 2] = _mm_unpacklo_epi64(xmm1[j], xmm1[j + 2]); + xmm0[j * 2 + 1] = _mm_unpackhi_epi64(xmm1[j], xmm1[j + 2]); + } + /* Store the result vectors */ + dest_for_ith_element = dest + i; + for (j = 0; j < 4; j++) { + _mm_storeu_si128((__m128i*)(dest_for_ith_element + (j * total_elements)), xmm0[j]); + } + } +} + +/* Routine optimized for shuffling a buffer for a type size of 8 bytes. */ +static void +shuffle8_sse2(uint8_t* const dest, const uint8_t* const src, + const int32_t vectorizable_elements, const int32_t total_elements) { + static const int32_t bytesoftype = 8; + int32_t j; + int k, l; + uint8_t* dest_for_jth_element; + __m128i xmm0[8], xmm1[8]; + + for (j = 0; j < vectorizable_elements; j += sizeof(__m128i)) { + /* Fetch 16 elements (128 bytes) then transpose bytes. */ + for (k = 0; k < 8; k++) { + xmm0[k] = _mm_loadu_si128((__m128i*)(src + (j * bytesoftype) + (k * sizeof(__m128i)))); + xmm1[k] = _mm_shuffle_epi32(xmm0[k], 0x4e); + xmm1[k] = _mm_unpacklo_epi8(xmm0[k], xmm1[k]); + } + /* Transpose words */ + for (k = 0, l = 0; k < 4; k++, l += 2) { + xmm0[k * 2] = _mm_unpacklo_epi16(xmm1[l], xmm1[l + 1]); + xmm0[k * 2 + 1] = _mm_unpackhi_epi16(xmm1[l], xmm1[l + 1]); + } + /* Transpose double words */ + for (k = 0, l = 0; k < 4; k++, l++) { + if (k == 2) l += 2; + xmm1[k * 2] = _mm_unpacklo_epi32(xmm0[l], xmm0[l + 2]); + xmm1[k * 2 + 1] = _mm_unpackhi_epi32(xmm0[l], xmm0[l + 2]); + } + /* Transpose quad words */ + for (k = 0; k < 4; k++) { + xmm0[k * 2] = _mm_unpacklo_epi64(xmm1[k], xmm1[k + 4]); + xmm0[k * 2 + 1] = _mm_unpackhi_epi64(xmm1[k], xmm1[k + 4]); + } + /* Store the result vectors */ + dest_for_jth_element = dest + j; + for (k = 0; k < 8; k++) { + _mm_storeu_si128((__m128i*)(dest_for_jth_element + (k * total_elements)), xmm0[k]); + } + } +} + +/* Routine optimized for shuffling a buffer for a type size of 16 bytes. */ +static void +shuffle16_sse2(uint8_t* const dest, const uint8_t* const src, + const int32_t vectorizable_elements, const int32_t total_elements) { + static const int32_t bytesoftype = 16; + int32_t j; + int k, l; + uint8_t* dest_for_jth_element; + __m128i xmm0[16], xmm1[16]; + + for (j = 0; j < vectorizable_elements; j += sizeof(__m128i)) { + /* Fetch 16 elements (256 bytes). */ + for (k = 0; k < 16; k++) { + xmm0[k] = _mm_loadu_si128((__m128i*)(src + (j * bytesoftype) + (k * sizeof(__m128i)))); + } + /* Transpose bytes */ + for (k = 0, l = 0; k < 8; k++, l += 2) { + xmm1[k * 2] = _mm_unpacklo_epi8(xmm0[l], xmm0[l + 1]); + xmm1[k * 2 + 1] = _mm_unpackhi_epi8(xmm0[l], xmm0[l + 1]); + } + /* Transpose words */ + for (k = 0, l = -2; k < 8; k++, l++) { + if ((k % 2) == 0) l += 2; + xmm0[k * 2] = _mm_unpacklo_epi16(xmm1[l], xmm1[l + 2]); + xmm0[k * 2 + 1] = _mm_unpackhi_epi16(xmm1[l], xmm1[l + 2]); + } + /* Transpose double words */ + for (k = 0, l = -4; k < 8; k++, l++) { + if ((k % 4) == 0) l += 4; + xmm1[k * 2] = _mm_unpacklo_epi32(xmm0[l], xmm0[l + 4]); + xmm1[k * 2 + 1] = _mm_unpackhi_epi32(xmm0[l], xmm0[l + 4]); + } + /* Transpose quad words */ + for (k = 0; k < 8; k++) { + xmm0[k * 2] = _mm_unpacklo_epi64(xmm1[k], xmm1[k + 8]); + xmm0[k * 2 + 1] = _mm_unpackhi_epi64(xmm1[k], xmm1[k + 8]); + } + /* Store the result vectors */ + dest_for_jth_element = dest + j; + for (k = 0; k < 16; k++) { + _mm_storeu_si128((__m128i*)(dest_for_jth_element + (k * total_elements)), xmm0[k]); + } + } +} + +/* Routine optimized for shuffling a buffer for a type size larger than 16 bytes. */ +static void +shuffle16_tiled_sse2(uint8_t* const dest, const uint8_t* const src, + const int32_t vectorizable_elements, const int32_t total_elements, const int32_t bytesoftype) { + int32_t j; + const int32_t vecs_per_el_rem = bytesoftype % (int32_t)sizeof(__m128i); + int k, l; + uint8_t* dest_for_jth_element; + __m128i xmm0[16], xmm1[16]; + + for (j = 0; j < vectorizable_elements; j += sizeof(__m128i)) { + /* Advance the offset into the type by the vector size (in bytes), unless this is + the initial iteration and the type size is not a multiple of the vector size. + In that case, only advance by the number of bytes necessary so that the number + of remaining bytes in the type will be a multiple of the vector size. */ + int32_t offset_into_type; + for (offset_into_type = 0; offset_into_type < bytesoftype; + offset_into_type += (offset_into_type == 0 && + vecs_per_el_rem > 0 ? vecs_per_el_rem : (int32_t)sizeof(__m128i))) { + + /* Fetch elements in groups of 256 bytes */ + const uint8_t* const src_with_offset = src + offset_into_type; + for (k = 0; k < 16; k++) { + xmm0[k] = _mm_loadu_si128((__m128i*)(src_with_offset + (j + k) * bytesoftype)); + } + /* Transpose bytes */ + for (k = 0, l = 0; k < 8; k++, l += 2) { + xmm1[k * 2] = _mm_unpacklo_epi8(xmm0[l], xmm0[l + 1]); + xmm1[k * 2 + 1] = _mm_unpackhi_epi8(xmm0[l], xmm0[l + 1]); + } + /* Transpose words */ + for (k = 0, l = -2; k < 8; k++, l++) { + if ((k % 2) == 0) l += 2; + xmm0[k * 2] = _mm_unpacklo_epi16(xmm1[l], xmm1[l + 2]); + xmm0[k * 2 + 1] = _mm_unpackhi_epi16(xmm1[l], xmm1[l + 2]); + } + /* Transpose double words */ + for (k = 0, l = -4; k < 8; k++, l++) { + if ((k % 4) == 0) l += 4; + xmm1[k * 2] = _mm_unpacklo_epi32(xmm0[l], xmm0[l + 4]); + xmm1[k * 2 + 1] = _mm_unpackhi_epi32(xmm0[l], xmm0[l + 4]); + } + /* Transpose quad words */ + for (k = 0; k < 8; k++) { + xmm0[k * 2] = _mm_unpacklo_epi64(xmm1[k], xmm1[k + 8]); + xmm0[k * 2 + 1] = _mm_unpackhi_epi64(xmm1[k], xmm1[k + 8]); + } + /* Store the result vectors */ + dest_for_jth_element = dest + j; + for (k = 0; k < 16; k++) { + _mm_storeu_si128((__m128i*)(dest_for_jth_element + (total_elements * (offset_into_type + k))), xmm0[k]); + } + } + } +} + +/* Routine optimized for unshuffling a buffer for a type size of 2 bytes. */ +static void +unshuffle2_sse2(uint8_t* const dest, const uint8_t* const src, + const int32_t vectorizable_elements, const int32_t total_elements) { + static const int32_t bytesoftype = 2; + int32_t i; + int j; + __m128i xmm0[2], xmm1[2]; + + for (i = 0; i < vectorizable_elements; i += sizeof(__m128i)) { + /* Load 16 elements (32 bytes) into 2 XMM registers. */ + const uint8_t* const src_for_ith_element = src + i; + for (j = 0; j < 2; j++) { + xmm0[j] = _mm_loadu_si128((__m128i*)(src_for_ith_element + (j * total_elements))); + } + /* Shuffle bytes */ + /* Compute the low 32 bytes */ + xmm1[0] = _mm_unpacklo_epi8(xmm0[0], xmm0[1]); + /* Compute the hi 32 bytes */ + xmm1[1] = _mm_unpackhi_epi8(xmm0[0], xmm0[1]); + /* Store the result vectors in proper order */ + _mm_storeu_si128((__m128i*)(dest + (i * bytesoftype) + (0 * sizeof(__m128i))), xmm1[0]); + _mm_storeu_si128((__m128i*)(dest + (i * bytesoftype) + (1 * sizeof(__m128i))), xmm1[1]); + } +} + +/* Routine optimized for unshuffling a buffer for a type size of 4 bytes. */ +static void +unshuffle4_sse2(uint8_t* const dest, const uint8_t* const src, + const int32_t vectorizable_elements, const int32_t total_elements) { + static const int32_t bytesoftype = 4; + int32_t i; + int j; + __m128i xmm0[4], xmm1[4]; + + for (i = 0; i < vectorizable_elements; i += sizeof(__m128i)) { + /* Load 16 elements (64 bytes) into 4 XMM registers. */ + const uint8_t* const src_for_ith_element = src + i; + for (j = 0; j < 4; j++) { + xmm0[j] = _mm_loadu_si128((__m128i*)(src_for_ith_element + (j * total_elements))); + } + /* Shuffle bytes */ + for (j = 0; j < 2; j++) { + /* Compute the low 32 bytes */ + xmm1[j] = _mm_unpacklo_epi8(xmm0[j * 2], xmm0[j * 2 + 1]); + /* Compute the hi 32 bytes */ + xmm1[2 + j] = _mm_unpackhi_epi8(xmm0[j * 2], xmm0[j * 2 + 1]); + } + /* Shuffle 2-byte words */ + for (j = 0; j < 2; j++) { + /* Compute the low 32 bytes */ + xmm0[j] = _mm_unpacklo_epi16(xmm1[j * 2], xmm1[j * 2 + 1]); + /* Compute the hi 32 bytes */ + xmm0[2 + j] = _mm_unpackhi_epi16(xmm1[j * 2], xmm1[j * 2 + 1]); + } + /* Store the result vectors in proper order */ + _mm_storeu_si128((__m128i*)(dest + (i * bytesoftype) + (0 * sizeof(__m128i))), xmm0[0]); + _mm_storeu_si128((__m128i*)(dest + (i * bytesoftype) + (1 * sizeof(__m128i))), xmm0[2]); + _mm_storeu_si128((__m128i*)(dest + (i * bytesoftype) + (2 * sizeof(__m128i))), xmm0[1]); + _mm_storeu_si128((__m128i*)(dest + (i * bytesoftype) + (3 * sizeof(__m128i))), xmm0[3]); + } +} + +/* Routine optimized for unshuffling a buffer for a type size of 8 bytes. */ +static void +unshuffle8_sse2(uint8_t* const dest, const uint8_t* const src, + const int32_t vectorizable_elements, const int32_t total_elements) { + static const int32_t bytesoftype = 8; + int32_t i; + int j; + __m128i xmm0[8], xmm1[8]; + + for (i = 0; i < vectorizable_elements; i += sizeof(__m128i)) { + /* Load 16 elements (128 bytes) into 8 XMM registers. */ + const uint8_t* const src_for_ith_element = src + i; + for (j = 0; j < 8; j++) { + xmm0[j] = _mm_loadu_si128((__m128i*)(src_for_ith_element + (j * total_elements))); + } + /* Shuffle bytes */ + for (j = 0; j < 4; j++) { + /* Compute the low 32 bytes */ + xmm1[j] = _mm_unpacklo_epi8(xmm0[j * 2], xmm0[j * 2 + 1]); + /* Compute the hi 32 bytes */ + xmm1[4 + j] = _mm_unpackhi_epi8(xmm0[j * 2], xmm0[j * 2 + 1]); + } + /* Shuffle 2-byte words */ + for (j = 0; j < 4; j++) { + /* Compute the low 32 bytes */ + xmm0[j] = _mm_unpacklo_epi16(xmm1[j * 2], xmm1[j * 2 + 1]); + /* Compute the hi 32 bytes */ + xmm0[4 + j] = _mm_unpackhi_epi16(xmm1[j * 2], xmm1[j * 2 + 1]); + } + /* Shuffle 4-byte dwords */ + for (j = 0; j < 4; j++) { + /* Compute the low 32 bytes */ + xmm1[j] = _mm_unpacklo_epi32(xmm0[j * 2], xmm0[j * 2 + 1]); + /* Compute the hi 32 bytes */ + xmm1[4 + j] = _mm_unpackhi_epi32(xmm0[j * 2], xmm0[j * 2 + 1]); + } + /* Store the result vectors in proper order */ + _mm_storeu_si128((__m128i*)(dest + (i * bytesoftype) + (0 * sizeof(__m128i))), xmm1[0]); + _mm_storeu_si128((__m128i*)(dest + (i * bytesoftype) + (1 * sizeof(__m128i))), xmm1[4]); + _mm_storeu_si128((__m128i*)(dest + (i * bytesoftype) + (2 * sizeof(__m128i))), xmm1[2]); + _mm_storeu_si128((__m128i*)(dest + (i * bytesoftype) + (3 * sizeof(__m128i))), xmm1[6]); + _mm_storeu_si128((__m128i*)(dest + (i * bytesoftype) + (4 * sizeof(__m128i))), xmm1[1]); + _mm_storeu_si128((__m128i*)(dest + (i * bytesoftype) + (5 * sizeof(__m128i))), xmm1[5]); + _mm_storeu_si128((__m128i*)(dest + (i * bytesoftype) + (6 * sizeof(__m128i))), xmm1[3]); + _mm_storeu_si128((__m128i*)(dest + (i * bytesoftype) + (7 * sizeof(__m128i))), xmm1[7]); + } +} + +/* Routine optimized for unshuffling a buffer for a type size of 16 bytes. */ +static void +unshuffle16_sse2(uint8_t* const dest, const uint8_t* const src, + const int32_t vectorizable_elements, const int32_t total_elements) { + static const int32_t bytesoftype = 16; + int32_t i; + int j; + __m128i xmm1[16], xmm2[16]; + + for (i = 0; i < vectorizable_elements; i += sizeof(__m128i)) { + /* Load 16 elements (256 bytes) into 16 XMM registers. */ + const uint8_t* const src_for_ith_element = src + i; + for (j = 0; j < 16; j++) { + xmm1[j] = _mm_loadu_si128((__m128i*)(src_for_ith_element + (j * total_elements))); + } + /* Shuffle bytes */ + for (j = 0; j < 8; j++) { + /* Compute the low 32 bytes */ + xmm2[j] = _mm_unpacklo_epi8(xmm1[j * 2], xmm1[j * 2 + 1]); + /* Compute the hi 32 bytes */ + xmm2[8 + j] = _mm_unpackhi_epi8(xmm1[j * 2], xmm1[j * 2 + 1]); + } + /* Shuffle 2-byte words */ + for (j = 0; j < 8; j++) { + /* Compute the low 32 bytes */ + xmm1[j] = _mm_unpacklo_epi16(xmm2[j * 2], xmm2[j * 2 + 1]); + /* Compute the hi 32 bytes */ + xmm1[8 + j] = _mm_unpackhi_epi16(xmm2[j * 2], xmm2[j * 2 + 1]); + } + /* Shuffle 4-byte dwords */ + for (j = 0; j < 8; j++) { + /* Compute the low 32 bytes */ + xmm2[j] = _mm_unpacklo_epi32(xmm1[j * 2], xmm1[j * 2 + 1]); + /* Compute the hi 32 bytes */ + xmm2[8 + j] = _mm_unpackhi_epi32(xmm1[j * 2], xmm1[j * 2 + 1]); + } + /* Shuffle 8-byte qwords */ + for (j = 0; j < 8; j++) { + /* Compute the low 32 bytes */ + xmm1[j] = _mm_unpacklo_epi64(xmm2[j * 2], xmm2[j * 2 + 1]); + /* Compute the hi 32 bytes */ + xmm1[8 + j] = _mm_unpackhi_epi64(xmm2[j * 2], xmm2[j * 2 + 1]); + } + + /* Store the result vectors in proper order */ + _mm_storeu_si128((__m128i*)(dest + (i * bytesoftype) + (0 * sizeof(__m128i))), xmm1[0]); + _mm_storeu_si128((__m128i*)(dest + (i * bytesoftype) + (1 * sizeof(__m128i))), xmm1[8]); + _mm_storeu_si128((__m128i*)(dest + (i * bytesoftype) + (2 * sizeof(__m128i))), xmm1[4]); + _mm_storeu_si128((__m128i*)(dest + (i * bytesoftype) + (3 * sizeof(__m128i))), xmm1[12]); + _mm_storeu_si128((__m128i*)(dest + (i * bytesoftype) + (4 * sizeof(__m128i))), xmm1[2]); + _mm_storeu_si128((__m128i*)(dest + (i * bytesoftype) + (5 * sizeof(__m128i))), xmm1[10]); + _mm_storeu_si128((__m128i*)(dest + (i * bytesoftype) + (6 * sizeof(__m128i))), xmm1[6]); + _mm_storeu_si128((__m128i*)(dest + (i * bytesoftype) + (7 * sizeof(__m128i))), xmm1[14]); + _mm_storeu_si128((__m128i*)(dest + (i * bytesoftype) + (8 * sizeof(__m128i))), xmm1[1]); + _mm_storeu_si128((__m128i*)(dest + (i * bytesoftype) + (9 * sizeof(__m128i))), xmm1[9]); + _mm_storeu_si128((__m128i*)(dest + (i * bytesoftype) + (10 * sizeof(__m128i))), xmm1[5]); + _mm_storeu_si128((__m128i*)(dest + (i * bytesoftype) + (11 * sizeof(__m128i))), xmm1[13]); + _mm_storeu_si128((__m128i*)(dest + (i * bytesoftype) + (12 * sizeof(__m128i))), xmm1[3]); + _mm_storeu_si128((__m128i*)(dest + (i * bytesoftype) + (13 * sizeof(__m128i))), xmm1[11]); + _mm_storeu_si128((__m128i*)(dest + (i * bytesoftype) + (14 * sizeof(__m128i))), xmm1[7]); + _mm_storeu_si128((__m128i*)(dest + (i * bytesoftype) + (15 * sizeof(__m128i))), xmm1[15]); + } +} + +/* Routine optimized for unshuffling a buffer for a type size larger than 16 bytes. */ +static void +unshuffle16_tiled_sse2(uint8_t* const dest, const uint8_t* const orig, + const int32_t vectorizable_elements, const int32_t total_elements, const int32_t bytesoftype) { + int32_t i; + const int32_t vecs_per_el_rem = bytesoftype % (int32_t)sizeof(__m128i); + + int j; + uint8_t* dest_with_offset; + __m128i xmm1[16], xmm2[16]; + + /* The unshuffle loops are inverted (compared to shuffle_tiled16_sse2) + to optimize cache utilization. */ + int32_t offset_into_type; + for (offset_into_type = 0; offset_into_type < bytesoftype; + offset_into_type += (offset_into_type == 0 && + vecs_per_el_rem > 0 ? vecs_per_el_rem : (int32_t)sizeof(__m128i))) { + for (i = 0; i < vectorizable_elements; i += sizeof(__m128i)) { + /* Load the first 128 bytes in 16 XMM registers */ + const uint8_t* const src_for_ith_element = orig + i; + for (j = 0; j < 16; j++) { + xmm1[j] = _mm_loadu_si128((__m128i*)(src_for_ith_element + (total_elements * (offset_into_type + j)))); + } + /* Shuffle bytes */ + for (j = 0; j < 8; j++) { + /* Compute the low 32 bytes */ + xmm2[j] = _mm_unpacklo_epi8(xmm1[j * 2], xmm1[j * 2 + 1]); + /* Compute the hi 32 bytes */ + xmm2[8 + j] = _mm_unpackhi_epi8(xmm1[j * 2], xmm1[j * 2 + 1]); + } + /* Shuffle 2-byte words */ + for (j = 0; j < 8; j++) { + /* Compute the low 32 bytes */ + xmm1[j] = _mm_unpacklo_epi16(xmm2[j * 2], xmm2[j * 2 + 1]); + /* Compute the hi 32 bytes */ + xmm1[8 + j] = _mm_unpackhi_epi16(xmm2[j * 2], xmm2[j * 2 + 1]); + } + /* Shuffle 4-byte dwords */ + for (j = 0; j < 8; j++) { + /* Compute the low 32 bytes */ + xmm2[j] = _mm_unpacklo_epi32(xmm1[j * 2], xmm1[j * 2 + 1]); + /* Compute the hi 32 bytes */ + xmm2[8 + j] = _mm_unpackhi_epi32(xmm1[j * 2], xmm1[j * 2 + 1]); + } + /* Shuffle 8-byte qwords */ + for (j = 0; j < 8; j++) { + /* Compute the low 32 bytes */ + xmm1[j] = _mm_unpacklo_epi64(xmm2[j * 2], xmm2[j * 2 + 1]); + /* Compute the hi 32 bytes */ + xmm1[8 + j] = _mm_unpackhi_epi64(xmm2[j * 2], xmm2[j * 2 + 1]); + } + + /* Store the result vectors in proper order */ + dest_with_offset = dest + offset_into_type; + _mm_storeu_si128((__m128i*)(dest_with_offset + (i + 0) * bytesoftype), xmm1[0]); + _mm_storeu_si128((__m128i*)(dest_with_offset + (i + 1) * bytesoftype), xmm1[8]); + _mm_storeu_si128((__m128i*)(dest_with_offset + (i + 2) * bytesoftype), xmm1[4]); + _mm_storeu_si128((__m128i*)(dest_with_offset + (i + 3) * bytesoftype), xmm1[12]); + _mm_storeu_si128((__m128i*)(dest_with_offset + (i + 4) * bytesoftype), xmm1[2]); + _mm_storeu_si128((__m128i*)(dest_with_offset + (i + 5) * bytesoftype), xmm1[10]); + _mm_storeu_si128((__m128i*)(dest_with_offset + (i + 6) * bytesoftype), xmm1[6]); + _mm_storeu_si128((__m128i*)(dest_with_offset + (i + 7) * bytesoftype), xmm1[14]); + _mm_storeu_si128((__m128i*)(dest_with_offset + (i + 8) * bytesoftype), xmm1[1]); + _mm_storeu_si128((__m128i*)(dest_with_offset + (i + 9) * bytesoftype), xmm1[9]); + _mm_storeu_si128((__m128i*)(dest_with_offset + (i + 10) * bytesoftype), xmm1[5]); + _mm_storeu_si128((__m128i*)(dest_with_offset + (i + 11) * bytesoftype), xmm1[13]); + _mm_storeu_si128((__m128i*)(dest_with_offset + (i + 12) * bytesoftype), xmm1[3]); + _mm_storeu_si128((__m128i*)(dest_with_offset + (i + 13) * bytesoftype), xmm1[11]); + _mm_storeu_si128((__m128i*)(dest_with_offset + (i + 14) * bytesoftype), xmm1[7]); + _mm_storeu_si128((__m128i*)(dest_with_offset + (i + 15) * bytesoftype), xmm1[15]); + } + } +} + +/* Shuffle a block. This can never fail. */ +void +shuffle_sse2(const int32_t bytesoftype, const int32_t blocksize, + const uint8_t *_src, uint8_t *_dest) { + const int32_t vectorized_chunk_size = bytesoftype * (int32_t)sizeof(__m128i); + /* If the blocksize is not a multiple of both the typesize and + the vector size, round the blocksize down to the next value + which is a multiple of both. The vectorized shuffle can be + used for that portion of the data, and the naive implementation + can be used for the remaining portion. */ + const int32_t vectorizable_bytes = blocksize - (blocksize % vectorized_chunk_size); + const int32_t vectorizable_elements = vectorizable_bytes / bytesoftype; + const int32_t total_elements = blocksize / bytesoftype; + + /* If the block size is too small to be vectorized, + use the generic implementation. */ + if (blocksize < vectorized_chunk_size) { + shuffle_generic(bytesoftype, blocksize, _src, _dest); + return; + } + + /* Optimized shuffle implementations */ + switch (bytesoftype) { + case 2: + shuffle2_sse2(_dest, _src, vectorizable_elements, total_elements); + break; + case 4: + shuffle4_sse2(_dest, _src, vectorizable_elements, total_elements); + break; + case 8: + shuffle8_sse2(_dest, _src, vectorizable_elements, total_elements); + break; + case 16: + shuffle16_sse2(_dest, _src, vectorizable_elements, total_elements); + break; + default: + if (bytesoftype > (int32_t)sizeof(__m128i)) { + shuffle16_tiled_sse2(_dest, _src, vectorizable_elements, total_elements, bytesoftype); + } + else { + /* Non-optimized shuffle */ + shuffle_generic(bytesoftype, blocksize, _src, _dest); + /* The non-optimized function covers the whole buffer, + so we're done processing here. */ + return; + } + } + + /* If the buffer had any bytes at the end which couldn't be handled + by the vectorized implementations, use the non-optimized version + to finish them up. */ + if (vectorizable_bytes < blocksize) { + shuffle_generic_inline(bytesoftype, vectorizable_bytes, blocksize, _src, _dest); + } +} + +/* Unshuffle a block. This can never fail. */ +void +unshuffle_sse2(const int32_t bytesoftype, const int32_t blocksize, + const uint8_t *_src, uint8_t *_dest) { + const int32_t vectorized_chunk_size = bytesoftype * (int32_t)sizeof(__m128i); + /* If the blocksize is not a multiple of both the typesize and + the vector size, round the blocksize down to the next value + which is a multiple of both. The vectorized unshuffle can be + used for that portion of the data, and the naive implementation + can be used for the remaining portion. */ + const int32_t vectorizable_bytes = blocksize - (blocksize % vectorized_chunk_size); + const int32_t vectorizable_elements = vectorizable_bytes / bytesoftype; + const int32_t total_elements = blocksize / bytesoftype; + + /* If the block size is too small to be vectorized, + use the generic implementation. */ + if (blocksize < vectorized_chunk_size) { + unshuffle_generic(bytesoftype, blocksize, _src, _dest); + return; + } + + /* Optimized unshuffle implementations */ + switch (bytesoftype) { + case 2: + unshuffle2_sse2(_dest, _src, vectorizable_elements, total_elements); + break; + case 4: + unshuffle4_sse2(_dest, _src, vectorizable_elements, total_elements); + break; + case 8: + unshuffle8_sse2(_dest, _src, vectorizable_elements, total_elements); + break; + case 16: + unshuffle16_sse2(_dest, _src, vectorizable_elements, total_elements); + break; + default: + if (bytesoftype > (int32_t)sizeof(__m128i)) { + unshuffle16_tiled_sse2(_dest, _src, vectorizable_elements, total_elements, bytesoftype); + } + else { + /* Non-optimized unshuffle */ + unshuffle_generic(bytesoftype, blocksize, _src, _dest); + /* The non-optimized function covers the whole buffer, + so we're done processing here. */ + return; + } + } + + /* If the buffer had any bytes at the end which couldn't be handled + by the vectorized implementations, use the non-optimized version + to finish them up. */ + if (vectorizable_bytes < blocksize) { + unshuffle_generic_inline(bytesoftype, vectorizable_bytes, blocksize, _src, _dest); + } +} diff --git a/src/blosc2/blosc/shuffle-sse2.h b/src/blosc2/blosc/shuffle-sse2.h new file mode 100644 index 0000000..783641d --- /dev/null +++ b/src/blosc2/blosc/shuffle-sse2.h @@ -0,0 +1,38 @@ +/********************************************************************* + Blosc - Blocked Shuffling and Compression Library + + Copyright (C) 2021 The Blosc Developers + https://blosc.org + License: BSD 3-Clause (see LICENSE.txt) + + See LICENSE.txt for details about copyright and rights to use. +**********************************************************************/ + +/* SSE2-accelerated shuffle/unshuffle routines. */ + +#ifndef SHUFFLE_SSE2_H +#define SHUFFLE_SSE2_H + +#include "blosc2/blosc2-common.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/** + SSE2-accelerated shuffle routine. +*/ +BLOSC_NO_EXPORT void shuffle_sse2(const int32_t bytesoftype, const int32_t blocksize, + const uint8_t *_src, uint8_t *_dest); + +/** + SSE2-accelerated unshuffle routine. +*/ +BLOSC_NO_EXPORT void unshuffle_sse2(const int32_t bytesoftype, const int32_t blocksize, + const uint8_t *_src, uint8_t *_dest); + +#ifdef __cplusplus +} +#endif + +#endif /* SHUFFLE_SSE2_H */ diff --git a/src/blosc2/blosc/shuffle.c b/src/blosc2/blosc/shuffle.c new file mode 100644 index 0000000..b5530c1 --- /dev/null +++ b/src/blosc2/blosc/shuffle.c @@ -0,0 +1,493 @@ +/********************************************************************* + Blosc - Blocked Shuffling and Compression Library + + Copyright (C) 2021 The Blosc Developers + https://blosc.org + License: BSD 3-Clause (see LICENSE.txt) + + See LICENSE.txt for details about copyright and rights to use. +**********************************************************************/ + +#include "shuffle.h" +#include "blosc2/blosc2-common.h" +#include "shuffle-generic.h" +#include "bitshuffle-generic.h" +#include +#include +#include + + +#if !defined(__clang__) && defined(__GNUC__) && defined(__GNUC_MINOR__) && \ + __GNUC__ >= 5 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8) +#define HAVE_CPU_FEAT_INTRIN +#endif + +/* Include hardware-accelerated shuffle/unshuffle routines based on + the target architecture. Note that a target architecture may support + more than one type of acceleration!*/ +#if defined(SHUFFLE_AVX2_ENABLED) + #include "shuffle-avx2.h" + #include "bitshuffle-avx2.h" +#endif /* defined(SHUFFLE_AVX2_ENABLED) */ + +#if defined(SHUFFLE_SSE2_ENABLED) + #include "shuffle-sse2.h" + #include "bitshuffle-sse2.h" +#endif /* defined(SHUFFLE_SSE2_ENABLED) */ + +#if defined(SHUFFLE_NEON_ENABLED) + #if defined(__linux__) + #include + #ifdef ARM_ASM_HWCAP + #include + #endif + #endif + #include "shuffle-neon.h" + #include "bitshuffle-neon.h" +#endif /* defined(SHUFFLE_NEON_ENABLED) */ + +#if defined(SHUFFLE_ALTIVEC_ENABLED) + #include "shuffle-altivec.h" + #include "bitshuffle-altivec.h" +#endif /* defined(SHUFFLE_ALTIVEC_ENABLED) */ + + +/* Define function pointer types for shuffle/unshuffle routines. */ +typedef void(* shuffle_func)(const int32_t, const int32_t, const uint8_t*, const uint8_t*); +typedef void(* unshuffle_func)(const int32_t, const int32_t, const uint8_t*, const uint8_t*); +// For bitshuffle, everything is done in terms of size_t and int64_t (return value) +// and although this is not strictly necessary for Blosc, it does not hurt either +typedef int64_t(* bitshuffle_func)(void*, void*, const size_t, const size_t, void*); +typedef int64_t(* bitunshuffle_func)(void*, void*, const size_t, const size_t, void*); + +/* An implementation of shuffle/unshuffle routines. */ +typedef struct shuffle_implementation { + /* Name of this implementation. */ + const char* name; + /* Function pointer to the shuffle routine for this implementation. */ + shuffle_func shuffle; + /* Function pointer to the unshuffle routine for this implementation. */ + unshuffle_func unshuffle; + /* Function pointer to the bitshuffle routine for this implementation. */ + bitshuffle_func bitshuffle; + /* Function pointer to the bitunshuffle routine for this implementation. */ + bitunshuffle_func bitunshuffle; +} shuffle_implementation_t; + +typedef enum { + BLOSC_HAVE_NOTHING = 0, + BLOSC_HAVE_SSE2 = 1, + BLOSC_HAVE_AVX2 = 2, + BLOSC_HAVE_NEON = 4, + BLOSC_HAVE_ALTIVEC = 8 +} blosc_cpu_features; + +/* Detect hardware and set function pointers to the best shuffle/unshuffle + implementations supported by the host processor. */ +#if defined(SHUFFLE_AVX2_ENABLED) || defined(SHUFFLE_SSE2_ENABLED) /* Intel/i686 */ + +/* Disabled the __builtin_cpu_supports() call, as it has issues with + new versions of gcc (like 5.3.1 in forthcoming ubuntu/xenial: + "undefined symbol: __cpu_model" + For a similar report, see: + https://lists.fedoraproject.org/archives/list/devel@lists.fedoraproject.org/thread/ZM2L65WIZEEQHHLFERZYD5FAG7QY2OGB/ +*/ +#if defined(HAVE_CPU_FEAT_INTRIN) && 0 +static blosc_cpu_features blosc_get_cpu_features(void) { + blosc_cpu_features cpu_features = BLOSC_HAVE_NOTHING; + if (__builtin_cpu_supports("sse2")) { + cpu_features |= BLOSC_HAVE_SSE2; + } + if (__builtin_cpu_supports("avx2")) { + cpu_features |= BLOSC_HAVE_AVX2; + } + return cpu_features; +} +#else + +#if defined(_MSC_VER) && !defined(__clang__) + #include /* Needed for _xgetbv */ + #include /* Needed for __cpuid */ +#else + +/* Implement the __cpuid and __cpuidex intrinsics for GCC, Clang, + and others using inline assembly. */ +__attribute__((always_inline)) +static inline void +__cpuidex(int32_t cpuInfo[4], int32_t function_id, int32_t subfunction_id) { + __asm__ __volatile__ ( +# if defined(__i386__) && defined (__PIC__) + /* Can't clobber ebx with PIC running under 32-bit, so it needs to be manually restored. + https://software.intel.com/en-us/articles/how-to-detect-new-instruction-support-in-the-4th-generation-intel-core-processor-family + */ + "movl %%ebx, %%edi\n\t" + "cpuid\n\t" + "xchgl %%ebx, %%edi": + "=D" (cpuInfo[1]), +#else + "cpuid": + "=b" (cpuInfo[1]), +#endif /* defined(__i386) && defined(__PIC__) */ + "=a" (cpuInfo[0]), + "=c" (cpuInfo[2]), + "=d" (cpuInfo[3]) : + "a" (function_id), "c" (subfunction_id) + ); +} + +#define __cpuid(cpuInfo, function_id) __cpuidex(cpuInfo, function_id, 0) + +#define _XCR_XFEATURE_ENABLED_MASK 0 + +// GCC folks added _xgetbv in immintrin.h starting in GCC 9 +// See https://gcc.gnu.org/bugzilla/show_bug.cgi?id=71659 +#if !(defined(_IMMINTRIN_H_INCLUDED) && (BLOSC_GCC_VERSION >= 900)) +/* Reads the content of an extended control register. + https://software.intel.com/en-us/articles/how-to-detect-new-instruction-support-in-the-4th-generation-intel-core-processor-family +*/ +static inline uint64_t +_xgetbv(uint32_t xcr) { + uint32_t eax, edx; + __asm__ __volatile__ ( + /* "xgetbv" + This is specified as raw instruction bytes due to some older compilers + having issues with the mnemonic form. + */ + ".byte 0x0f, 0x01, 0xd0": + "=a" (eax), + "=d" (edx) : + "c" (xcr) + ); + return ((uint64_t)edx << 32) | eax; +} +#endif // !(defined(_IMMINTRIN_H_INCLUDED) && (BLOSC_GCC_VERSION >= 900)) +#endif /* defined(_MSC_VER) */ + +#ifndef _XCR_XFEATURE_ENABLED_MASK +#define _XCR_XFEATURE_ENABLED_MASK 0x0 +#endif + +static blosc_cpu_features blosc_get_cpu_features(void) { + blosc_cpu_features result = BLOSC_HAVE_NOTHING; + /* Holds the values of eax, ebx, ecx, edx set by the `cpuid` instruction */ + int32_t cpu_info[4]; + + /* Get the number of basic functions available. */ + __cpuid(cpu_info, 0); + int32_t max_basic_function_id = cpu_info[0]; + + /* Check for SSE-based features and required OS support */ + __cpuid(cpu_info, 1); + const bool sse2_available = (cpu_info[3] & (1 << 26)) != 0; + const bool sse3_available = (cpu_info[2] & (1 << 0)) != 0; + const bool ssse3_available = (cpu_info[2] & (1 << 9)) != 0; + const bool sse41_available = (cpu_info[2] & (1 << 19)) != 0; + const bool sse42_available = (cpu_info[2] & (1 << 20)) != 0; + + const bool xsave_available = (cpu_info[2] & (1 << 26)) != 0; + const bool xsave_enabled_by_os = (cpu_info[2] & (1 << 27)) != 0; + + /* Check for AVX-based features, if the processor supports extended features. */ + bool avx2_available = false; + bool avx512bw_available = false; + if (max_basic_function_id >= 7) { + __cpuid(cpu_info, 7); + avx2_available = (cpu_info[1] & (1 << 5)) != 0; + avx512bw_available = (cpu_info[1] & (1 << 30)) != 0; + } + + /* Even if certain features are supported by the CPU, they may not be supported + by the OS (in which case using them would crash the process or system). + If xsave is available and enabled by the OS, check the contents of the + extended control register XCR0 to see if the CPU features are enabled. */ + bool xmm_state_enabled = false; + bool ymm_state_enabled = false; + //bool zmm_state_enabled = false; // commented this out for avoiding an 'unused variable' warning + +#if defined(_XCR_XFEATURE_ENABLED_MASK) + if (xsave_available && xsave_enabled_by_os && ( + sse2_available || sse3_available || ssse3_available + || sse41_available || sse42_available + || avx2_available || avx512bw_available)) { + /* Determine which register states can be restored by the OS. */ + uint64_t xcr0_contents = _xgetbv(_XCR_XFEATURE_ENABLED_MASK); + + xmm_state_enabled = (xcr0_contents & (1UL << 1)) != 0; + ymm_state_enabled = (xcr0_contents & (1UL << 2)) != 0; + + /* Require support for both the upper 256-bits of zmm0-zmm15 to be + restored as well as all of zmm16-zmm31 and the opmask registers. */ + //zmm_state_enabled = (xcr0_contents & 0x70) == 0x70; + } +#endif /* defined(_XCR_XFEATURE_ENABLED_MASK) */ + +#if defined(BLOSC_DUMP_CPU_INFO) + printf("Shuffle CPU Information:\n"); + printf("SSE2 available: %s\n", sse2_available ? "True" : "False"); + printf("SSE3 available: %s\n", sse3_available ? "True" : "False"); + printf("SSSE3 available: %s\n", ssse3_available ? "True" : "False"); + printf("SSE4.1 available: %s\n", sse41_available ? "True" : "False"); + printf("SSE4.2 available: %s\n", sse42_available ? "True" : "False"); + printf("AVX2 available: %s\n", avx2_available ? "True" : "False"); + printf("AVX512BW available: %s\n", avx512bw_available ? "True" : "False"); + printf("XSAVE available: %s\n", xsave_available ? "True" : "False"); + printf("XSAVE enabled: %s\n", xsave_enabled_by_os ? "True" : "False"); + printf("XMM state enabled: %s\n", xmm_state_enabled ? "True" : "False"); + printf("YMM state enabled: %s\n", ymm_state_enabled ? "True" : "False"); + //printf("ZMM state enabled: %s\n", zmm_state_enabled ? "True" : "False"); +#endif /* defined(BLOSC_DUMP_CPU_INFO) */ + + /* Using the gathered CPU information, determine which implementation to use. */ + /* technically could fail on sse2 cpu on os without xmm support, but that + * shouldn't exist anymore */ + if (sse2_available) { + result |= BLOSC_HAVE_SSE2; + } + if (xmm_state_enabled && ymm_state_enabled && avx2_available) { + result |= BLOSC_HAVE_AVX2; + } + return result; +} +#endif /* HAVE_CPU_FEAT_INTRIN */ + +#elif defined(SHUFFLE_NEON_ENABLED) /* ARM-NEON */ +static blosc_cpu_features blosc_get_cpu_features(void) { + blosc_cpu_features cpu_features = BLOSC_HAVE_NOTHING; +#if defined(__aarch64__) + /* aarch64 always has NEON */ + cpu_features |= BLOSC_HAVE_NEON; +#else + if (getauxval(AT_HWCAP) & HWCAP_ARM_NEON) { + cpu_features |= BLOSC_HAVE_NEON; + } +#endif + return cpu_features; +} +#elif defined(SHUFFLE_ALTIVEC_ENABLED) /* POWER9-ALTIVEC preliminary test*/ +static blosc_cpu_features blosc_get_cpu_features(void) { + blosc_cpu_features cpu_features = BLOSC_HAVE_NOTHING; + cpu_features |= BLOSC_HAVE_ALTIVEC; + return cpu_features; +} +#else /* No hardware acceleration supported for the target architecture. */ + #if defined(_MSC_VER) + #pragma message("Hardware-acceleration detection not implemented for the target architecture. Only the generic shuffle/unshuffle routines will be available.") + #else + #warning Hardware-acceleration detection not implemented for the target architecture. Only the generic shuffle/unshuffle routines will be available. + #endif + +static blosc_cpu_features blosc_get_cpu_features(void) { +return BLOSC_HAVE_NOTHING; +} + +#endif /* defined(SHUFFLE_AVX2_ENABLED) || defined(SHUFFLE_SSE2_ENABLED) */ + +static shuffle_implementation_t get_shuffle_implementation(void) { + blosc_cpu_features cpu_features = blosc_get_cpu_features(); +#if defined(SHUFFLE_AVX2_ENABLED) + if (cpu_features & BLOSC_HAVE_AVX2) { + shuffle_implementation_t impl_avx2; + impl_avx2.name = "avx2"; + impl_avx2.shuffle = (shuffle_func)shuffle_avx2; + impl_avx2.unshuffle = (unshuffle_func)unshuffle_avx2; + impl_avx2.bitshuffle = (bitshuffle_func)bshuf_trans_bit_elem_avx2; + impl_avx2.bitunshuffle = (bitunshuffle_func)bshuf_untrans_bit_elem_avx2; + return impl_avx2; + } +#endif /* defined(SHUFFLE_AVX2_ENABLED) */ + +#if defined(SHUFFLE_SSE2_ENABLED) + if (cpu_features & BLOSC_HAVE_SSE2) { + shuffle_implementation_t impl_sse2; + impl_sse2.name = "sse2"; + impl_sse2.shuffle = (shuffle_func)shuffle_sse2; + impl_sse2.unshuffle = (unshuffle_func)unshuffle_sse2; + impl_sse2.bitshuffle = (bitshuffle_func)bshuf_trans_bit_elem_sse2; + impl_sse2.bitunshuffle = (bitunshuffle_func)bshuf_untrans_bit_elem_sse2; + return impl_sse2; + } +#endif /* defined(SHUFFLE_SSE2_ENABLED) */ + +#if defined(SHUFFLE_NEON_ENABLED) + if (cpu_features & BLOSC_HAVE_NEON) { + shuffle_implementation_t impl_neon; + impl_neon.name = "neon"; + impl_neon.shuffle = (shuffle_func)shuffle_neon; + impl_neon.unshuffle = (unshuffle_func)unshuffle_neon; + //impl_neon.shuffle = (shuffle_func)shuffle_generic; + //impl_neon.unshuffle = (unshuffle_func)unshuffle_generic; + //impl_neon.bitshuffle = (bitshuffle_func)bitshuffle_neon; + //impl_neon.bitunshuffle = (bitunshuffle_func)bitunshuffle_neon; + // The current bitshuffle optimized for NEON is not any faster + // (in fact, it is pretty much slower) than the scalar implementation. + // Also, bitshuffle_neon (forward direction) is broken for 1, 2 and 4 bytes. + // So, let's use the the scalar one, which is pretty fast, at least on a M1 CPU. + impl_neon.bitshuffle = (bitshuffle_func)bshuf_trans_bit_elem_scal; + impl_neon.bitunshuffle = (bitunshuffle_func)bshuf_untrans_bit_elem_scal; + return impl_neon; + } +#endif /* defined(SHUFFLE_NEON_ENABLED) */ + +#if defined(SHUFFLE_ALTIVEC_ENABLED) + if (cpu_features & BLOSC_HAVE_ALTIVEC) { + shuffle_implementation_t impl_altivec; + impl_altivec.name = "altivec"; + impl_altivec.shuffle = (shuffle_func)shuffle_altivec; + impl_altivec.unshuffle = (unshuffle_func)unshuffle_altivec; + impl_altivec.bitshuffle = (bitshuffle_func)bshuf_trans_bit_elem_altivec; + impl_altivec.bitunshuffle = (bitunshuffle_func)bshuf_untrans_bit_elem_altivec; + return impl_altivec; + } +#endif /* defined(SHUFFLE_ALTIVEC_ENABLED) */ + + /* Processor doesn't support any of the hardware-accelerated implementations, + so use the generic implementation. */ + shuffle_implementation_t impl_generic; + impl_generic.name = "generic"; + impl_generic.shuffle = (shuffle_func)shuffle_generic; + impl_generic.unshuffle = (unshuffle_func)unshuffle_generic; + impl_generic.bitshuffle = (bitshuffle_func)bshuf_trans_bit_elem_scal; + impl_generic.bitunshuffle = (bitunshuffle_func)bshuf_untrans_bit_elem_scal; + return impl_generic; +} + + +/* Flag indicating whether the implementation has been initialized. + Zero means it hasn't been initialized, non-zero means it has. */ +static int32_t implementation_initialized; + +/* The dynamically-chosen shuffle/unshuffle implementation. + This is only safe to use once `implementation_initialized` is set. */ +static shuffle_implementation_t host_implementation; + +/* Initialize the shuffle implementation, if necessary. */ +#if defined(__GNUC__) || defined(__clang__) +__attribute__((always_inline)) +#endif +static +#if defined(_MSC_VER) +__forceinline +#else +inline +#endif +void init_shuffle_implementation(void) { + /* Initialization could (in rare cases) take place concurrently on + multiple threads, but it shouldn't matter because the + initialization should return the same result on each thread (so + the implementation will be the same). Since that's the case we + can avoid complicated synchronization here and get a small + performance benefit because we don't need to perform a volatile + load on the initialization variable each time this function is + called. */ +#if defined(__GNUC__) || defined(__clang__) + if (__builtin_expect(!implementation_initialized, 0)) { +#else + if (!implementation_initialized) { +#endif + /* Initialize the implementation. */ + host_implementation = get_shuffle_implementation(); + + /* Set the flag indicating the implementation has been initialized. */ + implementation_initialized = 1; + } +} + +/* Shuffle a block by dynamically dispatching to the appropriate + hardware-accelerated routine at run-time. */ +void +shuffle(const int32_t bytesoftype, const int32_t blocksize, + const uint8_t* _src, const uint8_t* _dest) { + /* Initialize the shuffle implementation if necessary. */ + init_shuffle_implementation(); + + /* The implementation is initialized. + Dispatch to it's shuffle routine. */ + (host_implementation.shuffle)(bytesoftype, blocksize, _src, _dest); +} + +/* Unshuffle a block by dynamically dispatching to the appropriate + hardware-accelerated routine at run-time. */ +void +unshuffle(const int32_t bytesoftype, const int32_t blocksize, + const uint8_t* _src, const uint8_t* _dest) { + /* Initialize the shuffle implementation if necessary. */ + init_shuffle_implementation(); + + /* The implementation is initialized. + Dispatch to it's unshuffle routine. */ + (host_implementation.unshuffle)(bytesoftype, blocksize, _src, _dest); +} + +/* Bit-shuffle a block by dynamically dispatching to the appropriate + hardware-accelerated routine at run-time. */ +int32_t +bitshuffle(const int32_t bytesoftype, const int32_t blocksize, + const uint8_t *_src, const uint8_t *_dest, + const uint8_t *_tmp) { + /* Initialize the shuffle implementation if necessary. */ + init_shuffle_implementation(); + size_t size = blocksize / bytesoftype; + /* bitshuffle only supports a number of elements that is a multiple of 8. */ + size -= size % 8; + int ret = (int) (host_implementation.bitshuffle)((void *) _src, (void *) _dest, + size, bytesoftype, (void *) _tmp); + if (ret < 0) { + // Some error in bitshuffle (should not happen) + fprintf(stderr, "the impossible happened: the bitshuffle filter failed!"); + return ret; + } + + // Copy the leftovers + size_t offset = size * bytesoftype; + memcpy((void *) (_dest + offset), (void *) (_src + offset), blocksize - offset); + + return blocksize; +} + +/* Bit-unshuffle a block by dynamically dispatching to the appropriate + hardware-accelerated routine at run-time. */ +int32_t bitunshuffle(const int32_t bytesoftype, const int32_t blocksize, + const uint8_t *_src, const uint8_t *_dest, + const uint8_t *_tmp, const uint8_t format_version) { + /* Initialize the shuffle implementation if necessary. */ + init_shuffle_implementation(); + size_t size = blocksize / bytesoftype; + + if (format_version == 2) { + /* Starting from version 3, bitshuffle() works differently */ + if ((size % 8) == 0) { + /* The number of elems is a multiple of 8 which is supported by + bitshuffle. */ + int ret = (int) (host_implementation.bitunshuffle)((void *) _src, (void *) _dest, + blocksize / bytesoftype, + bytesoftype, (void *) _tmp); + if (ret < 0) { + // Some error in bitshuffle (should not happen) + fprintf(stderr, "the impossible happened: the bitunshuffle filter failed!"); + return ret; + } + /* Copy the leftovers (we do so starting from c-blosc 1.18 on) */ + size_t offset = size * bytesoftype; + memcpy((void *) (_dest + offset), (void *) (_src + offset), blocksize - offset); + } + else { + memcpy((void *) _dest, (void *) _src, blocksize); + } + } + else { + /* bitshuffle only supports a number of bytes that is a multiple of 8. */ + size -= size % 8; + int ret = (int) (host_implementation.bitunshuffle)((void *) _src, (void *) _dest, + size, bytesoftype, (void *) _tmp); + if (ret < 0) { + fprintf(stderr, "the impossible happened: the bitunshuffle filter failed!"); + return ret; + } + + /* Copy the leftovers */ + size_t offset = size * bytesoftype; + memcpy((void *) (_dest + offset), (void *) (_src + offset), blocksize - offset); + } + + return blocksize; +} diff --git a/src/blosc2/blosc/shuffle.h b/src/blosc2/blosc/shuffle.h new file mode 100644 index 0000000..ba195cd --- /dev/null +++ b/src/blosc2/blosc/shuffle.h @@ -0,0 +1,72 @@ +/********************************************************************* + Blosc - Blocked Shuffling and Compression Library + + Copyright (C) 2021 The Blosc Developers + https://blosc.org + License: BSD 3-Clause (see LICENSE.txt) + + See LICENSE.txt for details about copyright and rights to use. +**********************************************************************/ + +/********************************************************************* + Shuffle/unshuffle routines which dynamically dispatch to hardware- + accelerated routines based on the processor's architecture. + Consumers should almost always prefer to call these routines instead + of directly calling one of the hardware-accelerated routines, since + these are cross-platform and future-proof. +**********************************************************************/ + + +#ifndef SHUFFLE_H +#define SHUFFLE_H + +#include "blosc2/blosc2-common.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/** + Primary shuffle and bitshuffle routines. + This function dynamically dispatches to the appropriate hardware-accelerated + routine based on the host processor's architecture. If the host processor is + not supported by any of the hardware-accelerated routines, the generic + (non-accelerated) implementation is used instead. + Consumers should almost always prefer to call this routine instead of directly + calling the hardware-accelerated routines because this method is both cross- + platform and future-proof. +*/ +BLOSC_NO_EXPORT void + shuffle(const int32_t bytesoftype, const int32_t blocksize, + const uint8_t* _src, const uint8_t* _dest); + +BLOSC_NO_EXPORT int32_t + bitshuffle(const int32_t bytesoftype, const int32_t blocksize, + const uint8_t *_src, const uint8_t *_dest, + const uint8_t *_tmp); + +/** + Primary unshuffle and bitunshuffle routine. + This function dynamically dispatches to the appropriate hardware-accelerated + routine based on the host processor's architecture. If the host processor is + not supported by any of the hardware-accelerated routines, the generic + (non-accelerated) implementation is used instead. + Consumers should almost always prefer to call this routine instead of directly + calling the hardware-accelerated routines because this method is both cross- + platform and future-proof. +*/ +BLOSC_NO_EXPORT void + unshuffle(const int32_t bytesoftype, const int32_t blocksize, + const uint8_t* _src, const uint8_t* _dest); + + +BLOSC_NO_EXPORT int32_t + bitunshuffle(const int32_t bytesoftype, const int32_t blocksize, + const uint8_t *_src, const uint8_t *_dest, + const uint8_t *_tmp, const uint8_t format_version); + +#ifdef __cplusplus +} +#endif + +#endif /* SHUFFLE_H */ diff --git a/src/blosc2/blosc/stune.c b/src/blosc2/blosc/stune.c new file mode 100644 index 0000000..8b62cea --- /dev/null +++ b/src/blosc2/blosc/stune.c @@ -0,0 +1,185 @@ +/********************************************************************* + Blosc - Blocked Shuffling and Compression Library + + Copyright (C) 2021 The Blosc Developers + https://blosc.org + License: BSD 3-Clause (see LICENSE.txt) + + See LICENSE.txt for details about copyright and rights to use. +**********************************************************************/ + +#include +#include +#include "stune.h" + + +/* Whether a codec is meant for High Compression Ratios + Includes LZ4 + BITSHUFFLE here, but not BloscLZ + BITSHUFFLE because, + for some reason, the latter does not work too well */ +static bool is_HCR(blosc2_context * context) { + switch (context->compcode) { + case BLOSC_BLOSCLZ : + return false; + case BLOSC_LZ4 : + return (context->filter_flags & BLOSC_DOBITSHUFFLE) ? true : false; + case BLOSC_LZ4HC : + case BLOSC_ZLIB : + case BLOSC_ZSTD : + return true; + default : + return false; + } +} + +void blosc_stune_init(void * config, blosc2_context* cctx, blosc2_context* dctx) { + BLOSC_UNUSED_PARAM(config); + BLOSC_UNUSED_PARAM(cctx); + BLOSC_UNUSED_PARAM(dctx); +} + +// Set the automatic blocksize 0 to its real value +void blosc_stune_next_blocksize(blosc2_context *context) { + int32_t clevel = context->clevel; + int32_t typesize = context->typesize; + int32_t nbytes = context->sourcesize; + int32_t user_blocksize = context->blocksize; + int32_t blocksize = nbytes; + + // Protection against very small buffers + if (nbytes < typesize) { + context->blocksize = 1; + return; + } + + if (user_blocksize) { + blocksize = user_blocksize; + goto last; + } + + if (nbytes >= L1) { + blocksize = L1; + + /* For HCR codecs, increase the block sizes by a factor of 2 because they + are meant for compressing large blocks (i.e. they show a big overhead + when compressing small ones). */ + if (is_HCR(context)) { + blocksize *= 2; + } + + // Choose a different blocksize depending on the compression level + switch (clevel) { + case 0: + // Case of plain copy + blocksize /= 4; + break; + case 1: + blocksize /= 2; + break; + case 2: + blocksize *= 1; + break; + case 3: + blocksize *= 2; + break; + case 4: + case 5: + blocksize *= 4; + break; + case 6: + case 7: + case 8: + blocksize *= 8; + break; + case 9: + // Do not exceed 256 KB for non HCR codecs + blocksize *= 8; + if (is_HCR(context)) { + blocksize *= 2; + } + break; + default: + break; + } + } + + /* Now the blocksize for splittable codecs */ + if (clevel > 0 && split_block(context, typesize, blocksize)) { + // For performance reasons, do not exceed 256 KB (it must fit in L2 cache) + switch (clevel) { + case 1: + case 2: + case 3: + blocksize = 32 * 1024; + break; + case 4: + case 5: + case 6: + case 7: + case 8: + blocksize = 256 * 1024; + break; + case 9: + default: + blocksize = 512 * 1024; + break; + } + // Multiply by typesize to get proper split sizes + blocksize *= typesize; + // But do not exceed 4 MB per thread (having this capacity in L3 is normal in modern CPUs) + if (blocksize > 4 * 1024 * 1024) { + blocksize = 4 * 1024 * 1024; + } + if (blocksize < 32 * 1024) { + /* Do not use a too small blocksize (< 32 KB) when typesize is small */ + blocksize = 32 * 1024; + } + } + + last: + /* Check that blocksize is not too large */ + if (blocksize > nbytes) { + blocksize = nbytes; + } + + // blocksize *must absolutely* be a multiple of the typesize + if (blocksize > typesize) { + blocksize = blocksize / typesize * typesize; + } + + context->blocksize = blocksize; +} + +void blosc_stune_next_cparams(blosc2_context * context) { + BLOSC_UNUSED_PARAM(context); +} + +void blosc_stune_update(blosc2_context * context, double ctime) { + BLOSC_UNUSED_PARAM(context); + BLOSC_UNUSED_PARAM(ctime); +} + +void blosc_stune_free(blosc2_context * context) { + BLOSC_UNUSED_PARAM(context); +} + +int split_block(blosc2_context *context, int32_t typesize, int32_t blocksize) { + switch (context->splitmode) { + case BLOSC_ALWAYS_SPLIT: + return 1; + case BLOSC_NEVER_SPLIT: + return 0; + case BLOSC_FORWARD_COMPAT_SPLIT: + case BLOSC_AUTO_SPLIT: + // These cases will be handled later + break; + default: + BLOSC_TRACE_WARNING("Unrecognized split mode. Default to BLOSC_FORWARD_COMPAT_SPLIT"); + } + + int compcode = context->compcode; + return ( + // Fast codecs like blosclz and lz4 always prefer to always split + ((compcode == BLOSC_BLOSCLZ) || (compcode == BLOSC_LZ4)) && + (typesize <= MAX_STREAMS) && + (blocksize / typesize) >= BLOSC_MIN_BUFFERSIZE); +} diff --git a/src/blosc2/blosc/stune.h b/src/blosc2/blosc/stune.h new file mode 100644 index 0000000..cf78cbf --- /dev/null +++ b/src/blosc2/blosc/stune.h @@ -0,0 +1,48 @@ +/********************************************************************* + Blosc - Blocked Shuffling and Compression Library + + Copyright (C) 2021 The Blosc Developers + https://blosc.org + License: BSD 3-Clause (see LICENSE.txt) + + See LICENSE.txt for details about copyright and rights to use. +**********************************************************************/ + +#ifndef STUNE_H +#define STUNE_H + +#include "context.h" + +/* The size of L1 cache. 32 KB is quite common nowadays. */ +#define L1 (32 * 1024) +/* The size of L2 cache. 256 KB is quite common nowadays. */ +#define L2 (256 * 1024) + +/* The maximum number of compressed data streams in a block for compression */ +#define MAX_STREAMS 16 /* Cannot be larger than 128 */ + + +void blosc_stune_init(void * config, blosc2_context* cctx, blosc2_context* dctx); + +void blosc_stune_next_blocksize(blosc2_context * context); + +void blosc_stune_next_cparams(blosc2_context * context); + +void blosc_stune_update(blosc2_context * context, double ctime); + +void blosc_stune_free(blosc2_context * context); + +static blosc2_btune BTUNE_DEFAULTS = { + .btune_init = blosc_stune_init, + .btune_free = blosc_stune_free, + .btune_update = blosc_stune_update, + .btune_next_cparams = blosc_stune_next_cparams, + .btune_next_blocksize = blosc_stune_next_blocksize, + .btune_config = NULL, +}; + + +/* Conditions for splitting a block before compressing with a codec. */ +int split_block(blosc2_context *context, int32_t typesize, int32_t blocksize); + +#endif /* STUNE_H */ diff --git a/src/blosc2/blosc/timestamp.c b/src/blosc2/blosc/timestamp.c new file mode 100644 index 0000000..c143971 --- /dev/null +++ b/src/blosc2/blosc/timestamp.c @@ -0,0 +1,61 @@ +/********************************************************************* + Blosc - Blocked Shuffling and Compression Library + + Copyright (C) 2021 The Blosc Developers + https://blosc.org + License: BSD 3-Clause (see LICENSE.txt) + + See LICENSE.txt for details about copyright and rights to use. +**********************************************************************/ + +#include "blosc2.h" + +/* System-specific high-precision timing functions. */ +#if defined(_WIN32) + +/* Set a timestamp value to the current time. */ +void blosc_set_timestamp(blosc_timestamp_t* timestamp) { + /* Ignore the return value, assume the call always succeeds. */ + QueryPerformanceCounter(timestamp); +} + +/* Given two timestamp values, return the difference in nanoseconds. */ +double blosc_elapsed_nsecs(blosc_timestamp_t start_time, + blosc_timestamp_t end_time) { + LARGE_INTEGER CounterFreq; + QueryPerformanceFrequency(&CounterFreq); + + return (double)(end_time.QuadPart - start_time.QuadPart) / + ((double)CounterFreq.QuadPart / 1e9); +} + +#else + +/* Set a timestamp value to the current time. */ +void blosc_set_timestamp(blosc_timestamp_t* timestamp) { +#ifdef __MACH__ // OS X does not have clock_gettime, use clock_get_time + clock_serv_t cclock; + mach_timespec_t mts; + host_get_clock_service(mach_host_self(), CALENDAR_CLOCK, &cclock); + clock_get_time(cclock, &mts); + mach_port_deallocate(mach_task_self(), cclock); + timestamp->tv_sec = mts.tv_sec; + timestamp->tv_nsec = mts.tv_nsec; +#else + clock_gettime(CLOCK_MONOTONIC, timestamp); +#endif +} + +/* Given two timestamp values, return the difference in nanoseconds. */ +double blosc_elapsed_nsecs(blosc_timestamp_t start_time, + blosc_timestamp_t end_time) { + return (1e9 * (double)(end_time.tv_sec - start_time.tv_sec)) + + (double)(end_time.tv_nsec - start_time.tv_nsec); +} + +#endif + +/* Given two timeval stamps, return the difference in seconds */ +double blosc_elapsed_secs(blosc_timestamp_t last, blosc_timestamp_t current) { + return 1e-9 * blosc_elapsed_nsecs(last, current); +} diff --git a/src/blosc2/blosc/transpose-altivec.h b/src/blosc2/blosc/transpose-altivec.h new file mode 100644 index 0000000..fab0e5d --- /dev/null +++ b/src/blosc2/blosc/transpose-altivec.h @@ -0,0 +1,121 @@ +/********************************************************************* + Blosc - Blocked Shuffling and Compression Library + + Copyright (C) 2021 The Blosc developers and Jerome Kieffer + https://blosc.org + License: BSD 3-Clause (see LICENSE.txt) + + See LICENSE.txt for details about copyright and rights to use. +**********************************************************************/ + +#ifndef BLOSC_TRANSPOSE_ALTIVEC_H +#define BLOSC_TRANSPOSE_ALTIVEC_H + +#ifdef __cplusplus +extern "C" { +#endif + +static const __vector uint8_t even = (const __vector uint8_t) { + 0x00, 0x02, 0x04, 0x06, 0x08, 0x0a, 0x0c, 0x0e, + 0x10, 0x12, 0x14, 0x16, 0x18, 0x1a, 0x1c, 0x1e}; + +static const __vector uint8_t odd = (const __vector uint8_t) { + 0x01, 0x03, 0x05, 0x07, 0x09, 0x0b, 0x0d, 0x0f, + 0x11, 0x13, 0x15, 0x17, 0x19, 0x1b, 0x1d, 0x1f}; + + +/* Transpose inplace 2 vectors of 16 bytes in src into dst. */ +static void transpose2x16(__vector uint8_t *xmm0) { + __vector uint8_t xmm1[2]; + xmm1[0] = vec_perm(xmm0[0], xmm0[1], even); + xmm1[1] = vec_perm(xmm0[0], xmm0[1], odd); + + for (int i = 0; i < 2; i++) { + xmm0[i] = xmm1[i]; + } +} + + +/* Transpose inplace 4 vectors of 16 bytes in src into dst. + * Total cost: 8 calls to vec_perm. */ +static void transpose4x16(__vector uint8_t *xmm0) { + __vector uint8_t xmm1[4]; + + /* Transpose vectors 0-1*/ + xmm1[0] = vec_perm(xmm0[0], xmm0[1], even); + xmm1[1] = vec_perm(xmm0[0], xmm0[1], odd); + xmm1[2] = vec_perm(xmm0[2], xmm0[3], even); + xmm1[3] = vec_perm(xmm0[2], xmm0[3], odd); + /* Transpose vectors 0-2*/ + xmm0[0] = vec_perm(xmm1[0], xmm1[2], even); + xmm0[1] = vec_perm(xmm1[1], xmm1[3], even); + xmm0[2] = vec_perm(xmm1[0], xmm1[2], odd); + xmm0[3] = vec_perm(xmm1[1], xmm1[3], odd); +} + + +/* Transpose inplace 8 vectors of 16 bytes in src into dst. + * Total cost: 24 calls to vec_perm. */ +static void transpose8x16(__vector uint8_t *xmm0) { + __vector uint8_t xmm1[8]; + + /* Transpose vectors 0-1*/ + for (int i = 0; i < 8; i += 2){ + xmm1[i] = vec_perm(xmm0[i], xmm0[i+1], even); + xmm1[i+1] = vec_perm(xmm0[i], xmm0[i+1], odd); + } + /* Transpose vectors 0-2*/ + for (int i = 0; i < 8; i += 4){ + for (int k = 0; k < 2; k++){ + xmm0[i+k] = vec_perm(xmm1[i+k], xmm1[i+k+2], even); + xmm0[i+k+2] = vec_perm(xmm1[i+k], xmm1[i+k+2], odd); + } + } + /* Transpose vectors 0-4*/ + for (int k = 0; k < 4; k++){ + xmm1[k] = vec_perm(xmm0[k], xmm0[k+4], even); + xmm1[k+4] = vec_perm(xmm0[k], xmm0[k+4], odd); + } + + for (int i = 0; i < 8; i++) { + xmm0[i] = xmm1[i]; + } +} + + +/* Transpose inplace 16 vectors of 16 bytes in src into dst. + * Total cost: 64 calls to vec_perm. */ +static void transpose16x16(__vector uint8_t * xmm0){ + __vector uint8_t xmm1[16]; + /* Transpose vectors 0-1*/ + for (int i = 0; i < 16; i += 2){ + xmm1[i] = vec_perm(xmm0[i], xmm0[i+1], even); + xmm1[i+1] = vec_perm(xmm0[i], xmm0[i+1], odd); + } + /* Transpose vectors 0-2*/ + for (int i = 0; i < 16; i += 4){ + for (int k = 0; k < 2; k++){ + xmm0[i+k] = vec_perm(xmm1[i+k], xmm1[i+k+2], even); + xmm0[i+k+2] = vec_perm(xmm1[i+k], xmm1[i+k+2], odd); + } + } + /* Transpose vectors 0-4*/ + for (int i = 0; i < 16; i += 8){ + for (int k = 0; k < 4; k++){ + xmm1[i+k] = vec_perm(xmm0[i+k], xmm0[i+k+4], even); + xmm1[i+k+4] = vec_perm(xmm0[i+k], xmm0[i+k+4], odd); + } + } + /* Transpose vectors 0-8*/ + for (int k = 0; k < 8; k++){ + xmm0[k] = vec_perm(xmm1[k], xmm1[k+8], even); + xmm0[k+8] = vec_perm(xmm1[k], xmm1[k+8], odd); + } +} + + +#ifdef __cplusplus +} +#endif + +#endif //BLOSC_TRANSPOSE_ALTIVEC_H diff --git a/src/blosc2/blosc/trunc-prec.c b/src/blosc2/blosc/trunc-prec.c new file mode 100644 index 0000000..f4891fc --- /dev/null +++ b/src/blosc2/blosc/trunc-prec.c @@ -0,0 +1,84 @@ +/********************************************************************* + Blosc - Blocked Shuffling and Compression Library + + Copyright (C) 2021 The Blosc Developers + https://blosc.org + License: BSD 3-Clause (see LICENSE.txt) + + See LICENSE.txt for details about copyright and rights to use. +**********************************************************************/ + +#include +#include +#include "assert.h" +#include "trunc-prec.h" +#include "blosc2.h" + +#define BITS_MANTISSA_FLOAT 23 +#define BITS_MANTISSA_DOUBLE 52 + + +int truncate_precision32(int8_t prec_bits, int32_t nelems, + const int32_t* src, int32_t* dest) { + // Make sure that we don't remove all the bits in mantissa so that we + // don't mess with NaNs or Infinite representation in IEEE 754: + // https://en.wikipedia.org/wiki/NaN + if ((abs(prec_bits) > BITS_MANTISSA_FLOAT)) { + BLOSC_TRACE_ERROR("The precision cannot be larger than %d bits for floats (asking for %d bits)", + BITS_MANTISSA_FLOAT, prec_bits); + return -1; + } + int zeroed_bits = (prec_bits >= 0) ? BITS_MANTISSA_FLOAT - prec_bits : -prec_bits; + if (zeroed_bits >= BITS_MANTISSA_FLOAT) { + BLOSC_TRACE_ERROR("The reduction in precision cannot be larger or equal than %d bits for floats (asking for %d bits)", + BITS_MANTISSA_FLOAT, zeroed_bits); + return -1; + } + int32_t mask = ~((1 << zeroed_bits) - 1); + for (int i = 0; i < nelems; i++) { + dest[i] = src[i] & mask; + } + return 0; +} + +int truncate_precision64(int8_t prec_bits, int32_t nelems, + const int64_t* src, int64_t* dest) { + // Make sure that we don't remove all the bits in mantissa so that we + // don't mess with NaNs or Infinite representation in IEEE 754: + // https://en.wikipedia.org/wiki/NaN + if ((abs(prec_bits) > BITS_MANTISSA_DOUBLE)) { + BLOSC_TRACE_ERROR("The precision cannot be larger than %d bits for floats (asking for %d bits)", + BITS_MANTISSA_DOUBLE, prec_bits); + return -1; + } + int zeroed_bits = (prec_bits >= 0) ? BITS_MANTISSA_DOUBLE - prec_bits : -prec_bits; + if (zeroed_bits >= BITS_MANTISSA_DOUBLE) { + BLOSC_TRACE_ERROR("The reduction in precision cannot be larger or equal than %d bits for floats (asking for %d bits)", + BITS_MANTISSA_DOUBLE, zeroed_bits); + return -1; + } + uint64_t mask = ~((1ULL << zeroed_bits) - 1ULL); + for (int i = 0; i < nelems; i++) { + dest[i] = (int64_t)(src[i] & mask); + } + return 0; +} + +/* Apply the truncate precision to src. This can never fail. */ +int truncate_precision(int8_t prec_bits, int32_t typesize, int32_t nbytes, + const uint8_t* src, uint8_t* dest) { + // Positive values of prec_bits will set absolute precision bits, whereas negative + // values will reduce the precision bits (similar to Python slicing convention). + switch (typesize) { + case 4: + return truncate_precision32(prec_bits, nbytes / typesize, + (int32_t *)src, (int32_t *)dest); + case 8: + return truncate_precision64(prec_bits, nbytes / typesize, + (int64_t *)src, (int64_t *)dest); + default: + fprintf(stderr, "Error in trunc-prec filter: Precision for typesize %d " + "not handled", (int)typesize); + return -1; + } +} diff --git a/src/blosc2/blosc/trunc-prec.h b/src/blosc2/blosc/trunc-prec.h new file mode 100644 index 0000000..50d53de --- /dev/null +++ b/src/blosc2/blosc/trunc-prec.h @@ -0,0 +1,20 @@ +/********************************************************************* + Blosc - Blocked Shuffling and Compression Library + + Copyright (C) 2021 The Blosc Developers + https://blosc.org + License: BSD 3-Clause (see LICENSE.txt) + + See LICENSE.txt for details about copyright and rights to use. +**********************************************************************/ + +#ifndef BLOSC_TRUNC_PREC_H +#define BLOSC_TRUNC_PREC_H + +#include +#include + +int truncate_precision(int8_t prec_bits, int32_t typesize, int32_t nbytes, + const uint8_t* src, uint8_t* dest); + +#endif //BLOSC_TRUNC_PREC_H diff --git a/src/blosc2/blosc/win32/pthread.c b/src/blosc2/blosc/win32/pthread.c new file mode 100644 index 0000000..c6ff3a4 --- /dev/null +++ b/src/blosc2/blosc/win32/pthread.c @@ -0,0 +1,227 @@ +/* + * Code for simulating pthreads API on Windows. This is Git-specific, + * but it is enough for Numexpr needs too. + * + * Copyright (C) 2009 Andrzej K. Haczewski + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * DISCLAIMER: The implementation is Git-specific, it is subset of original + * Pthreads API, without lots of other features that Git doesn't use. + * Git also makes sure that the passed arguments are valid, so there's + * no need for double-checking. + */ + +#ifndef PTHREAD_C +#define PTHREAD_C + +#include "pthread.h" + +#include +#include +#include +#include +#include + + +#define PTHREAD_UNUSED_PARAM(x) ((void)(x)) + +void die(const char *err, ...) +{ + printf("%s", err); + exit(-1); +} + +static unsigned __stdcall win32_start_routine(void *arg) +{ + pthread_t *thread = (pthread_t*)arg; + thread->arg = thread->start_routine(thread->arg); + return 0; +} + +int pthread_create(pthread_t *thread, const void *unused, + void *(*start_routine)(void*), void *arg) +{ + PTHREAD_UNUSED_PARAM(unused); + thread->arg = arg; + thread->start_routine = start_routine; + thread->handle = (HANDLE) + _beginthreadex(NULL, 0, win32_start_routine, thread, 0, NULL); + + if (!thread->handle) + return errno; + else + return 0; +} + +int win32_pthread_join(pthread_t *thread, void **value_ptr) +{ + DWORD result = WaitForSingleObject(thread->handle, INFINITE); + switch (result) { + case WAIT_OBJECT_0: + if (value_ptr) + *value_ptr = thread->arg; + return 0; + case WAIT_ABANDONED: + return EINVAL; + default: + return GetLastError(); + } +} + +int pthread_cond_init(pthread_cond_t *cond, const void *unused) +{ + PTHREAD_UNUSED_PARAM(unused); + cond->waiters = 0; + cond->was_broadcast = 0; + InitializeCriticalSection(&cond->waiters_lock); + + cond->sema = CreateSemaphore(NULL, 0, LONG_MAX, NULL); + if (!cond->sema) + die("CreateSemaphore() failed"); + + cond->continue_broadcast = CreateEvent(NULL, /* security */ + FALSE, /* auto-reset */ + FALSE, /* not signaled */ + NULL); /* name */ + if (!cond->continue_broadcast) + die("CreateEvent() failed"); + + return 0; +} + +int pthread_cond_destroy(pthread_cond_t *cond) +{ + CloseHandle(cond->sema); + CloseHandle(cond->continue_broadcast); + DeleteCriticalSection(&cond->waiters_lock); + return 0; +} + +int pthread_cond_wait(pthread_cond_t *cond, CRITICAL_SECTION *mutex) +{ + int last_waiter; + + EnterCriticalSection(&cond->waiters_lock); + cond->waiters++; + LeaveCriticalSection(&cond->waiters_lock); + + /* + * Unlock external mutex and wait for signal. + * NOTE: we've held mutex locked long enough to increment + * waiters count above, so there's no problem with + * leaving mutex unlocked before we wait on semaphore. + */ + LeaveCriticalSection(mutex); + + /* let's wait - ignore return value */ + WaitForSingleObject(cond->sema, INFINITE); + + /* + * Decrease waiters count. If we are the last waiter, then we must + * notify the broadcasting thread that it can continue. + * But if we continued due to cond_signal, we do not have to do that + * because the signaling thread knows that only one waiter continued. + */ + EnterCriticalSection(&cond->waiters_lock); + cond->waiters--; + last_waiter = cond->was_broadcast && cond->waiters == 0; + LeaveCriticalSection(&cond->waiters_lock); + + if (last_waiter) { + /* + * cond_broadcast was issued while mutex was held. This means + * that all other waiters have continued, but are contending + * for the mutex at the end of this function because the + * broadcasting thread did not leave cond_broadcast, yet. + * (This is so that it can be sure that each waiter has + * consumed exactly one slice of the semaphore.) + * The last waiter must tell the broadcasting thread that it + * can go on. + */ + SetEvent(cond->continue_broadcast); + /* + * Now we go on to contend with all other waiters for + * the mutex. Auf in den Kampf! + */ + } + /* lock external mutex again */ + EnterCriticalSection(mutex); + + return 0; +} + +/* + * IMPORTANT: This implementation requires that pthread_cond_signal + * is called while the mutex is held that is used in the corresponding + * pthread_cond_wait calls! + */ +int pthread_cond_signal(pthread_cond_t *cond) +{ + int have_waiters; + + EnterCriticalSection(&cond->waiters_lock); + have_waiters = cond->waiters > 0; + LeaveCriticalSection(&cond->waiters_lock); + + /* + * Signal only when there are waiters + */ + if (have_waiters) + return ReleaseSemaphore(cond->sema, 1, NULL) ? + 0 : GetLastError(); + else + return 0; +} + +/* + * DOUBLY IMPORTANT: This implementation requires that pthread_cond_broadcast + * is called while the mutex is held that is used in the corresponding + * pthread_cond_wait calls! + */ +int pthread_cond_broadcast(pthread_cond_t *cond) +{ + EnterCriticalSection(&cond->waiters_lock); + + if ((cond->was_broadcast = cond->waiters > 0)) { + /* wake up all waiters */ + ReleaseSemaphore(cond->sema, cond->waiters, NULL); + LeaveCriticalSection(&cond->waiters_lock); + /* + * At this point all waiters continue. Each one takes its + * slice of the semaphore. Now it's our turn to wait: Since + * the external mutex is held, no thread can leave cond_wait, + * yet. For this reason, we can be sure that no thread gets + * a chance to eat *more* than one slice. OTOH, it means + * that the last waiter must send us a wake-up. + */ + WaitForSingleObject(cond->continue_broadcast, INFINITE); + /* + * Since the external mutex is held, no thread can enter + * cond_wait, and, hence, it is safe to reset this flag + * without cond->waiters_lock held. + */ + cond->was_broadcast = 0; + } else { + LeaveCriticalSection(&cond->waiters_lock); + } + return 0; +} + +#endif /* PTHREAD_C */ diff --git a/src/blosc2/blosc/win32/pthread.h b/src/blosc2/blosc/win32/pthread.h new file mode 100644 index 0000000..e1a7ffb --- /dev/null +++ b/src/blosc2/blosc/win32/pthread.h @@ -0,0 +1,88 @@ +/* + * Code for simulating pthreads API on Windows. This is Git-specific, + * but it is enough for Numexpr needs too. + * + * Copyright (C) 2009 Andrzej K. Haczewski + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * DISCLAIMER: The implementation is Git-specific, it is subset of original + * Pthreads API, without lots of other features that Git doesn't use. + * Git also makes sure that the passed arguments are valid, so there's + * no need for double-checking. + */ + +#ifndef PTHREAD_H +#define PTHREAD_H + +#ifndef WIN32_LEAN_AND_MEAN +#define WIN32_LEAN_AND_MEAN +#endif + +#include + +/* + * Defines that adapt Windows API threads to pthreads API + */ +#define pthread_mutex_t CRITICAL_SECTION + +#define pthread_mutex_init(a,b) InitializeCriticalSection((a)) +#define pthread_mutex_destroy(a) DeleteCriticalSection((a)) +#define pthread_mutex_lock EnterCriticalSection +#define pthread_mutex_unlock LeaveCriticalSection + +/* + * Implement simple condition variable for Windows threads, based on ACE + * implementation. + */ +typedef struct { + LONG waiters; + int was_broadcast; + CRITICAL_SECTION waiters_lock; + HANDLE sema; + HANDLE continue_broadcast; +} pthread_cond_t; + +extern int pthread_cond_init(pthread_cond_t *cond, const void *unused); +extern int pthread_cond_destroy(pthread_cond_t *cond); +extern int pthread_cond_wait(pthread_cond_t *cond, CRITICAL_SECTION *mutex); +extern int pthread_cond_signal(pthread_cond_t *cond); +extern int pthread_cond_broadcast(pthread_cond_t *cond); + +/* + * Simple thread creation implementation using pthread API + */ +typedef struct { + HANDLE handle; + void *(*start_routine)(void*); + void *arg; +} pthread_t; + +extern int pthread_create(pthread_t *thread, const void *unused, + void *(*start_routine)(void*), void *arg); + +/* + * To avoid the need of copying a struct, we use small macro wrapper to pass + * pointer to win32_pthread_join instead. + */ +#define pthread_join(a, b) win32_pthread_join(&(a), (b)) + +extern int win32_pthread_join(pthread_t *thread, void **value_ptr); + +#endif /* PTHREAD_H */ diff --git a/src/blosc2/blosc/win32/stdint-windows.h b/src/blosc2/blosc/win32/stdint-windows.h new file mode 100644 index 0000000..4fe0ef9 --- /dev/null +++ b/src/blosc2/blosc/win32/stdint-windows.h @@ -0,0 +1,259 @@ +// ISO C9x compliant stdint.h for Microsoft Visual Studio +// Based on ISO/IEC 9899:TC2 Committee draft (May 6, 2005) WG14/N1124 +// +// Copyright (c) 2006-2013 Alexander Chemeris +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the product nor the names of its contributors may +// be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED +// WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO +// EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; +// OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +// WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR +// OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF +// ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +/////////////////////////////////////////////////////////////////////////////// + +#ifndef _MSC_VER // [ +#error "Use this header only with Microsoft Visual C++ compilers!" +#endif // _MSC_VER ] + +#ifndef _MSC_STDINT_H_ // [ +#define _MSC_STDINT_H_ + +#if _MSC_VER > 1000 +#pragma once +#endif + +#if _MSC_VER >= 1600 // [ +#include +#else // ] _MSC_VER >= 1600 [ + +#include + +// For Visual Studio 6 in C++ mode and for many Visual Studio versions when +// compiling for ARM we should wrap include with 'extern "C++" {}' +// or compiler give many errors like this: +// error C2733: second C linkage of overloaded function 'wmemchr' not allowed +#ifdef __cplusplus +extern "C" { +#endif +# include +#ifdef __cplusplus +} +#endif + +// Define _W64 macros to mark types changing their size, like intptr_t. +#ifndef _W64 +# if !defined(__midl) && (defined(_X86_) || defined(_M_IX86)) && _MSC_VER >= 1300 +# define _W64 __w64 +# else +# define _W64 +# endif +#endif + + +// 7.18.1 Integer types + +// 7.18.1.1 Exact-width integer types + +// Visual Studio 6 and Embedded Visual C++ 4 doesn't +// realize that, e.g. char has the same size as __int8 +// so we give up on __intX for them. +#if (_MSC_VER < 1300) + typedef signed char int8_t; + typedef signed short int16_t; + typedef signed int int32_t; + typedef unsigned char uint8_t; + typedef unsigned short uint16_t; + typedef unsigned int uint32_t; +#else + typedef signed __int8 int8_t; + typedef signed __int16 int16_t; + typedef signed __int32 int32_t; + typedef unsigned __int8 uint8_t; + typedef unsigned __int16 uint16_t; + typedef unsigned __int32 uint32_t; +#endif +typedef signed __int64 int64_t; +typedef unsigned __int64 uint64_t; + + +// 7.18.1.2 Minimum-width integer types +typedef int8_t int_least8_t; +typedef int16_t int_least16_t; +typedef int32_t int_least32_t; +typedef int64_t int_least64_t; +typedef uint8_t uint_least8_t; +typedef uint16_t uint_least16_t; +typedef uint32_t uint_least32_t; +typedef uint64_t uint_least64_t; + +// 7.18.1.3 Fastest minimum-width integer types +typedef int8_t int_fast8_t; +typedef int16_t int_fast16_t; +typedef int32_t int_fast32_t; +typedef int64_t int_fast64_t; +typedef uint8_t uint_fast8_t; +typedef uint16_t uint_fast16_t; +typedef uint32_t uint_fast32_t; +typedef uint64_t uint_fast64_t; + +// 7.18.1.4 Integer types capable of holding object pointers +#ifdef _WIN64 // [ + typedef signed __int64 intptr_t; + typedef unsigned __int64 uintptr_t; +#else // _WIN64 ][ + typedef _W64 signed int intptr_t; + typedef _W64 unsigned int uintptr_t; +#endif // _WIN64 ] + +// 7.18.1.5 Greatest-width integer types +typedef int64_t intmax_t; +typedef uint64_t uintmax_t; + + +// 7.18.2 Limits of specified-width integer types + +#if !defined(__cplusplus) || defined(__STDC_LIMIT_MACROS) // [ See footnote 220 at page 257 and footnote 221 at page 259 + +// 7.18.2.1 Limits of exact-width integer types +#define INT8_MIN ((int8_t)_I8_MIN) +#define INT8_MAX _I8_MAX +#define INT16_MIN ((int16_t)_I16_MIN) +#define INT16_MAX _I16_MAX +#define INT32_MIN ((int32_t)_I32_MIN) +#define INT32_MAX _I32_MAX +#define INT64_MIN ((int64_t)_I64_MIN) +#define INT64_MAX _I64_MAX +#define UINT8_MAX _UI8_MAX +#define UINT16_MAX _UI16_MAX +#define UINT32_MAX _UI32_MAX +#define UINT64_MAX _UI64_MAX + +// 7.18.2.2 Limits of minimum-width integer types +#define INT_LEAST8_MIN INT8_MIN +#define INT_LEAST8_MAX INT8_MAX +#define INT_LEAST16_MIN INT16_MIN +#define INT_LEAST16_MAX INT16_MAX +#define INT_LEAST32_MIN INT32_MIN +#define INT_LEAST32_MAX INT32_MAX +#define INT_LEAST64_MIN INT64_MIN +#define INT_LEAST64_MAX INT64_MAX +#define UINT_LEAST8_MAX UINT8_MAX +#define UINT_LEAST16_MAX UINT16_MAX +#define UINT_LEAST32_MAX UINT32_MAX +#define UINT_LEAST64_MAX UINT64_MAX + +// 7.18.2.3 Limits of fastest minimum-width integer types +#define INT_FAST8_MIN INT8_MIN +#define INT_FAST8_MAX INT8_MAX +#define INT_FAST16_MIN INT16_MIN +#define INT_FAST16_MAX INT16_MAX +#define INT_FAST32_MIN INT32_MIN +#define INT_FAST32_MAX INT32_MAX +#define INT_FAST64_MIN INT64_MIN +#define INT_FAST64_MAX INT64_MAX +#define UINT_FAST8_MAX UINT8_MAX +#define UINT_FAST16_MAX UINT16_MAX +#define UINT_FAST32_MAX UINT32_MAX +#define UINT_FAST64_MAX UINT64_MAX + +// 7.18.2.4 Limits of integer types capable of holding object pointers +#ifdef _WIN64 // [ +# define INTPTR_MIN INT64_MIN +# define INTPTR_MAX INT64_MAX +# define UINTPTR_MAX UINT64_MAX +#else // _WIN64 ][ +# define INTPTR_MIN INT32_MIN +# define INTPTR_MAX INT32_MAX +# define UINTPTR_MAX UINT32_MAX +#endif // _WIN64 ] + +// 7.18.2.5 Limits of greatest-width integer types +#define INTMAX_MIN INT64_MIN +#define INTMAX_MAX INT64_MAX +#define UINTMAX_MAX UINT64_MAX + +// 7.18.3 Limits of other integer types + +#ifdef _WIN64 // [ +# define PTRDIFF_MIN _I64_MIN +# define PTRDIFF_MAX _I64_MAX +#else // _WIN64 ][ +# define PTRDIFF_MIN _I32_MIN +# define PTRDIFF_MAX _I32_MAX +#endif // _WIN64 ] + +#define SIG_ATOMIC_MIN INT_MIN +#define SIG_ATOMIC_MAX INT_MAX + +#ifndef SIZE_MAX // [ +# ifdef _WIN64 // [ +# define SIZE_MAX _UI64_MAX +# else // _WIN64 ][ +# define SIZE_MAX _UI32_MAX +# endif // _WIN64 ] +#endif // SIZE_MAX ] + +// WCHAR_MIN and WCHAR_MAX are also defined in +#ifndef WCHAR_MIN // [ +# define WCHAR_MIN 0 +#endif // WCHAR_MIN ] +#ifndef WCHAR_MAX // [ +# define WCHAR_MAX _UI16_MAX +#endif // WCHAR_MAX ] + +#define WINT_MIN 0 +#define WINT_MAX _UI16_MAX + +#endif // __STDC_LIMIT_MACROS ] + + +// 7.18.4 Limits of other integer types + +#if !defined(__cplusplus) || defined(__STDC_CONSTANT_MACROS) // [ See footnote 224 at page 260 + +// 7.18.4.1 Macros for minimum-width integer constants + +#define INT8_C(val) val##i8 +#define INT16_C(val) val##i16 +#define INT32_C(val) val##i32 +#define INT64_C(val) val##i64 + +#define UINT8_C(val) val##ui8 +#define UINT16_C(val) val##ui16 +#define UINT32_C(val) val##ui32 +#define UINT64_C(val) val##ui64 + +// 7.18.4.2 Macros for greatest-width integer constants +// These #ifndef's are needed to prevent collisions with . +// Check out Issue 9 for the details. +#ifndef INTMAX_C // [ +# define INTMAX_C INT64_C +#endif // INTMAX_C ] +#ifndef UINTMAX_C // [ +# define UINTMAX_C UINT64_C +#endif // UINTMAX_C ] + +#endif // __STDC_CONSTANT_MACROS ] + +#endif // _MSC_VER >= 1600 ] + +#endif // _MSC_STDINT_H_ ] diff --git a/src/blosc2/include/blosc2.h b/src/blosc2/include/blosc2.h new file mode 100644 index 0000000..d3f4309 --- /dev/null +++ b/src/blosc2/include/blosc2.h @@ -0,0 +1,2214 @@ +/********************************************************************* + Blosc - Blocked Shuffling and Compression Library + + Copyright (C) 2021 The Blosc Developers + https://blosc.org + License: BSD 3-Clause (see LICENSE.txt) + + See LICENSE.txt for details about copyright and rights to use. +**********************************************************************/ + +/********************************************************************* + @file blosc2.h + @brief Blosc2 header file. + + This file contains Blosc2 public API and the structures needed to use it. + @author The Blosc Developers +**********************************************************************/ + + +#ifndef BLOSC2_H +#define BLOSC2_H + +#include +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif +#include "blosc2/blosc2-export.h" +#include "blosc2/blosc2-common.h" +#include "blosc2/blosc2-stdio.h" +#ifdef __cplusplus +} +#endif + +#if defined(_WIN32) && !defined(__MINGW32__) +#include + #include + + #include + #define getpid _getpid +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +// For compatibility with the Blosc 1.x series +#ifdef BLOSC1_COMPAT + // Blosc2 symbols that should be accessible from Blosc 1.x API + #define BLOSC2_VERSION_MAJOR BLOSC_VERSION_MAJOR + #define BLOSC2_VERSION_MINOR BLOSC_VERSION_MINOR + #define BLOSC2_VERSION_RELEASE BLOSC_VERSION_RELEASE + #define BLOSC2_VERSION_STRING BLOSC_VERSION_STRING + #define BLOSC2_VERSION_DATE BLOSC_VERSION_DATE + #define BLOSC2_MAX_OVERHEAD BLOSC_MAX_OVERHEAD + #define BLOSC2_MAX_BUFFERSIZE BLOSC_MAX_BUFFERSIZE + + // API that keeps the blosc1_ prefix + #define blosc1_compress blosc_compress + #define blosc1_decompress blosc_decompress + #define blosc1_getitem blosc_getitem + #define blosc1_get_compressor blosc_get_compressor + #define blosc1_set_compressor blosc_set_compressor + #define blosc1_cbuffer_sizes blosc_cbuffer_sizes + #define blosc1_cbuffer_validate blosc_cbuffer_validate + #define blosc1_cbuffer_metainfo blosc_cbuffer_metainfo + #define blosc1_get_blocksize blosc_get_blocksize + #define blosc1_set_blocksize blosc_set_blocksize + #define blosc1_set_splitmode blosc_set_splitmode + + // API that has been migrated to blosc2_ prefix + #define blosc2_init blosc_init + #define blosc2_destroy blosc_destroy + #define blosc2_free_resources blosc_free_resources + #define blosc2_get_nthreads blosc_get_nthreads + #define blosc2_set_nthreads blosc_set_nthreads + #define blosc2_compcode_to_compname blosc_compcode_to_compname + #define blosc2_compname_to_compcode blosc_compname_to_compcode + #define blosc2_list_compressors blosc_list_compressors + #define blosc2_get_version_string blosc_get_version_string + #define blosc2_get_complib_info blosc_get_complib_info + #define blosc2_cbuffer_versions blosc_cbuffer_versions + #define blosc2_cbuffer_complib blosc_cbuffer_complib +#endif + + +/* Version numbers */ +#define BLOSC2_VERSION_MAJOR 2 /* for major interface/format changes */ +#define BLOSC2_VERSION_MINOR 3 /* for minor interface/format changes */ +#define BLOSC2_VERSION_RELEASE 2 /* for tweaks, bug-fixes, or development */ + +#define BLOSC2_VERSION_STRING "2.3.2.dev" /* string version. Sync with above! */ +#define BLOSC2_VERSION_DATE "$Date:: 2022-08-24 #$" /* date version */ + + +/* The maximum number of dimensions for caterva arrays */ +#define BLOSC2_MAX_DIM 8 + + +/* Tracing macros */ +#define BLOSC_TRACE_ERROR(msg, ...) BLOSC_TRACE(error, msg, ##__VA_ARGS__) +#define BLOSC_TRACE_WARNING(msg, ...) BLOSC_TRACE(warning, msg, ##__VA_ARGS__) +#define BLOSC_TRACE(cat, msg, ...) \ + do { \ + const char *__e = getenv("BLOSC_TRACE"); \ + if (!__e) { break; } \ + fprintf(stderr, "[%s] - " msg " (%s:%d)\n", #cat, ##__VA_ARGS__, __FILE__, __LINE__); \ + } while(0) + + +/* The VERSION_FORMAT symbols below should be just 1-byte long */ +enum { + /* Blosc format version, starting at 1 + 1 -> Blosc pre-1.0 + 2 -> Blosc 1.x stable series + 3 -> Blosc 2-alpha.x series + 4 -> Blosc 2.x beta.1 series + 5 -> Blosc 2.x stable series + */ + BLOSC1_VERSION_FORMAT_PRE1 = 1, + BLOSC1_VERSION_FORMAT = 2, + BLOSC2_VERSION_FORMAT_ALPHA = 3, + BLOSC2_VERSION_FORMAT_BETA1 = 4, + BLOSC2_VERSION_FORMAT_STABLE = 5, + BLOSC2_VERSION_FORMAT = BLOSC2_VERSION_FORMAT_STABLE, +}; + + +/* The FRAME_FORMAT_VERSION symbols below should be just 4-bit long */ +enum { + /* Blosc format version + * 1 -> First version (introduced in beta.2) + * 2 -> Second version (introduced in rc.1) + * + */ + BLOSC2_VERSION_FRAME_FORMAT_BETA2 = 1, // for 2.0.0-beta2 and after + BLOSC2_VERSION_FRAME_FORMAT_RC1 = 2, // for 2.0.0-rc1 and after + BLOSC2_VERSION_FRAME_FORMAT = BLOSC2_VERSION_FRAME_FORMAT_RC1, +}; + + +//!< Struct for storing data from instrumentation of codecs +// This can be flexible because it is typically used mainly for development +typedef struct { + float cratio; + float cspeed; + float filter_speed; + //float memory; + //float power; + uint8_t flags[4]; +} blosc2_instr; + + +enum { +#ifndef BLOSC_H + BLOSC_MIN_HEADER_LENGTH = 16, + //!< Minimum header length (Blosc1) +#endif // BLOSC_H + BLOSC_EXTENDED_HEADER_LENGTH = 32, + //!< Extended header length (Blosc2, see README_HEADER) + BLOSC2_MAX_OVERHEAD = BLOSC_EXTENDED_HEADER_LENGTH, + //!< The maximum overhead during compression in bytes. This equals + //!< to @ref BLOSC_EXTENDED_HEADER_LENGTH now, but can be higher in future + //!< implementations. + BLOSC2_MAX_BUFFERSIZE = (INT_MAX - BLOSC2_MAX_OVERHEAD), + //!< Maximum source buffer size to be compressed +#ifndef BLOSC_H + BLOSC_MAX_TYPESIZE = UINT8_MAX, + //!< Maximum typesize before considering source buffer as a stream of bytes. + //!< Cannot be larger than 255. +#endif // BLOSC_H + BLOSC_MIN_BUFFERSIZE = 32, + //!< Minimum buffer size to be compressed. +}; + + +enum { + BLOSC2_DEFINED_FILTERS_START = 0, + BLOSC2_DEFINED_FILTERS_STOP = 31, + //!< Blosc-defined filters must be between 0 - 31. + BLOSC2_GLOBAL_REGISTERED_FILTERS_START = 32, + BLOSC2_GLOBAL_REGISTERED_FILTERS_STOP = 159, + //!< Blosc-registered filters must be between 32 - 159. + BLOSC2_GLOBAL_REGISTERED_FILTERS = 2, + //!< Number of Blosc-registered filters at the moment. + BLOSC2_USER_REGISTERED_FILTERS_START = 128, + BLOSC2_USER_REGISTERED_FILTERS_STOP = 255, + //!< User-defined filters must be between 128 - 255. + BLOSC2_MAX_FILTERS = 6, + //!< Maximum number of filters in the filter pipeline + BLOSC2_MAX_UDFILTERS = 16, + //!< Maximum number of filters that a user can register. +}; + + +/** + * @brief Codes for filters. + * + * @sa #blosc1_compress + */ +enum { +#ifndef BLOSC_H + BLOSC_NOSHUFFLE = 0, //!< No shuffle (for compatibility with Blosc1). + BLOSC_NOFILTER = 0, //!< No filter. + BLOSC_SHUFFLE = 1, //!< Byte-wise shuffle. + BLOSC_BITSHUFFLE = 2, //!< Bit-wise shuffle. +#endif // BLOSC_H + BLOSC_DELTA = 3, //!< Delta filter. + BLOSC_TRUNC_PREC = 4, //!< Truncate mantissa precision; positive values in cparams.filters_meta will keep bits; negative values will reduce bits. + BLOSC_LAST_FILTER = 5, //!< sentinel + BLOSC_LAST_REGISTERED_FILTER = BLOSC2_GLOBAL_REGISTERED_FILTERS_START + BLOSC2_GLOBAL_REGISTERED_FILTERS - 1, + //!< Determine the last registered filter. It is used to check if a filter is registered or not. +}; + +/** + * @brief Codes for internal flags (see blosc1_cbuffer_metainfo) + */ +enum { +#ifndef BLOSC_H + BLOSC_DOSHUFFLE = 0x1, //!< byte-wise shuffle + BLOSC_MEMCPYED = 0x2, //!< plain copy + BLOSC_DOBITSHUFFLE = 0x4, //!< bit-wise shuffle +#endif // BLOSC_H + BLOSC_DODELTA = 0x8, //!< delta coding +}; + +/** + * @brief Codes for new internal flags in Blosc2 + */ +enum { + BLOSC2_USEDICT = 0x1, //!< use dictionaries with codec + BLOSC2_BIGENDIAN = 0x2, //!< data is in big-endian ordering + BLOSC2_INSTR_CODEC = 0x80, //!< codec is instrumented (mainly for development) +}; + +/** + * @brief Values for different Blosc2 capabilities + */ +enum { + BLOSC2_MAXDICTSIZE = 128 * 1024, //!< maximum size for compression dicts + BLOSC2_MAXBLOCKSIZE = 536866816 //!< maximum size for blocks +}; + + +enum { + BLOSC2_DEFINED_CODECS_START = 0, + BLOSC2_DEFINED_CODECS_STOP = 31, + //!< Blosc-defined codecs must be between 0 - 31. + BLOSC2_GLOBAL_REGISTERED_CODECS_START = 32, + BLOSC2_GLOBAL_REGISTERED_CODECS_STOP = 159, + //!< Blosc-registered codecs must be between 31 - 159. + BLOSC2_GLOBAL_REGISTERED_CODECS = 1, + //!< Number of Blosc-registered codecs at the moment. + BLOSC2_USER_REGISTERED_CODECS_START = 160, + BLOSC2_USER_REGISTERED_CODECS_STOP = 255, + //!< User-defined codecs must be between 160 - 255. +}; + +/** + * @brief Codes for the different compressors shipped with Blosc + */ +enum { +#ifndef BLOSC_H + BLOSC_BLOSCLZ = 0, + BLOSC_LZ4 = 1, + BLOSC_LZ4HC = 2, + BLOSC_ZLIB = 4, + BLOSC_ZSTD = 5, +#endif // BLOSC_H + BLOSC_LAST_CODEC = 6, + //!< Determine the last codec defined by Blosc. + BLOSC_LAST_REGISTERED_CODEC = BLOSC2_GLOBAL_REGISTERED_CODECS_START + BLOSC2_GLOBAL_REGISTERED_CODECS - 1, + //!< Determine the last registered codec. It is used to check if a codec is registered or not. +}; + + +// Names for the different compressors shipped with Blosc + +#ifndef BLOSC_H +#define BLOSC_BLOSCLZ_COMPNAME "blosclz" +#define BLOSC_LZ4_COMPNAME "lz4" +#define BLOSC_LZ4HC_COMPNAME "lz4hc" +#define BLOSC_ZLIB_COMPNAME "zlib" +#define BLOSC_ZSTD_COMPNAME "zstd" +#endif // BLOSC_H + +/** + * @brief Codes for compression libraries shipped with Blosc (code must be < 8) + */ +enum { +#ifndef BLOSC_H + BLOSC_BLOSCLZ_LIB = 0, + BLOSC_LZ4_LIB = 1, + BLOSC_ZLIB_LIB = 3, + BLOSC_ZSTD_LIB = 4, +#endif // BLOSC_H + BLOSC_UDCODEC_LIB = 6, + BLOSC_SCHUNK_LIB = 7, //!< compressor library in super-chunk header +}; + +/** + * @brief Names for the different compression libraries shipped with Blosc + */ +#ifndef BLOSC_H +#define BLOSC_BLOSCLZ_LIBNAME "BloscLZ" +#define BLOSC_LZ4_LIBNAME "LZ4" +#define BLOSC_ZLIB_LIBNAME "Zlib" +#define BLOSC_ZSTD_LIBNAME "Zstd" +#endif // BLOSC_H + +/** + * @brief The codes for compressor formats shipped with Blosc + */ +enum { +#ifndef BLOSC_H + BLOSC_BLOSCLZ_FORMAT = BLOSC_BLOSCLZ_LIB, + BLOSC_LZ4_FORMAT = BLOSC_LZ4_LIB, + //!< LZ4HC and LZ4 share the same format + BLOSC_LZ4HC_FORMAT = BLOSC_LZ4_LIB, + BLOSC_ZLIB_FORMAT = BLOSC_ZLIB_LIB, + BLOSC_ZSTD_FORMAT = BLOSC_ZSTD_LIB, +#endif // BLOSC_H + BLOSC_UDCODEC_FORMAT = BLOSC_UDCODEC_LIB, +}; + +/** + * @brief The version formats for compressors shipped with Blosc. + * All versions here starts at 1 + */ +enum { +#ifndef BLOSC_H + BLOSC_BLOSCLZ_VERSION_FORMAT = 1, + BLOSC_LZ4_VERSION_FORMAT = 1, + BLOSC_LZ4HC_VERSION_FORMAT = 1, /* LZ4HC and LZ4 share the same format */ + BLOSC_ZLIB_VERSION_FORMAT = 1, + BLOSC_ZSTD_VERSION_FORMAT = 1, +#endif // BLOSC_H + BLOSC_UDCODEC_VERSION_FORMAT = 1, +}; + +/** + * @brief Split mode for blocks. + * NEVER and ALWAYS are for experimenting with compression ratio. + * AUTO for nearly optimal behaviour (based on heuristics). + * FORWARD_COMPAT provides best forward compatibility (default). + */ +#ifndef BLOSC_H +enum { + BLOSC_ALWAYS_SPLIT = 1, + BLOSC_NEVER_SPLIT = 2, + BLOSC_AUTO_SPLIT = 3, + BLOSC_FORWARD_COMPAT_SPLIT = 4, +}; +#endif // BLOSC_H + +/** + * @brief Offsets for fields in Blosc2 chunk header. + */ +enum { + BLOSC2_CHUNK_VERSION = 0x0, //!< the version for the chunk format + BLOSC2_CHUNK_VERSIONLZ = 0x1, //!< the version for the format of internal codec + BLOSC2_CHUNK_FLAGS = 0x2, //!< flags and codec info + BLOSC2_CHUNK_TYPESIZE = 0x3, //!< (uint8) the number of bytes of the atomic type + BLOSC2_CHUNK_NBYTES = 0x4, //!< (int32) uncompressed size of the buffer (this header is not included) + BLOSC2_CHUNK_BLOCKSIZE = 0x8, //!< (int32) size of internal blocks + BLOSC2_CHUNK_CBYTES = 0xc, //!< (int32) compressed size of the buffer (including this header) + BLOSC2_CHUNK_FILTER_CODES = 0x10, //!< the codecs for the filter pipeline (1 byte per code) + BLOSC2_CHUNK_FILTER_META = 0x18, //!< meta info for the filter pipeline (1 byte per code) + BLOSC2_CHUNK_BLOSC2_FLAGS = 0x1F, //!< flags specific for Blosc2 functionality +}; + +/** + * @brief Run lengths for special values for chunks/frames + */ +enum { + BLOSC2_NO_SPECIAL = 0x0, //!< no special value + BLOSC2_SPECIAL_ZERO = 0x1, //!< zero special value + BLOSC2_SPECIAL_NAN = 0x2, //!< NaN special value + BLOSC2_SPECIAL_VALUE = 0x3, //!< generic special value + BLOSC2_SPECIAL_UNINIT = 0x4, //!< non initialized values + BLOSC2_SPECIAL_LASTID = 0x4, //!< last valid ID for special value (update this adequately) + BLOSC2_SPECIAL_MASK = 0x7 //!< special value mask (prev IDs cannot be larger than this) +}; + +/** + * @brief Error codes + */ +enum { + BLOSC2_ERROR_SUCCESS = 0, //=0). In case the compressor + * is not recognized, or there is not support for it in this build, + * it returns a -1. + */ +BLOSC_EXPORT int blosc1_set_compressor(const char* compname); + + +/** + * @brief Select the delta coding filter to be used. + * + * @param dodelta A value >0 will activate the delta filter. + * If 0, it will be de-activated + * + * This call should always succeed. + */ +BLOSC_EXPORT void blosc2_set_delta(int dodelta); + + +/** + * @brief Get the compressor name associated with the compressor code. + * + * @param compcode The code identifying the compressor + * @param compname The pointer to a string where the compressor name will be put. + * + * @return The compressor code. If the compressor code is not recognized, + * or there is not support for it in this build, -1 is returned. + */ +BLOSC_EXPORT int blosc2_compcode_to_compname(int compcode, const char** compname); + + +/** + * @brief Get the compressor code associated with the compressor name. + * + * @param compname The string containing the compressor name. + * + * @return The compressor code. If the compressor name is not recognized, + * or there is not support for it in this build, -1 is returned instead. + */ +BLOSC_EXPORT int blosc2_compname_to_compcode(const char* compname); + + +/** + * @brief Get a list of compressors supported in the current build. + * + * @return The comma separated string with the list of compressor names + * supported. + * + * This function does not leak, so you should not free() the returned + * list. + * + * This function should always succeed. + */ +BLOSC_EXPORT const char* blosc2_list_compressors(void); + + +/** + * @brief Get the version of Blosc in string format. + * + * @return The string with the current Blosc version. + * Useful for dynamic libraries. + */ +BLOSC_EXPORT const char* blosc2_get_version_string(void); + + +/** + * @brief Get info from compression libraries included in the current build. + * + * @param compname The compressor name that you want info from. + * @param complib The pointer to a string where the + * compression library name, if available, will be put. + * @param version The pointer to a string where the + * compression library version, if available, will be put. + * + * @warning You are in charge of the @p complib and @p version strings, + * you should free() them so as to avoid leaks. + * + * @return The code for the compression library (>=0). If it is not supported, + * this function returns -1. + */ +BLOSC_EXPORT int blosc2_get_complib_info(const char* compname, char** complib, + char** version); + + +/** + * @brief Free possible memory temporaries and thread resources. Use this + * when you are not going to use Blosc for a long while. + * + * @return A 0 if succeeds, in case of problems releasing the resources, + * it returns a negative number. + */ +BLOSC_EXPORT int blosc2_free_resources(void); + + +/** + * @brief Get information about a compressed buffer, namely the number of + * uncompressed bytes (@p nbytes) and compressed (@p cbytes). It also + * returns the @p blocksize (which is used internally for doing the + * compression by blocks). + * + * @param cbuffer The buffer of compressed data. + * @param nbytes The pointer where the number of uncompressed bytes will be put. + * @param cbytes The pointer where the number of compressed bytes will be put. + * @param blocksize The pointer where the block size will be put. + * + * You only need to pass the first BLOSC_MIN_HEADER_LENGTH bytes of a + * compressed buffer for this call to work. + * + * This function should always succeed. + */ +BLOSC_EXPORT void blosc1_cbuffer_sizes(const void* cbuffer, size_t* nbytes, + size_t* cbytes, size_t* blocksize); +/** + * @brief Get information about a compressed buffer, namely the number of + * uncompressed bytes (@p nbytes) and compressed (@p cbytes). It also + * returns the @p blocksize (which is used internally for doing the + * compression by blocks). + * + * @param cbuffer The buffer of compressed data. + * @param nbytes The pointer where the number of uncompressed bytes will be put. + * @param cbytes The pointer where the number of compressed bytes will be put. + * @param blocksize The pointer where the block size will be put. + * + * @note: if any of the nbytes, cbytes or blocksize is NULL, it will not be returned. + * + * You only need to pass the first BLOSC_MIN_HEADER_LENGTH bytes of a + * compressed buffer for this call to work. + * + * @return On failure, returns negative value. + */ +BLOSC_EXPORT int blosc2_cbuffer_sizes(const void* cbuffer, int32_t* nbytes, + int32_t* cbytes, int32_t* blocksize); + +/** + * @brief Checks that the compressed buffer starting at @p cbuffer of length @p cbytes + * may contain valid blosc compressed data, and that it is safe to call + * blosc1_decompress/blosc1_getitem. + * On success, returns 0 and sets @p nbytes to the size of the uncompressed data. + * This does not guarantee that the decompression function won't return an error, + * but does guarantee that it is safe to attempt decompression. + * + * @param cbuffer The buffer of compressed data. + * @param cbytes The number of compressed bytes. + * @param nbytes The pointer where the number of uncompressed bytes will be put. + * + * @return On failure, returns negative value. + */ +BLOSC_EXPORT int blosc1_cbuffer_validate(const void* cbuffer, size_t cbytes, + size_t* nbytes); + +/** + * @brief Get information about a compressed buffer, namely the type size + * (@p typesize), as well as some internal @p flags. + * + * @param cbuffer The buffer of compressed data. + * @param typesize The pointer where the type size will be put. + * @param flags The pointer of the integer where the additional info is encoded. + * The @p flags is a set of bits, where the currently used ones are: + * * bit 0: whether the shuffle filter has been applied or not + * * bit 1: whether the internal buffer is a pure memcpy or not + * * bit 2: whether the bitshuffle filter has been applied or not + * * bit 3: whether the delta coding filter has been applied or not + * + * You can use the @p BLOSC_DOSHUFFLE, @p BLOSC_DOBITSHUFFLE, @p BLOSC_DODELTA + * and @p BLOSC_MEMCPYED symbols for extracting the interesting bits + * (e.g. @p flags & @p BLOSC_DOSHUFFLE says whether the buffer is byte-shuffled + * or not). + * + * This function should always succeed. + */ +BLOSC_EXPORT void blosc1_cbuffer_metainfo(const void* cbuffer, size_t* typesize, + int* flags); + + +/** + * @brief Get information about a compressed buffer, namely the internal + * Blosc format version (@p version) and the format for the internal + * Lempel-Ziv compressor used (@p versionlz). + * + * @param cbuffer The buffer of compressed data. + * @param version The pointer where the Blosc format version will be put. + * @param versionlz The pointer where the Lempel-Ziv version will be put. + * + * This function should always succeed. + */ +BLOSC_EXPORT void blosc2_cbuffer_versions(const void* cbuffer, int* version, + int* versionlz); + + +/** + * @brief Get the compressor library/format used in a compressed buffer. + * + * @param cbuffer The buffer of compressed data. + * + * @return The string identifying the compressor library/format used. + * + * This function should always succeed. + */ +BLOSC_EXPORT const char* blosc2_cbuffer_complib(const void* cbuffer); + +/********************************************************************* + Structures and functions related with user-defined input/output. +*********************************************************************/ + +enum { + BLOSC2_IO_FILESYSTEM = 0, + BLOSC_IO_LAST_BLOSC_DEFINED = 1, // sentinel + BLOSC_IO_LAST_REGISTERED = 32, // sentinel +}; + +enum { + BLOSC2_IO_BLOSC_DEFINED = 32, + BLOSC2_IO_REGISTERED = 160, + BLOSC2_IO_USER_DEFINED = 256 +}; + +typedef void* (*blosc2_open_cb)(const char *urlpath, const char *mode, void *params); +typedef int (*blosc2_close_cb)(void *stream); +typedef int64_t (*blosc2_tell_cb)(void *stream); +typedef int (*blosc2_seek_cb)(void *stream, int64_t offset, int whence); +typedef int64_t (*blosc2_write_cb)(const void *ptr, int64_t size, int64_t nitems, void *stream); +typedef int64_t (*blosc2_read_cb)(void *ptr, int64_t size, int64_t nitems, void *stream); +typedef int (*blosc2_truncate_cb)(void *stream, int64_t size); + + +/* + * Input/Output callbacks. + */ +typedef struct { + uint8_t id; + //!< The IO identifier. + blosc2_open_cb open; + //!< The IO open callback. + blosc2_close_cb close; + //!< The IO close callback. + blosc2_tell_cb tell; + //!< The IO tell callback. + blosc2_seek_cb seek; + //!< The IO seek callback. + blosc2_write_cb write; + //!< The IO write callback. + blosc2_read_cb read; + //!< The IO read callback. + blosc2_truncate_cb truncate; + //!< The IO truncate callback. +} blosc2_io_cb; + + +/* + * Input/Output parameters. + */ +typedef struct { + uint8_t id; + //!< The IO identifier. + void *params; + //!< The IO parameters. +} blosc2_io; + +static const blosc2_io_cb BLOSC2_IO_CB_DEFAULTS = { + .id = BLOSC2_IO_FILESYSTEM, + .open = (blosc2_open_cb) blosc2_stdio_open, + .close = (blosc2_close_cb) blosc2_stdio_close, + .tell = (blosc2_tell_cb) blosc2_stdio_tell, + .seek = (blosc2_seek_cb) blosc2_stdio_seek, + .write = (blosc2_write_cb) blosc2_stdio_write, + .read = (blosc2_read_cb) blosc2_stdio_read, + .truncate = (blosc2_truncate_cb) blosc2_stdio_truncate, +}; + +static const blosc2_io BLOSC2_IO_DEFAULTS = { + .id = BLOSC2_IO_FILESYSTEM, + .params = NULL, +}; + + +/** + * @brief Register a user-defined input/output callbacks in Blosc. + * + * @param io The callbacks API to register. + * + * @return 0 if succeeds. Else a negative code is returned. + */ +BLOSC_EXPORT int blosc2_register_io_cb(const blosc2_io_cb *io); + +BLOSC_EXPORT blosc2_io_cb *blosc2_get_io_cb(uint8_t id); + +/********************************************************************* + Structures and functions related with contexts. +*********************************************************************/ + +typedef struct blosc2_context_s blosc2_context; /* opaque type */ + +typedef struct { + void (*btune_init)(void * config, blosc2_context* cctx, blosc2_context* dctx); + //!< Initialize BTune. + void (*btune_next_blocksize)(blosc2_context * context); + //!< Only compute the next blocksize. Only it is executed if BTune is not initialized. + void (*btune_next_cparams)(blosc2_context * context); + //!< Compute the next cparams. Only is executed if BTune is initialized. + void (*btune_update)(blosc2_context * context, double ctime); + //!< Update the BTune parameters. + void (*btune_free)(blosc2_context * context); + //!< Free the BTune. + void *btune_config; + //!> BTune configuration. +}blosc2_btune; + + +/** + * @brief The parameters for a prefilter function. + * + */ +typedef struct { + void *user_data; // user-provided info (optional) + const uint8_t *in; // the input buffer + uint8_t *out; // the output buffer + int32_t out_size; // the output size (in bytes) + int32_t out_typesize; // the output typesize + int32_t out_offset; // offset to reach the start of the output buffer + int64_t nchunk; // the current nchunk in associated schunk (if exists; if not -1) + int32_t nblock; // the current nblock in associated chunk + int32_t tid; // thread id + uint8_t *ttmp; // a temporary that is able to hold several blocks for the output and is private for each thread + size_t ttmp_nbytes; // the size of the temporary in bytes + blosc2_context *ctx; // the compression context +} blosc2_prefilter_params; + +/** + * @brief The parameters for a postfilter function. + * + */ +typedef struct { + void *user_data; // user-provided info (optional) + const uint8_t *in; // the input buffer + uint8_t *out; // the output buffer + int32_t size; // the input size (in bytes) + int32_t typesize; // the input typesize + int32_t offset; // offset to reach the start of the input buffer + int64_t nchunk; // the current nchunk in associated schunk (if exists; if not -1) + int32_t nblock; // the current nblock in associated chunk + int32_t tid; // thread id + uint8_t *ttmp; // a temporary that is able to hold several blocks for the output and is private for each thread + size_t ttmp_nbytes; // the size of the temporary in bytes + blosc2_context *ctx; // the decompression context +} blosc2_postfilter_params; + +/** + * @brief The type of the prefilter function. + * + * If the function call is successful, the return value should be 0; else, a negative value. + */ +typedef int (*blosc2_prefilter_fn)(blosc2_prefilter_params* params); + +/** + * @brief The type of the postfilter function. + * + * If the function call is successful, the return value should be 0; else, a negative value. + */ +typedef int (*blosc2_postfilter_fn)(blosc2_postfilter_params* params); + +/** + * @brief The parameters for creating a context for compression purposes. + * + * In parenthesis it is shown the default value used internally when a 0 + * (zero) in the fields of the struct is passed to a function. + */ +typedef struct { + uint8_t compcode; + //!< The compressor codec. + uint8_t compcode_meta; + //!< The metadata for the compressor codec. + uint8_t clevel; + //!< The compression level (5). + int use_dict; + //!< Use dicts or not when compressing (only for ZSTD). + int32_t typesize; + //!< The type size (8). + int16_t nthreads; + //!< The number of threads to use internally (1). + int32_t blocksize; + //!< The requested size of the compressed blocks (0 means automatic). + int32_t splitmode; + //!< Whether the blocks should be split or not. + void* schunk; + //!< The associated schunk, if any (NULL). + uint8_t filters[BLOSC2_MAX_FILTERS]; + //!< The (sequence of) filters. + uint8_t filters_meta[BLOSC2_MAX_FILTERS]; + //!< The metadata for filters. + blosc2_prefilter_fn prefilter; + //!< The prefilter function. + blosc2_prefilter_params *preparams; + //!< The prefilter parameters. + blosc2_btune *udbtune; + //!< The user-defined BTune parameters. + bool instr_codec; + //!< Whether the codec is instrumented or not +} blosc2_cparams; + +/** + * @brief Default struct for compression params meant for user initialization. + */ +static const blosc2_cparams BLOSC2_CPARAMS_DEFAULTS = { + BLOSC_BLOSCLZ, 0, 5, 0, 8, 1, 0, + BLOSC_FORWARD_COMPAT_SPLIT, NULL, + .filters={0, 0, 0, 0, 0, BLOSC_SHUFFLE}, + .filters_meta={0, 0, 0, 0, 0, 0}, + NULL, NULL, NULL, 0}; + + +/** + @brief The parameters for creating a context for decompression purposes. + + In parenthesis it is shown the default value used internally when a 0 + (zero) in the fields of the struct is passed to a function. + */ +typedef struct { + int16_t nthreads; + //!< The number of threads to use internally (1). + void* schunk; + //!< The associated schunk, if any (NULL). + blosc2_postfilter_fn postfilter; + //!< The postfilter function. + blosc2_postfilter_params *postparams; + //!< The postfilter parameters. +} blosc2_dparams; + +/** + * @brief Default struct for decompression params meant for user initialization. + */ +static const blosc2_dparams BLOSC2_DPARAMS_DEFAULTS = {1, NULL, NULL, NULL}; + +/** + * @brief Create a context for @a *_ctx() compression functions. + * + * @param cparams The blosc2_cparams struct with the compression parameters. + * + * @return A pointer to the new context. NULL is returned if this fails. + */ +BLOSC_EXPORT blosc2_context* blosc2_create_cctx(blosc2_cparams cparams); + +/** + * @brief Create a context for *_ctx() decompression functions. + * + * @param dparams The blosc2_dparams struct with the decompression parameters. + * + * @return A pointer to the new context. NULL is returned if this fails. + */ +BLOSC_EXPORT blosc2_context* blosc2_create_dctx(blosc2_dparams dparams); + +/** + * @brief Free the resources associated with a context. + * + * @param context The context to free. + * + * This function should always succeed and is valid for contexts meant for + * both compression and decompression. + */ +BLOSC_EXPORT void blosc2_free_ctx(blosc2_context* context); + +/** + * @brief Create a @p cparams associated to a context. + * + * @param ctx The context from where to extract the compression parameters. + * @param cparams The pointer where the compression params will be stored. + * + * @return 0 if succeeds. Else a negative code is returned. + */ +BLOSC_EXPORT int blosc2_ctx_get_cparams(blosc2_context *ctx, blosc2_cparams *cparams); + +/** + * @brief Create a @p dparams associated to a context. + * + * @param ctx The context from where to extract the decompression parameters. + * @param dparams The pointer where the decompression params will be stored. + * + * @return 0 if succeeds. Else a negative code is returned. + */ +BLOSC_EXPORT int blosc2_ctx_get_dparams(blosc2_context *ctx, blosc2_dparams *dparams); + +/** + * @brief Set a maskout so as to avoid decompressing specified blocks. + * + * @param ctx The decompression context to update. + * + * @param maskout The boolean mask for the blocks where decompression + * is to be avoided. + * + * @remark The maskout is valid for contexts *only* meant for decompressing + * a chunk via #blosc2_decompress_ctx. Once a call to #blosc2_decompress_ctx + * is done, this mask is reset so that next call to #blosc2_decompress_ctx + * will decompress the whole chunk. + * + * @param nblocks The number of blocks in maskout above. + * + * @return If success, a 0 is returned. An error is signaled with a negative int. + * + */ +BLOSC_EXPORT int blosc2_set_maskout(blosc2_context *ctx, bool *maskout, int nblocks); + +/** + * @brief Compress a block of data in the @p src buffer and returns the size of + * compressed block. + * + * @remark Compression is memory safe and guaranteed not to write @p dest + * more than what is specified in @p destsize. + * There is not a minimum for @p src buffer size @p nbytes. + * + * @warning The @p src buffer and the @p dest buffer can not overlap. + * + * @param clevel The desired compression level and must be a number + * between 0 (no compression) and 9 (maximum compression). + * @param doshuffle Specifies whether the shuffle compression preconditioner + * should be applied or not. #BLOSC_NOFILTER means not applying filters, + * #BLOSC_SHUFFLE means applying shuffle at a byte level and + * #BLOSC_BITSHUFFLE at a bit level (slower but *may* achieve better + * compression). + * @param typesize Is the number of bytes for the atomic type in binary + * @p src buffer. This is mainly useful for the shuffle preconditioner. + * For implementation reasons, only a 1 < typesize < 256 will allow the + * shuffle filter to work. When typesize is not in this range, shuffle + * will be silently disabled. + * @param src The buffer containing the data to compress. + * @param srcsize The number of bytes to compress in the @p src buffer. + * @param dest The buffer where the compressed data will be put, + * must have at least the size of @p destsize. + * @param destsize The size of the dest buffer. Blosc + * guarantees that if you set @p destsize to, at least, + * (@p nbytes + #BLOSC2_MAX_OVERHEAD), the compression will always succeed. + * + * @return The number of bytes compressed. + * If @p src buffer cannot be compressed into @p destsize, the return + * value is zero and you should discard the contents of the @p dest + * buffer. A negative return value means that an internal error happened. This + * should never happen. If you see this, please report it back + * together with the buffer data causing this and compression settings. +*/ +/* + * Environment variables + * _____________________ + * + * *blosc1_compress()* honors different environment variables to control + * internal parameters without the need of doing that programmatically. + * Here are the ones supported: + * + * **BLOSC_CLEVEL=(INTEGER)**: This will overwrite the @p clevel parameter + * before the compression process starts. + * + * **BLOSC_SHUFFLE=[NOSHUFFLE | SHUFFLE | BITSHUFFLE]**: This will + * overwrite the *doshuffle* parameter before the compression process + * starts. + * + * **BLOSC_DELTA=(1|0)**: This will call *blosc2_set_delta()^* before the + * compression process starts. + * + * **BLOSC_TYPESIZE=(INTEGER)**: This will overwrite the *typesize* + * parameter before the compression process starts. + * + * **BLOSC_COMPRESSOR=[BLOSCLZ | LZ4 | LZ4HC | ZLIB | ZSTD]**: + * This will call #blosc_set_compressor before the compression process starts. + * + * **BLOSC_NTHREADS=(INTEGER)**: This will call + * #blosc_set_nthreads before the compression process + * starts. + * + * **BLOSC_BLOCKSIZE=(INTEGER)**: This will call + * #blosc_set_blocksize before the compression process starts. + * *NOTE:* The blocksize is a critical parameter with + * important restrictions in the allowed values, so use this with care. + * + * **BLOSC_NOLOCK=(ANY VALUE)**: This will call #blosc2_compress_ctx under + * the hood, with the *compressor*, *blocksize* and + * *numinternalthreads* parameters set to the same as the last calls to + * #blosc1_set_compressor, #blosc1_set_blocksize and + * #blosc2_set_nthreads. *BLOSC_CLEVEL*, *BLOSC_SHUFFLE*, *BLOSC_DELTA* and + * *BLOSC_TYPESIZE* environment vars will also be honored. + * + */ +BLOSC_EXPORT int blosc2_compress(int clevel, int doshuffle, int32_t typesize, + const void* src, int32_t srcsize, void* dest, + int32_t destsize); + + +/** + * @brief Decompress a block of compressed data in @p src, put the result in + * @p dest and returns the size of the decompressed block. + * + * @warning The @p src buffer and the @p dest buffer can not overlap. + * + * @remark Decompression is memory safe and guaranteed not to write the @p dest + * buffer more than what is specified in @p destsize. + * + * @remark In case you want to keep under control the number of bytes read from + * source, you can call #blosc1_cbuffer_sizes first to check whether the + * @p nbytes (i.e. the number of bytes to be read from @p src buffer by this + * function) in the compressed buffer is ok with you. + * + * @param src The buffer to be decompressed. + * @param srcsize The size of the buffer to be decompressed. + * @param dest The buffer where the decompressed data will be put. + * @param destsize The size of the @p dest buffer. + * + * @return The number of bytes decompressed. + * If an error occurs, e.g. the compressed data is corrupted or the + * output buffer is not large enough, then a negative value + * will be returned instead. +*/ +/* + * Environment variables + * _____________________ + * + * *blosc1_decompress* honors different environment variables to control + * internal parameters without the need of doing that programmatically. + * Here are the ones supported: + * + * **BLOSC_NTHREADS=(INTEGER)**: This will call + * *blosc_set_nthreads(BLOSC_NTHREADS)* before the proper decompression + * process starts. + * + * **BLOSC_NOLOCK=(ANY VALUE)**: This will call *blosc2_decompress_ctx* + * under the hood, with the *numinternalthreads* parameter set to the + * same value as the last call to *blosc2_set_nthreads*. + * + */ +BLOSC_EXPORT int blosc2_decompress(const void* src, int32_t srcsize, + void* dest, int32_t destsize); + +/** + * @brief Context interface to Blosc compression. This does not require a call + * to #blosc2_init and can be called from multithreaded applications + * without the global lock being used, so allowing Blosc be executed + * simultaneously in those scenarios. + * + * @param context A blosc2_context struct with the different compression params. + * @param src The buffer containing the data to be compressed. + * @param srcsize The number of bytes to be compressed from the @p src buffer. + * @param dest The buffer where the compressed data will be put. + * @param destsize The size in bytes of the @p dest buffer. + * + * @return The number of bytes compressed. + * If @p src buffer cannot be compressed into @p destsize, the return + * value is zero and you should discard the contents of the @p dest + * buffer. A negative return value means that an internal error happened. + * It could happen that context is not meant for compression (which is stated in stderr). + * Otherwise, please report it back together with the buffer data causing this + * and compression settings. + */ +BLOSC_EXPORT int blosc2_compress_ctx( + blosc2_context* context, const void* src, int32_t srcsize, void* dest, + int32_t destsize); + + +/** + * @brief Context interface to Blosc decompression. This does not require a + * call to #blosc2_init and can be called from multithreaded + * applications without the global lock being used, so allowing Blosc + * be executed simultaneously in those scenarios. + * + * @param context The blosc2_context struct with the different compression params. + * @param src The buffer of compressed data. + * @param srcsize The length of buffer of compressed data. + * @param dest The buffer where the decompressed data will be put. + * @param destsize The size in bytes of the @p dest buffer. + * + * @warning The @p src buffer and the @p dest buffer can not overlap. + * + * @remark Decompression is memory safe and guaranteed not to write the @p dest + * buffer more than what is specified in @p destsize. + * + * @remark In case you want to keep under control the number of bytes read from + * source, you can call #blosc1_cbuffer_sizes first to check the @p nbytes + * (i.e. the number of bytes to be read from @p src buffer by this function) + * in the compressed buffer. + * + * @remark If #blosc2_set_maskout is called prior to this function, its + * @p block_maskout parameter will be honored for just *one single* shot; + * i.e. the maskout in context will be automatically reset to NULL, so + * mask won't be used next time (unless #blosc2_set_maskout is called again). + * + * @return The number of bytes decompressed (i.e. the maskout blocks are not + * counted). If an error occurs, e.g. the compressed data is corrupted, + * @p destsize is not large enough or context is not meant for decompression, + * then a negative value will be returned instead. + */ +BLOSC_EXPORT int blosc2_decompress_ctx(blosc2_context* context, const void* src, + int32_t srcsize, void* dest, int32_t destsize); + +/** + * @brief Create a chunk made of zeros. + * + * @param cparams The compression parameters. + * @param nbytes The size (in bytes) of the chunk. + * @param dest The buffer where the data chunk will be put. + * @param destsize The size (in bytes) of the @p dest buffer; + * must be BLOSC_EXTENDED_HEADER_LENGTH at least. + * + * @return The number of bytes compressed (BLOSC_EXTENDED_HEADER_LENGTH). + * If negative, there has been an error and @p dest is unusable. + * */ +BLOSC_EXPORT int blosc2_chunk_zeros(blosc2_cparams cparams, int32_t nbytes, + void* dest, int32_t destsize); + + +/** + * @brief Create a chunk made of nans. + * + * @param cparams The compression parameters; + * only 4 bytes (float) and 8 bytes (double) are supported. + * @param nbytes The size (in bytes) of the chunk. + * @param dest The buffer where the data chunk will be put. + * @param destsize The size (in bytes) of the @p dest buffer; + * must be BLOSC_EXTENDED_HEADER_LENGTH at least. + * + * @note Whether the NaNs are floats or doubles will be given by the typesize. + * + * @return The number of bytes compressed (BLOSC_EXTENDED_HEADER_LENGTH). + * If negative, there has been an error and @p dest is unusable. + * */ +BLOSC_EXPORT int blosc2_chunk_nans(blosc2_cparams cparams, int32_t nbytes, + void* dest, int32_t destsize); + + +/** + * @brief Create a chunk made of repeated values. + * + * @param cparams The compression parameters. + * @param nbytes The size (in bytes) of the chunk. + * @param dest The buffer where the data chunk will be put. + * @param destsize The size (in bytes) of the @p dest buffer. + * @param repeatval A pointer to the repeated value (little endian). + * The size of the value is given by @p cparams.typesize param. + * + * @return The number of bytes compressed (BLOSC_EXTENDED_HEADER_LENGTH + typesize). + * If negative, there has been an error and @p dest is unusable. + * */ +BLOSC_EXPORT int blosc2_chunk_repeatval(blosc2_cparams cparams, int32_t nbytes, + void* dest, int32_t destsize, void* repeatval); + + +/** + * @brief Create a chunk made of uninitialized values. + * + * @param cparams The compression parameters. + * @param nbytes The size (in bytes) of the chunk. + * @param dest The buffer where the data chunk will be put. + * @param destsize The size (in bytes) of the @p dest buffer; + * must be BLOSC_EXTENDED_HEADER_LENGTH at least. + * + * @return The number of bytes compressed (BLOSC_EXTENDED_HEADER_LENGTH). + * If negative, there has been an error and @p dest is unusable. + * */ +BLOSC_EXPORT int blosc2_chunk_uninit(blosc2_cparams cparams, int32_t nbytes, + void* dest, int32_t destsize); + + +/** + * @brief Context interface counterpart for #blosc1_getitem. + * + * @param context Context pointer. + * @param src The compressed buffer from data will be decompressed. + * @param srcsize Compressed buffer length. + * @param start The position of the first item (of @p typesize size) from where data + * will be retrieved. + * @param nitems The number of items (of @p typesize size) that will be retrieved. + * @param dest The buffer where the decompressed data retrieved will be put. + * @param destsize Output buffer length. + * + * @return The number of bytes copied to @p dest or a negative value if + * some error happens. + */ +BLOSC_EXPORT int blosc2_getitem_ctx(blosc2_context* context, const void* src, + int32_t srcsize, int start, int nitems, void* dest, + int32_t destsize); + + +/********************************************************************* + Super-chunk related structures and functions. +*********************************************************************/ + +#define BLOSC2_MAX_METALAYERS 16 +#define BLOSC2_METALAYER_NAME_MAXLEN 31 + +// Allow for a reasonable number of vl metalayers +// max is 64 * 1024 due to msgpack map 16 in frame +// mem usage 8 * 1024 entries for blosc2_schunk.vlmetalayers[] is 64 KB +#define BLOSC2_MAX_VLMETALAYERS (8 * 1024) +#define BLOSC2_VLMETALAYERS_NAME_MAXLEN BLOSC2_METALAYER_NAME_MAXLEN + +/** + * @brief This struct is meant for holding storage parameters for a + * for a blosc2 container, allowing to specify, for example, how to interpret + * the contents included in the schunk. + */ +typedef struct { + bool contiguous; + //!< Whether the chunks are contiguous or sparse. + char* urlpath; + //!< The path for persistent storage. If NULL, that means in-memory. + blosc2_cparams* cparams; + //!< The compression params when creating a schunk. + //!< If NULL, sensible defaults are used depending on the context. + blosc2_dparams* dparams; + //!< The decompression params when creating a schunk. + //!< If NULL, sensible defaults are used depending on the context. + blosc2_io *io; + //!< Input/output backend. +} blosc2_storage; + +/** + * @brief Default struct for #blosc2_storage meant for user initialization. + */ +static const blosc2_storage BLOSC2_STORAGE_DEFAULTS = {false, NULL, NULL, NULL, NULL}; + +typedef struct blosc2_frame_s blosc2_frame; /* opaque type */ + +/** + * @brief This struct is meant to store metadata information inside + * a #blosc2_schunk, allowing to specify, for example, how to interpret + * the contents included in the schunk. + */ +typedef struct blosc2_metalayer { + char* name; //!< The metalayer identifier for Blosc client (e.g. Caterva). + uint8_t* content; //!< The serialized (msgpack preferably) content of the metalayer. + int32_t content_len; //!< The length in bytes of the content. +} blosc2_metalayer; + +/** + * @brief This struct is the standard container for Blosc 2 compressed data. + * + * This is essentially a container for Blosc 1 chunks of compressed data, + * and it allows to overcome the 32-bit limitation in Blosc 1. Optionally, + * a #blosc2_frame can be attached so as to store the compressed chunks contiguously. + */ +typedef struct blosc2_schunk { + uint8_t version; + uint8_t compcode; + //!< The default compressor. Each chunk can override this. + uint8_t compcode_meta; + //!< The default compressor metadata. Each chunk can override this. + uint8_t clevel; + //!< The compression level and other compress params. + int32_t typesize; + //!< The type size. + int32_t blocksize; + //!< The requested size of the compressed blocks (0; meaning automatic). + int32_t chunksize; + //!< Size of each chunk. 0 if not a fixed chunksize. + uint8_t filters[BLOSC2_MAX_FILTERS]; + //!< The (sequence of) filters. 8-bit per filter. + uint8_t filters_meta[BLOSC2_MAX_FILTERS]; + //!< Metadata for filters. 8-bit per meta-slot. + int64_t nchunks; + //!< Number of chunks in super-chunk. + int64_t current_nchunk; + //!< The current chunk that is being accessed + int64_t nbytes; + //!< The data size + metadata size + header size (uncompressed). + int64_t cbytes; + //!< The data size + metadata size + header size (compressed). + uint8_t** data; + //!< Pointer to chunk data pointers buffer. + size_t data_len; + //!< Length of the chunk data pointers buffer. + blosc2_storage* storage; + //!< Pointer to storage info. + blosc2_frame* frame; + //!< Pointer to frame used as store for chunks. + //!nvlmetalayers * sizeof(char*). + * + * @return The number of the variable-length metalayers in the super-chunk. + * This cannot fail unless the user does not pass a @p names which is large enough to + * keep pointers to all names, in which case funny things (seg faults and such) will happen. + */ +BLOSC_EXPORT int blosc2_vlmeta_get_names(blosc2_schunk *schunk, char **names); + + +/********************************************************************* + Time measurement utilities. +*********************************************************************/ + +#if defined(_WIN32) +/* For QueryPerformanceCounter(), etc. */ + #include +#elif defined(__MACH__) +#include +#include +#include +#elif defined(__unix__) +#if defined(__linux__) + #include + #else + #include + #endif +#else + #error Unable to detect platform. +#endif + +/* The type of timestamp used on this system. */ +#if defined(_WIN32) +#define blosc_timestamp_t LARGE_INTEGER +#else +#define blosc_timestamp_t struct timespec +#endif + +/* + * Set a timestamp. + */ +BLOSC_EXPORT void blosc_set_timestamp(blosc_timestamp_t* timestamp); + +/* + * Return the nanoseconds between 2 timestamps. + */ +BLOSC_EXPORT double blosc_elapsed_nsecs(blosc_timestamp_t start_time, + blosc_timestamp_t end_time); + +/* + * Return the seconds between 2 timestamps. + */ +BLOSC_EXPORT double blosc_elapsed_secs(blosc_timestamp_t start_time, + blosc_timestamp_t end_time); + + +/********************************************************************* + Low-level functions follows. Use them only if you are an expert! +*********************************************************************/ + +/** + * @brief Get the internal blocksize to be used during compression. 0 means + * that an automatic blocksize is computed internally. + * + * @return The size in bytes of the internal block size. + */ +BLOSC_EXPORT int blosc1_get_blocksize(void); + +/** + * @brief Force the use of a specific blocksize. If 0, an automatic + * blocksize will be used (the default). + * + * @warning The blocksize is a critical parameter with important + * restrictions in the allowed values, so use this with care. + */ +BLOSC_EXPORT void blosc1_set_blocksize(size_t blocksize); + + +/** + * @brief Set the split mode. + + * @param splitmode It can take the next values: + * BLOSC_FORWARD_COMPAT_SPLIT + * BLOSC_AUTO_SPLIT + * BLOSC_NEVER_SPLIT + * BLOSC_ALWAYS_SPLIT + * + * BLOSC_FORWARD_COMPAT offers reasonably forward compatibility, + * BLOSC_AUTO_SPLIT is for nearly optimal results (based on heuristics), + * BLOSC_NEVER_SPLIT and BLOSC_ALWAYS_SPLIT are for the user experimenting + * when trying to get best compression ratios and/or speed. + * + * If not called, the default mode is BLOSC_FORWARD_COMPAT_SPLIT. + * + * This function should always succeed. + */ +BLOSC_EXPORT void blosc1_set_splitmode(int splitmode); + + +/** + * @brief Get the offsets of a frame in a super-chunk. + * + * @param schunk The super-chunk containing the frame. + * + * @return If successful, return a pointer to a buffer of the decompressed offsets. + * The number of offsets is equal to schunk->nchunks; the user is + * responsible to free this buffer. Else, return a NULL value. + */ +BLOSC_EXPORT int64_t* blosc2_frame_get_offsets(blosc2_schunk *schunk); + + +/********************************************************************* + Structures and functions related with compression codecs. +*********************************************************************/ + +typedef int (* blosc2_codec_encoder_cb) (const uint8_t *input, int32_t input_len, uint8_t *output, int32_t output_len, uint8_t meta, blosc2_cparams *cparams, const void* chunk); +typedef int (* blosc2_codec_decoder_cb) (const uint8_t *input, int32_t input_len, uint8_t *output, int32_t output_len, uint8_t meta, blosc2_dparams *dparams, const void* chunk); + +typedef struct { + uint8_t compcode; + //!< The codec identifier. + char *compname; + //!< The codec name. + uint8_t complib; + //!< The codec library format. + uint8_t compver; + //!< The codec version. + blosc2_codec_encoder_cb encoder; + //!< The codec encoder that is used during compression. + blosc2_codec_decoder_cb decoder; + //!< The codec decoder that is used during decompression. +} blosc2_codec; + +/** + * @brief Register locally a user-defined codec in Blosc. + * + * @param codec The codec to register. + * + * @return 0 if succeeds. Else a negative code is returned. + */ +BLOSC_EXPORT int blosc2_register_codec(blosc2_codec *codec); + + +/********************************************************************* + Structures and functions related with filters plugins. +*********************************************************************/ + +typedef int (* blosc2_filter_forward_cb) (const uint8_t *, uint8_t *, int32_t, uint8_t, blosc2_cparams *); +typedef int (* blosc2_filter_backward_cb) (const uint8_t *, uint8_t *, int32_t, uint8_t, blosc2_dparams *); + +/** + * @brief The parameters for a user-defined filter. + */ +typedef struct { + uint8_t id; + //!< The filter identifier. + blosc2_filter_forward_cb forward; + //!< The filter function that is used during compression. + blosc2_filter_backward_cb backward; + //!< The filter function that is used during decompression. +} blosc2_filter; + +/** + * @brief Register locally a user-defined filter in Blosc. + * + * @param filter The filter to register. + * + * @return 0 if succeeds. Else a negative code is returned. + */ +BLOSC_EXPORT int blosc2_register_filter(blosc2_filter *filter); + + +/********************************************************************* + Directory utilities. +*********************************************************************/ + +/* + * @brief Remove a directory and its files. + */ +BLOSC_EXPORT int blosc2_remove_dir(const char *path); + +/* + * @brief Remove a file or a directory given by path. + */ +BLOSC_EXPORT int blosc2_remove_urlpath(const char *path); + +/* + * @brief Rename a file or a directory given by old_urlpath to new_path. + */ +BLOSC_EXPORT int blosc2_rename_urlpath(char* old_urlpath, char* new_path); + + +/********************************************************************* + Index utilities. +*********************************************************************/ + +/* + * @brief Convert a sequential index into a multidimensional index + */ +BLOSC_EXPORT void blosc2_unidim_to_multidim(uint8_t ndim, int64_t *shape, int64_t i, int64_t *index); + +/* + * @brief Convert a multidimensional index into a sequential index + */ +BLOSC_EXPORT void blosc2_multidim_to_unidim(const int64_t *index, int8_t ndim, const int64_t *strides, int64_t *i); + +#ifdef __cplusplus +} +#endif + + +#endif /* BLOSC2_H */ diff --git a/src/blosc2/include/blosc2/blosc2-common.h b/src/blosc2/include/blosc2/blosc2-common.h new file mode 100644 index 0000000..9acffdb --- /dev/null +++ b/src/blosc2/include/blosc2/blosc2-common.h @@ -0,0 +1,92 @@ +/********************************************************************* + Blosc - Blocked Shuffling and Compression Library + + Copyright (C) 2021 The Blosc Developers + https://blosc.org + License: BSD 3-Clause (see LICENSE.txt) + + See LICENSE.txt for details about copyright and rights to use. +**********************************************************************/ + +#ifndef SHUFFLE_COMMON_H +#define SHUFFLE_COMMON_H + +#include "blosc2-export.h" +#include + +// For shutting up stupid compiler warning about some 'unused' variables in GCC +#ifdef __GNUC__ +#define BLOSC_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) +#define BLOSC_UNUSED_VAR __attribute__ ((unused)) +#else +#define BLOSC_UNUSED_VAR +#endif // __GNUC__ + +// For shutting up compiler warning about unused parameters +#define BLOSC_UNUSED_PARAM(x) ((void)(x)) + +/* Import standard integer type definitions */ +#if defined(_WIN32) && !defined(__MINGW32__) + + /* stdint.h only available in VS2010 (VC++ 16.0) and newer */ + #if defined(_MSC_VER) && _MSC_VER < 1600 + #include "win32/stdint-windows.h" + #else + #include + #endif + + /* Use inlined functions for supported systems */ + #if defined(_MSC_VER) && !defined(__cplusplus) /* Visual Studio */ + #define inline __inline /* Visual C is not C99, but supports some kind of inline */ + #endif + +#else + #include +#endif /* _WIN32 */ + + +/* Define the __SSE2__ symbol if compiling with Visual C++ and + targeting the minimum architecture level supporting SSE2. + Other compilers define this as expected and emit warnings + when it is re-defined. */ +#if !defined(__SSE2__) && defined(_MSC_VER) && \ + (defined(_M_X64) || (defined(_M_IX86) && _M_IX86_FP >= 2)) + #define __SSE2__ +#endif + +/* + * Detect if the architecture is fine with unaligned access. + */ +#if !defined(BLOSC_STRICT_ALIGN) +#define BLOSC_STRICT_ALIGN +#if defined(__i386__) || defined(__386) || defined (__amd64) /* GNU C, Sun Studio */ +#undef BLOSC_STRICT_ALIGN +#elif defined(__i486__) || defined(__i586__) || defined(__i686__) /* GNU C */ +#undef BLOSC_STRICT_ALIGN +#elif defined(_M_IX86) || defined(_M_X64) /* Intel, MSVC */ +#undef BLOSC_STRICT_ALIGN +#elif defined(__386) +#undef BLOSC_STRICT_ALIGN +#elif defined(_X86_) /* MinGW */ +#undef BLOSC_STRICT_ALIGN +#elif defined(__I86__) /* Digital Mars */ +#undef BLOSC_STRICT_ALIGN +/* Modern ARM systems (like ARM64) should support unaligned access + quite efficiently. */ +#elif defined(__ARM_FEATURE_UNALIGNED) /* ARM, GNU C */ +#undef BLOSC_STRICT_ALIGN +#elif defined(_ARCH_PPC) || defined(__PPC__) +/* Modern PowerPC systems (like POWER8) should support unaligned access + quite efficiently. */ +#undef BLOSC_STRICT_ALIGN +#endif +#endif + +#if defined(__SSE2__) + #include +#endif +#if defined(__AVX2__) + #include +#endif + +#endif /* SHUFFLE_COMMON_H */ diff --git a/src/blosc2/include/blosc2/blosc2-export.h b/src/blosc2/include/blosc2/blosc2-export.h new file mode 100644 index 0000000..3923651 --- /dev/null +++ b/src/blosc2/include/blosc2/blosc2-export.h @@ -0,0 +1,48 @@ +/********************************************************************* + Blosc - Blocked Shuffling and Compression Library + + Copyright (C) 2021 The Blosc Developers + https://blosc.org + License: BSD 3-Clause (see LICENSE.txt) + + See LICENSE.txt for details about copyright and rights to use. +**********************************************************************/ + +#ifndef BLOSC_EXPORT_H +#define BLOSC_EXPORT_H + +/* Macros for specifying exported symbols. + BLOSC_EXPORT is used to decorate symbols that should be + exported by the blosc shared library. + BLOSC_NO_EXPORT is used to decorate symbols that should NOT + be exported by the blosc shared library. +*/ +#if defined(BLOSC_SHARED_LIBRARY) + #if defined(_MSC_VER) + #define BLOSC_EXPORT __declspec(dllexport) + #elif (defined(__GNUC__) && __GNUC__ >= 4) || defined(__clang__) + #if defined(_WIN32) || defined(__CYGWIN__) || defined(__MINGW32__) + #define BLOSC_EXPORT __attribute__((dllexport)) + #else + #define BLOSC_EXPORT __attribute__((visibility("default"))) + #endif /* defined(_WIN32) || defined(__CYGWIN__) */ + #else + #error Cannot determine how to define BLOSC_EXPORT for this compiler. + #endif +#else + #define BLOSC_EXPORT +#endif /* defined(BLOSC_SHARED_LIBRARY) */ + +#if defined(__GNUC__) || defined(__clang__) + #define BLOSC_NO_EXPORT __attribute__((visibility("hidden"))) +#else + #define BLOSC_NO_EXPORT +#endif /* defined(__GNUC__) || defined(__clang__) */ + +/* When testing, export everything to make it easier to implement tests. */ +#if defined(BLOSC_TESTING) + #undef BLOSC_NO_EXPORT + #define BLOSC_NO_EXPORT BLOSC_EXPORT +#endif /* defined(BLOSC_TESTING) */ + +#endif /* BLOSC_EXPORT_H */ diff --git a/src/blosc2/include/blosc2/blosc2-stdio.h b/src/blosc2/include/blosc2/blosc2-stdio.h new file mode 100644 index 0000000..09368cf --- /dev/null +++ b/src/blosc2/include/blosc2/blosc2-stdio.h @@ -0,0 +1,52 @@ +/********************************************************************* + Blosc - Blocked Shuffling and Compression Library + + Copyright (C) 2021 The Blosc Developers + https://blosc.org + License: BSD 3-Clause (see LICENSE.txt) + + See LICENSE.txt for details about copyright and rights to use. +**********************************************************************/ + + +#ifndef BLOSC_BLOSC2_STDIO_H +#define BLOSC_BLOSC2_STDIO_H + + +#include +#include +#include "blosc2-export.h" + + +#if defined(_WIN32) && !defined(__MINGW32__) + +/* stdint.h only available in VS2010 (VC++ 16.0) and newer */ + #if defined(_MSC_VER) && _MSC_VER < 1600 + #include "win32/stdint-windows.h" + #else + #include + #endif + +#else +#include +#endif + +#if defined(_MSC_VER) && (_MSC_VER >= 1400) +#include +#else +#include +#endif + +typedef struct { + FILE *file; +} blosc2_stdio_file; + +BLOSC_EXPORT void *blosc2_stdio_open(const char *urlpath, const char *mode, void* params); +BLOSC_EXPORT int blosc2_stdio_close(void *stream); +BLOSC_EXPORT int64_t blosc2_stdio_tell(void *stream); +BLOSC_EXPORT int blosc2_stdio_seek(void *stream, int64_t offset, int whence); +BLOSC_EXPORT int64_t blosc2_stdio_write(const void *ptr, int64_t size, int64_t nitems, void *stream); +BLOSC_EXPORT int64_t blosc2_stdio_read(void *ptr, int64_t size, int64_t nitems, void *stream); +BLOSC_EXPORT int blosc2_stdio_truncate(void *stream, int64_t size); + +#endif //BLOSC_BLOSC2_STDIO_H diff --git a/src/blosc2/include/blosc2/codecs-registry.h b/src/blosc2/include/blosc2/codecs-registry.h new file mode 100644 index 0000000..8b5e78d --- /dev/null +++ b/src/blosc2/include/blosc2/codecs-registry.h @@ -0,0 +1,18 @@ +/********************************************************************* + Blosc - Blocked Shuffling and Compression Library + + Copyright (C) 2021 The Blosc Developers + https://blosc.org + License: BSD 3-Clause (see LICENSE.txt) + + See LICENSE.txt for details about copyright and rights to use. +**********************************************************************/ + +enum { + BLOSC_CODEC_NDLZ = 32, + BLOSC_CODEC_ZFP_FIXED_ACCURACY = 33, + BLOSC_CODEC_ZFP_FIXED_PRECISION = 34, + BLOSC_CODEC_ZFP_FIXED_RATE = 35, +}; + +void register_codecs(void); diff --git a/src/blosc2/include/blosc2/filters-registry.h b/src/blosc2/include/blosc2/filters-registry.h new file mode 100644 index 0000000..60e2796 --- /dev/null +++ b/src/blosc2/include/blosc2/filters-registry.h @@ -0,0 +1,16 @@ +/********************************************************************* + Blosc - Blocked Shuffling and Compression Library + + Copyright (C) 2021 The Blosc Developers + https://blosc.org + License: BSD 3-Clause (see LICENSE.txt) + + See LICENSE.txt for details about copyright and rights to use. +**********************************************************************/ + +enum { + BLOSC_FILTER_NDCELL = 32, + BLOSC_FILTER_NDMEAN = 33, +}; + +void register_filters(void); diff --git a/src/blosc2/internal-complibs/zstd-1.5.2/.gitignore b/src/blosc2/internal-complibs/zstd-1.5.2/.gitignore new file mode 100644 index 0000000..4cd50ac --- /dev/null +++ b/src/blosc2/internal-complibs/zstd-1.5.2/.gitignore @@ -0,0 +1,3 @@ +# make install artefact +libzstd.pc +libzstd-nomt diff --git a/src/blosc2/internal-complibs/zstd-1.5.2/BUCK b/src/blosc2/internal-complibs/zstd-1.5.2/BUCK new file mode 100644 index 0000000..60c6bbb --- /dev/null +++ b/src/blosc2/internal-complibs/zstd-1.5.2/BUCK @@ -0,0 +1,232 @@ +cxx_library( + name='zstd', + header_namespace='', + exported_headers=['zstd.h'], + visibility=['PUBLIC'], + deps=[ + ':common', + ':compress', + ':decompress', + ':deprecated', + ], +) + +cxx_library( + name='compress', + header_namespace='', + visibility=['PUBLIC'], + exported_headers=subdir_glob([ + ('compress', 'zstd*.h'), + ]), + srcs=glob(['compress/zstd*.c', 'compress/hist.c']), + deps=[':common'], +) + +cxx_library( + name='decompress', + header_namespace='', + visibility=['PUBLIC'], + headers=subdir_glob([ + ('decompress', '*_impl.h'), + ]), + srcs=glob(['decompress/zstd*.c']), + deps=[ + ':common', + ':legacy', + ], +) + +cxx_library( + name='deprecated', + header_namespace='', + visibility=['PUBLIC'], + exported_headers=subdir_glob([ + ('deprecated', '*.h'), + ]), + srcs=glob(['deprecated/*.c']), + deps=[':common'], +) + +cxx_library( + name='legacy', + header_namespace='', + visibility=['PUBLIC'], + exported_headers=subdir_glob([ + ('legacy', '*.h'), + ]), + srcs=glob(['legacy/*.c']), + deps=[':common'], + exported_preprocessor_flags=[ + '-DZSTD_LEGACY_SUPPORT=4', + ], +) + +cxx_library( + name='zdict', + header_namespace='', + visibility=['PUBLIC'], + exported_headers=['zdict.h'], + headers=subdir_glob([ + ('dictBuilder', 'divsufsort.h'), + ('dictBuilder', 'cover.h'), + ]), + srcs=glob(['dictBuilder/*.c']), + deps=[':common'], +) + +cxx_library( + name='compiler', + header_namespace='', + visibility=['PUBLIC'], + exported_headers=subdir_glob([ + ('common', 'compiler.h'), + ]), +) + +cxx_library( + name='cpu', + header_namespace='', + visibility=['PUBLIC'], + exported_headers=subdir_glob([ + ('common', 'cpu.h'), + ]), +) + +cxx_library( + name='bitstream', + header_namespace='', + visibility=['PUBLIC'], + exported_headers=subdir_glob([ + ('common', 'bitstream.h'), + ]), +) + +cxx_library( + name='entropy', + header_namespace='', + visibility=['PUBLIC'], + exported_headers=subdir_glob([ + ('common', 'fse.h'), + ('common', 'huf.h'), + ]), + srcs=[ + 'common/entropy_common.c', + 'common/fse_decompress.c', + 'compress/fse_compress.c', + 'compress/huf_compress.c', + 'decompress/huf_decompress.c', + ], + deps=[ + ':debug', + ':bitstream', + ':compiler', + ':errors', + ':mem', + ], +) + +cxx_library( + name='errors', + header_namespace='', + visibility=['PUBLIC'], + exported_headers=[ + 'zstd_errors.h', + 'common/error_private.h', + ] + srcs=['common/error_private.c'], +) + +cxx_library( + name='mem', + header_namespace='', + visibility=['PUBLIC'], + exported_headers=subdir_glob([ + ('common', 'mem.h'), + ]), +) + +cxx_library( + name='pool', + header_namespace='', + visibility=['PUBLIC'], + exported_headers=subdir_glob([ + ('common', 'pool.h'), + ]), + srcs=['common/pool.c'], + deps=[ + ':threading', + ':zstd_common', + ], +) + +cxx_library( + name='threading', + header_namespace='', + visibility=['PUBLIC'], + exported_headers=subdir_glob([ + ('common', 'threading.h'), + ]), + srcs=['common/threading.c'], + exported_preprocessor_flags=[ + '-DZSTD_MULTITHREAD', + ], + exported_linker_flags=[ + '-pthread', + ], +) + +cxx_library( + name='xxhash', + header_namespace='', + visibility=['PUBLIC'], + exported_headers=subdir_glob([ + ('common', 'xxhash.h'), + ]), + srcs=['common/xxhash.c'], + exported_preprocessor_flags=[ + '-DXXH_NAMESPACE=ZSTD_', + ], +) + +cxx_library( + name='zstd_common', + header_namespace='', + visibility=['PUBLIC'], + exported_headers=subdir_glob([ + ('', 'zstd.h'), + ('common', 'zstd_internal.h'), + ]), + srcs=['common/zstd_common.c'], + deps=[ + ':compiler', + ':errors', + ':mem', + ], +) + +cxx_library( + name='debug', + header_namespace='', + visibility=['PUBLIC'], + exported_headers=subdir_glob([ + ('common', 'debug.h'), + ]), + srcs=['common/debug.c'], +) + +cxx_library( + name='common', + deps=[ + ':debug', + ':bitstream', + ':compiler', + ':cpu', + ':entropy', + ':errors', + ':mem', + ':pool', + ':threading', + ':xxhash', + ':zstd_common', + ] +) diff --git a/src/blosc2/internal-complibs/zstd-1.5.2/Makefile b/src/blosc2/internal-complibs/zstd-1.5.2/Makefile new file mode 100644 index 0000000..ef20218 --- /dev/null +++ b/src/blosc2/internal-complibs/zstd-1.5.2/Makefile @@ -0,0 +1,357 @@ +# ################################################################ +# Copyright (c) Yann Collet, Facebook, Inc. +# All rights reserved. +# +# This source code is licensed under both the BSD-style license (found in the +# LICENSE file in the root directory of this source tree) and the GPLv2 (found +# in the COPYING file in the root directory of this source tree). +# You may select, at your option, one of the above-listed licenses. +# ################################################################ + +# Modules +ZSTD_LIB_COMPRESSION ?= 1 +ZSTD_LIB_DECOMPRESSION ?= 1 +ZSTD_LIB_DICTBUILDER ?= 1 +ZSTD_LIB_DEPRECATED ?= 0 + +# Input variables for libzstd.mk +ifeq ($(ZSTD_LIB_COMPRESSION), 0) + ZSTD_LIB_DICTBUILDER = 0 + ZSTD_LIB_DEPRECATED = 0 +endif + +ifeq ($(ZSTD_LIB_DECOMPRESSION), 0) + ZSTD_LEGACY_SUPPORT = 0 + ZSTD_LIB_DEPRECATED = 0 +endif + +include libzstd.mk + +ZSTD_FILES := $(ZSTD_COMMON_FILES) $(ZSTD_LEGACY_FILES) + +ifneq ($(ZSTD_LIB_COMPRESSION), 0) + ZSTD_FILES += $(ZSTD_COMPRESS_FILES) +endif + +ifneq ($(ZSTD_LIB_DECOMPRESSION), 0) + ZSTD_FILES += $(ZSTD_DECOMPRESS_FILES) +endif + +ifneq ($(ZSTD_LIB_DEPRECATED), 0) + ZSTD_FILES += $(ZSTD_DEPRECATED_FILES) +endif + +ifneq ($(ZSTD_LIB_DICTBUILDER), 0) + ZSTD_FILES += $(ZSTD_DICTBUILDER_FILES) +endif + +ZSTD_LOCAL_SRC := $(notdir $(ZSTD_FILES)) +ZSTD_LOCAL_OBJ0 := $(ZSTD_LOCAL_SRC:.c=.o) +ZSTD_LOCAL_OBJ := $(ZSTD_LOCAL_OBJ0:.S=.o) + +VERSION := $(ZSTD_VERSION) + +# Note: by default, the static library is built single-threaded and dynamic library is built +# multi-threaded. It is possible to force multi or single threaded builds by appending +# -mt or -nomt to the build target (like lib-mt for multi-threaded, lib-nomt for single-threaded). +.PHONY: default +default: lib-release + +CPPFLAGS_DYNLIB += -DZSTD_MULTITHREAD # dynamic library build defaults to multi-threaded +LDFLAGS_DYNLIB += -pthread +CPPFLAGS_STATLIB += # static library build defaults to single-threaded + + +ifeq ($(findstring GCC,$(CCVER)),GCC) +decompress/zstd_decompress_block.o : CFLAGS+=-fno-tree-vectorize +endif + + +# macOS linker doesn't support -soname, and use different extension +# see : https://developer.apple.com/library/mac/documentation/DeveloperTools/Conceptual/DynamicLibraries/100-Articles/DynamicLibraryDesignGuidelines.html +ifeq ($(UNAME), Darwin) + SHARED_EXT = dylib + SHARED_EXT_MAJOR = $(LIBVER_MAJOR).$(SHARED_EXT) + SHARED_EXT_VER = $(LIBVER).$(SHARED_EXT) + SONAME_FLAGS = -install_name $(LIBDIR)/libzstd.$(SHARED_EXT_MAJOR) -compatibility_version $(LIBVER_MAJOR) -current_version $(LIBVER) +else + ifeq ($(UNAME), AIX) + SONAME_FLAGS = + else + SONAME_FLAGS = -Wl,-soname=libzstd.$(SHARED_EXT).$(LIBVER_MAJOR) + endif + SHARED_EXT = so + SHARED_EXT_MAJOR = $(SHARED_EXT).$(LIBVER_MAJOR) + SHARED_EXT_VER = $(SHARED_EXT).$(LIBVER) +endif + + +.PHONY: all +all: lib + + +.PHONY: libzstd.a # must be run every time +libzstd.a: CPPFLAGS += $(CPPFLAGS_STATLIB) + +SET_CACHE_DIRECTORY = \ + +$(MAKE) --no-print-directory $@ \ + BUILD_DIR=obj/$(HASH_DIR) \ + CPPFLAGS="$(CPPFLAGS)" \ + CFLAGS="$(CFLAGS)" \ + LDFLAGS="$(LDFLAGS)" + +ifndef BUILD_DIR +# determine BUILD_DIR from compilation flags + +libzstd.a: + $(SET_CACHE_DIRECTORY) + +else +# BUILD_DIR is defined + +ZSTD_STATLIB_DIR := $(BUILD_DIR)/static +ZSTD_STATLIB := $(ZSTD_STATLIB_DIR)/libzstd.a +ZSTD_STATLIB_OBJ := $(addprefix $(ZSTD_STATLIB_DIR)/,$(ZSTD_LOCAL_OBJ)) +$(ZSTD_STATLIB): ARFLAGS = rcs +$(ZSTD_STATLIB): | $(ZSTD_STATLIB_DIR) +$(ZSTD_STATLIB): $(ZSTD_STATLIB_OBJ) + # Check for multithread flag at target execution time + $(if $(filter -DZSTD_MULTITHREAD,$(CPPFLAGS)),\ + @echo compiling multi-threaded static library $(LIBVER),\ + @echo compiling single-threaded static library $(LIBVER)) + $(AR) $(ARFLAGS) $@ $^ + +libzstd.a: $(ZSTD_STATLIB) + cp -f $< $@ + +endif + +ifneq (,$(filter Windows%,$(TARGET_SYSTEM))) + +LIBZSTD = dll/libzstd.dll +$(LIBZSTD): $(ZSTD_FILES) + @echo compiling dynamic library $(LIBVER) + $(CC) $(FLAGS) -DZSTD_DLL_EXPORT=1 -Wl,--out-implib,dll/libzstd.dll.a -shared $^ -o $@ + +else # not Windows + +LIBZSTD = libzstd.$(SHARED_EXT_VER) +.PHONY: $(LIBZSTD) # must be run every time +$(LIBZSTD): CPPFLAGS += $(CPPFLAGS_DYNLIB) +$(LIBZSTD): CFLAGS += -fPIC -fvisibility=hidden +$(LIBZSTD): LDFLAGS += -shared $(LDFLAGS_DYNLIB) + +ifndef BUILD_DIR +# determine BUILD_DIR from compilation flags + +$(LIBZSTD): + $(SET_CACHE_DIRECTORY) + +else +# BUILD_DIR is defined + +ZSTD_DYNLIB_DIR := $(BUILD_DIR)/dynamic +ZSTD_DYNLIB := $(ZSTD_DYNLIB_DIR)/$(LIBZSTD) +ZSTD_DYNLIB_OBJ := $(addprefix $(ZSTD_DYNLIB_DIR)/,$(ZSTD_LOCAL_OBJ)) + +$(ZSTD_DYNLIB): | $(ZSTD_DYNLIB_DIR) +$(ZSTD_DYNLIB): $(ZSTD_DYNLIB_OBJ) +# Check for multithread flag at target execution time + $(if $(filter -DZSTD_MULTITHREAD,$(CPPFLAGS)),\ + @echo compiling multi-threaded dynamic library $(LIBVER),\ + @echo compiling single-threaded dynamic library $(LIBVER)) + $(CC) $(FLAGS) $^ $(LDFLAGS) $(SONAME_FLAGS) -o $@ + @echo creating versioned links + ln -sf $@ libzstd.$(SHARED_EXT_MAJOR) + ln -sf $@ libzstd.$(SHARED_EXT) + +$(LIBZSTD): $(ZSTD_DYNLIB) + cp -f $< $@ + +endif # ifndef BUILD_DIR +endif # if windows + +.PHONY: libzstd +libzstd : $(LIBZSTD) + +.PHONY: lib +lib : libzstd.a libzstd + + +# note : do not define lib-mt or lib-release as .PHONY +# make does not consider implicit pattern rule for .PHONY target + +%-mt : CPPFLAGS_DYNLIB := -DZSTD_MULTITHREAD +%-mt : CPPFLAGS_STATLIB := -DZSTD_MULTITHREAD +%-mt : LDFLAGS_DYNLIB := -pthread +%-mt : % + @echo multi-threaded build completed + +%-nomt : CPPFLAGS_DYNLIB := +%-nomt : LDFLAGS_DYNLIB := +%-nomt : CPPFLAGS_STATLIB := +%-nomt : % + @echo single-threaded build completed + +%-release : DEBUGFLAGS := +%-release : % + @echo release build completed + + +# Generate .h dependencies automatically + +DEPFLAGS = -MT $@ -MMD -MP -MF + +$(ZSTD_DYNLIB_DIR)/%.o : %.c $(ZSTD_DYNLIB_DIR)/%.d | $(ZSTD_DYNLIB_DIR) + @echo CC $@ + $(COMPILE.c) $(DEPFLAGS) $(ZSTD_DYNLIB_DIR)/$*.d $(OUTPUT_OPTION) $< + +$(ZSTD_STATLIB_DIR)/%.o : %.c $(ZSTD_STATLIB_DIR)/%.d | $(ZSTD_STATLIB_DIR) + @echo CC $@ + $(COMPILE.c) $(DEPFLAGS) $(ZSTD_STATLIB_DIR)/$*.d $(OUTPUT_OPTION) $< + +$(ZSTD_DYNLIB_DIR)/%.o : %.S | $(ZSTD_DYNLIB_DIR) + @echo AS $@ + $(COMPILE.S) $(OUTPUT_OPTION) $< + +$(ZSTD_STATLIB_DIR)/%.o : %.S | $(ZSTD_STATLIB_DIR) + @echo AS $@ + $(COMPILE.S) $(OUTPUT_OPTION) $< + +MKDIR ?= mkdir +$(BUILD_DIR) $(ZSTD_DYNLIB_DIR) $(ZSTD_STATLIB_DIR): + $(MKDIR) -p $@ + +DEPFILES := $(ZSTD_DYNLIB_OBJ:.o=.d) $(ZSTD_STATLIB_OBJ:.o=.d) +$(DEPFILES): + +include $(wildcard $(DEPFILES)) + + +# Special case : building library in single-thread mode _and_ without zstdmt_compress.c +ZSTDMT_FILES = compress/zstdmt_compress.c +ZSTD_NOMT_FILES = $(filter-out $(ZSTDMT_FILES),$(ZSTD_FILES)) +libzstd-nomt: CFLAGS += -fPIC -fvisibility=hidden +libzstd-nomt: LDFLAGS += -shared +libzstd-nomt: $(ZSTD_NOMT_FILES) + @echo compiling single-thread dynamic library $(LIBVER) + @echo files : $(ZSTD_NOMT_FILES) + $(CC) $(FLAGS) $^ $(LDFLAGS) $(SONAME_FLAGS) -o $@ + +.PHONY: clean +clean: + $(RM) -r *.dSYM # macOS-specific + $(RM) core *.o *.a *.gcda *.$(SHARED_EXT) *.$(SHARED_EXT).* libzstd.pc + $(RM) dll/libzstd.dll dll/libzstd.lib libzstd-nomt* + $(RM) -r obj/* + @echo Cleaning library completed + +#----------------------------------------------------------------------------- +# make install is validated only for below listed environments +#----------------------------------------------------------------------------- +ifneq (,$(filter $(UNAME),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS Haiku AIX)) + +lib: libzstd.pc + +HAS_EXPLICIT_EXEC_PREFIX := $(if $(or $(EXEC_PREFIX),$(exec_prefix)),1,) + +DESTDIR ?= +# directory variables : GNU conventions prefer lowercase +# see https://www.gnu.org/prep/standards/html_node/Makefile-Conventions.html +# support both lower and uppercase (BSD), use uppercase in script +prefix ?= /usr/local +PREFIX ?= $(prefix) +exec_prefix ?= $(PREFIX) +EXEC_PREFIX ?= $(exec_prefix) +libdir ?= $(EXEC_PREFIX)/lib +LIBDIR ?= $(libdir) +includedir ?= $(PREFIX)/include +INCLUDEDIR ?= $(includedir) + +PCINCDIR := $(patsubst $(PREFIX)%,%,$(INCLUDEDIR)) +PCLIBDIR := $(patsubst $(EXEC_PREFIX)%,%,$(LIBDIR)) + +# If we successfully stripped off a prefix, we'll add a reference to the +# relevant pc variable. +PCINCPREFIX := $(if $(findstring $(INCLUDEDIR),$(PCINCDIR)),,$${prefix}) +PCLIBPREFIX := $(if $(findstring $(LIBDIR),$(PCLIBDIR)),,$${exec_prefix}) + +# If no explicit EXEC_PREFIX was set by the caller, write it out as a reference +# to PREFIX, rather than as a resolved value. +PCEXEC_PREFIX := $(if $(HAS_EXPLICIT_EXEC_PREFIX),$(EXEC_PREFIX),$${prefix}) + +ifneq (,$(filter $(UNAME),FreeBSD NetBSD DragonFly)) + PKGCONFIGDIR ?= $(PREFIX)/libdata/pkgconfig +else + PKGCONFIGDIR ?= $(LIBDIR)/pkgconfig +endif + +ifneq (,$(filter $(UNAME),SunOS)) + INSTALL ?= ginstall +else + INSTALL ?= install +endif + +INSTALL_PROGRAM ?= $(INSTALL) +INSTALL_DATA ?= $(INSTALL) -m 644 + + +libzstd.pc: libzstd.pc.in + @echo creating pkgconfig + @sed $(SED_ERE_OPT) \ + -e 's|@PREFIX@|$(PREFIX)|' \ + -e 's|@EXEC_PREFIX@|$(PCEXEC_PREFIX)|' \ + -e 's|@INCLUDEDIR@|$(PCINCPREFIX)$(PCINCDIR)|' \ + -e 's|@LIBDIR@|$(PCLIBPREFIX)$(PCLIBDIR)|' \ + -e 's|@VERSION@|$(VERSION)|' \ + -e 's|@LIBS_PRIVATE@|$(LDFLAGS_DYNLIB)|' \ + $< >$@ + +.PHONY: install +install: install-pc install-static install-shared install-includes + @echo zstd static and shared library installed + +.PHONY: install-pc +install-pc: libzstd.pc + [ -e $(DESTDIR)$(PKGCONFIGDIR) ] || $(INSTALL) -d -m 755 $(DESTDIR)$(PKGCONFIGDIR)/ + $(INSTALL_DATA) libzstd.pc $(DESTDIR)$(PKGCONFIGDIR)/ + +.PHONY: install-static +install-static: + # only generate libzstd.a if it's not already present + [ -e libzstd.a ] || $(MAKE) libzstd.a-release + [ -e $(DESTDIR)$(LIBDIR) ] || $(INSTALL) -d -m 755 $(DESTDIR)$(LIBDIR)/ + @echo Installing static library + $(INSTALL_DATA) libzstd.a $(DESTDIR)$(LIBDIR) + +.PHONY: install-shared +install-shared: + # only generate libzstd.so if it's not already present + [ -e $(LIBZSTD) ] || $(MAKE) libzstd-release + [ -e $(DESTDIR)$(LIBDIR) ] || $(INSTALL) -d -m 755 $(DESTDIR)$(LIBDIR)/ + @echo Installing shared library + $(INSTALL_PROGRAM) $(LIBZSTD) $(DESTDIR)$(LIBDIR) + ln -sf $(LIBZSTD) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT_MAJOR) + ln -sf $(LIBZSTD) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT) + +.PHONY: install-includes +install-includes: + [ -e $(DESTDIR)$(INCLUDEDIR) ] || $(INSTALL) -d -m 755 $(DESTDIR)$(INCLUDEDIR)/ + @echo Installing includes + $(INSTALL_DATA) zstd.h $(DESTDIR)$(INCLUDEDIR) + $(INSTALL_DATA) zstd_errors.h $(DESTDIR)$(INCLUDEDIR) + $(INSTALL_DATA) zdict.h $(DESTDIR)$(INCLUDEDIR) + +.PHONY: uninstall +uninstall: + $(RM) $(DESTDIR)$(LIBDIR)/libzstd.a + $(RM) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT) + $(RM) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT_MAJOR) + $(RM) $(DESTDIR)$(LIBDIR)/$(LIBZSTD) + $(RM) $(DESTDIR)$(PKGCONFIGDIR)/libzstd.pc + $(RM) $(DESTDIR)$(INCLUDEDIR)/zstd.h + $(RM) $(DESTDIR)$(INCLUDEDIR)/zstd_errors.h + $(RM) $(DESTDIR)$(INCLUDEDIR)/zdict.h + @echo zstd libraries successfully uninstalled + +endif diff --git a/src/blosc2/internal-complibs/zstd-1.5.2/README.md b/src/blosc2/internal-complibs/zstd-1.5.2/README.md new file mode 100644 index 0000000..4c9d8f0 --- /dev/null +++ b/src/blosc2/internal-complibs/zstd-1.5.2/README.md @@ -0,0 +1,217 @@ +Zstandard library files +================================ + +The __lib__ directory is split into several sub-directories, +in order to make it easier to select or exclude features. + + +#### Building + +`Makefile` script is provided, supporting [Makefile conventions](https://www.gnu.org/prep/standards/html_node/Makefile-Conventions.html#Makefile-Conventions), +including commands variables, staged install, directory variables and standard targets. +- `make` : generates both static and dynamic libraries +- `make install` : install libraries and headers in target system directories + +`libzstd` default scope is pretty large, including compression, decompression, dictionary builder, +and support for decoding legacy formats >= v0.5.0. +The scope can be reduced on demand (see paragraph _modular build_). + + +#### Multithreading support + +When building with `make`, by default the dynamic library is multithreaded and static library is single-threaded (for compatibility reasons). + +Enabling multithreading requires 2 conditions : +- set build macro `ZSTD_MULTITHREAD` (`-DZSTD_MULTITHREAD` for `gcc`) +- for POSIX systems : compile with pthread (`-pthread` compilation flag for `gcc`) + +For convenience, we provide a build target to generate multi and single threaded libraries: +- Force enable multithreading on both dynamic and static libraries by appending `-mt` to the target, e.g. `make lib-mt`. +- Force disable multithreading on both dynamic and static libraries by appending `-nomt` to the target, e.g. `make lib-nomt`. +- By default, as mentioned before, dynamic library is multithreaded, and static library is single-threaded, e.g. `make lib`. + +When linking a POSIX program with a multithreaded version of `libzstd`, +note that it's necessary to invoke the `-pthread` flag during link stage. + +Multithreading capabilities are exposed +via the [advanced API defined in `lib/zstd.h`](https://github.com/facebook/zstd/blob/v1.4.3/lib/zstd.h#L351). + + +#### API + +Zstandard's stable API is exposed within [lib/zstd.h](zstd.h). + + +#### Advanced API + +Optional advanced features are exposed via : + +- `lib/zstd_errors.h` : translates `size_t` function results + into a `ZSTD_ErrorCode`, for accurate error handling. + +- `ZSTD_STATIC_LINKING_ONLY` : if this macro is defined _before_ including `zstd.h`, + it unlocks access to the experimental API, + exposed in the second part of `zstd.h`. + All definitions in the experimental APIs are unstable, + they may still change in the future, or even be removed. + As a consequence, experimental definitions shall ___never be used with dynamic library___ ! + Only static linking is allowed. + + +#### Modular build + +It's possible to compile only a limited set of features within `libzstd`. +The file structure is designed to make this selection manually achievable for any build system : + +- Directory `lib/common` is always required, for all variants. + +- Compression source code lies in `lib/compress` + +- Decompression source code lies in `lib/decompress` + +- It's possible to include only `compress` or only `decompress`, they don't depend on each other. + +- `lib/dictBuilder` : makes it possible to generate dictionaries from a set of samples. + The API is exposed in `lib/dictBuilder/zdict.h`. + This module depends on both `lib/common` and `lib/compress` . + +- `lib/legacy` : makes it possible to decompress legacy zstd formats, starting from `v0.1.0`. + This module depends on `lib/common` and `lib/decompress`. + To enable this feature, define `ZSTD_LEGACY_SUPPORT` during compilation. + Specifying a number limits versions supported to that version onward. + For example, `ZSTD_LEGACY_SUPPORT=2` means : "support legacy formats >= v0.2.0". + Conversely, `ZSTD_LEGACY_SUPPORT=0` means "do __not__ support legacy formats". + By default, this build macro is set as `ZSTD_LEGACY_SUPPORT=5`. + Decoding supported legacy format is a transparent capability triggered within decompression functions. + It's also allowed to invoke legacy API directly, exposed in `lib/legacy/zstd_legacy.h`. + Each version does also provide its own set of advanced API. + For example, advanced API for version `v0.4` is exposed in `lib/legacy/zstd_v04.h` . + +- While invoking `make libzstd`, it's possible to define build macros + `ZSTD_LIB_COMPRESSION, ZSTD_LIB_DECOMPRESSION`, `ZSTD_LIB_DICTBUILDER`, + and `ZSTD_LIB_DEPRECATED` as `0` to forgo compilation of the + corresponding features. This will also disable compilation of all + dependencies (eg. `ZSTD_LIB_COMPRESSION=0` will also disable + dictBuilder). + +- There are a number of options that can help minimize the binary size of + `libzstd`. + + The first step is to select the components needed (using the above-described + `ZSTD_LIB_COMPRESSION` etc.). + + The next step is to set `ZSTD_LIB_MINIFY` to `1` when invoking `make`. This + disables various optional components and changes the compilation flags to + prioritize space-saving. + + Detailed options: Zstandard's code and build environment is set up by default + to optimize above all else for performance. In pursuit of this goal, Zstandard + makes significant trade-offs in code size. For example, Zstandard often has + more than one implementation of a particular component, with each + implementation optimized for different scenarios. For example, the Huffman + decoder has complementary implementations that decode the stream one symbol at + a time or two symbols at a time. Zstd normally includes both (and dispatches + between them at runtime), but by defining `HUF_FORCE_DECOMPRESS_X1` or + `HUF_FORCE_DECOMPRESS_X2`, you can force the use of one or the other, avoiding + compilation of the other. Similarly, `ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT` + and `ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG` force the compilation and use of + only one or the other of two decompression implementations. The smallest + binary is achieved by using `HUF_FORCE_DECOMPRESS_X1` and + `ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT` (implied by `ZSTD_LIB_MINIFY`). + + For squeezing the last ounce of size out, you can also define + `ZSTD_NO_INLINE`, which disables inlining, and `ZSTD_STRIP_ERROR_STRINGS`, + which removes the error messages that are otherwise returned by + `ZSTD_getErrorName` (implied by `ZSTD_LIB_MINIFY`). + + Finally, when integrating into your application, make sure you're doing link- + time optimization and unused symbol garbage collection (via some combination of, + e.g., `-flto`, `-ffat-lto-objects`, `-fuse-linker-plugin`, + `-ffunction-sections`, `-fdata-sections`, `-fmerge-all-constants`, + `-Wl,--gc-sections`, `-Wl,-z,norelro`, and an archiver that understands + the compiler's intermediate representation, e.g., `AR=gcc-ar`). Consult your + compiler's documentation. + +- While invoking `make libzstd`, the build macro `ZSTD_LEGACY_MULTITHREADED_API=1` + will expose the deprecated `ZSTDMT` API exposed by `zstdmt_compress.h` in + the shared library, which is now hidden by default. + +- The build macro `DYNAMIC_BMI2` can be set to 1 or 0 in order to generate binaries + which can detect at runtime the presence of BMI2 instructions, and use them only if present. + These instructions contribute to better performance, notably on the decoder side. + By default, this feature is automatically enabled on detecting + the right instruction set (x64) and compiler (clang or gcc >= 5). + It's obviously disabled for different cpus, + or when BMI2 instruction set is _required_ by the compiler command line + (in this case, only the BMI2 code path is generated). + Setting this macro will either force to generate the BMI2 dispatcher (1) + or prevent it (0). It overrides automatic detection. + +- The build macro `ZSTD_NO_UNUSED_FUNCTIONS` can be defined to hide the definitions of functions + that zstd does not use. Not all unused functions are hidden, but they can be if needed. + Currently, this macro will hide function definitions in FSE and HUF that use an excessive + amount of stack space. + +- The build macro `ZSTD_NO_INTRINSICS` can be defined to disable all explicit intrinsics. + Compiler builtins are still used. + +- The build macro `ZSTD_DECODER_INTERNAL_BUFFER` can be set to control + the amount of extra memory used during decompression to store literals. + This defaults to 64kB. Reducing this value reduces the memory footprint of + `ZSTD_DCtx` decompression contexts, + but might also result in a small decompression speed cost. + + +#### Windows : using MinGW+MSYS to create DLL + +DLL can be created using MinGW+MSYS with the `make libzstd` command. +This command creates `dll\libzstd.dll` and the import library `dll\libzstd.lib`. +The import library is only required with Visual C++. +The header file `zstd.h` and the dynamic library `dll\libzstd.dll` are required to +compile a project using gcc/MinGW. +The dynamic library has to be added to linking options. +It means that if a project that uses ZSTD consists of a single `test-dll.c` +file it should be linked with `dll\libzstd.dll`. For example: +``` + gcc $(CFLAGS) -Iinclude/ test-dll.c -o test-dll dll\libzstd.dll +``` +The compiled executable will require ZSTD DLL which is available at `dll\libzstd.dll`. + + +#### Advanced Build options + +The build system requires a hash function in order to +separate object files created with different compilation flags. +By default, it tries to use `md5sum` or equivalent. +The hash function can be manually switched by setting the `HASH` variable. +For example : `make HASH=xxhsum` +The hash function needs to generate at least 64-bit using hexadecimal format. +When no hash function is found, +the Makefile just generates all object files into the same default directory, +irrespective of compilation flags. +This functionality only matters if `libzstd` is compiled multiple times +with different build flags. + +The build directory, where object files are stored +can also be manually controlled using variable `BUILD_DIR`, +for example `make BUILD_DIR=objectDir/v1`. +In which case, the hash function doesn't matter. + + +#### Deprecated API + +Obsolete API on their way out are stored in directory `lib/deprecated`. +At this stage, it contains older streaming prototypes, in `lib/deprecated/zbuff.h`. +These prototypes will be removed in some future version. +Consider migrating code towards supported streaming API exposed in `zstd.h`. + + +#### Miscellaneous + +The other files are not source code. There are : + + - `BUCK` : support for `buck` build system (https://buckbuild.com/) + - `Makefile` : `make` script to build and install zstd library (static and dynamic) + - `README.md` : this file + - `dll/` : resources directory for Windows compilation + - `libzstd.pc.in` : script for `pkg-config` (used in `make install`) diff --git a/src/blosc2/internal-complibs/zstd-1.5.2/common/bitstream.h b/src/blosc2/internal-complibs/zstd-1.5.2/common/bitstream.h new file mode 100644 index 0000000..84b6062 --- /dev/null +++ b/src/blosc2/internal-complibs/zstd-1.5.2/common/bitstream.h @@ -0,0 +1,478 @@ +/* ****************************************************************** + * bitstream + * Part of FSE library + * Copyright (c) Yann Collet, Facebook, Inc. + * + * You can contact the author at : + * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. +****************************************************************** */ +#ifndef BITSTREAM_H_MODULE +#define BITSTREAM_H_MODULE + +#if defined (__cplusplus) +extern "C" { +#endif +/* +* This API consists of small unitary functions, which must be inlined for best performance. +* Since link-time-optimization is not available for all compilers, +* these functions are defined into a .h to be included. +*/ + +/*-**************************************** +* Dependencies +******************************************/ +#include "mem.h" /* unaligned access routines */ +#include "compiler.h" /* UNLIKELY() */ +#include "debug.h" /* assert(), DEBUGLOG(), RAWLOG() */ +#include "error_private.h" /* error codes and messages */ + + +/*========================================= +* Target specific +=========================================*/ +#ifndef ZSTD_NO_INTRINSICS +# if defined(__BMI__) && defined(__GNUC__) +# include /* support for bextr (experimental) */ +# elif defined(__ICCARM__) +# include +# endif +#endif + +#define STREAM_ACCUMULATOR_MIN_32 25 +#define STREAM_ACCUMULATOR_MIN_64 57 +#define STREAM_ACCUMULATOR_MIN ((U32)(MEM_32bits() ? STREAM_ACCUMULATOR_MIN_32 : STREAM_ACCUMULATOR_MIN_64)) + + +/*-****************************************** +* bitStream encoding API (write forward) +********************************************/ +/* bitStream can mix input from multiple sources. + * A critical property of these streams is that they encode and decode in **reverse** direction. + * So the first bit sequence you add will be the last to be read, like a LIFO stack. + */ +typedef struct { + size_t bitContainer; + unsigned bitPos; + char* startPtr; + char* ptr; + char* endPtr; +} BIT_CStream_t; + +MEM_STATIC size_t BIT_initCStream(BIT_CStream_t* bitC, void* dstBuffer, size_t dstCapacity); +MEM_STATIC void BIT_addBits(BIT_CStream_t* bitC, size_t value, unsigned nbBits); +MEM_STATIC void BIT_flushBits(BIT_CStream_t* bitC); +MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC); + +/* Start with initCStream, providing the size of buffer to write into. +* bitStream will never write outside of this buffer. +* `dstCapacity` must be >= sizeof(bitD->bitContainer), otherwise @return will be an error code. +* +* bits are first added to a local register. +* Local register is size_t, hence 64-bits on 64-bits systems, or 32-bits on 32-bits systems. +* Writing data into memory is an explicit operation, performed by the flushBits function. +* Hence keep track how many bits are potentially stored into local register to avoid register overflow. +* After a flushBits, a maximum of 7 bits might still be stored into local register. +* +* Avoid storing elements of more than 24 bits if you want compatibility with 32-bits bitstream readers. +* +* Last operation is to close the bitStream. +* The function returns the final size of CStream in bytes. +* If data couldn't fit into `dstBuffer`, it will return a 0 ( == not storable) +*/ + + +/*-******************************************** +* bitStream decoding API (read backward) +**********************************************/ +typedef struct { + size_t bitContainer; + unsigned bitsConsumed; + const char* ptr; + const char* start; + const char* limitPtr; +} BIT_DStream_t; + +typedef enum { BIT_DStream_unfinished = 0, + BIT_DStream_endOfBuffer = 1, + BIT_DStream_completed = 2, + BIT_DStream_overflow = 3 } BIT_DStream_status; /* result of BIT_reloadDStream() */ + /* 1,2,4,8 would be better for bitmap combinations, but slows down performance a bit ... :( */ + +MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, size_t srcSize); +MEM_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, unsigned nbBits); +MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD); +MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* bitD); + + +/* Start by invoking BIT_initDStream(). +* A chunk of the bitStream is then stored into a local register. +* Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t). +* You can then retrieve bitFields stored into the local register, **in reverse order**. +* Local register is explicitly reloaded from memory by the BIT_reloadDStream() method. +* A reload guarantee a minimum of ((8*sizeof(bitD->bitContainer))-7) bits when its result is BIT_DStream_unfinished. +* Otherwise, it can be less than that, so proceed accordingly. +* Checking if DStream has reached its end can be performed with BIT_endOfDStream(). +*/ + + +/*-**************************************** +* unsafe API +******************************************/ +MEM_STATIC void BIT_addBitsFast(BIT_CStream_t* bitC, size_t value, unsigned nbBits); +/* faster, but works only if value is "clean", meaning all high bits above nbBits are 0 */ + +MEM_STATIC void BIT_flushBitsFast(BIT_CStream_t* bitC); +/* unsafe version; does not check buffer overflow */ + +MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits); +/* faster, but works only if nbBits >= 1 */ + + + +/*-************************************************************** +* Internal functions +****************************************************************/ +MEM_STATIC unsigned BIT_highbit32 (U32 val) +{ + assert(val != 0); + { +# if defined(_MSC_VER) /* Visual */ +# if STATIC_BMI2 == 1 + return _lzcnt_u32(val) ^ 31; +# else + if (val != 0) { + unsigned long r; + _BitScanReverse(&r, val); + return (unsigned)r; + } else { + /* Should not reach this code path */ + __assume(0); + } +# endif +# elif defined(__GNUC__) && (__GNUC__ >= 3) /* Use GCC Intrinsic */ + return __builtin_clz (val) ^ 31; +# elif defined(__ICCARM__) /* IAR Intrinsic */ + return 31 - __CLZ(val); +# else /* Software version */ + static const unsigned DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, + 11, 14, 16, 18, 22, 25, 3, 30, + 8, 12, 20, 28, 15, 17, 24, 7, + 19, 27, 23, 6, 26, 5, 4, 31 }; + U32 v = val; + v |= v >> 1; + v |= v >> 2; + v |= v >> 4; + v |= v >> 8; + v |= v >> 16; + return DeBruijnClz[ (U32) (v * 0x07C4ACDDU) >> 27]; +# endif + } +} + +/*===== Local Constants =====*/ +static const unsigned BIT_mask[] = { + 0, 1, 3, 7, 0xF, 0x1F, + 0x3F, 0x7F, 0xFF, 0x1FF, 0x3FF, 0x7FF, + 0xFFF, 0x1FFF, 0x3FFF, 0x7FFF, 0xFFFF, 0x1FFFF, + 0x3FFFF, 0x7FFFF, 0xFFFFF, 0x1FFFFF, 0x3FFFFF, 0x7FFFFF, + 0xFFFFFF, 0x1FFFFFF, 0x3FFFFFF, 0x7FFFFFF, 0xFFFFFFF, 0x1FFFFFFF, + 0x3FFFFFFF, 0x7FFFFFFF}; /* up to 31 bits */ +#define BIT_MASK_SIZE (sizeof(BIT_mask) / sizeof(BIT_mask[0])) + +/*-************************************************************** +* bitStream encoding +****************************************************************/ +/*! BIT_initCStream() : + * `dstCapacity` must be > sizeof(size_t) + * @return : 0 if success, + * otherwise an error code (can be tested using ERR_isError()) */ +MEM_STATIC size_t BIT_initCStream(BIT_CStream_t* bitC, + void* startPtr, size_t dstCapacity) +{ + bitC->bitContainer = 0; + bitC->bitPos = 0; + bitC->startPtr = (char*)startPtr; + bitC->ptr = bitC->startPtr; + bitC->endPtr = bitC->startPtr + dstCapacity - sizeof(bitC->bitContainer); + if (dstCapacity <= sizeof(bitC->bitContainer)) return ERROR(dstSize_tooSmall); + return 0; +} + +/*! BIT_addBits() : + * can add up to 31 bits into `bitC`. + * Note : does not check for register overflow ! */ +MEM_STATIC void BIT_addBits(BIT_CStream_t* bitC, + size_t value, unsigned nbBits) +{ + DEBUG_STATIC_ASSERT(BIT_MASK_SIZE == 32); + assert(nbBits < BIT_MASK_SIZE); + assert(nbBits + bitC->bitPos < sizeof(bitC->bitContainer) * 8); + bitC->bitContainer |= (value & BIT_mask[nbBits]) << bitC->bitPos; + bitC->bitPos += nbBits; +} + +/*! BIT_addBitsFast() : + * works only if `value` is _clean_, + * meaning all high bits above nbBits are 0 */ +MEM_STATIC void BIT_addBitsFast(BIT_CStream_t* bitC, + size_t value, unsigned nbBits) +{ + assert((value>>nbBits) == 0); + assert(nbBits + bitC->bitPos < sizeof(bitC->bitContainer) * 8); + bitC->bitContainer |= value << bitC->bitPos; + bitC->bitPos += nbBits; +} + +/*! BIT_flushBitsFast() : + * assumption : bitContainer has not overflowed + * unsafe version; does not check buffer overflow */ +MEM_STATIC void BIT_flushBitsFast(BIT_CStream_t* bitC) +{ + size_t const nbBytes = bitC->bitPos >> 3; + assert(bitC->bitPos < sizeof(bitC->bitContainer) * 8); + assert(bitC->ptr <= bitC->endPtr); + MEM_writeLEST(bitC->ptr, bitC->bitContainer); + bitC->ptr += nbBytes; + bitC->bitPos &= 7; + bitC->bitContainer >>= nbBytes*8; +} + +/*! BIT_flushBits() : + * assumption : bitContainer has not overflowed + * safe version; check for buffer overflow, and prevents it. + * note : does not signal buffer overflow. + * overflow will be revealed later on using BIT_closeCStream() */ +MEM_STATIC void BIT_flushBits(BIT_CStream_t* bitC) +{ + size_t const nbBytes = bitC->bitPos >> 3; + assert(bitC->bitPos < sizeof(bitC->bitContainer) * 8); + assert(bitC->ptr <= bitC->endPtr); + MEM_writeLEST(bitC->ptr, bitC->bitContainer); + bitC->ptr += nbBytes; + if (bitC->ptr > bitC->endPtr) bitC->ptr = bitC->endPtr; + bitC->bitPos &= 7; + bitC->bitContainer >>= nbBytes*8; +} + +/*! BIT_closeCStream() : + * @return : size of CStream, in bytes, + * or 0 if it could not fit into dstBuffer */ +MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC) +{ + BIT_addBitsFast(bitC, 1, 1); /* endMark */ + BIT_flushBits(bitC); + if (bitC->ptr >= bitC->endPtr) return 0; /* overflow detected */ + return (bitC->ptr - bitC->startPtr) + (bitC->bitPos > 0); +} + + +/*-******************************************************** +* bitStream decoding +**********************************************************/ +/*! BIT_initDStream() : + * Initialize a BIT_DStream_t. + * `bitD` : a pointer to an already allocated BIT_DStream_t structure. + * `srcSize` must be the *exact* size of the bitStream, in bytes. + * @return : size of stream (== srcSize), or an errorCode if a problem is detected + */ +MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, size_t srcSize) +{ + if (srcSize < 1) { ZSTD_memset(bitD, 0, sizeof(*bitD)); return ERROR(srcSize_wrong); } + + bitD->start = (const char*)srcBuffer; + bitD->limitPtr = bitD->start + sizeof(bitD->bitContainer); + + if (srcSize >= sizeof(bitD->bitContainer)) { /* normal case */ + bitD->ptr = (const char*)srcBuffer + srcSize - sizeof(bitD->bitContainer); + bitD->bitContainer = MEM_readLEST(bitD->ptr); + { BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize-1]; + bitD->bitsConsumed = lastByte ? 8 - BIT_highbit32(lastByte) : 0; /* ensures bitsConsumed is always set */ + if (lastByte == 0) return ERROR(GENERIC); /* endMark not present */ } + } else { + bitD->ptr = bitD->start; + bitD->bitContainer = *(const BYTE*)(bitD->start); + switch(srcSize) + { + case 7: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[6]) << (sizeof(bitD->bitContainer)*8 - 16); + ZSTD_FALLTHROUGH; + + case 6: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[5]) << (sizeof(bitD->bitContainer)*8 - 24); + ZSTD_FALLTHROUGH; + + case 5: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[4]) << (sizeof(bitD->bitContainer)*8 - 32); + ZSTD_FALLTHROUGH; + + case 4: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[3]) << 24; + ZSTD_FALLTHROUGH; + + case 3: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[2]) << 16; + ZSTD_FALLTHROUGH; + + case 2: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[1]) << 8; + ZSTD_FALLTHROUGH; + + default: break; + } + { BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize-1]; + bitD->bitsConsumed = lastByte ? 8 - BIT_highbit32(lastByte) : 0; + if (lastByte == 0) return ERROR(corruption_detected); /* endMark not present */ + } + bitD->bitsConsumed += (U32)(sizeof(bitD->bitContainer) - srcSize)*8; + } + + return srcSize; +} + +MEM_STATIC FORCE_INLINE_ATTR size_t BIT_getUpperBits(size_t bitContainer, U32 const start) +{ + return bitContainer >> start; +} + +MEM_STATIC FORCE_INLINE_ATTR size_t BIT_getMiddleBits(size_t bitContainer, U32 const start, U32 const nbBits) +{ + U32 const regMask = sizeof(bitContainer)*8 - 1; + /* if start > regMask, bitstream is corrupted, and result is undefined */ + assert(nbBits < BIT_MASK_SIZE); + /* x86 transform & ((1 << nbBits) - 1) to bzhi instruction, it is better + * than accessing memory. When bmi2 instruction is not present, we consider + * such cpus old (pre-Haswell, 2013) and their performance is not of that + * importance. + */ +#if defined(__x86_64__) || defined(_M_X86) + return (bitContainer >> (start & regMask)) & ((((U64)1) << nbBits) - 1); +#else + return (bitContainer >> (start & regMask)) & BIT_mask[nbBits]; +#endif +} + +MEM_STATIC FORCE_INLINE_ATTR size_t BIT_getLowerBits(size_t bitContainer, U32 const nbBits) +{ +#if defined(STATIC_BMI2) && STATIC_BMI2 == 1 + return _bzhi_u64(bitContainer, nbBits); +#else + assert(nbBits < BIT_MASK_SIZE); + return bitContainer & BIT_mask[nbBits]; +#endif +} + +/*! BIT_lookBits() : + * Provides next n bits from local register. + * local register is not modified. + * On 32-bits, maxNbBits==24. + * On 64-bits, maxNbBits==56. + * @return : value extracted */ +MEM_STATIC FORCE_INLINE_ATTR size_t BIT_lookBits(const BIT_DStream_t* bitD, U32 nbBits) +{ + /* arbitrate between double-shift and shift+mask */ +#if 1 + /* if bitD->bitsConsumed + nbBits > sizeof(bitD->bitContainer)*8, + * bitstream is likely corrupted, and result is undefined */ + return BIT_getMiddleBits(bitD->bitContainer, (sizeof(bitD->bitContainer)*8) - bitD->bitsConsumed - nbBits, nbBits); +#else + /* this code path is slower on my os-x laptop */ + U32 const regMask = sizeof(bitD->bitContainer)*8 - 1; + return ((bitD->bitContainer << (bitD->bitsConsumed & regMask)) >> 1) >> ((regMask-nbBits) & regMask); +#endif +} + +/*! BIT_lookBitsFast() : + * unsafe version; only works if nbBits >= 1 */ +MEM_STATIC size_t BIT_lookBitsFast(const BIT_DStream_t* bitD, U32 nbBits) +{ + U32 const regMask = sizeof(bitD->bitContainer)*8 - 1; + assert(nbBits >= 1); + return (bitD->bitContainer << (bitD->bitsConsumed & regMask)) >> (((regMask+1)-nbBits) & regMask); +} + +MEM_STATIC FORCE_INLINE_ATTR void BIT_skipBits(BIT_DStream_t* bitD, U32 nbBits) +{ + bitD->bitsConsumed += nbBits; +} + +/*! BIT_readBits() : + * Read (consume) next n bits from local register and update. + * Pay attention to not read more than nbBits contained into local register. + * @return : extracted value. */ +MEM_STATIC FORCE_INLINE_ATTR size_t BIT_readBits(BIT_DStream_t* bitD, unsigned nbBits) +{ + size_t const value = BIT_lookBits(bitD, nbBits); + BIT_skipBits(bitD, nbBits); + return value; +} + +/*! BIT_readBitsFast() : + * unsafe version; only works only if nbBits >= 1 */ +MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits) +{ + size_t const value = BIT_lookBitsFast(bitD, nbBits); + assert(nbBits >= 1); + BIT_skipBits(bitD, nbBits); + return value; +} + +/*! BIT_reloadDStreamFast() : + * Similar to BIT_reloadDStream(), but with two differences: + * 1. bitsConsumed <= sizeof(bitD->bitContainer)*8 must hold! + * 2. Returns BIT_DStream_overflow when bitD->ptr < bitD->limitPtr, at this + * point you must use BIT_reloadDStream() to reload. + */ +MEM_STATIC BIT_DStream_status BIT_reloadDStreamFast(BIT_DStream_t* bitD) +{ + if (UNLIKELY(bitD->ptr < bitD->limitPtr)) + return BIT_DStream_overflow; + assert(bitD->bitsConsumed <= sizeof(bitD->bitContainer)*8); + bitD->ptr -= bitD->bitsConsumed >> 3; + bitD->bitsConsumed &= 7; + bitD->bitContainer = MEM_readLEST(bitD->ptr); + return BIT_DStream_unfinished; +} + +/*! BIT_reloadDStream() : + * Refill `bitD` from buffer previously set in BIT_initDStream() . + * This function is safe, it guarantees it will not read beyond src buffer. + * @return : status of `BIT_DStream_t` internal register. + * when status == BIT_DStream_unfinished, internal register is filled with at least 25 or 57 bits */ +MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD) +{ + if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8)) /* overflow detected, like end of stream */ + return BIT_DStream_overflow; + + if (bitD->ptr >= bitD->limitPtr) { + return BIT_reloadDStreamFast(bitD); + } + if (bitD->ptr == bitD->start) { + if (bitD->bitsConsumed < sizeof(bitD->bitContainer)*8) return BIT_DStream_endOfBuffer; + return BIT_DStream_completed; + } + /* start < ptr < limitPtr */ + { U32 nbBytes = bitD->bitsConsumed >> 3; + BIT_DStream_status result = BIT_DStream_unfinished; + if (bitD->ptr - nbBytes < bitD->start) { + nbBytes = (U32)(bitD->ptr - bitD->start); /* ptr > start */ + result = BIT_DStream_endOfBuffer; + } + bitD->ptr -= nbBytes; + bitD->bitsConsumed -= nbBytes*8; + bitD->bitContainer = MEM_readLEST(bitD->ptr); /* reminder : srcSize > sizeof(bitD->bitContainer), otherwise bitD->ptr == bitD->start */ + return result; + } +} + +/*! BIT_endOfDStream() : + * @return : 1 if DStream has _exactly_ reached its end (all bits consumed). + */ +MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* DStream) +{ + return ((DStream->ptr == DStream->start) && (DStream->bitsConsumed == sizeof(DStream->bitContainer)*8)); +} + +#if defined (__cplusplus) +} +#endif + +#endif /* BITSTREAM_H_MODULE */ diff --git a/src/blosc2/internal-complibs/zstd-1.5.2/common/compiler.h b/src/blosc2/internal-complibs/zstd-1.5.2/common/compiler.h new file mode 100644 index 0000000..516930c --- /dev/null +++ b/src/blosc2/internal-complibs/zstd-1.5.2/common/compiler.h @@ -0,0 +1,335 @@ +/* + * Copyright (c) Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_COMPILER_H +#define ZSTD_COMPILER_H + +#include "portability_macros.h" + +/*-******************************************************* +* Compiler specifics +*********************************************************/ +/* force inlining */ + +#if !defined(ZSTD_NO_INLINE) +#if (defined(__GNUC__) && !defined(__STRICT_ANSI__)) || defined(__cplusplus) || defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ +# define INLINE_KEYWORD inline +#else +# define INLINE_KEYWORD +#endif + +#if defined(__GNUC__) || defined(__ICCARM__) +# define FORCE_INLINE_ATTR __attribute__((always_inline)) +#elif defined(_MSC_VER) +# define FORCE_INLINE_ATTR __forceinline +#else +# define FORCE_INLINE_ATTR +#endif + +#else + +#define INLINE_KEYWORD +#define FORCE_INLINE_ATTR + +#endif + +/** + On MSVC qsort requires that functions passed into it use the __cdecl calling conversion(CC). + This explicitly marks such functions as __cdecl so that the code will still compile + if a CC other than __cdecl has been made the default. +*/ +#if defined(_MSC_VER) +# define WIN_CDECL __cdecl +#else +# define WIN_CDECL +#endif + +/** + * FORCE_INLINE_TEMPLATE is used to define C "templates", which take constant + * parameters. They must be inlined for the compiler to eliminate the constant + * branches. + */ +#define FORCE_INLINE_TEMPLATE static INLINE_KEYWORD FORCE_INLINE_ATTR +/** + * HINT_INLINE is used to help the compiler generate better code. It is *not* + * used for "templates", so it can be tweaked based on the compilers + * performance. + * + * gcc-4.8 and gcc-4.9 have been shown to benefit from leaving off the + * always_inline attribute. + * + * clang up to 5.0.0 (trunk) benefit tremendously from the always_inline + * attribute. + */ +#if !defined(__clang__) && defined(__GNUC__) && __GNUC__ >= 4 && __GNUC_MINOR__ >= 8 && __GNUC__ < 5 +# define HINT_INLINE static INLINE_KEYWORD +#else +# define HINT_INLINE static INLINE_KEYWORD FORCE_INLINE_ATTR +#endif + +/* UNUSED_ATTR tells the compiler it is okay if the function is unused. */ +#if defined(__GNUC__) +# define UNUSED_ATTR __attribute__((unused)) +#else +# define UNUSED_ATTR +#endif + +/* force no inlining */ +#ifdef _MSC_VER +# define FORCE_NOINLINE static __declspec(noinline) +#else +# if defined(__GNUC__) || defined(__ICCARM__) +# define FORCE_NOINLINE static __attribute__((__noinline__)) +# else +# define FORCE_NOINLINE static +# endif +#endif + + +/* target attribute */ +#if defined(__GNUC__) || defined(__ICCARM__) +# define TARGET_ATTRIBUTE(target) __attribute__((__target__(target))) +#else +# define TARGET_ATTRIBUTE(target) +#endif + +/* Target attribute for BMI2 dynamic dispatch. + * Enable lzcnt, bmi, and bmi2. + * We test for bmi1 & bmi2. lzcnt is included in bmi1. + */ +#define BMI2_TARGET_ATTRIBUTE TARGET_ATTRIBUTE("lzcnt,bmi,bmi2") + +/* prefetch + * can be disabled, by declaring NO_PREFETCH build macro */ +#if defined(NO_PREFETCH) +# define PREFETCH_L1(ptr) (void)(ptr) /* disabled */ +# define PREFETCH_L2(ptr) (void)(ptr) /* disabled */ +#else +# if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_I86)) /* _mm_prefetch() is not defined outside of x86/x64 */ +# include /* https://msdn.microsoft.com/fr-fr/library/84szxsww(v=vs.90).aspx */ +# define PREFETCH_L1(ptr) _mm_prefetch((const char*)(ptr), _MM_HINT_T0) +# define PREFETCH_L2(ptr) _mm_prefetch((const char*)(ptr), _MM_HINT_T1) +# elif defined(__GNUC__) && ( (__GNUC__ >= 4) || ( (__GNUC__ == 3) && (__GNUC_MINOR__ >= 1) ) ) +# define PREFETCH_L1(ptr) __builtin_prefetch((ptr), 0 /* rw==read */, 3 /* locality */) +# define PREFETCH_L2(ptr) __builtin_prefetch((ptr), 0 /* rw==read */, 2 /* locality */) +# elif defined(__aarch64__) +# define PREFETCH_L1(ptr) __asm__ __volatile__("prfm pldl1keep, %0" ::"Q"(*(ptr))) +# define PREFETCH_L2(ptr) __asm__ __volatile__("prfm pldl2keep, %0" ::"Q"(*(ptr))) +# else +# define PREFETCH_L1(ptr) (void)(ptr) /* disabled */ +# define PREFETCH_L2(ptr) (void)(ptr) /* disabled */ +# endif +#endif /* NO_PREFETCH */ + +#define CACHELINE_SIZE 64 + +#define PREFETCH_AREA(p, s) { \ + const char* const _ptr = (const char*)(p); \ + size_t const _size = (size_t)(s); \ + size_t _pos; \ + for (_pos=0; _pos<_size; _pos+=CACHELINE_SIZE) { \ + PREFETCH_L2(_ptr + _pos); \ + } \ +} + +/* vectorization + * older GCC (pre gcc-4.3 picked as the cutoff) uses a different syntax, + * and some compilers, like Intel ICC and MCST LCC, do not support it at all. */ +#if !defined(__INTEL_COMPILER) && !defined(__clang__) && defined(__GNUC__) && !defined(__LCC__) +# if (__GNUC__ == 4 && __GNUC_MINOR__ > 3) || (__GNUC__ >= 5) +# define DONT_VECTORIZE __attribute__((optimize("no-tree-vectorize"))) +# else +# define DONT_VECTORIZE _Pragma("GCC optimize(\"no-tree-vectorize\")") +# endif +#else +# define DONT_VECTORIZE +#endif + +/* Tell the compiler that a branch is likely or unlikely. + * Only use these macros if it causes the compiler to generate better code. + * If you can remove a LIKELY/UNLIKELY annotation without speed changes in gcc + * and clang, please do. + */ +#if defined(__GNUC__) +#define LIKELY(x) (__builtin_expect((x), 1)) +#define UNLIKELY(x) (__builtin_expect((x), 0)) +#else +#define LIKELY(x) (x) +#define UNLIKELY(x) (x) +#endif + +/* disable warnings */ +#ifdef _MSC_VER /* Visual Studio */ +# include /* For Visual 2005 */ +# pragma warning(disable : 4100) /* disable: C4100: unreferenced formal parameter */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +# pragma warning(disable : 4204) /* disable: C4204: non-constant aggregate initializer */ +# pragma warning(disable : 4214) /* disable: C4214: non-int bitfields */ +# pragma warning(disable : 4324) /* disable: C4324: padded structure */ +#endif + +/*Like DYNAMIC_BMI2 but for compile time determination of BMI2 support*/ +#ifndef STATIC_BMI2 +# if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_I86)) +# ifdef __AVX2__ //MSVC does not have a BMI2 specific flag, but every CPU that supports AVX2 also supports BMI2 +# define STATIC_BMI2 1 +# endif +# endif +#endif + +#ifndef STATIC_BMI2 + #define STATIC_BMI2 0 +#endif + +/* compile time determination of SIMD support */ +#if !defined(ZSTD_NO_INTRINSICS) +# if defined(__SSE2__) || defined(_M_AMD64) || (defined (_M_IX86) && defined(_M_IX86_FP) && (_M_IX86_FP >= 2)) +# define ZSTD_ARCH_X86_SSE2 +# endif +# if defined(__ARM_NEON) || defined(_M_ARM64) +# define ZSTD_ARCH_ARM_NEON +# endif +# +# if defined(ZSTD_ARCH_X86_SSE2) +# include +# elif defined(ZSTD_ARCH_ARM_NEON) +# include +# endif +#endif + +/* C-language Attributes are added in C23. */ +#if defined(__STDC_VERSION__) && (__STDC_VERSION__ > 201710L) && defined(__has_c_attribute) +# define ZSTD_HAS_C_ATTRIBUTE(x) __has_c_attribute(x) +#else +# define ZSTD_HAS_C_ATTRIBUTE(x) 0 +#endif + +/* Only use C++ attributes in C++. Some compilers report support for C++ + * attributes when compiling with C. + */ +#if defined(__cplusplus) && defined(__has_cpp_attribute) +# define ZSTD_HAS_CPP_ATTRIBUTE(x) __has_cpp_attribute(x) +#else +# define ZSTD_HAS_CPP_ATTRIBUTE(x) 0 +#endif + +/* Define ZSTD_FALLTHROUGH macro for annotating switch case with the 'fallthrough' attribute. + * - C23: https://en.cppreference.com/w/c/language/attributes/fallthrough + * - CPP17: https://en.cppreference.com/w/cpp/language/attributes/fallthrough + * - Else: __attribute__((__fallthrough__)) + */ +#ifndef ZSTD_FALLTHROUGH +# if ZSTD_HAS_C_ATTRIBUTE(fallthrough) +# define ZSTD_FALLTHROUGH [[fallthrough]] +# elif ZSTD_HAS_CPP_ATTRIBUTE(fallthrough) +# define ZSTD_FALLTHROUGH [[fallthrough]] +# elif __has_attribute(__fallthrough__) +/* Leading semicolon is to satisfy gcc-11 with -pedantic. Without the semicolon + * gcc complains about: a label can only be part of a statement and a declaration is not a statement. + */ +# define ZSTD_FALLTHROUGH ; __attribute__((__fallthrough__)) +# else +# define ZSTD_FALLTHROUGH +# endif +#endif + +/*-************************************************************** +* Alignment check +*****************************************************************/ + +/* this test was initially positioned in mem.h, + * but this file is removed (or replaced) for linux kernel + * so it's now hosted in compiler.h, + * which remains valid for both user & kernel spaces. + */ + +#ifndef ZSTD_ALIGNOF +# if defined(__GNUC__) || defined(_MSC_VER) +/* covers gcc, clang & MSVC */ +/* note : this section must come first, before C11, + * due to a limitation in the kernel source generator */ +# define ZSTD_ALIGNOF(T) __alignof(T) + +# elif defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) +/* C11 support */ +# include +# define ZSTD_ALIGNOF(T) alignof(T) + +# else +/* No known support for alignof() - imperfect backup */ +# define ZSTD_ALIGNOF(T) (sizeof(void*) < sizeof(T) ? sizeof(void*) : sizeof(T)) + +# endif +#endif /* ZSTD_ALIGNOF */ + +/*-************************************************************** +* Sanitizer +*****************************************************************/ + +#if ZSTD_MEMORY_SANITIZER +/* Not all platforms that support msan provide sanitizers/msan_interface.h. + * We therefore declare the functions we need ourselves, rather than trying to + * include the header file... */ +#include /* size_t */ +#define ZSTD_DEPS_NEED_STDINT +#include "zstd_deps.h" /* intptr_t */ + +/* Make memory region fully initialized (without changing its contents). */ +void __msan_unpoison(const volatile void *a, size_t size); + +/* Make memory region fully uninitialized (without changing its contents). + This is a legacy interface that does not update origin information. Use + __msan_allocated_memory() instead. */ +void __msan_poison(const volatile void *a, size_t size); + +/* Returns the offset of the first (at least partially) poisoned byte in the + memory range, or -1 if the whole range is good. */ +intptr_t __msan_test_shadow(const volatile void *x, size_t size); +#endif + +#if ZSTD_ADDRESS_SANITIZER +/* Not all platforms that support asan provide sanitizers/asan_interface.h. + * We therefore declare the functions we need ourselves, rather than trying to + * include the header file... */ +#include /* size_t */ + +/** + * Marks a memory region ([addr, addr+size)) as unaddressable. + * + * This memory must be previously allocated by your program. Instrumented + * code is forbidden from accessing addresses in this region until it is + * unpoisoned. This function is not guaranteed to poison the entire region - + * it could poison only a subregion of [addr, addr+size) due to ASan + * alignment restrictions. + * + * \note This function is not thread-safe because no two threads can poison or + * unpoison memory in the same memory region simultaneously. + * + * \param addr Start of memory region. + * \param size Size of memory region. */ +void __asan_poison_memory_region(void const volatile *addr, size_t size); + +/** + * Marks a memory region ([addr, addr+size)) as addressable. + * + * This memory must be previously allocated by your program. Accessing + * addresses in this region is allowed until this region is poisoned again. + * This function could unpoison a super-region of [addr, addr+size) due + * to ASan alignment restrictions. + * + * \note This function is not thread-safe because no two threads can + * poison or unpoison memory in the same memory region simultaneously. + * + * \param addr Start of memory region. + * \param size Size of memory region. */ +void __asan_unpoison_memory_region(void const volatile *addr, size_t size); +#endif + +#endif /* ZSTD_COMPILER_H */ diff --git a/src/blosc2/internal-complibs/zstd-1.5.2/common/cpu.h b/src/blosc2/internal-complibs/zstd-1.5.2/common/cpu.h new file mode 100644 index 0000000..8acd33b --- /dev/null +++ b/src/blosc2/internal-complibs/zstd-1.5.2/common/cpu.h @@ -0,0 +1,213 @@ +/* + * Copyright (c) Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_COMMON_CPU_H +#define ZSTD_COMMON_CPU_H + +/** + * Implementation taken from folly/CpuId.h + * https://github.com/facebook/folly/blob/master/folly/CpuId.h + */ + +#include "mem.h" + +#ifdef _MSC_VER +#include +#endif + +typedef struct { + U32 f1c; + U32 f1d; + U32 f7b; + U32 f7c; +} ZSTD_cpuid_t; + +MEM_STATIC ZSTD_cpuid_t ZSTD_cpuid(void) { + U32 f1c = 0; + U32 f1d = 0; + U32 f7b = 0; + U32 f7c = 0; +#if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_IX86)) + int reg[4]; + __cpuid((int*)reg, 0); + { + int const n = reg[0]; + if (n >= 1) { + __cpuid((int*)reg, 1); + f1c = (U32)reg[2]; + f1d = (U32)reg[3]; + } + if (n >= 7) { + __cpuidex((int*)reg, 7, 0); + f7b = (U32)reg[1]; + f7c = (U32)reg[2]; + } + } +#elif defined(__i386__) && defined(__PIC__) && !defined(__clang__) && defined(__GNUC__) + /* The following block like the normal cpuid branch below, but gcc + * reserves ebx for use of its pic register so we must specially + * handle the save and restore to avoid clobbering the register + */ + U32 n; + __asm__( + "pushl %%ebx\n\t" + "cpuid\n\t" + "popl %%ebx\n\t" + : "=a"(n) + : "a"(0) + : "ecx", "edx"); + if (n >= 1) { + U32 f1a; + __asm__( + "pushl %%ebx\n\t" + "cpuid\n\t" + "popl %%ebx\n\t" + : "=a"(f1a), "=c"(f1c), "=d"(f1d) + : "a"(1)); + } + if (n >= 7) { + __asm__( + "pushl %%ebx\n\t" + "cpuid\n\t" + "movl %%ebx, %%eax\n\t" + "popl %%ebx" + : "=a"(f7b), "=c"(f7c) + : "a"(7), "c"(0) + : "edx"); + } +#elif defined(__x86_64__) || defined(_M_X64) || defined(__i386__) + U32 n; + __asm__("cpuid" : "=a"(n) : "a"(0) : "ebx", "ecx", "edx"); + if (n >= 1) { + U32 f1a; + __asm__("cpuid" : "=a"(f1a), "=c"(f1c), "=d"(f1d) : "a"(1) : "ebx"); + } + if (n >= 7) { + U32 f7a; + __asm__("cpuid" + : "=a"(f7a), "=b"(f7b), "=c"(f7c) + : "a"(7), "c"(0) + : "edx"); + } +#endif + { + ZSTD_cpuid_t cpuid; + cpuid.f1c = f1c; + cpuid.f1d = f1d; + cpuid.f7b = f7b; + cpuid.f7c = f7c; + return cpuid; + } +} + +#define X(name, r, bit) \ + MEM_STATIC int ZSTD_cpuid_##name(ZSTD_cpuid_t const cpuid) { \ + return ((cpuid.r) & (1U << bit)) != 0; \ + } + +/* cpuid(1): Processor Info and Feature Bits. */ +#define C(name, bit) X(name, f1c, bit) + C(sse3, 0) + C(pclmuldq, 1) + C(dtes64, 2) + C(monitor, 3) + C(dscpl, 4) + C(vmx, 5) + C(smx, 6) + C(eist, 7) + C(tm2, 8) + C(ssse3, 9) + C(cnxtid, 10) + C(fma, 12) + C(cx16, 13) + C(xtpr, 14) + C(pdcm, 15) + C(pcid, 17) + C(dca, 18) + C(sse41, 19) + C(sse42, 20) + C(x2apic, 21) + C(movbe, 22) + C(popcnt, 23) + C(tscdeadline, 24) + C(aes, 25) + C(xsave, 26) + C(osxsave, 27) + C(avx, 28) + C(f16c, 29) + C(rdrand, 30) +#undef C +#define D(name, bit) X(name, f1d, bit) + D(fpu, 0) + D(vme, 1) + D(de, 2) + D(pse, 3) + D(tsc, 4) + D(msr, 5) + D(pae, 6) + D(mce, 7) + D(cx8, 8) + D(apic, 9) + D(sep, 11) + D(mtrr, 12) + D(pge, 13) + D(mca, 14) + D(cmov, 15) + D(pat, 16) + D(pse36, 17) + D(psn, 18) + D(clfsh, 19) + D(ds, 21) + D(acpi, 22) + D(mmx, 23) + D(fxsr, 24) + D(sse, 25) + D(sse2, 26) + D(ss, 27) + D(htt, 28) + D(tm, 29) + D(pbe, 31) +#undef D + +/* cpuid(7): Extended Features. */ +#define B(name, bit) X(name, f7b, bit) + B(bmi1, 3) + B(hle, 4) + B(avx2, 5) + B(smep, 7) + B(bmi2, 8) + B(erms, 9) + B(invpcid, 10) + B(rtm, 11) + B(mpx, 14) + B(avx512f, 16) + B(avx512dq, 17) + B(rdseed, 18) + B(adx, 19) + B(smap, 20) + B(avx512ifma, 21) + B(pcommit, 22) + B(clflushopt, 23) + B(clwb, 24) + B(avx512pf, 26) + B(avx512er, 27) + B(avx512cd, 28) + B(sha, 29) + B(avx512bw, 30) + B(avx512vl, 31) +#undef B +#define C(name, bit) X(name, f7c, bit) + C(prefetchwt1, 0) + C(avx512vbmi, 1) +#undef C + +#undef X + +#endif /* ZSTD_COMMON_CPU_H */ diff --git a/src/blosc2/internal-complibs/zstd-1.5.2/common/debug.c b/src/blosc2/internal-complibs/zstd-1.5.2/common/debug.c new file mode 100644 index 0000000..bb863c9 --- /dev/null +++ b/src/blosc2/internal-complibs/zstd-1.5.2/common/debug.c @@ -0,0 +1,24 @@ +/* ****************************************************************** + * debug + * Part of FSE library + * Copyright (c) Yann Collet, Facebook, Inc. + * + * You can contact the author at : + * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. +****************************************************************** */ + + +/* + * This module only hosts one global variable + * which can be used to dynamically influence the verbosity of traces, + * such as DEBUGLOG and RAWLOG + */ + +#include "debug.h" + +int g_debuglevel = DEBUGLEVEL; diff --git a/src/blosc2/internal-complibs/zstd-1.5.2/common/debug.h b/src/blosc2/internal-complibs/zstd-1.5.2/common/debug.h new file mode 100644 index 0000000..3b2a320 --- /dev/null +++ b/src/blosc2/internal-complibs/zstd-1.5.2/common/debug.h @@ -0,0 +1,107 @@ +/* ****************************************************************** + * debug + * Part of FSE library + * Copyright (c) Yann Collet, Facebook, Inc. + * + * You can contact the author at : + * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. +****************************************************************** */ + + +/* + * The purpose of this header is to enable debug functions. + * They regroup assert(), DEBUGLOG() and RAWLOG() for run-time, + * and DEBUG_STATIC_ASSERT() for compile-time. + * + * By default, DEBUGLEVEL==0, which means run-time debug is disabled. + * + * Level 1 enables assert() only. + * Starting level 2, traces can be generated and pushed to stderr. + * The higher the level, the more verbose the traces. + * + * It's possible to dynamically adjust level using variable g_debug_level, + * which is only declared if DEBUGLEVEL>=2, + * and is a global variable, not multi-thread protected (use with care) + */ + +#ifndef DEBUG_H_12987983217 +#define DEBUG_H_12987983217 + +#if defined (__cplusplus) +extern "C" { +#endif + + +/* static assert is triggered at compile time, leaving no runtime artefact. + * static assert only works with compile-time constants. + * Also, this variant can only be used inside a function. */ +#define DEBUG_STATIC_ASSERT(c) (void)sizeof(char[(c) ? 1 : -1]) + + +/* DEBUGLEVEL is expected to be defined externally, + * typically through compiler command line. + * Value must be a number. */ +#ifndef DEBUGLEVEL +# define DEBUGLEVEL 0 +#endif + + +/* recommended values for DEBUGLEVEL : + * 0 : release mode, no debug, all run-time checks disabled + * 1 : enables assert() only, no display + * 2 : reserved, for currently active debug path + * 3 : events once per object lifetime (CCtx, CDict, etc.) + * 4 : events once per frame + * 5 : events once per block + * 6 : events once per sequence (verbose) + * 7+: events at every position (*very* verbose) + * + * It's generally inconvenient to output traces > 5. + * In which case, it's possible to selectively trigger high verbosity levels + * by modifying g_debug_level. + */ + +#if (DEBUGLEVEL>=1) +# define ZSTD_DEPS_NEED_ASSERT +# include "zstd_deps.h" +#else +# ifndef assert /* assert may be already defined, due to prior #include */ +# define assert(condition) ((void)0) /* disable assert (default) */ +# endif +#endif + +#if (DEBUGLEVEL>=2) +# define ZSTD_DEPS_NEED_IO +# include "zstd_deps.h" +extern int g_debuglevel; /* the variable is only declared, + it actually lives in debug.c, + and is shared by the whole process. + It's not thread-safe. + It's useful when enabling very verbose levels + on selective conditions (such as position in src) */ + +# define RAWLOG(l, ...) { \ + if (l<=g_debuglevel) { \ + ZSTD_DEBUG_PRINT(__VA_ARGS__); \ + } } +# define DEBUGLOG(l, ...) { \ + if (l<=g_debuglevel) { \ + ZSTD_DEBUG_PRINT(__FILE__ ": " __VA_ARGS__); \ + ZSTD_DEBUG_PRINT(" \n"); \ + } } +#else +# define RAWLOG(l, ...) {} /* disabled */ +# define DEBUGLOG(l, ...) {} /* disabled */ +#endif + + +#if defined (__cplusplus) +} +#endif + +#endif /* DEBUG_H_12987983217 */ diff --git a/src/blosc2/internal-complibs/zstd-1.5.2/common/entropy_common.c b/src/blosc2/internal-complibs/zstd-1.5.2/common/entropy_common.c new file mode 100644 index 0000000..4229b40 --- /dev/null +++ b/src/blosc2/internal-complibs/zstd-1.5.2/common/entropy_common.c @@ -0,0 +1,368 @@ +/* ****************************************************************** + * Common functions of New Generation Entropy library + * Copyright (c) Yann Collet, Facebook, Inc. + * + * You can contact the author at : + * - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy + * - Public forum : https://groups.google.com/forum/#!forum/lz4c + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. +****************************************************************** */ + +/* ************************************* +* Dependencies +***************************************/ +#include "mem.h" +#include "error_private.h" /* ERR_*, ERROR */ +#define FSE_STATIC_LINKING_ONLY /* FSE_MIN_TABLELOG */ +#include "fse.h" +#define HUF_STATIC_LINKING_ONLY /* HUF_TABLELOG_ABSOLUTEMAX */ +#include "huf.h" + + +/*=== Version ===*/ +unsigned FSE_versionNumber(void) { return FSE_VERSION_NUMBER; } + + +/*=== Error Management ===*/ +unsigned FSE_isError(size_t code) { return ERR_isError(code); } +const char* FSE_getErrorName(size_t code) { return ERR_getErrorName(code); } + +unsigned HUF_isError(size_t code) { return ERR_isError(code); } +const char* HUF_getErrorName(size_t code) { return ERR_getErrorName(code); } + + +/*-************************************************************** +* FSE NCount encoding-decoding +****************************************************************/ +static U32 FSE_ctz(U32 val) +{ + assert(val != 0); + { +# if defined(_MSC_VER) /* Visual */ + if (val != 0) { + unsigned long r; + _BitScanForward(&r, val); + return (unsigned)r; + } else { + /* Should not reach this code path */ + __assume(0); + } +# elif defined(__GNUC__) && (__GNUC__ >= 3) /* GCC Intrinsic */ + return __builtin_ctz(val); +# elif defined(__ICCARM__) /* IAR Intrinsic */ + return __CTZ(val); +# else /* Software version */ + U32 count = 0; + while ((val & 1) == 0) { + val >>= 1; + ++count; + } + return count; +# endif + } +} + +FORCE_INLINE_TEMPLATE +size_t FSE_readNCount_body(short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr, + const void* headerBuffer, size_t hbSize) +{ + const BYTE* const istart = (const BYTE*) headerBuffer; + const BYTE* const iend = istart + hbSize; + const BYTE* ip = istart; + int nbBits; + int remaining; + int threshold; + U32 bitStream; + int bitCount; + unsigned charnum = 0; + unsigned const maxSV1 = *maxSVPtr + 1; + int previous0 = 0; + + if (hbSize < 8) { + /* This function only works when hbSize >= 8 */ + char buffer[8] = {0}; + ZSTD_memcpy(buffer, headerBuffer, hbSize); + { size_t const countSize = FSE_readNCount(normalizedCounter, maxSVPtr, tableLogPtr, + buffer, sizeof(buffer)); + if (FSE_isError(countSize)) return countSize; + if (countSize > hbSize) return ERROR(corruption_detected); + return countSize; + } } + assert(hbSize >= 8); + + /* init */ + ZSTD_memset(normalizedCounter, 0, (*maxSVPtr+1) * sizeof(normalizedCounter[0])); /* all symbols not present in NCount have a frequency of 0 */ + bitStream = MEM_readLE32(ip); + nbBits = (bitStream & 0xF) + FSE_MIN_TABLELOG; /* extract tableLog */ + if (nbBits > FSE_TABLELOG_ABSOLUTE_MAX) return ERROR(tableLog_tooLarge); + bitStream >>= 4; + bitCount = 4; + *tableLogPtr = nbBits; + remaining = (1<> 1; + while (repeats >= 12) { + charnum += 3 * 12; + if (LIKELY(ip <= iend-7)) { + ip += 3; + } else { + bitCount -= (int)(8 * (iend - 7 - ip)); + bitCount &= 31; + ip = iend - 4; + } + bitStream = MEM_readLE32(ip) >> bitCount; + repeats = FSE_ctz(~bitStream | 0x80000000) >> 1; + } + charnum += 3 * repeats; + bitStream >>= 2 * repeats; + bitCount += 2 * repeats; + + /* Add the final repeat which isn't 0b11. */ + assert((bitStream & 3) < 3); + charnum += bitStream & 3; + bitCount += 2; + + /* This is an error, but break and return an error + * at the end, because returning out of a loop makes + * it harder for the compiler to optimize. + */ + if (charnum >= maxSV1) break; + + /* We don't need to set the normalized count to 0 + * because we already memset the whole buffer to 0. + */ + + if (LIKELY(ip <= iend-7) || (ip + (bitCount>>3) <= iend-4)) { + assert((bitCount >> 3) <= 3); /* For first condition to work */ + ip += bitCount>>3; + bitCount &= 7; + } else { + bitCount -= (int)(8 * (iend - 4 - ip)); + bitCount &= 31; + ip = iend - 4; + } + bitStream = MEM_readLE32(ip) >> bitCount; + } + { + int const max = (2*threshold-1) - remaining; + int count; + + if ((bitStream & (threshold-1)) < (U32)max) { + count = bitStream & (threshold-1); + bitCount += nbBits-1; + } else { + count = bitStream & (2*threshold-1); + if (count >= threshold) count -= max; + bitCount += nbBits; + } + + count--; /* extra accuracy */ + /* When it matters (small blocks), this is a + * predictable branch, because we don't use -1. + */ + if (count >= 0) { + remaining -= count; + } else { + assert(count == -1); + remaining += count; + } + normalizedCounter[charnum++] = (short)count; + previous0 = !count; + + assert(threshold > 1); + if (remaining < threshold) { + /* This branch can be folded into the + * threshold update condition because we + * know that threshold > 1. + */ + if (remaining <= 1) break; + nbBits = BIT_highbit32(remaining) + 1; + threshold = 1 << (nbBits - 1); + } + if (charnum >= maxSV1) break; + + if (LIKELY(ip <= iend-7) || (ip + (bitCount>>3) <= iend-4)) { + ip += bitCount>>3; + bitCount &= 7; + } else { + bitCount -= (int)(8 * (iend - 4 - ip)); + bitCount &= 31; + ip = iend - 4; + } + bitStream = MEM_readLE32(ip) >> bitCount; + } } + if (remaining != 1) return ERROR(corruption_detected); + /* Only possible when there are too many zeros. */ + if (charnum > maxSV1) return ERROR(maxSymbolValue_tooSmall); + if (bitCount > 32) return ERROR(corruption_detected); + *maxSVPtr = charnum-1; + + ip += (bitCount+7)>>3; + return ip-istart; +} + +/* Avoids the FORCE_INLINE of the _body() function. */ +static size_t FSE_readNCount_body_default( + short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr, + const void* headerBuffer, size_t hbSize) +{ + return FSE_readNCount_body(normalizedCounter, maxSVPtr, tableLogPtr, headerBuffer, hbSize); +} + +#if DYNAMIC_BMI2 +BMI2_TARGET_ATTRIBUTE static size_t FSE_readNCount_body_bmi2( + short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr, + const void* headerBuffer, size_t hbSize) +{ + return FSE_readNCount_body(normalizedCounter, maxSVPtr, tableLogPtr, headerBuffer, hbSize); +} +#endif + +size_t FSE_readNCount_bmi2( + short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr, + const void* headerBuffer, size_t hbSize, int bmi2) +{ +#if DYNAMIC_BMI2 + if (bmi2) { + return FSE_readNCount_body_bmi2(normalizedCounter, maxSVPtr, tableLogPtr, headerBuffer, hbSize); + } +#endif + (void)bmi2; + return FSE_readNCount_body_default(normalizedCounter, maxSVPtr, tableLogPtr, headerBuffer, hbSize); +} + +size_t FSE_readNCount( + short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr, + const void* headerBuffer, size_t hbSize) +{ + return FSE_readNCount_bmi2(normalizedCounter, maxSVPtr, tableLogPtr, headerBuffer, hbSize, /* bmi2 */ 0); +} + + +/*! HUF_readStats() : + Read compact Huffman tree, saved by HUF_writeCTable(). + `huffWeight` is destination buffer. + `rankStats` is assumed to be a table of at least HUF_TABLELOG_MAX U32. + @return : size read from `src` , or an error Code . + Note : Needed by HUF_readCTable() and HUF_readDTableX?() . +*/ +size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats, + U32* nbSymbolsPtr, U32* tableLogPtr, + const void* src, size_t srcSize) +{ + U32 wksp[HUF_READ_STATS_WORKSPACE_SIZE_U32]; + return HUF_readStats_wksp(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, wksp, sizeof(wksp), /* bmi2 */ 0); +} + +FORCE_INLINE_TEMPLATE size_t +HUF_readStats_body(BYTE* huffWeight, size_t hwSize, U32* rankStats, + U32* nbSymbolsPtr, U32* tableLogPtr, + const void* src, size_t srcSize, + void* workSpace, size_t wkspSize, + int bmi2) +{ + U32 weightTotal; + const BYTE* ip = (const BYTE*) src; + size_t iSize; + size_t oSize; + + if (!srcSize) return ERROR(srcSize_wrong); + iSize = ip[0]; + /* ZSTD_memset(huffWeight, 0, hwSize); *//* is not necessary, even though some analyzer complain ... */ + + if (iSize >= 128) { /* special header */ + oSize = iSize - 127; + iSize = ((oSize+1)/2); + if (iSize+1 > srcSize) return ERROR(srcSize_wrong); + if (oSize >= hwSize) return ERROR(corruption_detected); + ip += 1; + { U32 n; + for (n=0; n> 4; + huffWeight[n+1] = ip[n/2] & 15; + } } } + else { /* header compressed with FSE (normal case) */ + if (iSize+1 > srcSize) return ERROR(srcSize_wrong); + /* max (hwSize-1) values decoded, as last one is implied */ + oSize = FSE_decompress_wksp_bmi2(huffWeight, hwSize-1, ip+1, iSize, 6, workSpace, wkspSize, bmi2); + if (FSE_isError(oSize)) return oSize; + } + + /* collect weight stats */ + ZSTD_memset(rankStats, 0, (HUF_TABLELOG_MAX + 1) * sizeof(U32)); + weightTotal = 0; + { U32 n; for (n=0; n HUF_TABLELOG_MAX) return ERROR(corruption_detected); + rankStats[huffWeight[n]]++; + weightTotal += (1 << huffWeight[n]) >> 1; + } } + if (weightTotal == 0) return ERROR(corruption_detected); + + /* get last non-null symbol weight (implied, total must be 2^n) */ + { U32 const tableLog = BIT_highbit32(weightTotal) + 1; + if (tableLog > HUF_TABLELOG_MAX) return ERROR(corruption_detected); + *tableLogPtr = tableLog; + /* determine last weight */ + { U32 const total = 1 << tableLog; + U32 const rest = total - weightTotal; + U32 const verif = 1 << BIT_highbit32(rest); + U32 const lastWeight = BIT_highbit32(rest) + 1; + if (verif != rest) return ERROR(corruption_detected); /* last value must be a clean power of 2 */ + huffWeight[oSize] = (BYTE)lastWeight; + rankStats[lastWeight]++; + } } + + /* check tree construction validity */ + if ((rankStats[1] < 2) || (rankStats[1] & 1)) return ERROR(corruption_detected); /* by construction : at least 2 elts of rank 1, must be even */ + + /* results */ + *nbSymbolsPtr = (U32)(oSize+1); + return iSize+1; +} + +/* Avoids the FORCE_INLINE of the _body() function. */ +static size_t HUF_readStats_body_default(BYTE* huffWeight, size_t hwSize, U32* rankStats, + U32* nbSymbolsPtr, U32* tableLogPtr, + const void* src, size_t srcSize, + void* workSpace, size_t wkspSize) +{ + return HUF_readStats_body(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, workSpace, wkspSize, 0); +} + +#if DYNAMIC_BMI2 +static BMI2_TARGET_ATTRIBUTE size_t HUF_readStats_body_bmi2(BYTE* huffWeight, size_t hwSize, U32* rankStats, + U32* nbSymbolsPtr, U32* tableLogPtr, + const void* src, size_t srcSize, + void* workSpace, size_t wkspSize) +{ + return HUF_readStats_body(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, workSpace, wkspSize, 1); +} +#endif + +size_t HUF_readStats_wksp(BYTE* huffWeight, size_t hwSize, U32* rankStats, + U32* nbSymbolsPtr, U32* tableLogPtr, + const void* src, size_t srcSize, + void* workSpace, size_t wkspSize, + int bmi2) +{ +#if DYNAMIC_BMI2 + if (bmi2) { + return HUF_readStats_body_bmi2(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, workSpace, wkspSize); + } +#endif + (void)bmi2; + return HUF_readStats_body_default(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, workSpace, wkspSize); +} diff --git a/src/blosc2/internal-complibs/zstd-1.5.2/common/error_private.c b/src/blosc2/internal-complibs/zstd-1.5.2/common/error_private.c new file mode 100644 index 0000000..6d1135f --- /dev/null +++ b/src/blosc2/internal-complibs/zstd-1.5.2/common/error_private.c @@ -0,0 +1,56 @@ +/* + * Copyright (c) Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/* The purpose of this file is to have a single list of error strings embedded in binary */ + +#include "error_private.h" + +const char* ERR_getErrorString(ERR_enum code) +{ +#ifdef ZSTD_STRIP_ERROR_STRINGS + (void)code; + return "Error strings stripped"; +#else + static const char* const notErrorCode = "Unspecified error code"; + switch( code ) + { + case PREFIX(no_error): return "No error detected"; + case PREFIX(GENERIC): return "Error (generic)"; + case PREFIX(prefix_unknown): return "Unknown frame descriptor"; + case PREFIX(version_unsupported): return "Version not supported"; + case PREFIX(frameParameter_unsupported): return "Unsupported frame parameter"; + case PREFIX(frameParameter_windowTooLarge): return "Frame requires too much memory for decoding"; + case PREFIX(corruption_detected): return "Corrupted block detected"; + case PREFIX(checksum_wrong): return "Restored data doesn't match checksum"; + case PREFIX(parameter_unsupported): return "Unsupported parameter"; + case PREFIX(parameter_outOfBound): return "Parameter is out of bound"; + case PREFIX(init_missing): return "Context should be init first"; + case PREFIX(memory_allocation): return "Allocation error : not enough memory"; + case PREFIX(workSpace_tooSmall): return "workSpace buffer is not large enough"; + case PREFIX(stage_wrong): return "Operation not authorized at current processing stage"; + case PREFIX(tableLog_tooLarge): return "tableLog requires too much memory : unsupported"; + case PREFIX(maxSymbolValue_tooLarge): return "Unsupported max Symbol Value : too large"; + case PREFIX(maxSymbolValue_tooSmall): return "Specified maxSymbolValue is too small"; + case PREFIX(dictionary_corrupted): return "Dictionary is corrupted"; + case PREFIX(dictionary_wrong): return "Dictionary mismatch"; + case PREFIX(dictionaryCreation_failed): return "Cannot create Dictionary from provided samples"; + case PREFIX(dstSize_tooSmall): return "Destination buffer is too small"; + case PREFIX(srcSize_wrong): return "Src size is incorrect"; + case PREFIX(dstBuffer_null): return "Operation on NULL destination buffer"; + /* following error codes are not stable and may be removed or changed in a future version */ + case PREFIX(frameIndex_tooLarge): return "Frame index is too large"; + case PREFIX(seekableIO): return "An I/O error occurred when reading/seeking"; + case PREFIX(dstBuffer_wrong): return "Destination buffer is wrong"; + case PREFIX(srcBuffer_wrong): return "Source buffer is wrong"; + case PREFIX(maxCode): + default: return notErrorCode; + } +#endif +} diff --git a/src/blosc2/internal-complibs/zstd-1.5.2/common/error_private.h b/src/blosc2/internal-complibs/zstd-1.5.2/common/error_private.h new file mode 100644 index 0000000..007d810 --- /dev/null +++ b/src/blosc2/internal-complibs/zstd-1.5.2/common/error_private.h @@ -0,0 +1,159 @@ +/* + * Copyright (c) Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/* Note : this module is expected to remain private, do not expose it */ + +#ifndef ERROR_H_MODULE +#define ERROR_H_MODULE + +#if defined (__cplusplus) +extern "C" { +#endif + + +/* **************************************** +* Dependencies +******************************************/ +#include "../zstd_errors.h" /* enum list */ +#include "compiler.h" +#include "debug.h" +#include "zstd_deps.h" /* size_t */ + + +/* **************************************** +* Compiler-specific +******************************************/ +#if defined(__GNUC__) +# define ERR_STATIC static __attribute__((unused)) +#elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) +# define ERR_STATIC static inline +#elif defined(_MSC_VER) +# define ERR_STATIC static __inline +#else +# define ERR_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */ +#endif + + +/*-**************************************** +* Customization (error_public.h) +******************************************/ +typedef ZSTD_ErrorCode ERR_enum; +#define PREFIX(name) ZSTD_error_##name + + +/*-**************************************** +* Error codes handling +******************************************/ +#undef ERROR /* already defined on Visual Studio */ +#define ERROR(name) ZSTD_ERROR(name) +#define ZSTD_ERROR(name) ((size_t)-PREFIX(name)) + +ERR_STATIC unsigned ERR_isError(size_t code) { return (code > ERROR(maxCode)); } + +ERR_STATIC ERR_enum ERR_getErrorCode(size_t code) { if (!ERR_isError(code)) return (ERR_enum)0; return (ERR_enum) (0-code); } + +/* check and forward error code */ +#define CHECK_V_F(e, f) size_t const e = f; if (ERR_isError(e)) return e +#define CHECK_F(f) { CHECK_V_F(_var_err__, f); } + + +/*-**************************************** +* Error Strings +******************************************/ + +const char* ERR_getErrorString(ERR_enum code); /* error_private.c */ + +ERR_STATIC const char* ERR_getErrorName(size_t code) +{ + return ERR_getErrorString(ERR_getErrorCode(code)); +} + +/** + * Ignore: this is an internal helper. + * + * This is a helper function to help force C99-correctness during compilation. + * Under strict compilation modes, variadic macro arguments can't be empty. + * However, variadic function arguments can be. Using a function therefore lets + * us statically check that at least one (string) argument was passed, + * independent of the compilation flags. + */ +static INLINE_KEYWORD UNUSED_ATTR +void _force_has_format_string(const char *format, ...) { + (void)format; +} + +/** + * Ignore: this is an internal helper. + * + * We want to force this function invocation to be syntactically correct, but + * we don't want to force runtime evaluation of its arguments. + */ +#define _FORCE_HAS_FORMAT_STRING(...) \ + if (0) { \ + _force_has_format_string(__VA_ARGS__); \ + } + +#define ERR_QUOTE(str) #str + +/** + * Return the specified error if the condition evaluates to true. + * + * In debug modes, prints additional information. + * In order to do that (particularly, printing the conditional that failed), + * this can't just wrap RETURN_ERROR(). + */ +#define RETURN_ERROR_IF(cond, err, ...) \ + if (cond) { \ + RAWLOG(3, "%s:%d: ERROR!: check %s failed, returning %s", \ + __FILE__, __LINE__, ERR_QUOTE(cond), ERR_QUOTE(ERROR(err))); \ + _FORCE_HAS_FORMAT_STRING(__VA_ARGS__); \ + RAWLOG(3, ": " __VA_ARGS__); \ + RAWLOG(3, "\n"); \ + return ERROR(err); \ + } + +/** + * Unconditionally return the specified error. + * + * In debug modes, prints additional information. + */ +#define RETURN_ERROR(err, ...) \ + do { \ + RAWLOG(3, "%s:%d: ERROR!: unconditional check failed, returning %s", \ + __FILE__, __LINE__, ERR_QUOTE(ERROR(err))); \ + _FORCE_HAS_FORMAT_STRING(__VA_ARGS__); \ + RAWLOG(3, ": " __VA_ARGS__); \ + RAWLOG(3, "\n"); \ + return ERROR(err); \ + } while(0); + +/** + * If the provided expression evaluates to an error code, returns that error code. + * + * In debug modes, prints additional information. + */ +#define FORWARD_IF_ERROR(err, ...) \ + do { \ + size_t const err_code = (err); \ + if (ERR_isError(err_code)) { \ + RAWLOG(3, "%s:%d: ERROR!: forwarding error in %s: %s", \ + __FILE__, __LINE__, ERR_QUOTE(err), ERR_getErrorName(err_code)); \ + _FORCE_HAS_FORMAT_STRING(__VA_ARGS__); \ + RAWLOG(3, ": " __VA_ARGS__); \ + RAWLOG(3, "\n"); \ + return err_code; \ + } \ + } while(0); + +#if defined (__cplusplus) +} +#endif + +#endif /* ERROR_H_MODULE */ diff --git a/src/blosc2/internal-complibs/zstd-1.5.2/common/fse.h b/src/blosc2/internal-complibs/zstd-1.5.2/common/fse.h new file mode 100644 index 0000000..714bfd3 --- /dev/null +++ b/src/blosc2/internal-complibs/zstd-1.5.2/common/fse.h @@ -0,0 +1,717 @@ +/* ****************************************************************** + * FSE : Finite State Entropy codec + * Public Prototypes declaration + * Copyright (c) Yann Collet, Facebook, Inc. + * + * You can contact the author at : + * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. +****************************************************************** */ + +#if defined (__cplusplus) +extern "C" { +#endif + +#ifndef FSE_H +#define FSE_H + + +/*-***************************************** +* Dependencies +******************************************/ +#include "zstd_deps.h" /* size_t, ptrdiff_t */ + + +/*-***************************************** +* FSE_PUBLIC_API : control library symbols visibility +******************************************/ +#if defined(FSE_DLL_EXPORT) && (FSE_DLL_EXPORT==1) && defined(__GNUC__) && (__GNUC__ >= 4) +# define FSE_PUBLIC_API __attribute__ ((visibility ("default"))) +#elif defined(FSE_DLL_EXPORT) && (FSE_DLL_EXPORT==1) /* Visual expected */ +# define FSE_PUBLIC_API __declspec(dllexport) +#elif defined(FSE_DLL_IMPORT) && (FSE_DLL_IMPORT==1) +# define FSE_PUBLIC_API __declspec(dllimport) /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/ +#else +# define FSE_PUBLIC_API +#endif + +/*------ Version ------*/ +#define FSE_VERSION_MAJOR 0 +#define FSE_VERSION_MINOR 9 +#define FSE_VERSION_RELEASE 0 + +#define FSE_LIB_VERSION FSE_VERSION_MAJOR.FSE_VERSION_MINOR.FSE_VERSION_RELEASE +#define FSE_QUOTE(str) #str +#define FSE_EXPAND_AND_QUOTE(str) FSE_QUOTE(str) +#define FSE_VERSION_STRING FSE_EXPAND_AND_QUOTE(FSE_LIB_VERSION) + +#define FSE_VERSION_NUMBER (FSE_VERSION_MAJOR *100*100 + FSE_VERSION_MINOR *100 + FSE_VERSION_RELEASE) +FSE_PUBLIC_API unsigned FSE_versionNumber(void); /**< library version number; to be used when checking dll version */ + + +/*-**************************************** +* FSE simple functions +******************************************/ +/*! FSE_compress() : + Compress content of buffer 'src', of size 'srcSize', into destination buffer 'dst'. + 'dst' buffer must be already allocated. Compression runs faster is dstCapacity >= FSE_compressBound(srcSize). + @return : size of compressed data (<= dstCapacity). + Special values : if return == 0, srcData is not compressible => Nothing is stored within dst !!! + if return == 1, srcData is a single byte symbol * srcSize times. Use RLE compression instead. + if FSE_isError(return), compression failed (more details using FSE_getErrorName()) +*/ +FSE_PUBLIC_API size_t FSE_compress(void* dst, size_t dstCapacity, + const void* src, size_t srcSize); + +/*! FSE_decompress(): + Decompress FSE data from buffer 'cSrc', of size 'cSrcSize', + into already allocated destination buffer 'dst', of size 'dstCapacity'. + @return : size of regenerated data (<= maxDstSize), + or an error code, which can be tested using FSE_isError() . + + ** Important ** : FSE_decompress() does not decompress non-compressible nor RLE data !!! + Why ? : making this distinction requires a header. + Header management is intentionally delegated to the user layer, which can better manage special cases. +*/ +FSE_PUBLIC_API size_t FSE_decompress(void* dst, size_t dstCapacity, + const void* cSrc, size_t cSrcSize); + + +/*-***************************************** +* Tool functions +******************************************/ +FSE_PUBLIC_API size_t FSE_compressBound(size_t size); /* maximum compressed size */ + +/* Error Management */ +FSE_PUBLIC_API unsigned FSE_isError(size_t code); /* tells if a return value is an error code */ +FSE_PUBLIC_API const char* FSE_getErrorName(size_t code); /* provides error code string (useful for debugging) */ + + +/*-***************************************** +* FSE advanced functions +******************************************/ +/*! FSE_compress2() : + Same as FSE_compress(), but allows the selection of 'maxSymbolValue' and 'tableLog' + Both parameters can be defined as '0' to mean : use default value + @return : size of compressed data + Special values : if return == 0, srcData is not compressible => Nothing is stored within cSrc !!! + if return == 1, srcData is a single byte symbol * srcSize times. Use RLE compression. + if FSE_isError(return), it's an error code. +*/ +FSE_PUBLIC_API size_t FSE_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog); + + +/*-***************************************** +* FSE detailed API +******************************************/ +/*! +FSE_compress() does the following: +1. count symbol occurrence from source[] into table count[] (see hist.h) +2. normalize counters so that sum(count[]) == Power_of_2 (2^tableLog) +3. save normalized counters to memory buffer using writeNCount() +4. build encoding table 'CTable' from normalized counters +5. encode the data stream using encoding table 'CTable' + +FSE_decompress() does the following: +1. read normalized counters with readNCount() +2. build decoding table 'DTable' from normalized counters +3. decode the data stream using decoding table 'DTable' + +The following API allows targeting specific sub-functions for advanced tasks. +For example, it's possible to compress several blocks using the same 'CTable', +or to save and provide normalized distribution using external method. +*/ + +/* *** COMPRESSION *** */ + +/*! FSE_optimalTableLog(): + dynamically downsize 'tableLog' when conditions are met. + It saves CPU time, by using smaller tables, while preserving or even improving compression ratio. + @return : recommended tableLog (necessarily <= 'maxTableLog') */ +FSE_PUBLIC_API unsigned FSE_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue); + +/*! FSE_normalizeCount(): + normalize counts so that sum(count[]) == Power_of_2 (2^tableLog) + 'normalizedCounter' is a table of short, of minimum size (maxSymbolValue+1). + useLowProbCount is a boolean parameter which trades off compressed size for + faster header decoding. When it is set to 1, the compressed data will be slightly + smaller. And when it is set to 0, FSE_readNCount() and FSE_buildDTable() will be + faster. If you are compressing a small amount of data (< 2 KB) then useLowProbCount=0 + is a good default, since header deserialization makes a big speed difference. + Otherwise, useLowProbCount=1 is a good default, since the speed difference is small. + @return : tableLog, + or an errorCode, which can be tested using FSE_isError() */ +FSE_PUBLIC_API size_t FSE_normalizeCount(short* normalizedCounter, unsigned tableLog, + const unsigned* count, size_t srcSize, unsigned maxSymbolValue, unsigned useLowProbCount); + +/*! FSE_NCountWriteBound(): + Provides the maximum possible size of an FSE normalized table, given 'maxSymbolValue' and 'tableLog'. + Typically useful for allocation purpose. */ +FSE_PUBLIC_API size_t FSE_NCountWriteBound(unsigned maxSymbolValue, unsigned tableLog); + +/*! FSE_writeNCount(): + Compactly save 'normalizedCounter' into 'buffer'. + @return : size of the compressed table, + or an errorCode, which can be tested using FSE_isError(). */ +FSE_PUBLIC_API size_t FSE_writeNCount (void* buffer, size_t bufferSize, + const short* normalizedCounter, + unsigned maxSymbolValue, unsigned tableLog); + +/*! Constructor and Destructor of FSE_CTable. + Note that FSE_CTable size depends on 'tableLog' and 'maxSymbolValue' */ +typedef unsigned FSE_CTable; /* don't allocate that. It's only meant to be more restrictive than void* */ +FSE_PUBLIC_API FSE_CTable* FSE_createCTable (unsigned maxSymbolValue, unsigned tableLog); +FSE_PUBLIC_API void FSE_freeCTable (FSE_CTable* ct); + +/*! FSE_buildCTable(): + Builds `ct`, which must be already allocated, using FSE_createCTable(). + @return : 0, or an errorCode, which can be tested using FSE_isError() */ +FSE_PUBLIC_API size_t FSE_buildCTable(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog); + +/*! FSE_compress_usingCTable(): + Compress `src` using `ct` into `dst` which must be already allocated. + @return : size of compressed data (<= `dstCapacity`), + or 0 if compressed data could not fit into `dst`, + or an errorCode, which can be tested using FSE_isError() */ +FSE_PUBLIC_API size_t FSE_compress_usingCTable (void* dst, size_t dstCapacity, const void* src, size_t srcSize, const FSE_CTable* ct); + +/*! +Tutorial : +---------- +The first step is to count all symbols. FSE_count() does this job very fast. +Result will be saved into 'count', a table of unsigned int, which must be already allocated, and have 'maxSymbolValuePtr[0]+1' cells. +'src' is a table of bytes of size 'srcSize'. All values within 'src' MUST be <= maxSymbolValuePtr[0] +maxSymbolValuePtr[0] will be updated, with its real value (necessarily <= original value) +FSE_count() will return the number of occurrence of the most frequent symbol. +This can be used to know if there is a single symbol within 'src', and to quickly evaluate its compressibility. +If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError()). + +The next step is to normalize the frequencies. +FSE_normalizeCount() will ensure that sum of frequencies is == 2 ^'tableLog'. +It also guarantees a minimum of 1 to any Symbol with frequency >= 1. +You can use 'tableLog'==0 to mean "use default tableLog value". +If you are unsure of which tableLog value to use, you can ask FSE_optimalTableLog(), +which will provide the optimal valid tableLog given sourceSize, maxSymbolValue, and a user-defined maximum (0 means "default"). + +The result of FSE_normalizeCount() will be saved into a table, +called 'normalizedCounter', which is a table of signed short. +'normalizedCounter' must be already allocated, and have at least 'maxSymbolValue+1' cells. +The return value is tableLog if everything proceeded as expected. +It is 0 if there is a single symbol within distribution. +If there is an error (ex: invalid tableLog value), the function will return an ErrorCode (which can be tested using FSE_isError()). + +'normalizedCounter' can be saved in a compact manner to a memory area using FSE_writeNCount(). +'buffer' must be already allocated. +For guaranteed success, buffer size must be at least FSE_headerBound(). +The result of the function is the number of bytes written into 'buffer'. +If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError(); ex : buffer size too small). + +'normalizedCounter' can then be used to create the compression table 'CTable'. +The space required by 'CTable' must be already allocated, using FSE_createCTable(). +You can then use FSE_buildCTable() to fill 'CTable'. +If there is an error, both functions will return an ErrorCode (which can be tested using FSE_isError()). + +'CTable' can then be used to compress 'src', with FSE_compress_usingCTable(). +Similar to FSE_count(), the convention is that 'src' is assumed to be a table of char of size 'srcSize' +The function returns the size of compressed data (without header), necessarily <= `dstCapacity`. +If it returns '0', compressed data could not fit into 'dst'. +If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError()). +*/ + + +/* *** DECOMPRESSION *** */ + +/*! FSE_readNCount(): + Read compactly saved 'normalizedCounter' from 'rBuffer'. + @return : size read from 'rBuffer', + or an errorCode, which can be tested using FSE_isError(). + maxSymbolValuePtr[0] and tableLogPtr[0] will also be updated with their respective values */ +FSE_PUBLIC_API size_t FSE_readNCount (short* normalizedCounter, + unsigned* maxSymbolValuePtr, unsigned* tableLogPtr, + const void* rBuffer, size_t rBuffSize); + +/*! FSE_readNCount_bmi2(): + * Same as FSE_readNCount() but pass bmi2=1 when your CPU supports BMI2 and 0 otherwise. + */ +FSE_PUBLIC_API size_t FSE_readNCount_bmi2(short* normalizedCounter, + unsigned* maxSymbolValuePtr, unsigned* tableLogPtr, + const void* rBuffer, size_t rBuffSize, int bmi2); + +/*! Constructor and Destructor of FSE_DTable. + Note that its size depends on 'tableLog' */ +typedef unsigned FSE_DTable; /* don't allocate that. It's just a way to be more restrictive than void* */ +FSE_PUBLIC_API FSE_DTable* FSE_createDTable(unsigned tableLog); +FSE_PUBLIC_API void FSE_freeDTable(FSE_DTable* dt); + +/*! FSE_buildDTable(): + Builds 'dt', which must be already allocated, using FSE_createDTable(). + return : 0, or an errorCode, which can be tested using FSE_isError() */ +FSE_PUBLIC_API size_t FSE_buildDTable (FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog); + +/*! FSE_decompress_usingDTable(): + Decompress compressed source `cSrc` of size `cSrcSize` using `dt` + into `dst` which must be already allocated. + @return : size of regenerated data (necessarily <= `dstCapacity`), + or an errorCode, which can be tested using FSE_isError() */ +FSE_PUBLIC_API size_t FSE_decompress_usingDTable(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, const FSE_DTable* dt); + +/*! +Tutorial : +---------- +(Note : these functions only decompress FSE-compressed blocks. + If block is uncompressed, use memcpy() instead + If block is a single repeated byte, use memset() instead ) + +The first step is to obtain the normalized frequencies of symbols. +This can be performed by FSE_readNCount() if it was saved using FSE_writeNCount(). +'normalizedCounter' must be already allocated, and have at least 'maxSymbolValuePtr[0]+1' cells of signed short. +In practice, that means it's necessary to know 'maxSymbolValue' beforehand, +or size the table to handle worst case situations (typically 256). +FSE_readNCount() will provide 'tableLog' and 'maxSymbolValue'. +The result of FSE_readNCount() is the number of bytes read from 'rBuffer'. +Note that 'rBufferSize' must be at least 4 bytes, even if useful information is less than that. +If there is an error, the function will return an error code, which can be tested using FSE_isError(). + +The next step is to build the decompression tables 'FSE_DTable' from 'normalizedCounter'. +This is performed by the function FSE_buildDTable(). +The space required by 'FSE_DTable' must be already allocated using FSE_createDTable(). +If there is an error, the function will return an error code, which can be tested using FSE_isError(). + +`FSE_DTable` can then be used to decompress `cSrc`, with FSE_decompress_usingDTable(). +`cSrcSize` must be strictly correct, otherwise decompression will fail. +FSE_decompress_usingDTable() result will tell how many bytes were regenerated (<=`dstCapacity`). +If there is an error, the function will return an error code, which can be tested using FSE_isError(). (ex: dst buffer too small) +*/ + +#endif /* FSE_H */ + +#if defined(FSE_STATIC_LINKING_ONLY) && !defined(FSE_H_FSE_STATIC_LINKING_ONLY) +#define FSE_H_FSE_STATIC_LINKING_ONLY + +/* *** Dependency *** */ +#include "bitstream.h" + + +/* ***************************************** +* Static allocation +*******************************************/ +/* FSE buffer bounds */ +#define FSE_NCOUNTBOUND 512 +#define FSE_BLOCKBOUND(size) ((size) + ((size)>>7) + 4 /* fse states */ + sizeof(size_t) /* bitContainer */) +#define FSE_COMPRESSBOUND(size) (FSE_NCOUNTBOUND + FSE_BLOCKBOUND(size)) /* Macro version, useful for static allocation */ + +/* It is possible to statically allocate FSE CTable/DTable as a table of FSE_CTable/FSE_DTable using below macros */ +#define FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) (1 + (1<<((maxTableLog)-1)) + (((maxSymbolValue)+1)*2)) +#define FSE_DTABLE_SIZE_U32(maxTableLog) (1 + (1<<(maxTableLog))) + +/* or use the size to malloc() space directly. Pay attention to alignment restrictions though */ +#define FSE_CTABLE_SIZE(maxTableLog, maxSymbolValue) (FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) * sizeof(FSE_CTable)) +#define FSE_DTABLE_SIZE(maxTableLog) (FSE_DTABLE_SIZE_U32(maxTableLog) * sizeof(FSE_DTable)) + + +/* ***************************************** + * FSE advanced API + ***************************************** */ + +unsigned FSE_optimalTableLog_internal(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, unsigned minus); +/**< same as FSE_optimalTableLog(), which used `minus==2` */ + +/* FSE_compress_wksp() : + * Same as FSE_compress2(), but using an externally allocated scratch buffer (`workSpace`). + * FSE_COMPRESS_WKSP_SIZE_U32() provides the minimum size required for `workSpace` as a table of FSE_CTable. + */ +#define FSE_COMPRESS_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) ( FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) + ((maxTableLog > 12) ? (1 << (maxTableLog - 2)) : 1024) ) +size_t FSE_compress_wksp (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize); + +size_t FSE_buildCTable_raw (FSE_CTable* ct, unsigned nbBits); +/**< build a fake FSE_CTable, designed for a flat distribution, where each symbol uses nbBits */ + +size_t FSE_buildCTable_rle (FSE_CTable* ct, unsigned char symbolValue); +/**< build a fake FSE_CTable, designed to compress always the same symbolValue */ + +/* FSE_buildCTable_wksp() : + * Same as FSE_buildCTable(), but using an externally allocated scratch buffer (`workSpace`). + * `wkspSize` must be >= `FSE_BUILD_CTABLE_WORKSPACE_SIZE_U32(maxSymbolValue, tableLog)` of `unsigned`. + * See FSE_buildCTable_wksp() for breakdown of workspace usage. + */ +#define FSE_BUILD_CTABLE_WORKSPACE_SIZE_U32(maxSymbolValue, tableLog) (((maxSymbolValue + 2) + (1ull << (tableLog)))/2 + sizeof(U64)/sizeof(U32) /* additional 8 bytes for potential table overwrite */) +#define FSE_BUILD_CTABLE_WORKSPACE_SIZE(maxSymbolValue, tableLog) (sizeof(unsigned) * FSE_BUILD_CTABLE_WORKSPACE_SIZE_U32(maxSymbolValue, tableLog)) +size_t FSE_buildCTable_wksp(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize); + +#define FSE_BUILD_DTABLE_WKSP_SIZE(maxTableLog, maxSymbolValue) (sizeof(short) * (maxSymbolValue + 1) + (1ULL << maxTableLog) + 8) +#define FSE_BUILD_DTABLE_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) ((FSE_BUILD_DTABLE_WKSP_SIZE(maxTableLog, maxSymbolValue) + sizeof(unsigned) - 1) / sizeof(unsigned)) +FSE_PUBLIC_API size_t FSE_buildDTable_wksp(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize); +/**< Same as FSE_buildDTable(), using an externally allocated `workspace` produced with `FSE_BUILD_DTABLE_WKSP_SIZE_U32(maxSymbolValue)` */ + +size_t FSE_buildDTable_raw (FSE_DTable* dt, unsigned nbBits); +/**< build a fake FSE_DTable, designed to read a flat distribution where each symbol uses nbBits */ + +size_t FSE_buildDTable_rle (FSE_DTable* dt, unsigned char symbolValue); +/**< build a fake FSE_DTable, designed to always generate the same symbolValue */ + +#define FSE_DECOMPRESS_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) (FSE_DTABLE_SIZE_U32(maxTableLog) + FSE_BUILD_DTABLE_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) + (FSE_MAX_SYMBOL_VALUE + 1) / 2 + 1) +#define FSE_DECOMPRESS_WKSP_SIZE(maxTableLog, maxSymbolValue) (FSE_DECOMPRESS_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) * sizeof(unsigned)) +size_t FSE_decompress_wksp(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize); +/**< same as FSE_decompress(), using an externally allocated `workSpace` produced with `FSE_DECOMPRESS_WKSP_SIZE_U32(maxLog, maxSymbolValue)` */ + +size_t FSE_decompress_wksp_bmi2(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize, int bmi2); +/**< Same as FSE_decompress_wksp() but with dynamic BMI2 support. Pass 1 if your CPU supports BMI2 or 0 if it doesn't. */ + +typedef enum { + FSE_repeat_none, /**< Cannot use the previous table */ + FSE_repeat_check, /**< Can use the previous table but it must be checked */ + FSE_repeat_valid /**< Can use the previous table and it is assumed to be valid */ + } FSE_repeat; + +/* ***************************************** +* FSE symbol compression API +*******************************************/ +/*! + This API consists of small unitary functions, which highly benefit from being inlined. + Hence their body are included in next section. +*/ +typedef struct { + ptrdiff_t value; + const void* stateTable; + const void* symbolTT; + unsigned stateLog; +} FSE_CState_t; + +static void FSE_initCState(FSE_CState_t* CStatePtr, const FSE_CTable* ct); + +static void FSE_encodeSymbol(BIT_CStream_t* bitC, FSE_CState_t* CStatePtr, unsigned symbol); + +static void FSE_flushCState(BIT_CStream_t* bitC, const FSE_CState_t* CStatePtr); + +/**< +These functions are inner components of FSE_compress_usingCTable(). +They allow the creation of custom streams, mixing multiple tables and bit sources. + +A key property to keep in mind is that encoding and decoding are done **in reverse direction**. +So the first symbol you will encode is the last you will decode, like a LIFO stack. + +You will need a few variables to track your CStream. They are : + +FSE_CTable ct; // Provided by FSE_buildCTable() +BIT_CStream_t bitStream; // bitStream tracking structure +FSE_CState_t state; // State tracking structure (can have several) + + +The first thing to do is to init bitStream and state. + size_t errorCode = BIT_initCStream(&bitStream, dstBuffer, maxDstSize); + FSE_initCState(&state, ct); + +Note that BIT_initCStream() can produce an error code, so its result should be tested, using FSE_isError(); +You can then encode your input data, byte after byte. +FSE_encodeSymbol() outputs a maximum of 'tableLog' bits at a time. +Remember decoding will be done in reverse direction. + FSE_encodeByte(&bitStream, &state, symbol); + +At any time, you can also add any bit sequence. +Note : maximum allowed nbBits is 25, for compatibility with 32-bits decoders + BIT_addBits(&bitStream, bitField, nbBits); + +The above methods don't commit data to memory, they just store it into local register, for speed. +Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t). +Writing data to memory is a manual operation, performed by the flushBits function. + BIT_flushBits(&bitStream); + +Your last FSE encoding operation shall be to flush your last state value(s). + FSE_flushState(&bitStream, &state); + +Finally, you must close the bitStream. +The function returns the size of CStream in bytes. +If data couldn't fit into dstBuffer, it will return a 0 ( == not compressible) +If there is an error, it returns an errorCode (which can be tested using FSE_isError()). + size_t size = BIT_closeCStream(&bitStream); +*/ + + +/* ***************************************** +* FSE symbol decompression API +*******************************************/ +typedef struct { + size_t state; + const void* table; /* precise table may vary, depending on U16 */ +} FSE_DState_t; + + +static void FSE_initDState(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD, const FSE_DTable* dt); + +static unsigned char FSE_decodeSymbol(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD); + +static unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr); + +/**< +Let's now decompose FSE_decompress_usingDTable() into its unitary components. +You will decode FSE-encoded symbols from the bitStream, +and also any other bitFields you put in, **in reverse order**. + +You will need a few variables to track your bitStream. They are : + +BIT_DStream_t DStream; // Stream context +FSE_DState_t DState; // State context. Multiple ones are possible +FSE_DTable* DTablePtr; // Decoding table, provided by FSE_buildDTable() + +The first thing to do is to init the bitStream. + errorCode = BIT_initDStream(&DStream, srcBuffer, srcSize); + +You should then retrieve your initial state(s) +(in reverse flushing order if you have several ones) : + errorCode = FSE_initDState(&DState, &DStream, DTablePtr); + +You can then decode your data, symbol after symbol. +For information the maximum number of bits read by FSE_decodeSymbol() is 'tableLog'. +Keep in mind that symbols are decoded in reverse order, like a LIFO stack (last in, first out). + unsigned char symbol = FSE_decodeSymbol(&DState, &DStream); + +You can retrieve any bitfield you eventually stored into the bitStream (in reverse order) +Note : maximum allowed nbBits is 25, for 32-bits compatibility + size_t bitField = BIT_readBits(&DStream, nbBits); + +All above operations only read from local register (which size depends on size_t). +Refueling the register from memory is manually performed by the reload method. + endSignal = FSE_reloadDStream(&DStream); + +BIT_reloadDStream() result tells if there is still some more data to read from DStream. +BIT_DStream_unfinished : there is still some data left into the DStream. +BIT_DStream_endOfBuffer : Dstream reached end of buffer. Its container may no longer be completely filled. +BIT_DStream_completed : Dstream reached its exact end, corresponding in general to decompression completed. +BIT_DStream_tooFar : Dstream went too far. Decompression result is corrupted. + +When reaching end of buffer (BIT_DStream_endOfBuffer), progress slowly, notably if you decode multiple symbols per loop, +to properly detect the exact end of stream. +After each decoded symbol, check if DStream is fully consumed using this simple test : + BIT_reloadDStream(&DStream) >= BIT_DStream_completed + +When it's done, verify decompression is fully completed, by checking both DStream and the relevant states. +Checking if DStream has reached its end is performed by : + BIT_endOfDStream(&DStream); +Check also the states. There might be some symbols left there, if some high probability ones (>50%) are possible. + FSE_endOfDState(&DState); +*/ + + +/* ***************************************** +* FSE unsafe API +*******************************************/ +static unsigned char FSE_decodeSymbolFast(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD); +/* faster, but works only if nbBits is always >= 1 (otherwise, result will be corrupted) */ + + +/* ***************************************** +* Implementation of inlined functions +*******************************************/ +typedef struct { + int deltaFindState; + U32 deltaNbBits; +} FSE_symbolCompressionTransform; /* total 8 bytes */ + +MEM_STATIC void FSE_initCState(FSE_CState_t* statePtr, const FSE_CTable* ct) +{ + const void* ptr = ct; + const U16* u16ptr = (const U16*) ptr; + const U32 tableLog = MEM_read16(ptr); + statePtr->value = (ptrdiff_t)1<stateTable = u16ptr+2; + statePtr->symbolTT = ct + 1 + (tableLog ? (1<<(tableLog-1)) : 1); + statePtr->stateLog = tableLog; +} + + +/*! FSE_initCState2() : +* Same as FSE_initCState(), but the first symbol to include (which will be the last to be read) +* uses the smallest state value possible, saving the cost of this symbol */ +MEM_STATIC void FSE_initCState2(FSE_CState_t* statePtr, const FSE_CTable* ct, U32 symbol) +{ + FSE_initCState(statePtr, ct); + { const FSE_symbolCompressionTransform symbolTT = ((const FSE_symbolCompressionTransform*)(statePtr->symbolTT))[symbol]; + const U16* stateTable = (const U16*)(statePtr->stateTable); + U32 nbBitsOut = (U32)((symbolTT.deltaNbBits + (1<<15)) >> 16); + statePtr->value = (nbBitsOut << 16) - symbolTT.deltaNbBits; + statePtr->value = stateTable[(statePtr->value >> nbBitsOut) + symbolTT.deltaFindState]; + } +} + +MEM_STATIC void FSE_encodeSymbol(BIT_CStream_t* bitC, FSE_CState_t* statePtr, unsigned symbol) +{ + FSE_symbolCompressionTransform const symbolTT = ((const FSE_symbolCompressionTransform*)(statePtr->symbolTT))[symbol]; + const U16* const stateTable = (const U16*)(statePtr->stateTable); + U32 const nbBitsOut = (U32)((statePtr->value + symbolTT.deltaNbBits) >> 16); + BIT_addBits(bitC, statePtr->value, nbBitsOut); + statePtr->value = stateTable[ (statePtr->value >> nbBitsOut) + symbolTT.deltaFindState]; +} + +MEM_STATIC void FSE_flushCState(BIT_CStream_t* bitC, const FSE_CState_t* statePtr) +{ + BIT_addBits(bitC, statePtr->value, statePtr->stateLog); + BIT_flushBits(bitC); +} + + +/* FSE_getMaxNbBits() : + * Approximate maximum cost of a symbol, in bits. + * Fractional get rounded up (i.e : a symbol with a normalized frequency of 3 gives the same result as a frequency of 2) + * note 1 : assume symbolValue is valid (<= maxSymbolValue) + * note 2 : if freq[symbolValue]==0, @return a fake cost of tableLog+1 bits */ +MEM_STATIC U32 FSE_getMaxNbBits(const void* symbolTTPtr, U32 symbolValue) +{ + const FSE_symbolCompressionTransform* symbolTT = (const FSE_symbolCompressionTransform*) symbolTTPtr; + return (symbolTT[symbolValue].deltaNbBits + ((1<<16)-1)) >> 16; +} + +/* FSE_bitCost() : + * Approximate symbol cost, as fractional value, using fixed-point format (accuracyLog fractional bits) + * note 1 : assume symbolValue is valid (<= maxSymbolValue) + * note 2 : if freq[symbolValue]==0, @return a fake cost of tableLog+1 bits */ +MEM_STATIC U32 FSE_bitCost(const void* symbolTTPtr, U32 tableLog, U32 symbolValue, U32 accuracyLog) +{ + const FSE_symbolCompressionTransform* symbolTT = (const FSE_symbolCompressionTransform*) symbolTTPtr; + U32 const minNbBits = symbolTT[symbolValue].deltaNbBits >> 16; + U32 const threshold = (minNbBits+1) << 16; + assert(tableLog < 16); + assert(accuracyLog < 31-tableLog); /* ensure enough room for renormalization double shift */ + { U32 const tableSize = 1 << tableLog; + U32 const deltaFromThreshold = threshold - (symbolTT[symbolValue].deltaNbBits + tableSize); + U32 const normalizedDeltaFromThreshold = (deltaFromThreshold << accuracyLog) >> tableLog; /* linear interpolation (very approximate) */ + U32 const bitMultiplier = 1 << accuracyLog; + assert(symbolTT[symbolValue].deltaNbBits + tableSize <= threshold); + assert(normalizedDeltaFromThreshold <= bitMultiplier); + return (minNbBits+1)*bitMultiplier - normalizedDeltaFromThreshold; + } +} + + +/* ====== Decompression ====== */ + +typedef struct { + U16 tableLog; + U16 fastMode; +} FSE_DTableHeader; /* sizeof U32 */ + +typedef struct +{ + unsigned short newState; + unsigned char symbol; + unsigned char nbBits; +} FSE_decode_t; /* size == U32 */ + +MEM_STATIC void FSE_initDState(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD, const FSE_DTable* dt) +{ + const void* ptr = dt; + const FSE_DTableHeader* const DTableH = (const FSE_DTableHeader*)ptr; + DStatePtr->state = BIT_readBits(bitD, DTableH->tableLog); + BIT_reloadDStream(bitD); + DStatePtr->table = dt + 1; +} + +MEM_STATIC BYTE FSE_peekSymbol(const FSE_DState_t* DStatePtr) +{ + FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state]; + return DInfo.symbol; +} + +MEM_STATIC void FSE_updateState(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD) +{ + FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state]; + U32 const nbBits = DInfo.nbBits; + size_t const lowBits = BIT_readBits(bitD, nbBits); + DStatePtr->state = DInfo.newState + lowBits; +} + +MEM_STATIC BYTE FSE_decodeSymbol(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD) +{ + FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state]; + U32 const nbBits = DInfo.nbBits; + BYTE const symbol = DInfo.symbol; + size_t const lowBits = BIT_readBits(bitD, nbBits); + + DStatePtr->state = DInfo.newState + lowBits; + return symbol; +} + +/*! FSE_decodeSymbolFast() : + unsafe, only works if no symbol has a probability > 50% */ +MEM_STATIC BYTE FSE_decodeSymbolFast(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD) +{ + FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state]; + U32 const nbBits = DInfo.nbBits; + BYTE const symbol = DInfo.symbol; + size_t const lowBits = BIT_readBitsFast(bitD, nbBits); + + DStatePtr->state = DInfo.newState + lowBits; + return symbol; +} + +MEM_STATIC unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr) +{ + return DStatePtr->state == 0; +} + + + +#ifndef FSE_COMMONDEFS_ONLY + +/* ************************************************************** +* Tuning parameters +****************************************************************/ +/*!MEMORY_USAGE : +* Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.) +* Increasing memory usage improves compression ratio +* Reduced memory usage can improve speed, due to cache effect +* Recommended max value is 14, for 16KB, which nicely fits into Intel x86 L1 cache */ +#ifndef FSE_MAX_MEMORY_USAGE +# define FSE_MAX_MEMORY_USAGE 14 +#endif +#ifndef FSE_DEFAULT_MEMORY_USAGE +# define FSE_DEFAULT_MEMORY_USAGE 13 +#endif +#if (FSE_DEFAULT_MEMORY_USAGE > FSE_MAX_MEMORY_USAGE) +# error "FSE_DEFAULT_MEMORY_USAGE must be <= FSE_MAX_MEMORY_USAGE" +#endif + +/*!FSE_MAX_SYMBOL_VALUE : +* Maximum symbol value authorized. +* Required for proper stack allocation */ +#ifndef FSE_MAX_SYMBOL_VALUE +# define FSE_MAX_SYMBOL_VALUE 255 +#endif + +/* ************************************************************** +* template functions type & suffix +****************************************************************/ +#define FSE_FUNCTION_TYPE BYTE +#define FSE_FUNCTION_EXTENSION +#define FSE_DECODE_TYPE FSE_decode_t + + +#endif /* !FSE_COMMONDEFS_ONLY */ + + +/* *************************************************************** +* Constants +*****************************************************************/ +#define FSE_MAX_TABLELOG (FSE_MAX_MEMORY_USAGE-2) +#define FSE_MAX_TABLESIZE (1U< FSE_TABLELOG_ABSOLUTE_MAX +# error "FSE_MAX_TABLELOG > FSE_TABLELOG_ABSOLUTE_MAX is not supported" +#endif + +#define FSE_TABLESTEP(tableSize) (((tableSize)>>1) + ((tableSize)>>3) + 3) + + +#endif /* FSE_STATIC_LINKING_ONLY */ + + +#if defined (__cplusplus) +} +#endif diff --git a/src/blosc2/internal-complibs/zstd-1.5.2/common/fse_decompress.c b/src/blosc2/internal-complibs/zstd-1.5.2/common/fse_decompress.c new file mode 100644 index 0000000..a5a3580 --- /dev/null +++ b/src/blosc2/internal-complibs/zstd-1.5.2/common/fse_decompress.c @@ -0,0 +1,403 @@ +/* ****************************************************************** + * FSE : Finite State Entropy decoder + * Copyright (c) Yann Collet, Facebook, Inc. + * + * You can contact the author at : + * - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy + * - Public forum : https://groups.google.com/forum/#!forum/lz4c + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. +****************************************************************** */ + + +/* ************************************************************** +* Includes +****************************************************************/ +#include "debug.h" /* assert */ +#include "bitstream.h" +#include "compiler.h" +#define FSE_STATIC_LINKING_ONLY +#include "fse.h" +#include "error_private.h" +#define ZSTD_DEPS_NEED_MALLOC +#include "zstd_deps.h" + + +/* ************************************************************** +* Error Management +****************************************************************/ +#define FSE_isError ERR_isError +#define FSE_STATIC_ASSERT(c) DEBUG_STATIC_ASSERT(c) /* use only *after* variable declarations */ + + +/* ************************************************************** +* Templates +****************************************************************/ +/* + designed to be included + for type-specific functions (template emulation in C) + Objective is to write these functions only once, for improved maintenance +*/ + +/* safety checks */ +#ifndef FSE_FUNCTION_EXTENSION +# error "FSE_FUNCTION_EXTENSION must be defined" +#endif +#ifndef FSE_FUNCTION_TYPE +# error "FSE_FUNCTION_TYPE must be defined" +#endif + +/* Function names */ +#define FSE_CAT(X,Y) X##Y +#define FSE_FUNCTION_NAME(X,Y) FSE_CAT(X,Y) +#define FSE_TYPE_NAME(X,Y) FSE_CAT(X,Y) + + +/* Function templates */ +FSE_DTable* FSE_createDTable (unsigned tableLog) +{ + if (tableLog > FSE_TABLELOG_ABSOLUTE_MAX) tableLog = FSE_TABLELOG_ABSOLUTE_MAX; + return (FSE_DTable*)ZSTD_malloc( FSE_DTABLE_SIZE_U32(tableLog) * sizeof (U32) ); +} + +void FSE_freeDTable (FSE_DTable* dt) +{ + ZSTD_free(dt); +} + +static size_t FSE_buildDTable_internal(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize) +{ + void* const tdPtr = dt+1; /* because *dt is unsigned, 32-bits aligned on 32-bits */ + FSE_DECODE_TYPE* const tableDecode = (FSE_DECODE_TYPE*) (tdPtr); + U16* symbolNext = (U16*)workSpace; + BYTE* spread = (BYTE*)(symbolNext + maxSymbolValue + 1); + + U32 const maxSV1 = maxSymbolValue + 1; + U32 const tableSize = 1 << tableLog; + U32 highThreshold = tableSize-1; + + /* Sanity Checks */ + if (FSE_BUILD_DTABLE_WKSP_SIZE(tableLog, maxSymbolValue) > wkspSize) return ERROR(maxSymbolValue_tooLarge); + if (maxSymbolValue > FSE_MAX_SYMBOL_VALUE) return ERROR(maxSymbolValue_tooLarge); + if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge); + + /* Init, lay down lowprob symbols */ + { FSE_DTableHeader DTableH; + DTableH.tableLog = (U16)tableLog; + DTableH.fastMode = 1; + { S16 const largeLimit= (S16)(1 << (tableLog-1)); + U32 s; + for (s=0; s= largeLimit) DTableH.fastMode=0; + symbolNext[s] = normalizedCounter[s]; + } } } + ZSTD_memcpy(dt, &DTableH, sizeof(DTableH)); + } + + /* Spread symbols */ + if (highThreshold == tableSize - 1) { + size_t const tableMask = tableSize-1; + size_t const step = FSE_TABLESTEP(tableSize); + /* First lay down the symbols in order. + * We use a uint64_t to lay down 8 bytes at a time. This reduces branch + * misses since small blocks generally have small table logs, so nearly + * all symbols have counts <= 8. We ensure we have 8 bytes at the end of + * our buffer to handle the over-write. + */ + { + U64 const add = 0x0101010101010101ull; + size_t pos = 0; + U64 sv = 0; + U32 s; + for (s=0; s highThreshold) position = (position + step) & tableMask; /* lowprob area */ + } } + if (position!=0) return ERROR(GENERIC); /* position must reach all cells once, otherwise normalizedCounter is incorrect */ + } + + /* Build Decoding table */ + { U32 u; + for (u=0; utableLog = 0; + DTableH->fastMode = 0; + + cell->newState = 0; + cell->symbol = symbolValue; + cell->nbBits = 0; + + return 0; +} + + +size_t FSE_buildDTable_raw (FSE_DTable* dt, unsigned nbBits) +{ + void* ptr = dt; + FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)ptr; + void* dPtr = dt + 1; + FSE_decode_t* const dinfo = (FSE_decode_t*)dPtr; + const unsigned tableSize = 1 << nbBits; + const unsigned tableMask = tableSize - 1; + const unsigned maxSV1 = tableMask+1; + unsigned s; + + /* Sanity checks */ + if (nbBits < 1) return ERROR(GENERIC); /* min size */ + + /* Build Decoding Table */ + DTableH->tableLog = (U16)nbBits; + DTableH->fastMode = 1; + for (s=0; s sizeof(bitD.bitContainer)*8) /* This test must be static */ + BIT_reloadDStream(&bitD); + + op[1] = FSE_GETSYMBOL(&state2); + + if (FSE_MAX_TABLELOG*4+7 > sizeof(bitD.bitContainer)*8) /* This test must be static */ + { if (BIT_reloadDStream(&bitD) > BIT_DStream_unfinished) { op+=2; break; } } + + op[2] = FSE_GETSYMBOL(&state1); + + if (FSE_MAX_TABLELOG*2+7 > sizeof(bitD.bitContainer)*8) /* This test must be static */ + BIT_reloadDStream(&bitD); + + op[3] = FSE_GETSYMBOL(&state2); + } + + /* tail */ + /* note : BIT_reloadDStream(&bitD) >= FSE_DStream_partiallyFilled; Ends at exactly BIT_DStream_completed */ + while (1) { + if (op>(omax-2)) return ERROR(dstSize_tooSmall); + *op++ = FSE_GETSYMBOL(&state1); + if (BIT_reloadDStream(&bitD)==BIT_DStream_overflow) { + *op++ = FSE_GETSYMBOL(&state2); + break; + } + + if (op>(omax-2)) return ERROR(dstSize_tooSmall); + *op++ = FSE_GETSYMBOL(&state2); + if (BIT_reloadDStream(&bitD)==BIT_DStream_overflow) { + *op++ = FSE_GETSYMBOL(&state1); + break; + } } + + return op-ostart; +} + + +size_t FSE_decompress_usingDTable(void* dst, size_t originalSize, + const void* cSrc, size_t cSrcSize, + const FSE_DTable* dt) +{ + const void* ptr = dt; + const FSE_DTableHeader* DTableH = (const FSE_DTableHeader*)ptr; + const U32 fastMode = DTableH->fastMode; + + /* select fast mode (static) */ + if (fastMode) return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 1); + return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 0); +} + + +size_t FSE_decompress_wksp(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize) +{ + return FSE_decompress_wksp_bmi2(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize, /* bmi2 */ 0); +} + +typedef struct { + short ncount[FSE_MAX_SYMBOL_VALUE + 1]; + FSE_DTable dtable[1]; /* Dynamically sized */ +} FSE_DecompressWksp; + + +FORCE_INLINE_TEMPLATE size_t FSE_decompress_wksp_body( + void* dst, size_t dstCapacity, + const void* cSrc, size_t cSrcSize, + unsigned maxLog, void* workSpace, size_t wkspSize, + int bmi2) +{ + const BYTE* const istart = (const BYTE*)cSrc; + const BYTE* ip = istart; + unsigned tableLog; + unsigned maxSymbolValue = FSE_MAX_SYMBOL_VALUE; + FSE_DecompressWksp* const wksp = (FSE_DecompressWksp*)workSpace; + + DEBUG_STATIC_ASSERT((FSE_MAX_SYMBOL_VALUE + 1) % 2 == 0); + if (wkspSize < sizeof(*wksp)) return ERROR(GENERIC); + + /* normal FSE decoding mode */ + { + size_t const NCountLength = FSE_readNCount_bmi2(wksp->ncount, &maxSymbolValue, &tableLog, istart, cSrcSize, bmi2); + if (FSE_isError(NCountLength)) return NCountLength; + if (tableLog > maxLog) return ERROR(tableLog_tooLarge); + assert(NCountLength <= cSrcSize); + ip += NCountLength; + cSrcSize -= NCountLength; + } + + if (FSE_DECOMPRESS_WKSP_SIZE(tableLog, maxSymbolValue) > wkspSize) return ERROR(tableLog_tooLarge); + workSpace = wksp->dtable + FSE_DTABLE_SIZE_U32(tableLog); + wkspSize -= sizeof(*wksp) + FSE_DTABLE_SIZE(tableLog); + + CHECK_F( FSE_buildDTable_internal(wksp->dtable, wksp->ncount, maxSymbolValue, tableLog, workSpace, wkspSize) ); + + { + const void* ptr = wksp->dtable; + const FSE_DTableHeader* DTableH = (const FSE_DTableHeader*)ptr; + const U32 fastMode = DTableH->fastMode; + + /* select fast mode (static) */ + if (fastMode) return FSE_decompress_usingDTable_generic(dst, dstCapacity, ip, cSrcSize, wksp->dtable, 1); + return FSE_decompress_usingDTable_generic(dst, dstCapacity, ip, cSrcSize, wksp->dtable, 0); + } +} + +/* Avoids the FORCE_INLINE of the _body() function. */ +static size_t FSE_decompress_wksp_body_default(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize) +{ + return FSE_decompress_wksp_body(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize, 0); +} + +#if DYNAMIC_BMI2 +BMI2_TARGET_ATTRIBUTE static size_t FSE_decompress_wksp_body_bmi2(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize) +{ + return FSE_decompress_wksp_body(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize, 1); +} +#endif + +size_t FSE_decompress_wksp_bmi2(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize, int bmi2) +{ +#if DYNAMIC_BMI2 + if (bmi2) { + return FSE_decompress_wksp_body_bmi2(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize); + } +#endif + (void)bmi2; + return FSE_decompress_wksp_body_default(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize); +} + + +typedef FSE_DTable DTable_max_t[FSE_DTABLE_SIZE_U32(FSE_MAX_TABLELOG)]; + +#ifndef ZSTD_NO_UNUSED_FUNCTIONS +size_t FSE_buildDTable(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog) { + U32 wksp[FSE_BUILD_DTABLE_WKSP_SIZE_U32(FSE_TABLELOG_ABSOLUTE_MAX, FSE_MAX_SYMBOL_VALUE)]; + return FSE_buildDTable_wksp(dt, normalizedCounter, maxSymbolValue, tableLog, wksp, sizeof(wksp)); +} + +size_t FSE_decompress(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize) +{ + /* Static analyzer seems unable to understand this table will be properly initialized later */ + U32 wksp[FSE_DECOMPRESS_WKSP_SIZE_U32(FSE_MAX_TABLELOG, FSE_MAX_SYMBOL_VALUE)]; + return FSE_decompress_wksp(dst, dstCapacity, cSrc, cSrcSize, FSE_MAX_TABLELOG, wksp, sizeof(wksp)); +} +#endif + + +#endif /* FSE_COMMONDEFS_ONLY */ diff --git a/src/blosc2/internal-complibs/zstd-1.5.2/common/huf.h b/src/blosc2/internal-complibs/zstd-1.5.2/common/huf.h new file mode 100644 index 0000000..8551848 --- /dev/null +++ b/src/blosc2/internal-complibs/zstd-1.5.2/common/huf.h @@ -0,0 +1,364 @@ +/* ****************************************************************** + * huff0 huffman codec, + * part of Finite State Entropy library + * Copyright (c) Yann Collet, Facebook, Inc. + * + * You can contact the author at : + * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. +****************************************************************** */ + +#if defined (__cplusplus) +extern "C" { +#endif + +#ifndef HUF_H_298734234 +#define HUF_H_298734234 + +/* *** Dependencies *** */ +#include "zstd_deps.h" /* size_t */ + + +/* *** library symbols visibility *** */ +/* Note : when linking with -fvisibility=hidden on gcc, or by default on Visual, + * HUF symbols remain "private" (internal symbols for library only). + * Set macro FSE_DLL_EXPORT to 1 if you want HUF symbols visible on DLL interface */ +#if defined(FSE_DLL_EXPORT) && (FSE_DLL_EXPORT==1) && defined(__GNUC__) && (__GNUC__ >= 4) +# define HUF_PUBLIC_API __attribute__ ((visibility ("default"))) +#elif defined(FSE_DLL_EXPORT) && (FSE_DLL_EXPORT==1) /* Visual expected */ +# define HUF_PUBLIC_API __declspec(dllexport) +#elif defined(FSE_DLL_IMPORT) && (FSE_DLL_IMPORT==1) +# define HUF_PUBLIC_API __declspec(dllimport) /* not required, just to generate faster code (saves a function pointer load from IAT and an indirect jump) */ +#else +# define HUF_PUBLIC_API +#endif + + +/* ========================== */ +/* *** simple functions *** */ +/* ========================== */ + +/** HUF_compress() : + * Compress content from buffer 'src', of size 'srcSize', into buffer 'dst'. + * 'dst' buffer must be already allocated. + * Compression runs faster if `dstCapacity` >= HUF_compressBound(srcSize). + * `srcSize` must be <= `HUF_BLOCKSIZE_MAX` == 128 KB. + * @return : size of compressed data (<= `dstCapacity`). + * Special values : if return == 0, srcData is not compressible => Nothing is stored within dst !!! + * if HUF_isError(return), compression failed (more details using HUF_getErrorName()) + */ +HUF_PUBLIC_API size_t HUF_compress(void* dst, size_t dstCapacity, + const void* src, size_t srcSize); + +/** HUF_decompress() : + * Decompress HUF data from buffer 'cSrc', of size 'cSrcSize', + * into already allocated buffer 'dst', of minimum size 'dstSize'. + * `originalSize` : **must** be the ***exact*** size of original (uncompressed) data. + * Note : in contrast with FSE, HUF_decompress can regenerate + * RLE (cSrcSize==1) and uncompressed (cSrcSize==dstSize) data, + * because it knows size to regenerate (originalSize). + * @return : size of regenerated data (== originalSize), + * or an error code, which can be tested using HUF_isError() + */ +HUF_PUBLIC_API size_t HUF_decompress(void* dst, size_t originalSize, + const void* cSrc, size_t cSrcSize); + + +/* *** Tool functions *** */ +#define HUF_BLOCKSIZE_MAX (128 * 1024) /**< maximum input size for a single block compressed with HUF_compress */ +HUF_PUBLIC_API size_t HUF_compressBound(size_t size); /**< maximum compressed size (worst case) */ + +/* Error Management */ +HUF_PUBLIC_API unsigned HUF_isError(size_t code); /**< tells if a return value is an error code */ +HUF_PUBLIC_API const char* HUF_getErrorName(size_t code); /**< provides error code string (useful for debugging) */ + + +/* *** Advanced function *** */ + +/** HUF_compress2() : + * Same as HUF_compress(), but offers control over `maxSymbolValue` and `tableLog`. + * `maxSymbolValue` must be <= HUF_SYMBOLVALUE_MAX . + * `tableLog` must be `<= HUF_TABLELOG_MAX` . */ +HUF_PUBLIC_API size_t HUF_compress2 (void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + unsigned maxSymbolValue, unsigned tableLog); + +/** HUF_compress4X_wksp() : + * Same as HUF_compress2(), but uses externally allocated `workSpace`. + * `workspace` must be at least as large as HUF_WORKSPACE_SIZE */ +#define HUF_WORKSPACE_SIZE ((8 << 10) + 512 /* sorting scratch space */) +#define HUF_WORKSPACE_SIZE_U64 (HUF_WORKSPACE_SIZE / sizeof(U64)) +HUF_PUBLIC_API size_t HUF_compress4X_wksp (void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + unsigned maxSymbolValue, unsigned tableLog, + void* workSpace, size_t wkspSize); + +#endif /* HUF_H_298734234 */ + +/* ****************************************************************** + * WARNING !! + * The following section contains advanced and experimental definitions + * which shall never be used in the context of a dynamic library, + * because they are not guaranteed to remain stable in the future. + * Only consider them in association with static linking. + * *****************************************************************/ +#if defined(HUF_STATIC_LINKING_ONLY) && !defined(HUF_H_HUF_STATIC_LINKING_ONLY) +#define HUF_H_HUF_STATIC_LINKING_ONLY + +/* *** Dependencies *** */ +#include "mem.h" /* U32 */ +#define FSE_STATIC_LINKING_ONLY +#include "fse.h" + + +/* *** Constants *** */ +#define HUF_TABLELOG_MAX 12 /* max runtime value of tableLog (due to static allocation); can be modified up to HUF_TABLELOG_ABSOLUTEMAX */ +#define HUF_TABLELOG_DEFAULT 11 /* default tableLog value when none specified */ +#define HUF_SYMBOLVALUE_MAX 255 + +#define HUF_TABLELOG_ABSOLUTEMAX 12 /* absolute limit of HUF_MAX_TABLELOG. Beyond that value, code does not work */ +#if (HUF_TABLELOG_MAX > HUF_TABLELOG_ABSOLUTEMAX) +# error "HUF_TABLELOG_MAX is too large !" +#endif + + +/* **************************************** +* Static allocation +******************************************/ +/* HUF buffer bounds */ +#define HUF_CTABLEBOUND 129 +#define HUF_BLOCKBOUND(size) (size + (size>>8) + 8) /* only true when incompressible is pre-filtered with fast heuristic */ +#define HUF_COMPRESSBOUND(size) (HUF_CTABLEBOUND + HUF_BLOCKBOUND(size)) /* Macro version, useful for static allocation */ + +/* static allocation of HUF's Compression Table */ +/* this is a private definition, just exposed for allocation and strict aliasing purpose. never EVER access its members directly */ +typedef size_t HUF_CElt; /* consider it an incomplete type */ +#define HUF_CTABLE_SIZE_ST(maxSymbolValue) ((maxSymbolValue)+2) /* Use tables of size_t, for proper alignment */ +#define HUF_CTABLE_SIZE(maxSymbolValue) (HUF_CTABLE_SIZE_ST(maxSymbolValue) * sizeof(size_t)) +#define HUF_CREATE_STATIC_CTABLE(name, maxSymbolValue) \ + HUF_CElt name[HUF_CTABLE_SIZE_ST(maxSymbolValue)] /* no final ; */ + +/* static allocation of HUF's DTable */ +typedef U32 HUF_DTable; +#define HUF_DTABLE_SIZE(maxTableLog) (1 + (1<<(maxTableLog))) +#define HUF_CREATE_STATIC_DTABLEX1(DTable, maxTableLog) \ + HUF_DTable DTable[HUF_DTABLE_SIZE((maxTableLog)-1)] = { ((U32)((maxTableLog)-1) * 0x01000001) } +#define HUF_CREATE_STATIC_DTABLEX2(DTable, maxTableLog) \ + HUF_DTable DTable[HUF_DTABLE_SIZE(maxTableLog)] = { ((U32)(maxTableLog) * 0x01000001) } + + +/* **************************************** +* Advanced decompression functions +******************************************/ +size_t HUF_decompress4X1 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */ +#ifndef HUF_FORCE_DECOMPRESS_X1 +size_t HUF_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */ +#endif + +size_t HUF_decompress4X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< decodes RLE and uncompressed */ +size_t HUF_decompress4X_hufOnly(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< considers RLE and uncompressed as errors */ +size_t HUF_decompress4X_hufOnly_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< considers RLE and uncompressed as errors */ +size_t HUF_decompress4X1_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */ +size_t HUF_decompress4X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< single-symbol decoder */ +#ifndef HUF_FORCE_DECOMPRESS_X1 +size_t HUF_decompress4X2_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */ +size_t HUF_decompress4X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< double-symbols decoder */ +#endif + + +/* **************************************** + * HUF detailed API + * ****************************************/ + +/*! HUF_compress() does the following: + * 1. count symbol occurrence from source[] into table count[] using FSE_count() (exposed within "fse.h") + * 2. (optional) refine tableLog using HUF_optimalTableLog() + * 3. build Huffman table from count using HUF_buildCTable() + * 4. save Huffman table to memory buffer using HUF_writeCTable() + * 5. encode the data stream using HUF_compress4X_usingCTable() + * + * The following API allows targeting specific sub-functions for advanced tasks. + * For example, it's possible to compress several blocks using the same 'CTable', + * or to save and regenerate 'CTable' using external methods. + */ +unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue); +size_t HUF_buildCTable (HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue, unsigned maxNbBits); /* @return : maxNbBits; CTable and count can overlap. In which case, CTable will overwrite count content */ +size_t HUF_writeCTable (void* dst, size_t maxDstSize, const HUF_CElt* CTable, unsigned maxSymbolValue, unsigned huffLog); +size_t HUF_writeCTable_wksp(void* dst, size_t maxDstSize, const HUF_CElt* CTable, unsigned maxSymbolValue, unsigned huffLog, void* workspace, size_t workspaceSize); +size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable); +size_t HUF_compress4X_usingCTable_bmi2(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable, int bmi2); +size_t HUF_estimateCompressedSize(const HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue); +int HUF_validateCTable(const HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue); + +typedef enum { + HUF_repeat_none, /**< Cannot use the previous table */ + HUF_repeat_check, /**< Can use the previous table but it must be checked. Note : The previous table must have been constructed by HUF_compress{1, 4}X_repeat */ + HUF_repeat_valid /**< Can use the previous table and it is assumed to be valid */ + } HUF_repeat; +/** HUF_compress4X_repeat() : + * Same as HUF_compress4X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none. + * If it uses hufTable it does not modify hufTable or repeat. + * If it doesn't, it sets *repeat = HUF_repeat_none, and it sets hufTable to the table used. + * If preferRepeat then the old table will always be used if valid. + * If suspectUncompressible then some sampling checks will be run to potentially skip huffman coding */ +size_t HUF_compress4X_repeat(void* dst, size_t dstSize, + const void* src, size_t srcSize, + unsigned maxSymbolValue, unsigned tableLog, + void* workSpace, size_t wkspSize, /**< `workSpace` must be aligned on 4-bytes boundaries, `wkspSize` must be >= HUF_WORKSPACE_SIZE */ + HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat, int bmi2, unsigned suspectUncompressible); + +/** HUF_buildCTable_wksp() : + * Same as HUF_buildCTable(), but using externally allocated scratch buffer. + * `workSpace` must be aligned on 4-bytes boundaries, and its size must be >= HUF_CTABLE_WORKSPACE_SIZE. + */ +#define HUF_CTABLE_WORKSPACE_SIZE_U32 (2*HUF_SYMBOLVALUE_MAX +1 +1) +#define HUF_CTABLE_WORKSPACE_SIZE (HUF_CTABLE_WORKSPACE_SIZE_U32 * sizeof(unsigned)) +size_t HUF_buildCTable_wksp (HUF_CElt* tree, + const unsigned* count, U32 maxSymbolValue, U32 maxNbBits, + void* workSpace, size_t wkspSize); + +/*! HUF_readStats() : + * Read compact Huffman tree, saved by HUF_writeCTable(). + * `huffWeight` is destination buffer. + * @return : size read from `src` , or an error Code . + * Note : Needed by HUF_readCTable() and HUF_readDTableXn() . */ +size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, + U32* rankStats, U32* nbSymbolsPtr, U32* tableLogPtr, + const void* src, size_t srcSize); + +/*! HUF_readStats_wksp() : + * Same as HUF_readStats() but takes an external workspace which must be + * 4-byte aligned and its size must be >= HUF_READ_STATS_WORKSPACE_SIZE. + * If the CPU has BMI2 support, pass bmi2=1, otherwise pass bmi2=0. + */ +#define HUF_READ_STATS_WORKSPACE_SIZE_U32 FSE_DECOMPRESS_WKSP_SIZE_U32(6, HUF_TABLELOG_MAX-1) +#define HUF_READ_STATS_WORKSPACE_SIZE (HUF_READ_STATS_WORKSPACE_SIZE_U32 * sizeof(unsigned)) +size_t HUF_readStats_wksp(BYTE* huffWeight, size_t hwSize, + U32* rankStats, U32* nbSymbolsPtr, U32* tableLogPtr, + const void* src, size_t srcSize, + void* workspace, size_t wkspSize, + int bmi2); + +/** HUF_readCTable() : + * Loading a CTable saved with HUF_writeCTable() */ +size_t HUF_readCTable (HUF_CElt* CTable, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize, unsigned *hasZeroWeights); + +/** HUF_getNbBitsFromCTable() : + * Read nbBits from CTable symbolTable, for symbol `symbolValue` presumed <= HUF_SYMBOLVALUE_MAX + * Note 1 : is not inlined, as HUF_CElt definition is private */ +U32 HUF_getNbBitsFromCTable(const HUF_CElt* symbolTable, U32 symbolValue); + +/* + * HUF_decompress() does the following: + * 1. select the decompression algorithm (X1, X2) based on pre-computed heuristics + * 2. build Huffman table from save, using HUF_readDTableX?() + * 3. decode 1 or 4 segments in parallel using HUF_decompress?X?_usingDTable() + */ + +/** HUF_selectDecoder() : + * Tells which decoder is likely to decode faster, + * based on a set of pre-computed metrics. + * @return : 0==HUF_decompress4X1, 1==HUF_decompress4X2 . + * Assumption : 0 < dstSize <= 128 KB */ +U32 HUF_selectDecoder (size_t dstSize, size_t cSrcSize); + +/** + * The minimum workspace size for the `workSpace` used in + * HUF_readDTableX1_wksp() and HUF_readDTableX2_wksp(). + * + * The space used depends on HUF_TABLELOG_MAX, ranging from ~1500 bytes when + * HUF_TABLE_LOG_MAX=12 to ~1850 bytes when HUF_TABLE_LOG_MAX=15. + * Buffer overflow errors may potentially occur if code modifications result in + * a required workspace size greater than that specified in the following + * macro. + */ +#define HUF_DECOMPRESS_WORKSPACE_SIZE ((2 << 10) + (1 << 9)) +#define HUF_DECOMPRESS_WORKSPACE_SIZE_U32 (HUF_DECOMPRESS_WORKSPACE_SIZE / sizeof(U32)) + +#ifndef HUF_FORCE_DECOMPRESS_X2 +size_t HUF_readDTableX1 (HUF_DTable* DTable, const void* src, size_t srcSize); +size_t HUF_readDTableX1_wksp (HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize); +#endif +#ifndef HUF_FORCE_DECOMPRESS_X1 +size_t HUF_readDTableX2 (HUF_DTable* DTable, const void* src, size_t srcSize); +size_t HUF_readDTableX2_wksp (HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize); +#endif + +size_t HUF_decompress4X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); +#ifndef HUF_FORCE_DECOMPRESS_X2 +size_t HUF_decompress4X1_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); +#endif +#ifndef HUF_FORCE_DECOMPRESS_X1 +size_t HUF_decompress4X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); +#endif + + +/* ====================== */ +/* single stream variants */ +/* ====================== */ + +size_t HUF_compress1X (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog); +size_t HUF_compress1X_wksp (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize); /**< `workSpace` must be a table of at least HUF_WORKSPACE_SIZE_U64 U64 */ +size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable); +size_t HUF_compress1X_usingCTable_bmi2(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable, int bmi2); +/** HUF_compress1X_repeat() : + * Same as HUF_compress1X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none. + * If it uses hufTable it does not modify hufTable or repeat. + * If it doesn't, it sets *repeat = HUF_repeat_none, and it sets hufTable to the table used. + * If preferRepeat then the old table will always be used if valid. + * If suspectUncompressible then some sampling checks will be run to potentially skip huffman coding */ +size_t HUF_compress1X_repeat(void* dst, size_t dstSize, + const void* src, size_t srcSize, + unsigned maxSymbolValue, unsigned tableLog, + void* workSpace, size_t wkspSize, /**< `workSpace` must be aligned on 4-bytes boundaries, `wkspSize` must be >= HUF_WORKSPACE_SIZE */ + HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat, int bmi2, unsigned suspectUncompressible); + +size_t HUF_decompress1X1 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* single-symbol decoder */ +#ifndef HUF_FORCE_DECOMPRESS_X1 +size_t HUF_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* double-symbol decoder */ +#endif + +size_t HUF_decompress1X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); +size_t HUF_decompress1X_DCtx_wksp (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); +#ifndef HUF_FORCE_DECOMPRESS_X2 +size_t HUF_decompress1X1_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */ +size_t HUF_decompress1X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< single-symbol decoder */ +#endif +#ifndef HUF_FORCE_DECOMPRESS_X1 +size_t HUF_decompress1X2_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */ +size_t HUF_decompress1X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< double-symbols decoder */ +#endif + +size_t HUF_decompress1X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); /**< automatic selection of sing or double symbol decoder, based on DTable */ +#ifndef HUF_FORCE_DECOMPRESS_X2 +size_t HUF_decompress1X1_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); +#endif +#ifndef HUF_FORCE_DECOMPRESS_X1 +size_t HUF_decompress1X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); +#endif + +/* BMI2 variants. + * If the CPU has BMI2 support, pass bmi2=1, otherwise pass bmi2=0. + */ +size_t HUF_decompress1X_usingDTable_bmi2(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int bmi2); +#ifndef HUF_FORCE_DECOMPRESS_X2 +size_t HUF_decompress1X1_DCtx_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int bmi2); +#endif +size_t HUF_decompress4X_usingDTable_bmi2(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int bmi2); +size_t HUF_decompress4X_hufOnly_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int bmi2); +#ifndef HUF_FORCE_DECOMPRESS_X2 +size_t HUF_readDTableX1_wksp_bmi2(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize, int bmi2); +#endif +#ifndef HUF_FORCE_DECOMPRESS_X1 +size_t HUF_readDTableX2_wksp_bmi2(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize, int bmi2); +#endif + +#endif /* HUF_STATIC_LINKING_ONLY */ + +#if defined (__cplusplus) +} +#endif diff --git a/src/blosc2/internal-complibs/zstd-1.5.2/common/mem.h b/src/blosc2/internal-complibs/zstd-1.5.2/common/mem.h new file mode 100644 index 0000000..85581c3 --- /dev/null +++ b/src/blosc2/internal-complibs/zstd-1.5.2/common/mem.h @@ -0,0 +1,442 @@ +/* + * Copyright (c) Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef MEM_H_MODULE +#define MEM_H_MODULE + +#if defined (__cplusplus) +extern "C" { +#endif + +/*-**************************************** +* Dependencies +******************************************/ +#include /* size_t, ptrdiff_t */ +#include "compiler.h" /* __has_builtin */ +#include "debug.h" /* DEBUG_STATIC_ASSERT */ +#include "zstd_deps.h" /* ZSTD_memcpy */ + + +/*-**************************************** +* Compiler specifics +******************************************/ +#if defined(_MSC_VER) /* Visual Studio */ +# include /* _byteswap_ulong */ +# include /* _byteswap_* */ +#endif +#if defined(__GNUC__) +# define MEM_STATIC static __inline __attribute__((unused)) +#elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) +# define MEM_STATIC static inline +#elif defined(_MSC_VER) +# define MEM_STATIC static __inline +#else +# define MEM_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */ +#endif + +/*-************************************************************** +* Basic Types +*****************************************************************/ +#if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) +# if defined(_AIX) +# include +# else +# include /* intptr_t */ +# endif + typedef uint8_t BYTE; + typedef uint8_t U8; + typedef int8_t S8; + typedef uint16_t U16; + typedef int16_t S16; + typedef uint32_t U32; + typedef int32_t S32; + typedef uint64_t U64; + typedef int64_t S64; +#else +# include +#if CHAR_BIT != 8 +# error "this implementation requires char to be exactly 8-bit type" +#endif + typedef unsigned char BYTE; + typedef unsigned char U8; + typedef signed char S8; +#if USHRT_MAX != 65535 +# error "this implementation requires short to be exactly 16-bit type" +#endif + typedef unsigned short U16; + typedef signed short S16; +#if UINT_MAX != 4294967295 +# error "this implementation requires int to be exactly 32-bit type" +#endif + typedef unsigned int U32; + typedef signed int S32; +/* note : there are no limits defined for long long type in C90. + * limits exist in C99, however, in such case, is preferred */ + typedef unsigned long long U64; + typedef signed long long S64; +#endif + + +/*-************************************************************** +* Memory I/O API +*****************************************************************/ +/*=== Static platform detection ===*/ +MEM_STATIC unsigned MEM_32bits(void); +MEM_STATIC unsigned MEM_64bits(void); +MEM_STATIC unsigned MEM_isLittleEndian(void); + +/*=== Native unaligned read/write ===*/ +MEM_STATIC U16 MEM_read16(const void* memPtr); +MEM_STATIC U32 MEM_read32(const void* memPtr); +MEM_STATIC U64 MEM_read64(const void* memPtr); +MEM_STATIC size_t MEM_readST(const void* memPtr); + +MEM_STATIC void MEM_write16(void* memPtr, U16 value); +MEM_STATIC void MEM_write32(void* memPtr, U32 value); +MEM_STATIC void MEM_write64(void* memPtr, U64 value); + +/*=== Little endian unaligned read/write ===*/ +MEM_STATIC U16 MEM_readLE16(const void* memPtr); +MEM_STATIC U32 MEM_readLE24(const void* memPtr); +MEM_STATIC U32 MEM_readLE32(const void* memPtr); +MEM_STATIC U64 MEM_readLE64(const void* memPtr); +MEM_STATIC size_t MEM_readLEST(const void* memPtr); + +MEM_STATIC void MEM_writeLE16(void* memPtr, U16 val); +MEM_STATIC void MEM_writeLE24(void* memPtr, U32 val); +MEM_STATIC void MEM_writeLE32(void* memPtr, U32 val32); +MEM_STATIC void MEM_writeLE64(void* memPtr, U64 val64); +MEM_STATIC void MEM_writeLEST(void* memPtr, size_t val); + +/*=== Big endian unaligned read/write ===*/ +MEM_STATIC U32 MEM_readBE32(const void* memPtr); +MEM_STATIC U64 MEM_readBE64(const void* memPtr); +MEM_STATIC size_t MEM_readBEST(const void* memPtr); + +MEM_STATIC void MEM_writeBE32(void* memPtr, U32 val32); +MEM_STATIC void MEM_writeBE64(void* memPtr, U64 val64); +MEM_STATIC void MEM_writeBEST(void* memPtr, size_t val); + +/*=== Byteswap ===*/ +MEM_STATIC U32 MEM_swap32(U32 in); +MEM_STATIC U64 MEM_swap64(U64 in); +MEM_STATIC size_t MEM_swapST(size_t in); + + +/*-************************************************************** +* Memory I/O Implementation +*****************************************************************/ +/* MEM_FORCE_MEMORY_ACCESS : + * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable. + * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal. + * The below switch allow to select different access method for improved performance. + * Method 0 (default) : use `memcpy()`. Safe and portable. + * Method 1 : `__packed` statement. It depends on compiler extension (i.e., not portable). + * This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`. + * Method 2 : direct access. This method is portable but violate C standard. + * It can generate buggy code on targets depending on alignment. + * In some circumstances, it's the only known way to get the most performance (i.e. GCC + ARMv6) + * See http://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details. + * Prefer these methods in priority order (0 > 1 > 2) + */ +#ifndef MEM_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */ +# if defined(__INTEL_COMPILER) || defined(__GNUC__) || defined(__ICCARM__) +# define MEM_FORCE_MEMORY_ACCESS 1 +# endif +#endif + +MEM_STATIC unsigned MEM_32bits(void) { return sizeof(size_t)==4; } +MEM_STATIC unsigned MEM_64bits(void) { return sizeof(size_t)==8; } + +MEM_STATIC unsigned MEM_isLittleEndian(void) +{ +#if defined(__BYTE_ORDER__) && defined(__ORDER_LITTLE_ENDIAN__) && (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) + return 1; +#elif defined(__BYTE_ORDER__) && defined(__ORDER_BIG_ENDIAN__) && (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) + return 0; +#elif defined(__clang__) && __LITTLE_ENDIAN__ + return 1; +#elif defined(__clang__) && __BIG_ENDIAN__ + return 0; +#elif defined(_MSC_VER) && (_M_AMD64 || _M_IX86) + return 1; +#elif defined(__DMC__) && defined(_M_IX86) + return 1; +#else + const union { U32 u; BYTE c[4]; } one = { 1 }; /* don't use static : performance detrimental */ + return one.c[0]; +#endif +} + +#if defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==2) + +/* violates C standard, by lying on structure alignment. +Only use if no other choice to achieve best performance on target platform */ +MEM_STATIC U16 MEM_read16(const void* memPtr) { return *(const U16*) memPtr; } +MEM_STATIC U32 MEM_read32(const void* memPtr) { return *(const U32*) memPtr; } +MEM_STATIC U64 MEM_read64(const void* memPtr) { return *(const U64*) memPtr; } +MEM_STATIC size_t MEM_readST(const void* memPtr) { return *(const size_t*) memPtr; } + +MEM_STATIC void MEM_write16(void* memPtr, U16 value) { *(U16*)memPtr = value; } +MEM_STATIC void MEM_write32(void* memPtr, U32 value) { *(U32*)memPtr = value; } +MEM_STATIC void MEM_write64(void* memPtr, U64 value) { *(U64*)memPtr = value; } + +#elif defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==1) + +/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */ +/* currently only defined for gcc and icc */ +#if defined(_MSC_VER) || (defined(__INTEL_COMPILER) && defined(WIN32)) + __pragma( pack(push, 1) ) + typedef struct { U16 v; } unalign16; + typedef struct { U32 v; } unalign32; + typedef struct { U64 v; } unalign64; + typedef struct { size_t v; } unalignArch; + __pragma( pack(pop) ) +#else + typedef struct { U16 v; } __attribute__((packed)) unalign16; + typedef struct { U32 v; } __attribute__((packed)) unalign32; + typedef struct { U64 v; } __attribute__((packed)) unalign64; + typedef struct { size_t v; } __attribute__((packed)) unalignArch; +#endif + +MEM_STATIC U16 MEM_read16(const void* ptr) { return ((const unalign16*)ptr)->v; } +MEM_STATIC U32 MEM_read32(const void* ptr) { return ((const unalign32*)ptr)->v; } +MEM_STATIC U64 MEM_read64(const void* ptr) { return ((const unalign64*)ptr)->v; } +MEM_STATIC size_t MEM_readST(const void* ptr) { return ((const unalignArch*)ptr)->v; } + +MEM_STATIC void MEM_write16(void* memPtr, U16 value) { ((unalign16*)memPtr)->v = value; } +MEM_STATIC void MEM_write32(void* memPtr, U32 value) { ((unalign32*)memPtr)->v = value; } +MEM_STATIC void MEM_write64(void* memPtr, U64 value) { ((unalign64*)memPtr)->v = value; } + +#else + +/* default method, safe and standard. + can sometimes prove slower */ + +MEM_STATIC U16 MEM_read16(const void* memPtr) +{ + U16 val; ZSTD_memcpy(&val, memPtr, sizeof(val)); return val; +} + +MEM_STATIC U32 MEM_read32(const void* memPtr) +{ + U32 val; ZSTD_memcpy(&val, memPtr, sizeof(val)); return val; +} + +MEM_STATIC U64 MEM_read64(const void* memPtr) +{ + U64 val; ZSTD_memcpy(&val, memPtr, sizeof(val)); return val; +} + +MEM_STATIC size_t MEM_readST(const void* memPtr) +{ + size_t val; ZSTD_memcpy(&val, memPtr, sizeof(val)); return val; +} + +MEM_STATIC void MEM_write16(void* memPtr, U16 value) +{ + ZSTD_memcpy(memPtr, &value, sizeof(value)); +} + +MEM_STATIC void MEM_write32(void* memPtr, U32 value) +{ + ZSTD_memcpy(memPtr, &value, sizeof(value)); +} + +MEM_STATIC void MEM_write64(void* memPtr, U64 value) +{ + ZSTD_memcpy(memPtr, &value, sizeof(value)); +} + +#endif /* MEM_FORCE_MEMORY_ACCESS */ + +MEM_STATIC U32 MEM_swap32(U32 in) +{ +#if defined(_MSC_VER) /* Visual Studio */ + return _byteswap_ulong(in); +#elif (defined (__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ >= 403)) \ + || (defined(__clang__) && __has_builtin(__builtin_bswap32)) + return __builtin_bswap32(in); +#else + return ((in << 24) & 0xff000000 ) | + ((in << 8) & 0x00ff0000 ) | + ((in >> 8) & 0x0000ff00 ) | + ((in >> 24) & 0x000000ff ); +#endif +} + +MEM_STATIC U64 MEM_swap64(U64 in) +{ +#if defined(_MSC_VER) /* Visual Studio */ + return _byteswap_uint64(in); +#elif (defined (__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ >= 403)) \ + || (defined(__clang__) && __has_builtin(__builtin_bswap64)) + return __builtin_bswap64(in); +#else + return ((in << 56) & 0xff00000000000000ULL) | + ((in << 40) & 0x00ff000000000000ULL) | + ((in << 24) & 0x0000ff0000000000ULL) | + ((in << 8) & 0x000000ff00000000ULL) | + ((in >> 8) & 0x00000000ff000000ULL) | + ((in >> 24) & 0x0000000000ff0000ULL) | + ((in >> 40) & 0x000000000000ff00ULL) | + ((in >> 56) & 0x00000000000000ffULL); +#endif +} + +MEM_STATIC size_t MEM_swapST(size_t in) +{ + if (MEM_32bits()) + return (size_t)MEM_swap32((U32)in); + else + return (size_t)MEM_swap64((U64)in); +} + +/*=== Little endian r/w ===*/ + +MEM_STATIC U16 MEM_readLE16(const void* memPtr) +{ + if (MEM_isLittleEndian()) + return MEM_read16(memPtr); + else { + const BYTE* p = (const BYTE*)memPtr; + return (U16)(p[0] + (p[1]<<8)); + } +} + +MEM_STATIC void MEM_writeLE16(void* memPtr, U16 val) +{ + if (MEM_isLittleEndian()) { + MEM_write16(memPtr, val); + } else { + BYTE* p = (BYTE*)memPtr; + p[0] = (BYTE)val; + p[1] = (BYTE)(val>>8); + } +} + +MEM_STATIC U32 MEM_readLE24(const void* memPtr) +{ + return (U32)MEM_readLE16(memPtr) + ((U32)(((const BYTE*)memPtr)[2]) << 16); +} + +MEM_STATIC void MEM_writeLE24(void* memPtr, U32 val) +{ + MEM_writeLE16(memPtr, (U16)val); + ((BYTE*)memPtr)[2] = (BYTE)(val>>16); +} + +MEM_STATIC U32 MEM_readLE32(const void* memPtr) +{ + if (MEM_isLittleEndian()) + return MEM_read32(memPtr); + else + return MEM_swap32(MEM_read32(memPtr)); +} + +MEM_STATIC void MEM_writeLE32(void* memPtr, U32 val32) +{ + if (MEM_isLittleEndian()) + MEM_write32(memPtr, val32); + else + MEM_write32(memPtr, MEM_swap32(val32)); +} + +MEM_STATIC U64 MEM_readLE64(const void* memPtr) +{ + if (MEM_isLittleEndian()) + return MEM_read64(memPtr); + else + return MEM_swap64(MEM_read64(memPtr)); +} + +MEM_STATIC void MEM_writeLE64(void* memPtr, U64 val64) +{ + if (MEM_isLittleEndian()) + MEM_write64(memPtr, val64); + else + MEM_write64(memPtr, MEM_swap64(val64)); +} + +MEM_STATIC size_t MEM_readLEST(const void* memPtr) +{ + if (MEM_32bits()) + return (size_t)MEM_readLE32(memPtr); + else + return (size_t)MEM_readLE64(memPtr); +} + +MEM_STATIC void MEM_writeLEST(void* memPtr, size_t val) +{ + if (MEM_32bits()) + MEM_writeLE32(memPtr, (U32)val); + else + MEM_writeLE64(memPtr, (U64)val); +} + +/*=== Big endian r/w ===*/ + +MEM_STATIC U32 MEM_readBE32(const void* memPtr) +{ + if (MEM_isLittleEndian()) + return MEM_swap32(MEM_read32(memPtr)); + else + return MEM_read32(memPtr); +} + +MEM_STATIC void MEM_writeBE32(void* memPtr, U32 val32) +{ + if (MEM_isLittleEndian()) + MEM_write32(memPtr, MEM_swap32(val32)); + else + MEM_write32(memPtr, val32); +} + +MEM_STATIC U64 MEM_readBE64(const void* memPtr) +{ + if (MEM_isLittleEndian()) + return MEM_swap64(MEM_read64(memPtr)); + else + return MEM_read64(memPtr); +} + +MEM_STATIC void MEM_writeBE64(void* memPtr, U64 val64) +{ + if (MEM_isLittleEndian()) + MEM_write64(memPtr, MEM_swap64(val64)); + else + MEM_write64(memPtr, val64); +} + +MEM_STATIC size_t MEM_readBEST(const void* memPtr) +{ + if (MEM_32bits()) + return (size_t)MEM_readBE32(memPtr); + else + return (size_t)MEM_readBE64(memPtr); +} + +MEM_STATIC void MEM_writeBEST(void* memPtr, size_t val) +{ + if (MEM_32bits()) + MEM_writeBE32(memPtr, (U32)val); + else + MEM_writeBE64(memPtr, (U64)val); +} + +/* code only tested on 32 and 64 bits systems */ +MEM_STATIC void MEM_check(void) { DEBUG_STATIC_ASSERT((sizeof(size_t)==4) || (sizeof(size_t)==8)); } + + +#if defined (__cplusplus) +} +#endif + +#endif /* MEM_H_MODULE */ diff --git a/src/blosc2/internal-complibs/zstd-1.5.2/common/pool.c b/src/blosc2/internal-complibs/zstd-1.5.2/common/pool.c new file mode 100644 index 0000000..2e37cdd --- /dev/null +++ b/src/blosc2/internal-complibs/zstd-1.5.2/common/pool.c @@ -0,0 +1,355 @@ +/* + * Copyright (c) Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + + +/* ====== Dependencies ======= */ +#include "zstd_deps.h" /* size_t */ +#include "debug.h" /* assert */ +#include "zstd_internal.h" /* ZSTD_customMalloc, ZSTD_customFree */ +#include "pool.h" + +/* ====== Compiler specifics ====== */ +#if defined(_MSC_VER) +# pragma warning(disable : 4204) /* disable: C4204: non-constant aggregate initializer */ +#endif + + +#ifdef ZSTD_MULTITHREAD + +#include "threading.h" /* pthread adaptation */ + +/* A job is a function and an opaque argument */ +typedef struct POOL_job_s { + POOL_function function; + void *opaque; +} POOL_job; + +struct POOL_ctx_s { + ZSTD_customMem customMem; + /* Keep track of the threads */ + ZSTD_pthread_t* threads; + size_t threadCapacity; + size_t threadLimit; + + /* The queue is a circular buffer */ + POOL_job *queue; + size_t queueHead; + size_t queueTail; + size_t queueSize; + + /* The number of threads working on jobs */ + size_t numThreadsBusy; + /* Indicates if the queue is empty */ + int queueEmpty; + + /* The mutex protects the queue */ + ZSTD_pthread_mutex_t queueMutex; + /* Condition variable for pushers to wait on when the queue is full */ + ZSTD_pthread_cond_t queuePushCond; + /* Condition variables for poppers to wait on when the queue is empty */ + ZSTD_pthread_cond_t queuePopCond; + /* Indicates if the queue is shutting down */ + int shutdown; +}; + +/* POOL_thread() : + * Work thread for the thread pool. + * Waits for jobs and executes them. + * @returns : NULL on failure else non-null. + */ +static void* POOL_thread(void* opaque) { + POOL_ctx* const ctx = (POOL_ctx*)opaque; + if (!ctx) { return NULL; } + for (;;) { + /* Lock the mutex and wait for a non-empty queue or until shutdown */ + ZSTD_pthread_mutex_lock(&ctx->queueMutex); + + while ( ctx->queueEmpty + || (ctx->numThreadsBusy >= ctx->threadLimit) ) { + if (ctx->shutdown) { + /* even if !queueEmpty, (possible if numThreadsBusy >= threadLimit), + * a few threads will be shutdown while !queueEmpty, + * but enough threads will remain active to finish the queue */ + ZSTD_pthread_mutex_unlock(&ctx->queueMutex); + return opaque; + } + ZSTD_pthread_cond_wait(&ctx->queuePopCond, &ctx->queueMutex); + } + /* Pop a job off the queue */ + { POOL_job const job = ctx->queue[ctx->queueHead]; + ctx->queueHead = (ctx->queueHead + 1) % ctx->queueSize; + ctx->numThreadsBusy++; + ctx->queueEmpty = (ctx->queueHead == ctx->queueTail); + /* Unlock the mutex, signal a pusher, and run the job */ + ZSTD_pthread_cond_signal(&ctx->queuePushCond); + ZSTD_pthread_mutex_unlock(&ctx->queueMutex); + + job.function(job.opaque); + + /* If the intended queue size was 0, signal after finishing job */ + ZSTD_pthread_mutex_lock(&ctx->queueMutex); + ctx->numThreadsBusy--; + if (ctx->queueSize == 1) { + ZSTD_pthread_cond_signal(&ctx->queuePushCond); + } + ZSTD_pthread_mutex_unlock(&ctx->queueMutex); + } + } /* for (;;) */ + assert(0); /* Unreachable */ +} + +/* ZSTD_createThreadPool() : public access point */ +POOL_ctx* ZSTD_createThreadPool(size_t numThreads) { + return POOL_create (numThreads, 0); +} + +POOL_ctx* POOL_create(size_t numThreads, size_t queueSize) { + return POOL_create_advanced(numThreads, queueSize, ZSTD_defaultCMem); +} + +POOL_ctx* POOL_create_advanced(size_t numThreads, size_t queueSize, + ZSTD_customMem customMem) +{ + POOL_ctx* ctx; + /* Check parameters */ + if (!numThreads) { return NULL; } + /* Allocate the context and zero initialize */ + ctx = (POOL_ctx*)ZSTD_customCalloc(sizeof(POOL_ctx), customMem); + if (!ctx) { return NULL; } + /* Initialize the job queue. + * It needs one extra space since one space is wasted to differentiate + * empty and full queues. + */ + ctx->queueSize = queueSize + 1; + ctx->queue = (POOL_job*)ZSTD_customMalloc(ctx->queueSize * sizeof(POOL_job), customMem); + ctx->queueHead = 0; + ctx->queueTail = 0; + ctx->numThreadsBusy = 0; + ctx->queueEmpty = 1; + { + int error = 0; + error |= ZSTD_pthread_mutex_init(&ctx->queueMutex, NULL); + error |= ZSTD_pthread_cond_init(&ctx->queuePushCond, NULL); + error |= ZSTD_pthread_cond_init(&ctx->queuePopCond, NULL); + if (error) { POOL_free(ctx); return NULL; } + } + ctx->shutdown = 0; + /* Allocate space for the thread handles */ + ctx->threads = (ZSTD_pthread_t*)ZSTD_customMalloc(numThreads * sizeof(ZSTD_pthread_t), customMem); + ctx->threadCapacity = 0; + ctx->customMem = customMem; + /* Check for errors */ + if (!ctx->threads || !ctx->queue) { POOL_free(ctx); return NULL; } + /* Initialize the threads */ + { size_t i; + for (i = 0; i < numThreads; ++i) { + if (ZSTD_pthread_create(&ctx->threads[i], NULL, &POOL_thread, ctx)) { + ctx->threadCapacity = i; + POOL_free(ctx); + return NULL; + } } + ctx->threadCapacity = numThreads; + ctx->threadLimit = numThreads; + } + return ctx; +} + +/*! POOL_join() : + Shutdown the queue, wake any sleeping threads, and join all of the threads. +*/ +static void POOL_join(POOL_ctx* ctx) { + /* Shut down the queue */ + ZSTD_pthread_mutex_lock(&ctx->queueMutex); + ctx->shutdown = 1; + ZSTD_pthread_mutex_unlock(&ctx->queueMutex); + /* Wake up sleeping threads */ + ZSTD_pthread_cond_broadcast(&ctx->queuePushCond); + ZSTD_pthread_cond_broadcast(&ctx->queuePopCond); + /* Join all of the threads */ + { size_t i; + for (i = 0; i < ctx->threadCapacity; ++i) { + ZSTD_pthread_join(ctx->threads[i], NULL); /* note : could fail */ + } } +} + +void POOL_free(POOL_ctx *ctx) { + if (!ctx) { return; } + POOL_join(ctx); + ZSTD_pthread_mutex_destroy(&ctx->queueMutex); + ZSTD_pthread_cond_destroy(&ctx->queuePushCond); + ZSTD_pthread_cond_destroy(&ctx->queuePopCond); + ZSTD_customFree(ctx->queue, ctx->customMem); + ZSTD_customFree(ctx->threads, ctx->customMem); + ZSTD_customFree(ctx, ctx->customMem); +} + +void ZSTD_freeThreadPool (ZSTD_threadPool* pool) { + POOL_free (pool); +} + +size_t POOL_sizeof(const POOL_ctx* ctx) { + if (ctx==NULL) return 0; /* supports sizeof NULL */ + return sizeof(*ctx) + + ctx->queueSize * sizeof(POOL_job) + + ctx->threadCapacity * sizeof(ZSTD_pthread_t); +} + + +/* @return : 0 on success, 1 on error */ +static int POOL_resize_internal(POOL_ctx* ctx, size_t numThreads) +{ + if (numThreads <= ctx->threadCapacity) { + if (!numThreads) return 1; + ctx->threadLimit = numThreads; + return 0; + } + /* numThreads > threadCapacity */ + { ZSTD_pthread_t* const threadPool = (ZSTD_pthread_t*)ZSTD_customMalloc(numThreads * sizeof(ZSTD_pthread_t), ctx->customMem); + if (!threadPool) return 1; + /* replace existing thread pool */ + ZSTD_memcpy(threadPool, ctx->threads, ctx->threadCapacity * sizeof(*threadPool)); + ZSTD_customFree(ctx->threads, ctx->customMem); + ctx->threads = threadPool; + /* Initialize additional threads */ + { size_t threadId; + for (threadId = ctx->threadCapacity; threadId < numThreads; ++threadId) { + if (ZSTD_pthread_create(&threadPool[threadId], NULL, &POOL_thread, ctx)) { + ctx->threadCapacity = threadId; + return 1; + } } + } } + /* successfully expanded */ + ctx->threadCapacity = numThreads; + ctx->threadLimit = numThreads; + return 0; +} + +/* @return : 0 on success, 1 on error */ +int POOL_resize(POOL_ctx* ctx, size_t numThreads) +{ + int result; + if (ctx==NULL) return 1; + ZSTD_pthread_mutex_lock(&ctx->queueMutex); + result = POOL_resize_internal(ctx, numThreads); + ZSTD_pthread_cond_broadcast(&ctx->queuePopCond); + ZSTD_pthread_mutex_unlock(&ctx->queueMutex); + return result; +} + +/** + * Returns 1 if the queue is full and 0 otherwise. + * + * When queueSize is 1 (pool was created with an intended queueSize of 0), + * then a queue is empty if there is a thread free _and_ no job is waiting. + */ +static int isQueueFull(POOL_ctx const* ctx) { + if (ctx->queueSize > 1) { + return ctx->queueHead == ((ctx->queueTail + 1) % ctx->queueSize); + } else { + return (ctx->numThreadsBusy == ctx->threadLimit) || + !ctx->queueEmpty; + } +} + + +static void +POOL_add_internal(POOL_ctx* ctx, POOL_function function, void *opaque) +{ + POOL_job const job = {function, opaque}; + assert(ctx != NULL); + if (ctx->shutdown) return; + + ctx->queueEmpty = 0; + ctx->queue[ctx->queueTail] = job; + ctx->queueTail = (ctx->queueTail + 1) % ctx->queueSize; + ZSTD_pthread_cond_signal(&ctx->queuePopCond); +} + +void POOL_add(POOL_ctx* ctx, POOL_function function, void* opaque) +{ + assert(ctx != NULL); + ZSTD_pthread_mutex_lock(&ctx->queueMutex); + /* Wait until there is space in the queue for the new job */ + while (isQueueFull(ctx) && (!ctx->shutdown)) { + ZSTD_pthread_cond_wait(&ctx->queuePushCond, &ctx->queueMutex); + } + POOL_add_internal(ctx, function, opaque); + ZSTD_pthread_mutex_unlock(&ctx->queueMutex); +} + + +int POOL_tryAdd(POOL_ctx* ctx, POOL_function function, void* opaque) +{ + assert(ctx != NULL); + ZSTD_pthread_mutex_lock(&ctx->queueMutex); + if (isQueueFull(ctx)) { + ZSTD_pthread_mutex_unlock(&ctx->queueMutex); + return 0; + } + POOL_add_internal(ctx, function, opaque); + ZSTD_pthread_mutex_unlock(&ctx->queueMutex); + return 1; +} + + +#else /* ZSTD_MULTITHREAD not defined */ + +/* ========================== */ +/* No multi-threading support */ +/* ========================== */ + + +/* We don't need any data, but if it is empty, malloc() might return NULL. */ +struct POOL_ctx_s { + int dummy; +}; +static POOL_ctx g_poolCtx; + +POOL_ctx* POOL_create(size_t numThreads, size_t queueSize) { + return POOL_create_advanced(numThreads, queueSize, ZSTD_defaultCMem); +} + +POOL_ctx* +POOL_create_advanced(size_t numThreads, size_t queueSize, ZSTD_customMem customMem) +{ + (void)numThreads; + (void)queueSize; + (void)customMem; + return &g_poolCtx; +} + +void POOL_free(POOL_ctx* ctx) { + assert(!ctx || ctx == &g_poolCtx); + (void)ctx; +} + +int POOL_resize(POOL_ctx* ctx, size_t numThreads) { + (void)ctx; (void)numThreads; + return 0; +} + +void POOL_add(POOL_ctx* ctx, POOL_function function, void* opaque) { + (void)ctx; + function(opaque); +} + +int POOL_tryAdd(POOL_ctx* ctx, POOL_function function, void* opaque) { + (void)ctx; + function(opaque); + return 1; +} + +size_t POOL_sizeof(const POOL_ctx* ctx) { + if (ctx==NULL) return 0; /* supports sizeof NULL */ + assert(ctx == &g_poolCtx); + return sizeof(*ctx); +} + +#endif /* ZSTD_MULTITHREAD */ diff --git a/src/blosc2/internal-complibs/zstd-1.5.2/common/pool.h b/src/blosc2/internal-complibs/zstd-1.5.2/common/pool.h new file mode 100644 index 0000000..0ebde18 --- /dev/null +++ b/src/blosc2/internal-complibs/zstd-1.5.2/common/pool.h @@ -0,0 +1,84 @@ +/* + * Copyright (c) Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef POOL_H +#define POOL_H + +#if defined (__cplusplus) +extern "C" { +#endif + + +#include "zstd_deps.h" +#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_customMem */ +#include "../zstd.h" + +typedef struct POOL_ctx_s POOL_ctx; + +/*! POOL_create() : + * Create a thread pool with at most `numThreads` threads. + * `numThreads` must be at least 1. + * The maximum number of queued jobs before blocking is `queueSize`. + * @return : POOL_ctx pointer on success, else NULL. +*/ +POOL_ctx* POOL_create(size_t numThreads, size_t queueSize); + +POOL_ctx* POOL_create_advanced(size_t numThreads, size_t queueSize, + ZSTD_customMem customMem); + +/*! POOL_free() : + * Free a thread pool returned by POOL_create(). + */ +void POOL_free(POOL_ctx* ctx); + +/*! POOL_resize() : + * Expands or shrinks pool's number of threads. + * This is more efficient than releasing + creating a new context, + * since it tries to preserve and re-use existing threads. + * `numThreads` must be at least 1. + * @return : 0 when resize was successful, + * !0 (typically 1) if there is an error. + * note : only numThreads can be resized, queueSize remains unchanged. + */ +int POOL_resize(POOL_ctx* ctx, size_t numThreads); + +/*! POOL_sizeof() : + * @return threadpool memory usage + * note : compatible with NULL (returns 0 in this case) + */ +size_t POOL_sizeof(const POOL_ctx* ctx); + +/*! POOL_function : + * The function type that can be added to a thread pool. + */ +typedef void (*POOL_function)(void*); + +/*! POOL_add() : + * Add the job `function(opaque)` to the thread pool. `ctx` must be valid. + * Possibly blocks until there is room in the queue. + * Note : The function may be executed asynchronously, + * therefore, `opaque` must live until function has been completed. + */ +void POOL_add(POOL_ctx* ctx, POOL_function function, void* opaque); + + +/*! POOL_tryAdd() : + * Add the job `function(opaque)` to thread pool _if_ a queue slot is available. + * Returns immediately even if not (does not block). + * @return : 1 if successful, 0 if not. + */ +int POOL_tryAdd(POOL_ctx* ctx, POOL_function function, void* opaque); + + +#if defined (__cplusplus) +} +#endif + +#endif diff --git a/src/blosc2/internal-complibs/zstd-1.5.2/common/portability_macros.h b/src/blosc2/internal-complibs/zstd-1.5.2/common/portability_macros.h new file mode 100644 index 0000000..2143817 --- /dev/null +++ b/src/blosc2/internal-complibs/zstd-1.5.2/common/portability_macros.h @@ -0,0 +1,137 @@ +/* + * Copyright (c) Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_PORTABILITY_MACROS_H +#define ZSTD_PORTABILITY_MACROS_H + +/** + * This header file contains macro defintions to support portability. + * This header is shared between C and ASM code, so it MUST only + * contain macro definitions. It MUST not contain any C code. + * + * This header ONLY defines macros to detect platforms/feature support. + * + */ + + +/* compat. with non-clang compilers */ +#ifndef __has_attribute + #define __has_attribute(x) 0 +#endif + +/* compat. with non-clang compilers */ +#ifndef __has_builtin +# define __has_builtin(x) 0 +#endif + +/* compat. with non-clang compilers */ +#ifndef __has_feature +# define __has_feature(x) 0 +#endif + +/* detects whether we are being compiled under msan */ +#ifndef ZSTD_MEMORY_SANITIZER +# if __has_feature(memory_sanitizer) +# define ZSTD_MEMORY_SANITIZER 1 +# else +# define ZSTD_MEMORY_SANITIZER 0 +# endif +#endif + +/* detects whether we are being compiled under asan */ +#ifndef ZSTD_ADDRESS_SANITIZER +# if __has_feature(address_sanitizer) +# define ZSTD_ADDRESS_SANITIZER 1 +# elif defined(__SANITIZE_ADDRESS__) +# define ZSTD_ADDRESS_SANITIZER 1 +# else +# define ZSTD_ADDRESS_SANITIZER 0 +# endif +#endif + +/* detects whether we are being compiled under dfsan */ +#ifndef ZSTD_DATAFLOW_SANITIZER +# if __has_feature(dataflow_sanitizer) +# define ZSTD_DATAFLOW_SANITIZER 1 +# else +# define ZSTD_DATAFLOW_SANITIZER 0 +# endif +#endif + +/* Mark the internal assembly functions as hidden */ +#ifdef __ELF__ +# define ZSTD_HIDE_ASM_FUNCTION(func) .hidden func +#else +# define ZSTD_HIDE_ASM_FUNCTION(func) +#endif + +/* Enable runtime BMI2 dispatch based on the CPU. + * Enabled for clang & gcc >=4.8 on x86 when BMI2 isn't enabled by default. + */ +#ifndef DYNAMIC_BMI2 + #if ((defined(__clang__) && __has_attribute(__target__)) \ + || (defined(__GNUC__) \ + && (__GNUC__ >= 5 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8)))) \ + && (defined(__x86_64__) || defined(_M_X64)) \ + && !defined(__BMI2__) + # define DYNAMIC_BMI2 1 + #else + # define DYNAMIC_BMI2 0 + #endif +#endif + +/** + * Only enable assembly for GNUC comptabile compilers, + * because other platforms may not support GAS assembly syntax. + * + * Only enable assembly for Linux / MacOS, other platforms may + * work, but they haven't been tested. This could likely be + * extended to BSD systems. + * + * Disable assembly when MSAN is enabled, because MSAN requires + * 100% of code to be instrumented to work. + */ +#if defined(__GNUC__) +# if defined(__linux__) || defined(__linux) || defined(__APPLE__) +# if ZSTD_MEMORY_SANITIZER +# define ZSTD_ASM_SUPPORTED 0 +# elif ZSTD_DATAFLOW_SANITIZER +# define ZSTD_ASM_SUPPORTED 0 +# else +# define ZSTD_ASM_SUPPORTED 1 +# endif +# else +# define ZSTD_ASM_SUPPORTED 0 +# endif +#else +# define ZSTD_ASM_SUPPORTED 0 +#endif + +/** + * Determines whether we should enable assembly for x86-64 + * with BMI2. + * + * Enable if all of the following conditions hold: + * - ASM hasn't been explicitly disabled by defining ZSTD_DISABLE_ASM + * - Assembly is supported + * - We are compiling for x86-64 and either: + * - DYNAMIC_BMI2 is enabled + * - BMI2 is supported at compile time + */ +#if !defined(ZSTD_DISABLE_ASM) && \ + ZSTD_ASM_SUPPORTED && \ + defined(__x86_64__) && \ + (DYNAMIC_BMI2 || defined(__BMI2__)) +# define ZSTD_ENABLE_ASM_X86_64_BMI2 1 +#else +# define ZSTD_ENABLE_ASM_X86_64_BMI2 0 +#endif + +#endif /* ZSTD_PORTABILITY_MACROS_H */ diff --git a/src/blosc2/internal-complibs/zstd-1.5.2/common/threading.c b/src/blosc2/internal-complibs/zstd-1.5.2/common/threading.c new file mode 100644 index 0000000..92cf57c --- /dev/null +++ b/src/blosc2/internal-complibs/zstd-1.5.2/common/threading.c @@ -0,0 +1,122 @@ +/** + * Copyright (c) 2016 Tino Reichardt + * All rights reserved. + * + * You can contact the author at: + * - zstdmt source repository: https://github.com/mcmilk/zstdmt + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/** + * This file will hold wrapper for systems, which do not support pthreads + */ + +#include "threading.h" + +/* create fake symbol to avoid empty translation unit warning */ +int g_ZSTD_threading_useless_symbol; + +#if defined(ZSTD_MULTITHREAD) && defined(_WIN32) + +/** + * Windows minimalist Pthread Wrapper, based on : + * http://www.cse.wustl.edu/~schmidt/win32-cv-1.html + */ + + +/* === Dependencies === */ +#include +#include + + +/* === Implementation === */ + +static unsigned __stdcall worker(void *arg) +{ + ZSTD_pthread_t* const thread = (ZSTD_pthread_t*) arg; + thread->arg = thread->start_routine(thread->arg); + return 0; +} + +int ZSTD_pthread_create(ZSTD_pthread_t* thread, const void* unused, + void* (*start_routine) (void*), void* arg) +{ + (void)unused; + thread->arg = arg; + thread->start_routine = start_routine; + thread->handle = (HANDLE) _beginthreadex(NULL, 0, worker, thread, 0, NULL); + + if (!thread->handle) + return errno; + else + return 0; +} + +int ZSTD_pthread_join(ZSTD_pthread_t thread, void **value_ptr) +{ + DWORD result; + + if (!thread.handle) return 0; + + result = WaitForSingleObject(thread.handle, INFINITE); + switch (result) { + case WAIT_OBJECT_0: + if (value_ptr) *value_ptr = thread.arg; + return 0; + case WAIT_ABANDONED: + return EINVAL; + default: + return GetLastError(); + } +} + +#endif /* ZSTD_MULTITHREAD */ + +#if defined(ZSTD_MULTITHREAD) && DEBUGLEVEL >= 1 && !defined(_WIN32) + +#define ZSTD_DEPS_NEED_MALLOC +#include "zstd_deps.h" + +int ZSTD_pthread_mutex_init(ZSTD_pthread_mutex_t* mutex, pthread_mutexattr_t const* attr) +{ + *mutex = (pthread_mutex_t*)ZSTD_malloc(sizeof(pthread_mutex_t)); + if (!*mutex) + return 1; + return pthread_mutex_init(*mutex, attr); +} + +int ZSTD_pthread_mutex_destroy(ZSTD_pthread_mutex_t* mutex) +{ + if (!*mutex) + return 0; + { + int const ret = pthread_mutex_destroy(*mutex); + ZSTD_free(*mutex); + return ret; + } +} + +int ZSTD_pthread_cond_init(ZSTD_pthread_cond_t* cond, pthread_condattr_t const* attr) +{ + *cond = (pthread_cond_t*)ZSTD_malloc(sizeof(pthread_cond_t)); + if (!*cond) + return 1; + return pthread_cond_init(*cond, attr); +} + +int ZSTD_pthread_cond_destroy(ZSTD_pthread_cond_t* cond) +{ + if (!*cond) + return 0; + { + int const ret = pthread_cond_destroy(*cond); + ZSTD_free(*cond); + return ret; + } +} + +#endif diff --git a/src/blosc2/internal-complibs/zstd-1.5.2/common/threading.h b/src/blosc2/internal-complibs/zstd-1.5.2/common/threading.h new file mode 100644 index 0000000..fd0060d --- /dev/null +++ b/src/blosc2/internal-complibs/zstd-1.5.2/common/threading.h @@ -0,0 +1,155 @@ +/** + * Copyright (c) 2016 Tino Reichardt + * All rights reserved. + * + * You can contact the author at: + * - zstdmt source repository: https://github.com/mcmilk/zstdmt + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef THREADING_H_938743 +#define THREADING_H_938743 + +#include "debug.h" + +#if defined (__cplusplus) +extern "C" { +#endif + +#if defined(ZSTD_MULTITHREAD) && defined(_WIN32) + +/** + * Windows minimalist Pthread Wrapper, based on : + * http://www.cse.wustl.edu/~schmidt/win32-cv-1.html + */ +#ifdef WINVER +# undef WINVER +#endif +#define WINVER 0x0600 + +#ifdef _WIN32_WINNT +# undef _WIN32_WINNT +#endif +#define _WIN32_WINNT 0x0600 + +#ifndef WIN32_LEAN_AND_MEAN +# define WIN32_LEAN_AND_MEAN +#endif + +#undef ERROR /* reported already defined on VS 2015 (Rich Geldreich) */ +#include +#undef ERROR +#define ERROR(name) ZSTD_ERROR(name) + + +/* mutex */ +#define ZSTD_pthread_mutex_t CRITICAL_SECTION +#define ZSTD_pthread_mutex_init(a, b) ((void)(b), InitializeCriticalSection((a)), 0) +#define ZSTD_pthread_mutex_destroy(a) DeleteCriticalSection((a)) +#define ZSTD_pthread_mutex_lock(a) EnterCriticalSection((a)) +#define ZSTD_pthread_mutex_unlock(a) LeaveCriticalSection((a)) + +/* condition variable */ +#define ZSTD_pthread_cond_t CONDITION_VARIABLE +#define ZSTD_pthread_cond_init(a, b) ((void)(b), InitializeConditionVariable((a)), 0) +#define ZSTD_pthread_cond_destroy(a) ((void)(a)) +#define ZSTD_pthread_cond_wait(a, b) SleepConditionVariableCS((a), (b), INFINITE) +#define ZSTD_pthread_cond_signal(a) WakeConditionVariable((a)) +#define ZSTD_pthread_cond_broadcast(a) WakeAllConditionVariable((a)) + +/* ZSTD_pthread_create() and ZSTD_pthread_join() */ +typedef struct { + HANDLE handle; + void* (*start_routine)(void*); + void* arg; +} ZSTD_pthread_t; + +int ZSTD_pthread_create(ZSTD_pthread_t* thread, const void* unused, + void* (*start_routine) (void*), void* arg); + +int ZSTD_pthread_join(ZSTD_pthread_t thread, void** value_ptr); + +/** + * add here more wrappers as required + */ + + +#elif defined(ZSTD_MULTITHREAD) /* posix assumed ; need a better detection method */ +/* === POSIX Systems === */ +# include + +#if DEBUGLEVEL < 1 + +#define ZSTD_pthread_mutex_t pthread_mutex_t +#define ZSTD_pthread_mutex_init(a, b) pthread_mutex_init((a), (b)) +#define ZSTD_pthread_mutex_destroy(a) pthread_mutex_destroy((a)) +#define ZSTD_pthread_mutex_lock(a) pthread_mutex_lock((a)) +#define ZSTD_pthread_mutex_unlock(a) pthread_mutex_unlock((a)) + +#define ZSTD_pthread_cond_t pthread_cond_t +#define ZSTD_pthread_cond_init(a, b) pthread_cond_init((a), (b)) +#define ZSTD_pthread_cond_destroy(a) pthread_cond_destroy((a)) +#define ZSTD_pthread_cond_wait(a, b) pthread_cond_wait((a), (b)) +#define ZSTD_pthread_cond_signal(a) pthread_cond_signal((a)) +#define ZSTD_pthread_cond_broadcast(a) pthread_cond_broadcast((a)) + +#define ZSTD_pthread_t pthread_t +#define ZSTD_pthread_create(a, b, c, d) pthread_create((a), (b), (c), (d)) +#define ZSTD_pthread_join(a, b) pthread_join((a),(b)) + +#else /* DEBUGLEVEL >= 1 */ + +/* Debug implementation of threading. + * In this implementation we use pointers for mutexes and condition variables. + * This way, if we forget to init/destroy them the program will crash or ASAN + * will report leaks. + */ + +#define ZSTD_pthread_mutex_t pthread_mutex_t* +int ZSTD_pthread_mutex_init(ZSTD_pthread_mutex_t* mutex, pthread_mutexattr_t const* attr); +int ZSTD_pthread_mutex_destroy(ZSTD_pthread_mutex_t* mutex); +#define ZSTD_pthread_mutex_lock(a) pthread_mutex_lock(*(a)) +#define ZSTD_pthread_mutex_unlock(a) pthread_mutex_unlock(*(a)) + +#define ZSTD_pthread_cond_t pthread_cond_t* +int ZSTD_pthread_cond_init(ZSTD_pthread_cond_t* cond, pthread_condattr_t const* attr); +int ZSTD_pthread_cond_destroy(ZSTD_pthread_cond_t* cond); +#define ZSTD_pthread_cond_wait(a, b) pthread_cond_wait(*(a), *(b)) +#define ZSTD_pthread_cond_signal(a) pthread_cond_signal(*(a)) +#define ZSTD_pthread_cond_broadcast(a) pthread_cond_broadcast(*(a)) + +#define ZSTD_pthread_t pthread_t +#define ZSTD_pthread_create(a, b, c, d) pthread_create((a), (b), (c), (d)) +#define ZSTD_pthread_join(a, b) pthread_join((a),(b)) + +#endif + +#else /* ZSTD_MULTITHREAD not defined */ +/* No multithreading support */ + +typedef int ZSTD_pthread_mutex_t; +#define ZSTD_pthread_mutex_init(a, b) ((void)(a), (void)(b), 0) +#define ZSTD_pthread_mutex_destroy(a) ((void)(a)) +#define ZSTD_pthread_mutex_lock(a) ((void)(a)) +#define ZSTD_pthread_mutex_unlock(a) ((void)(a)) + +typedef int ZSTD_pthread_cond_t; +#define ZSTD_pthread_cond_init(a, b) ((void)(a), (void)(b), 0) +#define ZSTD_pthread_cond_destroy(a) ((void)(a)) +#define ZSTD_pthread_cond_wait(a, b) ((void)(a), (void)(b)) +#define ZSTD_pthread_cond_signal(a) ((void)(a)) +#define ZSTD_pthread_cond_broadcast(a) ((void)(a)) + +/* do not use ZSTD_pthread_t */ + +#endif /* ZSTD_MULTITHREAD */ + +#if defined (__cplusplus) +} +#endif + +#endif /* THREADING_H_938743 */ diff --git a/src/blosc2/internal-complibs/zstd-1.5.2/common/xxhash.c b/src/blosc2/internal-complibs/zstd-1.5.2/common/xxhash.c new file mode 100644 index 0000000..d49497c --- /dev/null +++ b/src/blosc2/internal-complibs/zstd-1.5.2/common/xxhash.c @@ -0,0 +1,24 @@ +/* + * xxHash - Fast Hash algorithm + * Copyright (c) Yann Collet, Facebook, Inc. + * + * You can contact the author at : + * - xxHash homepage: http://www.xxhash.com + * - xxHash source repository : https://github.com/Cyan4973/xxHash + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. +*/ + + + +/* + * xxhash.c instantiates functions defined in xxhash.h + */ + +#define XXH_STATIC_LINKING_ONLY /* access advanced declarations */ +#define XXH_IMPLEMENTATION /* access definitions */ + +#include "xxhash.h" diff --git a/src/blosc2/internal-complibs/zstd-1.5.2/common/xxhash.h b/src/blosc2/internal-complibs/zstd-1.5.2/common/xxhash.h new file mode 100644 index 0000000..8ebbfdd --- /dev/null +++ b/src/blosc2/internal-complibs/zstd-1.5.2/common/xxhash.h @@ -0,0 +1,5686 @@ +/* + * xxHash - Fast Hash algorithm + * Copyright (c) Yann Collet, Facebook, Inc. + * + * You can contact the author at : + * - xxHash homepage: http://www.xxhash.com + * - xxHash source repository : https://github.com/Cyan4973/xxHash + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. +*/ + + +#ifndef XXH_NO_XXH3 +# define XXH_NO_XXH3 +#endif + +#ifndef XXH_NAMESPACE +# define XXH_NAMESPACE ZSTD_ +#endif + +/*! + * @mainpage xxHash + * + * @file xxhash.h + * xxHash prototypes and implementation + */ +/* TODO: update */ +/* Notice extracted from xxHash homepage: + +xxHash is an extremely fast hash algorithm, running at RAM speed limits. +It also successfully passes all tests from the SMHasher suite. + +Comparison (single thread, Windows Seven 32 bits, using SMHasher on a Core 2 Duo @3GHz) + +Name Speed Q.Score Author +xxHash 5.4 GB/s 10 +CrapWow 3.2 GB/s 2 Andrew +MurmurHash 3a 2.7 GB/s 10 Austin Appleby +SpookyHash 2.0 GB/s 10 Bob Jenkins +SBox 1.4 GB/s 9 Bret Mulvey +Lookup3 1.2 GB/s 9 Bob Jenkins +SuperFastHash 1.2 GB/s 1 Paul Hsieh +CityHash64 1.05 GB/s 10 Pike & Alakuijala +FNV 0.55 GB/s 5 Fowler, Noll, Vo +CRC32 0.43 GB/s 9 +MD5-32 0.33 GB/s 10 Ronald L. Rivest +SHA1-32 0.28 GB/s 10 + +Q.Score is a measure of quality of the hash function. +It depends on successfully passing SMHasher test set. +10 is a perfect score. + +Note: SMHasher's CRC32 implementation is not the fastest one. +Other speed-oriented implementations can be faster, +especially in combination with PCLMUL instruction: +https://fastcompression.blogspot.com/2019/03/presenting-xxh3.html?showComment=1552696407071#c3490092340461170735 + +A 64-bit version, named XXH64, is available since r35. +It offers much better speed, but for 64-bit applications only. +Name Speed on 64 bits Speed on 32 bits +XXH64 13.8 GB/s 1.9 GB/s +XXH32 6.8 GB/s 6.0 GB/s +*/ + +#if defined (__cplusplus) +extern "C" { +#endif + +/* **************************** + * INLINE mode + ******************************/ +/*! + * XXH_INLINE_ALL (and XXH_PRIVATE_API) + * Use these build macros to inline xxhash into the target unit. + * Inlining improves performance on small inputs, especially when the length is + * expressed as a compile-time constant: + * + * https://fastcompression.blogspot.com/2018/03/xxhash-for-small-keys-impressive-power.html + * + * It also keeps xxHash symbols private to the unit, so they are not exported. + * + * Usage: + * #define XXH_INLINE_ALL + * #include "xxhash.h" + * + * Do not compile and link xxhash.o as a separate object, as it is not useful. + */ +#if (defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API)) \ + && !defined(XXH_INLINE_ALL_31684351384) + /* this section should be traversed only once */ +# define XXH_INLINE_ALL_31684351384 + /* give access to the advanced API, required to compile implementations */ +# undef XXH_STATIC_LINKING_ONLY /* avoid macro redef */ +# define XXH_STATIC_LINKING_ONLY + /* make all functions private */ +# undef XXH_PUBLIC_API +# if defined(__GNUC__) +# define XXH_PUBLIC_API static __inline __attribute__((unused)) +# elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) +# define XXH_PUBLIC_API static inline +# elif defined(_MSC_VER) +# define XXH_PUBLIC_API static __inline +# else + /* note: this version may generate warnings for unused static functions */ +# define XXH_PUBLIC_API static +# endif + + /* + * This part deals with the special case where a unit wants to inline xxHash, + * but "xxhash.h" has previously been included without XXH_INLINE_ALL, + * such as part of some previously included *.h header file. + * Without further action, the new include would just be ignored, + * and functions would effectively _not_ be inlined (silent failure). + * The following macros solve this situation by prefixing all inlined names, + * avoiding naming collision with previous inclusions. + */ + /* Before that, we unconditionally #undef all symbols, + * in case they were already defined with XXH_NAMESPACE. + * They will then be redefined for XXH_INLINE_ALL + */ +# undef XXH_versionNumber + /* XXH32 */ +# undef XXH32 +# undef XXH32_createState +# undef XXH32_freeState +# undef XXH32_reset +# undef XXH32_update +# undef XXH32_digest +# undef XXH32_copyState +# undef XXH32_canonicalFromHash +# undef XXH32_hashFromCanonical + /* XXH64 */ +# undef XXH64 +# undef XXH64_createState +# undef XXH64_freeState +# undef XXH64_reset +# undef XXH64_update +# undef XXH64_digest +# undef XXH64_copyState +# undef XXH64_canonicalFromHash +# undef XXH64_hashFromCanonical + /* XXH3_64bits */ +# undef XXH3_64bits +# undef XXH3_64bits_withSecret +# undef XXH3_64bits_withSeed +# undef XXH3_64bits_withSecretandSeed +# undef XXH3_createState +# undef XXH3_freeState +# undef XXH3_copyState +# undef XXH3_64bits_reset +# undef XXH3_64bits_reset_withSeed +# undef XXH3_64bits_reset_withSecret +# undef XXH3_64bits_update +# undef XXH3_64bits_digest +# undef XXH3_generateSecret + /* XXH3_128bits */ +# undef XXH128 +# undef XXH3_128bits +# undef XXH3_128bits_withSeed +# undef XXH3_128bits_withSecret +# undef XXH3_128bits_reset +# undef XXH3_128bits_reset_withSeed +# undef XXH3_128bits_reset_withSecret +# undef XXH3_128bits_reset_withSecretandSeed +# undef XXH3_128bits_update +# undef XXH3_128bits_digest +# undef XXH128_isEqual +# undef XXH128_cmp +# undef XXH128_canonicalFromHash +# undef XXH128_hashFromCanonical + /* Finally, free the namespace itself */ +# undef XXH_NAMESPACE + + /* employ the namespace for XXH_INLINE_ALL */ +# define XXH_NAMESPACE XXH_INLINE_ + /* + * Some identifiers (enums, type names) are not symbols, + * but they must nonetheless be renamed to avoid redeclaration. + * Alternative solution: do not redeclare them. + * However, this requires some #ifdefs, and has a more dispersed impact. + * Meanwhile, renaming can be achieved in a single place. + */ +# define XXH_IPREF(Id) XXH_NAMESPACE ## Id +# define XXH_OK XXH_IPREF(XXH_OK) +# define XXH_ERROR XXH_IPREF(XXH_ERROR) +# define XXH_errorcode XXH_IPREF(XXH_errorcode) +# define XXH32_canonical_t XXH_IPREF(XXH32_canonical_t) +# define XXH64_canonical_t XXH_IPREF(XXH64_canonical_t) +# define XXH128_canonical_t XXH_IPREF(XXH128_canonical_t) +# define XXH32_state_s XXH_IPREF(XXH32_state_s) +# define XXH32_state_t XXH_IPREF(XXH32_state_t) +# define XXH64_state_s XXH_IPREF(XXH64_state_s) +# define XXH64_state_t XXH_IPREF(XXH64_state_t) +# define XXH3_state_s XXH_IPREF(XXH3_state_s) +# define XXH3_state_t XXH_IPREF(XXH3_state_t) +# define XXH128_hash_t XXH_IPREF(XXH128_hash_t) + /* Ensure the header is parsed again, even if it was previously included */ +# undef XXHASH_H_5627135585666179 +# undef XXHASH_H_STATIC_13879238742 +#endif /* XXH_INLINE_ALL || XXH_PRIVATE_API */ + + + +/* **************************************************************** + * Stable API + *****************************************************************/ +#ifndef XXHASH_H_5627135585666179 +#define XXHASH_H_5627135585666179 1 + + +/*! + * @defgroup public Public API + * Contains details on the public xxHash functions. + * @{ + */ +/* specific declaration modes for Windows */ +#if !defined(XXH_INLINE_ALL) && !defined(XXH_PRIVATE_API) +# if defined(WIN32) && defined(_MSC_VER) && (defined(XXH_IMPORT) || defined(XXH_EXPORT)) +# ifdef XXH_EXPORT +# define XXH_PUBLIC_API __declspec(dllexport) +# elif XXH_IMPORT +# define XXH_PUBLIC_API __declspec(dllimport) +# endif +# else +# define XXH_PUBLIC_API /* do nothing */ +# endif +#endif + +#ifdef XXH_DOXYGEN +/*! + * @brief Emulate a namespace by transparently prefixing all symbols. + * + * If you want to include _and expose_ xxHash functions from within your own + * library, but also want to avoid symbol collisions with other libraries which + * may also include xxHash, you can use XXH_NAMESPACE to automatically prefix + * any public symbol from xxhash library with the value of XXH_NAMESPACE + * (therefore, avoid empty or numeric values). + * + * Note that no change is required within the calling program as long as it + * includes `xxhash.h`: Regular symbol names will be automatically translated + * by this header. + */ +# define XXH_NAMESPACE /* YOUR NAME HERE */ +# undef XXH_NAMESPACE +#endif + +#ifdef XXH_NAMESPACE +# define XXH_CAT(A,B) A##B +# define XXH_NAME2(A,B) XXH_CAT(A,B) +# define XXH_versionNumber XXH_NAME2(XXH_NAMESPACE, XXH_versionNumber) +/* XXH32 */ +# define XXH32 XXH_NAME2(XXH_NAMESPACE, XXH32) +# define XXH32_createState XXH_NAME2(XXH_NAMESPACE, XXH32_createState) +# define XXH32_freeState XXH_NAME2(XXH_NAMESPACE, XXH32_freeState) +# define XXH32_reset XXH_NAME2(XXH_NAMESPACE, XXH32_reset) +# define XXH32_update XXH_NAME2(XXH_NAMESPACE, XXH32_update) +# define XXH32_digest XXH_NAME2(XXH_NAMESPACE, XXH32_digest) +# define XXH32_copyState XXH_NAME2(XXH_NAMESPACE, XXH32_copyState) +# define XXH32_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH32_canonicalFromHash) +# define XXH32_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH32_hashFromCanonical) +/* XXH64 */ +# define XXH64 XXH_NAME2(XXH_NAMESPACE, XXH64) +# define XXH64_createState XXH_NAME2(XXH_NAMESPACE, XXH64_createState) +# define XXH64_freeState XXH_NAME2(XXH_NAMESPACE, XXH64_freeState) +# define XXH64_reset XXH_NAME2(XXH_NAMESPACE, XXH64_reset) +# define XXH64_update XXH_NAME2(XXH_NAMESPACE, XXH64_update) +# define XXH64_digest XXH_NAME2(XXH_NAMESPACE, XXH64_digest) +# define XXH64_copyState XXH_NAME2(XXH_NAMESPACE, XXH64_copyState) +# define XXH64_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH64_canonicalFromHash) +# define XXH64_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH64_hashFromCanonical) +/* XXH3_64bits */ +# define XXH3_64bits XXH_NAME2(XXH_NAMESPACE, XXH3_64bits) +# define XXH3_64bits_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_withSecret) +# define XXH3_64bits_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_withSeed) +# define XXH3_64bits_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_withSecretandSeed) +# define XXH3_createState XXH_NAME2(XXH_NAMESPACE, XXH3_createState) +# define XXH3_freeState XXH_NAME2(XXH_NAMESPACE, XXH3_freeState) +# define XXH3_copyState XXH_NAME2(XXH_NAMESPACE, XXH3_copyState) +# define XXH3_64bits_reset XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset) +# define XXH3_64bits_reset_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset_withSeed) +# define XXH3_64bits_reset_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset_withSecret) +# define XXH3_64bits_reset_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset_withSecretandSeed) +# define XXH3_64bits_update XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_update) +# define XXH3_64bits_digest XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_digest) +# define XXH3_generateSecret XXH_NAME2(XXH_NAMESPACE, XXH3_generateSecret) +# define XXH3_generateSecret_fromSeed XXH_NAME2(XXH_NAMESPACE, XXH3_generateSecret_fromSeed) +/* XXH3_128bits */ +# define XXH128 XXH_NAME2(XXH_NAMESPACE, XXH128) +# define XXH3_128bits XXH_NAME2(XXH_NAMESPACE, XXH3_128bits) +# define XXH3_128bits_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_withSeed) +# define XXH3_128bits_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_withSecret) +# define XXH3_128bits_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_withSecretandSeed) +# define XXH3_128bits_reset XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset) +# define XXH3_128bits_reset_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset_withSeed) +# define XXH3_128bits_reset_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset_withSecret) +# define XXH3_128bits_reset_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset_withSecretandSeed) +# define XXH3_128bits_update XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_update) +# define XXH3_128bits_digest XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_digest) +# define XXH128_isEqual XXH_NAME2(XXH_NAMESPACE, XXH128_isEqual) +# define XXH128_cmp XXH_NAME2(XXH_NAMESPACE, XXH128_cmp) +# define XXH128_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH128_canonicalFromHash) +# define XXH128_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH128_hashFromCanonical) +#endif + + +/* ************************************* +* Version +***************************************/ +#define XXH_VERSION_MAJOR 0 +#define XXH_VERSION_MINOR 8 +#define XXH_VERSION_RELEASE 1 +#define XXH_VERSION_NUMBER (XXH_VERSION_MAJOR *100*100 + XXH_VERSION_MINOR *100 + XXH_VERSION_RELEASE) + +/*! + * @brief Obtains the xxHash version. + * + * This is mostly useful when xxHash is compiled as a shared library, + * since the returned value comes from the library, as opposed to header file. + * + * @return `XXH_VERSION_NUMBER` of the invoked library. + */ +XXH_PUBLIC_API unsigned XXH_versionNumber (void); + + +/* **************************** +* Common basic types +******************************/ +#include /* size_t */ +typedef enum { XXH_OK=0, XXH_ERROR } XXH_errorcode; + + +/*-********************************************************************** +* 32-bit hash +************************************************************************/ +#if defined(XXH_DOXYGEN) /* Don't show include */ +/*! + * @brief An unsigned 32-bit integer. + * + * Not necessarily defined to `uint32_t` but functionally equivalent. + */ +typedef uint32_t XXH32_hash_t; + +#elif !defined (__VMS) \ + && (defined (__cplusplus) \ + || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) +# include + typedef uint32_t XXH32_hash_t; + +#else +# include +# if UINT_MAX == 0xFFFFFFFFUL + typedef unsigned int XXH32_hash_t; +# else +# if ULONG_MAX == 0xFFFFFFFFUL + typedef unsigned long XXH32_hash_t; +# else +# error "unsupported platform: need a 32-bit type" +# endif +# endif +#endif + +/*! + * @} + * + * @defgroup xxh32_family XXH32 family + * @ingroup public + * Contains functions used in the classic 32-bit xxHash algorithm. + * + * @note + * XXH32 is useful for older platforms, with no or poor 64-bit performance. + * Note that @ref xxh3_family provides competitive speed + * for both 32-bit and 64-bit systems, and offers true 64/128 bit hash results. + * + * @see @ref xxh64_family, @ref xxh3_family : Other xxHash families + * @see @ref xxh32_impl for implementation details + * @{ + */ + +/*! + * @brief Calculates the 32-bit hash of @p input using xxHash32. + * + * Speed on Core 2 Duo @ 3 GHz (single thread, SMHasher benchmark): 5.4 GB/s + * + * @param input The block of data to be hashed, at least @p length bytes in size. + * @param length The length of @p input, in bytes. + * @param seed The 32-bit seed to alter the hash's output predictably. + * + * @pre + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * @return The calculated 32-bit hash value. + * + * @see + * XXH64(), XXH3_64bits_withSeed(), XXH3_128bits_withSeed(), XXH128(): + * Direct equivalents for the other variants of xxHash. + * @see + * XXH32_createState(), XXH32_update(), XXH32_digest(): Streaming version. + */ +XXH_PUBLIC_API XXH32_hash_t XXH32 (const void* input, size_t length, XXH32_hash_t seed); + +/*! + * Streaming functions generate the xxHash value from an incremental input. + * This method is slower than single-call functions, due to state management. + * For small inputs, prefer `XXH32()` and `XXH64()`, which are better optimized. + * + * An XXH state must first be allocated using `XXH*_createState()`. + * + * Start a new hash by initializing the state with a seed using `XXH*_reset()`. + * + * Then, feed the hash state by calling `XXH*_update()` as many times as necessary. + * + * The function returns an error code, with 0 meaning OK, and any other value + * meaning there is an error. + * + * Finally, a hash value can be produced anytime, by using `XXH*_digest()`. + * This function returns the nn-bits hash as an int or long long. + * + * It's still possible to continue inserting input into the hash state after a + * digest, and generate new hash values later on by invoking `XXH*_digest()`. + * + * When done, release the state using `XXH*_freeState()`. + * + * Example code for incrementally hashing a file: + * @code{.c} + * #include + * #include + * #define BUFFER_SIZE 256 + * + * // Note: XXH64 and XXH3 use the same interface. + * XXH32_hash_t + * hashFile(FILE* stream) + * { + * XXH32_state_t* state; + * unsigned char buf[BUFFER_SIZE]; + * size_t amt; + * XXH32_hash_t hash; + * + * state = XXH32_createState(); // Create a state + * assert(state != NULL); // Error check here + * XXH32_reset(state, 0xbaad5eed); // Reset state with our seed + * while ((amt = fread(buf, 1, sizeof(buf), stream)) != 0) { + * XXH32_update(state, buf, amt); // Hash the file in chunks + * } + * hash = XXH32_digest(state); // Finalize the hash + * XXH32_freeState(state); // Clean up + * return hash; + * } + * @endcode + */ + +/*! + * @typedef struct XXH32_state_s XXH32_state_t + * @brief The opaque state struct for the XXH32 streaming API. + * + * @see XXH32_state_s for details. + */ +typedef struct XXH32_state_s XXH32_state_t; + +/*! + * @brief Allocates an @ref XXH32_state_t. + * + * Must be freed with XXH32_freeState(). + * @return An allocated XXH32_state_t on success, `NULL` on failure. + */ +XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void); +/*! + * @brief Frees an @ref XXH32_state_t. + * + * Must be allocated with XXH32_createState(). + * @param statePtr A pointer to an @ref XXH32_state_t allocated with @ref XXH32_createState(). + * @return XXH_OK. + */ +XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr); +/*! + * @brief Copies one @ref XXH32_state_t to another. + * + * @param dst_state The state to copy to. + * @param src_state The state to copy from. + * @pre + * @p dst_state and @p src_state must not be `NULL` and must not overlap. + */ +XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* dst_state, const XXH32_state_t* src_state); + +/*! + * @brief Resets an @ref XXH32_state_t to begin a new hash. + * + * This function resets and seeds a state. Call it before @ref XXH32_update(). + * + * @param statePtr The state struct to reset. + * @param seed The 32-bit seed to alter the hash result predictably. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return @ref XXH_OK on success, @ref XXH_ERROR on failure. + */ +XXH_PUBLIC_API XXH_errorcode XXH32_reset (XXH32_state_t* statePtr, XXH32_hash_t seed); + +/*! + * @brief Consumes a block of @p input to an @ref XXH32_state_t. + * + * Call this to incrementally consume blocks of data. + * + * @param statePtr The state struct to update. + * @param input The block of data to be hashed, at least @p length bytes in size. + * @param length The length of @p input, in bytes. + * + * @pre + * @p statePtr must not be `NULL`. + * @pre + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * @return @ref XXH_OK on success, @ref XXH_ERROR on failure. + */ +XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* statePtr, const void* input, size_t length); + +/*! + * @brief Returns the calculated hash value from an @ref XXH32_state_t. + * + * @note + * Calling XXH32_digest() will not affect @p statePtr, so you can update, + * digest, and update again. + * + * @param statePtr The state struct to calculate the hash from. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return The calculated xxHash32 value from that state. + */ +XXH_PUBLIC_API XXH32_hash_t XXH32_digest (const XXH32_state_t* statePtr); + +/******* Canonical representation *******/ + +/* + * The default return values from XXH functions are unsigned 32 and 64 bit + * integers. + * This the simplest and fastest format for further post-processing. + * + * However, this leaves open the question of what is the order on the byte level, + * since little and big endian conventions will store the same number differently. + * + * The canonical representation settles this issue by mandating big-endian + * convention, the same convention as human-readable numbers (large digits first). + * + * When writing hash values to storage, sending them over a network, or printing + * them, it's highly recommended to use the canonical representation to ensure + * portability across a wider range of systems, present and future. + * + * The following functions allow transformation of hash values to and from + * canonical format. + */ + +/*! + * @brief Canonical (big endian) representation of @ref XXH32_hash_t. + */ +typedef struct { + unsigned char digest[4]; /*!< Hash bytes, big endian */ +} XXH32_canonical_t; + +/*! + * @brief Converts an @ref XXH32_hash_t to a big endian @ref XXH32_canonical_t. + * + * @param dst The @ref XXH32_canonical_t pointer to be stored to. + * @param hash The @ref XXH32_hash_t to be converted. + * + * @pre + * @p dst must not be `NULL`. + */ +XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash); + +/*! + * @brief Converts an @ref XXH32_canonical_t to a native @ref XXH32_hash_t. + * + * @param src The @ref XXH32_canonical_t to convert. + * + * @pre + * @p src must not be `NULL`. + * + * @return The converted hash. + */ +XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src); + + +#ifdef __has_attribute +# define XXH_HAS_ATTRIBUTE(x) __has_attribute(x) +#else +# define XXH_HAS_ATTRIBUTE(x) 0 +#endif + +/* C-language Attributes are added in C23. */ +#if defined(__STDC_VERSION__) && (__STDC_VERSION__ > 201710L) && defined(__has_c_attribute) +# define XXH_HAS_C_ATTRIBUTE(x) __has_c_attribute(x) +#else +# define XXH_HAS_C_ATTRIBUTE(x) 0 +#endif + +#if defined(__cplusplus) && defined(__has_cpp_attribute) +# define XXH_HAS_CPP_ATTRIBUTE(x) __has_cpp_attribute(x) +#else +# define XXH_HAS_CPP_ATTRIBUTE(x) 0 +#endif + +/* +Define XXH_FALLTHROUGH macro for annotating switch case with the 'fallthrough' attribute +introduced in CPP17 and C23. +CPP17 : https://en.cppreference.com/w/cpp/language/attributes/fallthrough +C23 : https://en.cppreference.com/w/c/language/attributes/fallthrough +*/ +#if XXH_HAS_C_ATTRIBUTE(x) +# define XXH_FALLTHROUGH [[fallthrough]] +#elif XXH_HAS_CPP_ATTRIBUTE(x) +# define XXH_FALLTHROUGH [[fallthrough]] +#elif XXH_HAS_ATTRIBUTE(__fallthrough__) +# define XXH_FALLTHROUGH __attribute__ ((fallthrough)) +#else +# define XXH_FALLTHROUGH +#endif + +/*! + * @} + * @ingroup public + * @{ + */ + +#ifndef XXH_NO_LONG_LONG +/*-********************************************************************** +* 64-bit hash +************************************************************************/ +#if defined(XXH_DOXYGEN) /* don't include */ +/*! + * @brief An unsigned 64-bit integer. + * + * Not necessarily defined to `uint64_t` but functionally equivalent. + */ +typedef uint64_t XXH64_hash_t; +#elif !defined (__VMS) \ + && (defined (__cplusplus) \ + || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) +# include + typedef uint64_t XXH64_hash_t; +#else +# include +# if defined(__LP64__) && ULONG_MAX == 0xFFFFFFFFFFFFFFFFULL + /* LP64 ABI says uint64_t is unsigned long */ + typedef unsigned long XXH64_hash_t; +# else + /* the following type must have a width of 64-bit */ + typedef unsigned long long XXH64_hash_t; +# endif +#endif + +/*! + * @} + * + * @defgroup xxh64_family XXH64 family + * @ingroup public + * @{ + * Contains functions used in the classic 64-bit xxHash algorithm. + * + * @note + * XXH3 provides competitive speed for both 32-bit and 64-bit systems, + * and offers true 64/128 bit hash results. + * It provides better speed for systems with vector processing capabilities. + */ + + +/*! + * @brief Calculates the 64-bit hash of @p input using xxHash64. + * + * This function usually runs faster on 64-bit systems, but slower on 32-bit + * systems (see benchmark). + * + * @param input The block of data to be hashed, at least @p length bytes in size. + * @param length The length of @p input, in bytes. + * @param seed The 64-bit seed to alter the hash's output predictably. + * + * @pre + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * @return The calculated 64-bit hash. + * + * @see + * XXH32(), XXH3_64bits_withSeed(), XXH3_128bits_withSeed(), XXH128(): + * Direct equivalents for the other variants of xxHash. + * @see + * XXH64_createState(), XXH64_update(), XXH64_digest(): Streaming version. + */ +XXH_PUBLIC_API XXH64_hash_t XXH64(const void* input, size_t length, XXH64_hash_t seed); + +/******* Streaming *******/ +/*! + * @brief The opaque state struct for the XXH64 streaming API. + * + * @see XXH64_state_s for details. + */ +typedef struct XXH64_state_s XXH64_state_t; /* incomplete type */ +XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void); +XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr); +XXH_PUBLIC_API void XXH64_copyState(XXH64_state_t* dst_state, const XXH64_state_t* src_state); + +XXH_PUBLIC_API XXH_errorcode XXH64_reset (XXH64_state_t* statePtr, XXH64_hash_t seed); +XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH64_state_t* statePtr, const void* input, size_t length); +XXH_PUBLIC_API XXH64_hash_t XXH64_digest (const XXH64_state_t* statePtr); + +/******* Canonical representation *******/ +typedef struct { unsigned char digest[sizeof(XXH64_hash_t)]; } XXH64_canonical_t; +XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t* dst, XXH64_hash_t hash); +XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src); + +#ifndef XXH_NO_XXH3 +/*! + * @} + * ************************************************************************ + * @defgroup xxh3_family XXH3 family + * @ingroup public + * @{ + * + * XXH3 is a more recent hash algorithm featuring: + * - Improved speed for both small and large inputs + * - True 64-bit and 128-bit outputs + * - SIMD acceleration + * - Improved 32-bit viability + * + * Speed analysis methodology is explained here: + * + * https://fastcompression.blogspot.com/2019/03/presenting-xxh3.html + * + * Compared to XXH64, expect XXH3 to run approximately + * ~2x faster on large inputs and >3x faster on small ones, + * exact differences vary depending on platform. + * + * XXH3's speed benefits greatly from SIMD and 64-bit arithmetic, + * but does not require it. + * Any 32-bit and 64-bit targets that can run XXH32 smoothly + * can run XXH3 at competitive speeds, even without vector support. + * Further details are explained in the implementation. + * + * Optimized implementations are provided for AVX512, AVX2, SSE2, NEON, POWER8, + * ZVector and scalar targets. This can be controlled via the XXH_VECTOR macro. + * + * XXH3 implementation is portable: + * it has a generic C90 formulation that can be compiled on any platform, + * all implementations generage exactly the same hash value on all platforms. + * Starting from v0.8.0, it's also labelled "stable", meaning that + * any future version will also generate the same hash value. + * + * XXH3 offers 2 variants, _64bits and _128bits. + * + * When only 64 bits are needed, prefer invoking the _64bits variant, as it + * reduces the amount of mixing, resulting in faster speed on small inputs. + * It's also generally simpler to manipulate a scalar return type than a struct. + * + * The API supports one-shot hashing, streaming mode, and custom secrets. + */ + +/*-********************************************************************** +* XXH3 64-bit variant +************************************************************************/ + +/* XXH3_64bits(): + * default 64-bit variant, using default secret and default seed of 0. + * It's the fastest variant. */ +XXH_PUBLIC_API XXH64_hash_t XXH3_64bits(const void* data, size_t len); + +/* + * XXH3_64bits_withSeed(): + * This variant generates a custom secret on the fly + * based on default secret altered using the `seed` value. + * While this operation is decently fast, note that it's not completely free. + * Note: seed==0 produces the same results as XXH3_64bits(). + */ +XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_withSeed(const void* data, size_t len, XXH64_hash_t seed); + +/*! + * The bare minimum size for a custom secret. + * + * @see + * XXH3_64bits_withSecret(), XXH3_64bits_reset_withSecret(), + * XXH3_128bits_withSecret(), XXH3_128bits_reset_withSecret(). + */ +#define XXH3_SECRET_SIZE_MIN 136 + +/* + * XXH3_64bits_withSecret(): + * It's possible to provide any blob of bytes as a "secret" to generate the hash. + * This makes it more difficult for an external actor to prepare an intentional collision. + * The main condition is that secretSize *must* be large enough (>= XXH3_SECRET_SIZE_MIN). + * However, the quality of the secret impacts the dispersion of the hash algorithm. + * Therefore, the secret _must_ look like a bunch of random bytes. + * Avoid "trivial" or structured data such as repeated sequences or a text document. + * Whenever in doubt about the "randomness" of the blob of bytes, + * consider employing "XXH3_generateSecret()" instead (see below). + * It will generate a proper high entropy secret derived from the blob of bytes. + * Another advantage of using XXH3_generateSecret() is that + * it guarantees that all bits within the initial blob of bytes + * will impact every bit of the output. + * This is not necessarily the case when using the blob of bytes directly + * because, when hashing _small_ inputs, only a portion of the secret is employed. + */ +XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_withSecret(const void* data, size_t len, const void* secret, size_t secretSize); + + +/******* Streaming *******/ +/* + * Streaming requires state maintenance. + * This operation costs memory and CPU. + * As a consequence, streaming is slower than one-shot hashing. + * For better performance, prefer one-shot functions whenever applicable. + */ + +/*! + * @brief The state struct for the XXH3 streaming API. + * + * @see XXH3_state_s for details. + */ +typedef struct XXH3_state_s XXH3_state_t; +XXH_PUBLIC_API XXH3_state_t* XXH3_createState(void); +XXH_PUBLIC_API XXH_errorcode XXH3_freeState(XXH3_state_t* statePtr); +XXH_PUBLIC_API void XXH3_copyState(XXH3_state_t* dst_state, const XXH3_state_t* src_state); + +/* + * XXH3_64bits_reset(): + * Initialize with default parameters. + * digest will be equivalent to `XXH3_64bits()`. + */ +XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset(XXH3_state_t* statePtr); +/* + * XXH3_64bits_reset_withSeed(): + * Generate a custom secret from `seed`, and store it into `statePtr`. + * digest will be equivalent to `XXH3_64bits_withSeed()`. + */ +XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset_withSeed(XXH3_state_t* statePtr, XXH64_hash_t seed); +/* + * XXH3_64bits_reset_withSecret(): + * `secret` is referenced, it _must outlive_ the hash streaming session. + * Similar to one-shot API, `secretSize` must be >= `XXH3_SECRET_SIZE_MIN`, + * and the quality of produced hash values depends on secret's entropy + * (secret's content should look like a bunch of random bytes). + * When in doubt about the randomness of a candidate `secret`, + * consider employing `XXH3_generateSecret()` instead (see below). + */ +XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset_withSecret(XXH3_state_t* statePtr, const void* secret, size_t secretSize); + +XXH_PUBLIC_API XXH_errorcode XXH3_64bits_update (XXH3_state_t* statePtr, const void* input, size_t length); +XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_digest (const XXH3_state_t* statePtr); + +/* note : canonical representation of XXH3 is the same as XXH64 + * since they both produce XXH64_hash_t values */ + + +/*-********************************************************************** +* XXH3 128-bit variant +************************************************************************/ + +/*! + * @brief The return value from 128-bit hashes. + * + * Stored in little endian order, although the fields themselves are in native + * endianness. + */ +typedef struct { + XXH64_hash_t low64; /*!< `value & 0xFFFFFFFFFFFFFFFF` */ + XXH64_hash_t high64; /*!< `value >> 64` */ +} XXH128_hash_t; + +XXH_PUBLIC_API XXH128_hash_t XXH3_128bits(const void* data, size_t len); +XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_withSeed(const void* data, size_t len, XXH64_hash_t seed); +XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_withSecret(const void* data, size_t len, const void* secret, size_t secretSize); + +/******* Streaming *******/ +/* + * Streaming requires state maintenance. + * This operation costs memory and CPU. + * As a consequence, streaming is slower than one-shot hashing. + * For better performance, prefer one-shot functions whenever applicable. + * + * XXH3_128bits uses the same XXH3_state_t as XXH3_64bits(). + * Use already declared XXH3_createState() and XXH3_freeState(). + * + * All reset and streaming functions have same meaning as their 64-bit counterpart. + */ + +XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset(XXH3_state_t* statePtr); +XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset_withSeed(XXH3_state_t* statePtr, XXH64_hash_t seed); +XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset_withSecret(XXH3_state_t* statePtr, const void* secret, size_t secretSize); + +XXH_PUBLIC_API XXH_errorcode XXH3_128bits_update (XXH3_state_t* statePtr, const void* input, size_t length); +XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_digest (const XXH3_state_t* statePtr); + +/* Following helper functions make it possible to compare XXH128_hast_t values. + * Since XXH128_hash_t is a structure, this capability is not offered by the language. + * Note: For better performance, these functions can be inlined using XXH_INLINE_ALL */ + +/*! + * XXH128_isEqual(): + * Return: 1 if `h1` and `h2` are equal, 0 if they are not. + */ +XXH_PUBLIC_API int XXH128_isEqual(XXH128_hash_t h1, XXH128_hash_t h2); + +/*! + * XXH128_cmp(): + * + * This comparator is compatible with stdlib's `qsort()`/`bsearch()`. + * + * return: >0 if *h128_1 > *h128_2 + * =0 if *h128_1 == *h128_2 + * <0 if *h128_1 < *h128_2 + */ +XXH_PUBLIC_API int XXH128_cmp(const void* h128_1, const void* h128_2); + + +/******* Canonical representation *******/ +typedef struct { unsigned char digest[sizeof(XXH128_hash_t)]; } XXH128_canonical_t; +XXH_PUBLIC_API void XXH128_canonicalFromHash(XXH128_canonical_t* dst, XXH128_hash_t hash); +XXH_PUBLIC_API XXH128_hash_t XXH128_hashFromCanonical(const XXH128_canonical_t* src); + + +#endif /* !XXH_NO_XXH3 */ +#endif /* XXH_NO_LONG_LONG */ + +/*! + * @} + */ +#endif /* XXHASH_H_5627135585666179 */ + + + +#if defined(XXH_STATIC_LINKING_ONLY) && !defined(XXHASH_H_STATIC_13879238742) +#define XXHASH_H_STATIC_13879238742 +/* **************************************************************************** + * This section contains declarations which are not guaranteed to remain stable. + * They may change in future versions, becoming incompatible with a different + * version of the library. + * These declarations should only be used with static linking. + * Never use them in association with dynamic linking! + ***************************************************************************** */ + +/* + * These definitions are only present to allow static allocation + * of XXH states, on stack or in a struct, for example. + * Never **ever** access their members directly. + */ + +/*! + * @internal + * @brief Structure for XXH32 streaming API. + * + * @note This is only defined when @ref XXH_STATIC_LINKING_ONLY, + * @ref XXH_INLINE_ALL, or @ref XXH_IMPLEMENTATION is defined. Otherwise it is + * an opaque type. This allows fields to safely be changed. + * + * Typedef'd to @ref XXH32_state_t. + * Do not access the members of this struct directly. + * @see XXH64_state_s, XXH3_state_s + */ +struct XXH32_state_s { + XXH32_hash_t total_len_32; /*!< Total length hashed, modulo 2^32 */ + XXH32_hash_t large_len; /*!< Whether the hash is >= 16 (handles @ref total_len_32 overflow) */ + XXH32_hash_t v[4]; /*!< Accumulator lanes */ + XXH32_hash_t mem32[4]; /*!< Internal buffer for partial reads. Treated as unsigned char[16]. */ + XXH32_hash_t memsize; /*!< Amount of data in @ref mem32 */ + XXH32_hash_t reserved; /*!< Reserved field. Do not read nor write to it. */ +}; /* typedef'd to XXH32_state_t */ + + +#ifndef XXH_NO_LONG_LONG /* defined when there is no 64-bit support */ + +/*! + * @internal + * @brief Structure for XXH64 streaming API. + * + * @note This is only defined when @ref XXH_STATIC_LINKING_ONLY, + * @ref XXH_INLINE_ALL, or @ref XXH_IMPLEMENTATION is defined. Otherwise it is + * an opaque type. This allows fields to safely be changed. + * + * Typedef'd to @ref XXH64_state_t. + * Do not access the members of this struct directly. + * @see XXH32_state_s, XXH3_state_s + */ +struct XXH64_state_s { + XXH64_hash_t total_len; /*!< Total length hashed. This is always 64-bit. */ + XXH64_hash_t v[4]; /*!< Accumulator lanes */ + XXH64_hash_t mem64[4]; /*!< Internal buffer for partial reads. Treated as unsigned char[32]. */ + XXH32_hash_t memsize; /*!< Amount of data in @ref mem64 */ + XXH32_hash_t reserved32; /*!< Reserved field, needed for padding anyways*/ + XXH64_hash_t reserved64; /*!< Reserved field. Do not read or write to it. */ +}; /* typedef'd to XXH64_state_t */ + + +#ifndef XXH_NO_XXH3 + +#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* >= C11 */ +# include +# define XXH_ALIGN(n) alignas(n) +#elif defined(__cplusplus) && (__cplusplus >= 201103L) /* >= C++11 */ +/* In C++ alignas() is a keyword */ +# define XXH_ALIGN(n) alignas(n) +#elif defined(__GNUC__) +# define XXH_ALIGN(n) __attribute__ ((aligned(n))) +#elif defined(_MSC_VER) +# define XXH_ALIGN(n) __declspec(align(n)) +#else +# define XXH_ALIGN(n) /* disabled */ +#endif + +/* Old GCC versions only accept the attribute after the type in structures. */ +#if !(defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L)) /* C11+ */ \ + && ! (defined(__cplusplus) && (__cplusplus >= 201103L)) /* >= C++11 */ \ + && defined(__GNUC__) +# define XXH_ALIGN_MEMBER(align, type) type XXH_ALIGN(align) +#else +# define XXH_ALIGN_MEMBER(align, type) XXH_ALIGN(align) type +#endif + +/*! + * @brief The size of the internal XXH3 buffer. + * + * This is the optimal update size for incremental hashing. + * + * @see XXH3_64b_update(), XXH3_128b_update(). + */ +#define XXH3_INTERNALBUFFER_SIZE 256 + +/*! + * @brief Default size of the secret buffer (and @ref XXH3_kSecret). + * + * This is the size used in @ref XXH3_kSecret and the seeded functions. + * + * Not to be confused with @ref XXH3_SECRET_SIZE_MIN. + */ +#define XXH3_SECRET_DEFAULT_SIZE 192 + +/*! + * @internal + * @brief Structure for XXH3 streaming API. + * + * @note This is only defined when @ref XXH_STATIC_LINKING_ONLY, + * @ref XXH_INLINE_ALL, or @ref XXH_IMPLEMENTATION is defined. + * Otherwise it is an opaque type. + * Never use this definition in combination with dynamic library. + * This allows fields to safely be changed in the future. + * + * @note ** This structure has a strict alignment requirement of 64 bytes!! ** + * Do not allocate this with `malloc()` or `new`, + * it will not be sufficiently aligned. + * Use @ref XXH3_createState() and @ref XXH3_freeState(), or stack allocation. + * + * Typedef'd to @ref XXH3_state_t. + * Do never access the members of this struct directly. + * + * @see XXH3_INITSTATE() for stack initialization. + * @see XXH3_createState(), XXH3_freeState(). + * @see XXH32_state_s, XXH64_state_s + */ +struct XXH3_state_s { + XXH_ALIGN_MEMBER(64, XXH64_hash_t acc[8]); + /*!< The 8 accumulators. Similar to `vN` in @ref XXH32_state_s::v1 and @ref XXH64_state_s */ + XXH_ALIGN_MEMBER(64, unsigned char customSecret[XXH3_SECRET_DEFAULT_SIZE]); + /*!< Used to store a custom secret generated from a seed. */ + XXH_ALIGN_MEMBER(64, unsigned char buffer[XXH3_INTERNALBUFFER_SIZE]); + /*!< The internal buffer. @see XXH32_state_s::mem32 */ + XXH32_hash_t bufferedSize; + /*!< The amount of memory in @ref buffer, @see XXH32_state_s::memsize */ + XXH32_hash_t useSeed; + /*!< Reserved field. Needed for padding on 64-bit. */ + size_t nbStripesSoFar; + /*!< Number or stripes processed. */ + XXH64_hash_t totalLen; + /*!< Total length hashed. 64-bit even on 32-bit targets. */ + size_t nbStripesPerBlock; + /*!< Number of stripes per block. */ + size_t secretLimit; + /*!< Size of @ref customSecret or @ref extSecret */ + XXH64_hash_t seed; + /*!< Seed for _withSeed variants. Must be zero otherwise, @see XXH3_INITSTATE() */ + XXH64_hash_t reserved64; + /*!< Reserved field. */ + const unsigned char* extSecret; + /*!< Reference to an external secret for the _withSecret variants, NULL + * for other variants. */ + /* note: there may be some padding at the end due to alignment on 64 bytes */ +}; /* typedef'd to XXH3_state_t */ + +#undef XXH_ALIGN_MEMBER + +/*! + * @brief Initializes a stack-allocated `XXH3_state_s`. + * + * When the @ref XXH3_state_t structure is merely emplaced on stack, + * it should be initialized with XXH3_INITSTATE() or a memset() + * in case its first reset uses XXH3_NNbits_reset_withSeed(). + * This init can be omitted if the first reset uses default or _withSecret mode. + * This operation isn't necessary when the state is created with XXH3_createState(). + * Note that this doesn't prepare the state for a streaming operation, + * it's still necessary to use XXH3_NNbits_reset*() afterwards. + */ +#define XXH3_INITSTATE(XXH3_state_ptr) { (XXH3_state_ptr)->seed = 0; } + + +/* XXH128() : + * simple alias to pre-selected XXH3_128bits variant + */ +XXH_PUBLIC_API XXH128_hash_t XXH128(const void* data, size_t len, XXH64_hash_t seed); + + +/* === Experimental API === */ +/* Symbols defined below must be considered tied to a specific library version. */ + +/* + * XXH3_generateSecret(): + * + * Derive a high-entropy secret from any user-defined content, named customSeed. + * The generated secret can be used in combination with `*_withSecret()` functions. + * The `_withSecret()` variants are useful to provide a higher level of protection than 64-bit seed, + * as it becomes much more difficult for an external actor to guess how to impact the calculation logic. + * + * The function accepts as input a custom seed of any length and any content, + * and derives from it a high-entropy secret of length @secretSize + * into an already allocated buffer @secretBuffer. + * @secretSize must be >= XXH3_SECRET_SIZE_MIN + * + * The generated secret can then be used with any `*_withSecret()` variant. + * Functions `XXH3_128bits_withSecret()`, `XXH3_64bits_withSecret()`, + * `XXH3_128bits_reset_withSecret()` and `XXH3_64bits_reset_withSecret()` + * are part of this list. They all accept a `secret` parameter + * which must be large enough for implementation reasons (>= XXH3_SECRET_SIZE_MIN) + * _and_ feature very high entropy (consist of random-looking bytes). + * These conditions can be a high bar to meet, so + * XXH3_generateSecret() can be employed to ensure proper quality. + * + * customSeed can be anything. It can have any size, even small ones, + * and its content can be anything, even "poor entropy" sources such as a bunch of zeroes. + * The resulting `secret` will nonetheless provide all required qualities. + * + * When customSeedSize > 0, supplying NULL as customSeed is undefined behavior. + */ +XXH_PUBLIC_API XXH_errorcode XXH3_generateSecret(void* secretBuffer, size_t secretSize, const void* customSeed, size_t customSeedSize); + + +/* + * XXH3_generateSecret_fromSeed(): + * + * Generate the same secret as the _withSeed() variants. + * + * The resulting secret has a length of XXH3_SECRET_DEFAULT_SIZE (necessarily). + * @secretBuffer must be already allocated, of size at least XXH3_SECRET_DEFAULT_SIZE bytes. + * + * The generated secret can be used in combination with + *`*_withSecret()` and `_withSecretandSeed()` variants. + * This generator is notably useful in combination with `_withSecretandSeed()`, + * as a way to emulate a faster `_withSeed()` variant. + */ +XXH_PUBLIC_API void XXH3_generateSecret_fromSeed(void* secretBuffer, XXH64_hash_t seed); + +/* + * *_withSecretandSeed() : + * These variants generate hash values using either + * @seed for "short" keys (< XXH3_MIDSIZE_MAX = 240 bytes) + * or @secret for "large" keys (>= XXH3_MIDSIZE_MAX). + * + * This generally benefits speed, compared to `_withSeed()` or `_withSecret()`. + * `_withSeed()` has to generate the secret on the fly for "large" keys. + * It's fast, but can be perceptible for "not so large" keys (< 1 KB). + * `_withSecret()` has to generate the masks on the fly for "small" keys, + * which requires more instructions than _withSeed() variants. + * Therefore, _withSecretandSeed variant combines the best of both worlds. + * + * When @secret has been generated by XXH3_generateSecret_fromSeed(), + * this variant produces *exactly* the same results as `_withSeed()` variant, + * hence offering only a pure speed benefit on "large" input, + * by skipping the need to regenerate the secret for every large input. + * + * Another usage scenario is to hash the secret to a 64-bit hash value, + * for example with XXH3_64bits(), which then becomes the seed, + * and then employ both the seed and the secret in _withSecretandSeed(). + * On top of speed, an added benefit is that each bit in the secret + * has a 50% chance to swap each bit in the output, + * via its impact to the seed. + * This is not guaranteed when using the secret directly in "small data" scenarios, + * because only portions of the secret are employed for small data. + */ +XXH_PUBLIC_API XXH64_hash_t +XXH3_64bits_withSecretandSeed(const void* data, size_t len, + const void* secret, size_t secretSize, + XXH64_hash_t seed); + +XXH_PUBLIC_API XXH128_hash_t +XXH3_128bits_withSecretandSeed(const void* data, size_t len, + const void* secret, size_t secretSize, + XXH64_hash_t seed64); + +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_reset_withSecretandSeed(XXH3_state_t* statePtr, + const void* secret, size_t secretSize, + XXH64_hash_t seed64); + +XXH_PUBLIC_API XXH_errorcode +XXH3_128bits_reset_withSecretandSeed(XXH3_state_t* statePtr, + const void* secret, size_t secretSize, + XXH64_hash_t seed64); + + +#endif /* XXH_NO_XXH3 */ +#endif /* XXH_NO_LONG_LONG */ +#if defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API) +# define XXH_IMPLEMENTATION +#endif + +#endif /* defined(XXH_STATIC_LINKING_ONLY) && !defined(XXHASH_H_STATIC_13879238742) */ + + +/* ======================================================================== */ +/* ======================================================================== */ +/* ======================================================================== */ + + +/*-********************************************************************** + * xxHash implementation + *-********************************************************************** + * xxHash's implementation used to be hosted inside xxhash.c. + * + * However, inlining requires implementation to be visible to the compiler, + * hence be included alongside the header. + * Previously, implementation was hosted inside xxhash.c, + * which was then #included when inlining was activated. + * This construction created issues with a few build and install systems, + * as it required xxhash.c to be stored in /include directory. + * + * xxHash implementation is now directly integrated within xxhash.h. + * As a consequence, xxhash.c is no longer needed in /include. + * + * xxhash.c is still available and is still useful. + * In a "normal" setup, when xxhash is not inlined, + * xxhash.h only exposes the prototypes and public symbols, + * while xxhash.c can be built into an object file xxhash.o + * which can then be linked into the final binary. + ************************************************************************/ + +#if ( defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API) \ + || defined(XXH_IMPLEMENTATION) ) && !defined(XXH_IMPLEM_13a8737387) +# define XXH_IMPLEM_13a8737387 + +/* ************************************* +* Tuning parameters +***************************************/ + +/*! + * @defgroup tuning Tuning parameters + * @{ + * + * Various macros to control xxHash's behavior. + */ +#ifdef XXH_DOXYGEN +/*! + * @brief Define this to disable 64-bit code. + * + * Useful if only using the @ref xxh32_family and you have a strict C90 compiler. + */ +# define XXH_NO_LONG_LONG +# undef XXH_NO_LONG_LONG /* don't actually */ +/*! + * @brief Controls how unaligned memory is accessed. + * + * By default, access to unaligned memory is controlled by `memcpy()`, which is + * safe and portable. + * + * Unfortunately, on some target/compiler combinations, the generated assembly + * is sub-optimal. + * + * The below switch allow selection of a different access method + * in the search for improved performance. + * + * @par Possible options: + * + * - `XXH_FORCE_MEMORY_ACCESS=0` (default): `memcpy` + * @par + * Use `memcpy()`. Safe and portable. Note that most modern compilers will + * eliminate the function call and treat it as an unaligned access. + * + * - `XXH_FORCE_MEMORY_ACCESS=1`: `__attribute__((packed))` + * @par + * Depends on compiler extensions and is therefore not portable. + * This method is safe _if_ your compiler supports it, + * and *generally* as fast or faster than `memcpy`. + * + * - `XXH_FORCE_MEMORY_ACCESS=2`: Direct cast + * @par + * Casts directly and dereferences. This method doesn't depend on the + * compiler, but it violates the C standard as it directly dereferences an + * unaligned pointer. It can generate buggy code on targets which do not + * support unaligned memory accesses, but in some circumstances, it's the + * only known way to get the most performance. + * + * - `XXH_FORCE_MEMORY_ACCESS=3`: Byteshift + * @par + * Also portable. This can generate the best code on old compilers which don't + * inline small `memcpy()` calls, and it might also be faster on big-endian + * systems which lack a native byteswap instruction. However, some compilers + * will emit literal byteshifts even if the target supports unaligned access. + * . + * + * @warning + * Methods 1 and 2 rely on implementation-defined behavior. Use these with + * care, as what works on one compiler/platform/optimization level may cause + * another to read garbage data or even crash. + * + * See http://fastcompression.blogspot.com/2015/08/accessing-unaligned-memory.html for details. + * + * Prefer these methods in priority order (0 > 3 > 1 > 2) + */ +# define XXH_FORCE_MEMORY_ACCESS 0 + +/*! + * @def XXH_FORCE_ALIGN_CHECK + * @brief If defined to non-zero, adds a special path for aligned inputs (XXH32() + * and XXH64() only). + * + * This is an important performance trick for architectures without decent + * unaligned memory access performance. + * + * It checks for input alignment, and when conditions are met, uses a "fast + * path" employing direct 32-bit/64-bit reads, resulting in _dramatically + * faster_ read speed. + * + * The check costs one initial branch per hash, which is generally negligible, + * but not zero. + * + * Moreover, it's not useful to generate an additional code path if memory + * access uses the same instruction for both aligned and unaligned + * addresses (e.g. x86 and aarch64). + * + * In these cases, the alignment check can be removed by setting this macro to 0. + * Then the code will always use unaligned memory access. + * Align check is automatically disabled on x86, x64 & arm64, + * which are platforms known to offer good unaligned memory accesses performance. + * + * This option does not affect XXH3 (only XXH32 and XXH64). + */ +# define XXH_FORCE_ALIGN_CHECK 0 + +/*! + * @def XXH_NO_INLINE_HINTS + * @brief When non-zero, sets all functions to `static`. + * + * By default, xxHash tries to force the compiler to inline almost all internal + * functions. + * + * This can usually improve performance due to reduced jumping and improved + * constant folding, but significantly increases the size of the binary which + * might not be favorable. + * + * Additionally, sometimes the forced inlining can be detrimental to performance, + * depending on the architecture. + * + * XXH_NO_INLINE_HINTS marks all internal functions as static, giving the + * compiler full control on whether to inline or not. + * + * When not optimizing (-O0), optimizing for size (-Os, -Oz), or using + * -fno-inline with GCC or Clang, this will automatically be defined. + */ +# define XXH_NO_INLINE_HINTS 0 + +/*! + * @def XXH32_ENDJMP + * @brief Whether to use a jump for `XXH32_finalize`. + * + * For performance, `XXH32_finalize` uses multiple branches in the finalizer. + * This is generally preferable for performance, + * but depending on exact architecture, a jmp may be preferable. + * + * This setting is only possibly making a difference for very small inputs. + */ +# define XXH32_ENDJMP 0 + +/*! + * @internal + * @brief Redefines old internal names. + * + * For compatibility with code that uses xxHash's internals before the names + * were changed to improve namespacing. There is no other reason to use this. + */ +# define XXH_OLD_NAMES +# undef XXH_OLD_NAMES /* don't actually use, it is ugly. */ +#endif /* XXH_DOXYGEN */ +/*! + * @} + */ + +#ifndef XXH_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */ + /* prefer __packed__ structures (method 1) for gcc on armv7+ and mips */ +# if !defined(__clang__) && \ +( \ + (defined(__INTEL_COMPILER) && !defined(_WIN32)) || \ + ( \ + defined(__GNUC__) && ( \ + (defined(__ARM_ARCH) && __ARM_ARCH >= 7) || \ + ( \ + defined(__mips__) && \ + (__mips <= 5 || __mips_isa_rev < 6) && \ + (!defined(__mips16) || defined(__mips_mips16e2)) \ + ) \ + ) \ + ) \ +) +# define XXH_FORCE_MEMORY_ACCESS 1 +# endif +#endif + +#ifndef XXH_FORCE_ALIGN_CHECK /* can be defined externally */ +# if defined(__i386) || defined(__x86_64__) || defined(__aarch64__) \ + || defined(_M_IX86) || defined(_M_X64) || defined(_M_ARM64) /* visual */ +# define XXH_FORCE_ALIGN_CHECK 0 +# else +# define XXH_FORCE_ALIGN_CHECK 1 +# endif +#endif + +#ifndef XXH_NO_INLINE_HINTS +# if defined(__OPTIMIZE_SIZE__) /* -Os, -Oz */ \ + || defined(__NO_INLINE__) /* -O0, -fno-inline */ +# define XXH_NO_INLINE_HINTS 1 +# else +# define XXH_NO_INLINE_HINTS 0 +# endif +#endif + +#ifndef XXH32_ENDJMP +/* generally preferable for performance */ +# define XXH32_ENDJMP 0 +#endif + +/*! + * @defgroup impl Implementation + * @{ + */ + + +/* ************************************* +* Includes & Memory related functions +***************************************/ +/* Modify the local functions below should you wish to use some other memory routines */ +/* for ZSTD_malloc(), ZSTD_free() */ +#define ZSTD_DEPS_NEED_MALLOC +#include "zstd_deps.h" /* size_t, ZSTD_malloc, ZSTD_free, ZSTD_memcpy */ +static void* XXH_malloc(size_t s) { return ZSTD_malloc(s); } +static void XXH_free (void* p) { ZSTD_free(p); } +static void* XXH_memcpy(void* dest, const void* src, size_t size) { return ZSTD_memcpy(dest,src,size); } + + +/* ************************************* +* Compiler Specific Options +***************************************/ +#ifdef _MSC_VER /* Visual Studio warning fix */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +#endif + +#if XXH_NO_INLINE_HINTS /* disable inlining hints */ +# if defined(__GNUC__) || defined(__clang__) +# define XXH_FORCE_INLINE static __attribute__((unused)) +# else +# define XXH_FORCE_INLINE static +# endif +# define XXH_NO_INLINE static +/* enable inlining hints */ +#elif defined(__GNUC__) || defined(__clang__) +# define XXH_FORCE_INLINE static __inline__ __attribute__((always_inline, unused)) +# define XXH_NO_INLINE static __attribute__((noinline)) +#elif defined(_MSC_VER) /* Visual Studio */ +# define XXH_FORCE_INLINE static __forceinline +# define XXH_NO_INLINE static __declspec(noinline) +#elif defined (__cplusplus) \ + || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)) /* C99 */ +# define XXH_FORCE_INLINE static inline +# define XXH_NO_INLINE static +#else +# define XXH_FORCE_INLINE static +# define XXH_NO_INLINE static +#endif + + + +/* ************************************* +* Debug +***************************************/ +/*! + * @ingroup tuning + * @def XXH_DEBUGLEVEL + * @brief Sets the debugging level. + * + * XXH_DEBUGLEVEL is expected to be defined externally, typically via the + * compiler's command line options. The value must be a number. + */ +#ifndef XXH_DEBUGLEVEL +# ifdef DEBUGLEVEL /* backwards compat */ +# define XXH_DEBUGLEVEL DEBUGLEVEL +# else +# define XXH_DEBUGLEVEL 0 +# endif +#endif + +#if (XXH_DEBUGLEVEL>=1) +# include /* note: can still be disabled with NDEBUG */ +# define XXH_ASSERT(c) assert(c) +#else +# define XXH_ASSERT(c) ((void)0) +#endif + +/* note: use after variable declarations */ +#ifndef XXH_STATIC_ASSERT +# if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* C11 */ +# include +# define XXH_STATIC_ASSERT_WITH_MESSAGE(c,m) do { static_assert((c),m); } while(0) +# elif defined(__cplusplus) && (__cplusplus >= 201103L) /* C++11 */ +# define XXH_STATIC_ASSERT_WITH_MESSAGE(c,m) do { static_assert((c),m); } while(0) +# else +# define XXH_STATIC_ASSERT_WITH_MESSAGE(c,m) do { struct xxh_sa { char x[(c) ? 1 : -1]; }; } while(0) +# endif +# define XXH_STATIC_ASSERT(c) XXH_STATIC_ASSERT_WITH_MESSAGE((c),#c) +#endif + +/*! + * @internal + * @def XXH_COMPILER_GUARD(var) + * @brief Used to prevent unwanted optimizations for @p var. + * + * It uses an empty GCC inline assembly statement with a register constraint + * which forces @p var into a general purpose register (eg eax, ebx, ecx + * on x86) and marks it as modified. + * + * This is used in a few places to avoid unwanted autovectorization (e.g. + * XXH32_round()). All vectorization we want is explicit via intrinsics, + * and _usually_ isn't wanted elsewhere. + * + * We also use it to prevent unwanted constant folding for AArch64 in + * XXH3_initCustomSecret_scalar(). + */ +#if defined(__GNUC__) || defined(__clang__) +# define XXH_COMPILER_GUARD(var) __asm__ __volatile__("" : "+r" (var)) +#else +# define XXH_COMPILER_GUARD(var) ((void)0) +#endif + +/* ************************************* +* Basic Types +***************************************/ +#if !defined (__VMS) \ + && (defined (__cplusplus) \ + || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) +# include + typedef uint8_t xxh_u8; +#else + typedef unsigned char xxh_u8; +#endif +typedef XXH32_hash_t xxh_u32; + +#ifdef XXH_OLD_NAMES +# define BYTE xxh_u8 +# define U8 xxh_u8 +# define U32 xxh_u32 +#endif + +/* *** Memory access *** */ + +/*! + * @internal + * @fn xxh_u32 XXH_read32(const void* ptr) + * @brief Reads an unaligned 32-bit integer from @p ptr in native endianness. + * + * Affected by @ref XXH_FORCE_MEMORY_ACCESS. + * + * @param ptr The pointer to read from. + * @return The 32-bit native endian integer from the bytes at @p ptr. + */ + +/*! + * @internal + * @fn xxh_u32 XXH_readLE32(const void* ptr) + * @brief Reads an unaligned 32-bit little endian integer from @p ptr. + * + * Affected by @ref XXH_FORCE_MEMORY_ACCESS. + * + * @param ptr The pointer to read from. + * @return The 32-bit little endian integer from the bytes at @p ptr. + */ + +/*! + * @internal + * @fn xxh_u32 XXH_readBE32(const void* ptr) + * @brief Reads an unaligned 32-bit big endian integer from @p ptr. + * + * Affected by @ref XXH_FORCE_MEMORY_ACCESS. + * + * @param ptr The pointer to read from. + * @return The 32-bit big endian integer from the bytes at @p ptr. + */ + +/*! + * @internal + * @fn xxh_u32 XXH_readLE32_align(const void* ptr, XXH_alignment align) + * @brief Like @ref XXH_readLE32(), but has an option for aligned reads. + * + * Affected by @ref XXH_FORCE_MEMORY_ACCESS. + * Note that when @ref XXH_FORCE_ALIGN_CHECK == 0, the @p align parameter is + * always @ref XXH_alignment::XXH_unaligned. + * + * @param ptr The pointer to read from. + * @param align Whether @p ptr is aligned. + * @pre + * If @p align == @ref XXH_alignment::XXH_aligned, @p ptr must be 4 byte + * aligned. + * @return The 32-bit little endian integer from the bytes at @p ptr. + */ + +#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3)) +/* + * Manual byteshift. Best for old compilers which don't inline memcpy. + * We actually directly use XXH_readLE32 and XXH_readBE32. + */ +#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2)) + +/* + * Force direct memory access. Only works on CPU which support unaligned memory + * access in hardware. + */ +static xxh_u32 XXH_read32(const void* memPtr) { return *(const xxh_u32*) memPtr; } + +#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1)) + +/* + * __pack instructions are safer but compiler specific, hence potentially + * problematic for some compilers. + * + * Currently only defined for GCC and ICC. + */ +#ifdef XXH_OLD_NAMES +typedef union { xxh_u32 u32; } __attribute__((packed)) unalign; +#endif +static xxh_u32 XXH_read32(const void* ptr) +{ + typedef union { xxh_u32 u32; } __attribute__((packed)) xxh_unalign; + return ((const xxh_unalign*)ptr)->u32; +} + +#else + +/* + * Portable and safe solution. Generally efficient. + * see: http://fastcompression.blogspot.com/2015/08/accessing-unaligned-memory.html + */ +static xxh_u32 XXH_read32(const void* memPtr) +{ + xxh_u32 val; + XXH_memcpy(&val, memPtr, sizeof(val)); + return val; +} + +#endif /* XXH_FORCE_DIRECT_MEMORY_ACCESS */ + + +/* *** Endianness *** */ + +/*! + * @ingroup tuning + * @def XXH_CPU_LITTLE_ENDIAN + * @brief Whether the target is little endian. + * + * Defined to 1 if the target is little endian, or 0 if it is big endian. + * It can be defined externally, for example on the compiler command line. + * + * If it is not defined, + * a runtime check (which is usually constant folded) is used instead. + * + * @note + * This is not necessarily defined to an integer constant. + * + * @see XXH_isLittleEndian() for the runtime check. + */ +#ifndef XXH_CPU_LITTLE_ENDIAN +/* + * Try to detect endianness automatically, to avoid the nonstandard behavior + * in `XXH_isLittleEndian()` + */ +# if defined(_WIN32) /* Windows is always little endian */ \ + || defined(__LITTLE_ENDIAN__) \ + || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) +# define XXH_CPU_LITTLE_ENDIAN 1 +# elif defined(__BIG_ENDIAN__) \ + || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) +# define XXH_CPU_LITTLE_ENDIAN 0 +# else +/*! + * @internal + * @brief Runtime check for @ref XXH_CPU_LITTLE_ENDIAN. + * + * Most compilers will constant fold this. + */ +static int XXH_isLittleEndian(void) +{ + /* + * Portable and well-defined behavior. + * Don't use static: it is detrimental to performance. + */ + const union { xxh_u32 u; xxh_u8 c[4]; } one = { 1 }; + return one.c[0]; +} +# define XXH_CPU_LITTLE_ENDIAN XXH_isLittleEndian() +# endif +#endif + + + + +/* **************************************** +* Compiler-specific Functions and Macros +******************************************/ +#define XXH_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) + +#ifdef __has_builtin +# define XXH_HAS_BUILTIN(x) __has_builtin(x) +#else +# define XXH_HAS_BUILTIN(x) 0 +#endif + +/*! + * @internal + * @def XXH_rotl32(x,r) + * @brief 32-bit rotate left. + * + * @param x The 32-bit integer to be rotated. + * @param r The number of bits to rotate. + * @pre + * @p r > 0 && @p r < 32 + * @note + * @p x and @p r may be evaluated multiple times. + * @return The rotated result. + */ +#if !defined(NO_CLANG_BUILTIN) && XXH_HAS_BUILTIN(__builtin_rotateleft32) \ + && XXH_HAS_BUILTIN(__builtin_rotateleft64) +# define XXH_rotl32 __builtin_rotateleft32 +# define XXH_rotl64 __builtin_rotateleft64 +/* Note: although _rotl exists for minGW (GCC under windows), performance seems poor */ +#elif defined(_MSC_VER) +# define XXH_rotl32(x,r) _rotl(x,r) +# define XXH_rotl64(x,r) _rotl64(x,r) +#else +# define XXH_rotl32(x,r) (((x) << (r)) | ((x) >> (32 - (r)))) +# define XXH_rotl64(x,r) (((x) << (r)) | ((x) >> (64 - (r)))) +#endif + +/*! + * @internal + * @fn xxh_u32 XXH_swap32(xxh_u32 x) + * @brief A 32-bit byteswap. + * + * @param x The 32-bit integer to byteswap. + * @return @p x, byteswapped. + */ +#if defined(_MSC_VER) /* Visual Studio */ +# define XXH_swap32 _byteswap_ulong +#elif XXH_GCC_VERSION >= 403 +# define XXH_swap32 __builtin_bswap32 +#else +static xxh_u32 XXH_swap32 (xxh_u32 x) +{ + return ((x << 24) & 0xff000000 ) | + ((x << 8) & 0x00ff0000 ) | + ((x >> 8) & 0x0000ff00 ) | + ((x >> 24) & 0x000000ff ); +} +#endif + + +/* *************************** +* Memory reads +*****************************/ + +/*! + * @internal + * @brief Enum to indicate whether a pointer is aligned. + */ +typedef enum { + XXH_aligned, /*!< Aligned */ + XXH_unaligned /*!< Possibly unaligned */ +} XXH_alignment; + +/* + * XXH_FORCE_MEMORY_ACCESS==3 is an endian-independent byteshift load. + * + * This is ideal for older compilers which don't inline memcpy. + */ +#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3)) + +XXH_FORCE_INLINE xxh_u32 XXH_readLE32(const void* memPtr) +{ + const xxh_u8* bytePtr = (const xxh_u8 *)memPtr; + return bytePtr[0] + | ((xxh_u32)bytePtr[1] << 8) + | ((xxh_u32)bytePtr[2] << 16) + | ((xxh_u32)bytePtr[3] << 24); +} + +XXH_FORCE_INLINE xxh_u32 XXH_readBE32(const void* memPtr) +{ + const xxh_u8* bytePtr = (const xxh_u8 *)memPtr; + return bytePtr[3] + | ((xxh_u32)bytePtr[2] << 8) + | ((xxh_u32)bytePtr[1] << 16) + | ((xxh_u32)bytePtr[0] << 24); +} + +#else +XXH_FORCE_INLINE xxh_u32 XXH_readLE32(const void* ptr) +{ + return XXH_CPU_LITTLE_ENDIAN ? XXH_read32(ptr) : XXH_swap32(XXH_read32(ptr)); +} + +static xxh_u32 XXH_readBE32(const void* ptr) +{ + return XXH_CPU_LITTLE_ENDIAN ? XXH_swap32(XXH_read32(ptr)) : XXH_read32(ptr); +} +#endif + +XXH_FORCE_INLINE xxh_u32 +XXH_readLE32_align(const void* ptr, XXH_alignment align) +{ + if (align==XXH_unaligned) { + return XXH_readLE32(ptr); + } else { + return XXH_CPU_LITTLE_ENDIAN ? *(const xxh_u32*)ptr : XXH_swap32(*(const xxh_u32*)ptr); + } +} + + +/* ************************************* +* Misc +***************************************/ +/*! @ingroup public */ +XXH_PUBLIC_API unsigned XXH_versionNumber (void) { return XXH_VERSION_NUMBER; } + + +/* ******************************************************************* +* 32-bit hash functions +*********************************************************************/ +/*! + * @} + * @defgroup xxh32_impl XXH32 implementation + * @ingroup impl + * @{ + */ + /* #define instead of static const, to be used as initializers */ +#define XXH_PRIME32_1 0x9E3779B1U /*!< 0b10011110001101110111100110110001 */ +#define XXH_PRIME32_2 0x85EBCA77U /*!< 0b10000101111010111100101001110111 */ +#define XXH_PRIME32_3 0xC2B2AE3DU /*!< 0b11000010101100101010111000111101 */ +#define XXH_PRIME32_4 0x27D4EB2FU /*!< 0b00100111110101001110101100101111 */ +#define XXH_PRIME32_5 0x165667B1U /*!< 0b00010110010101100110011110110001 */ + +#ifdef XXH_OLD_NAMES +# define PRIME32_1 XXH_PRIME32_1 +# define PRIME32_2 XXH_PRIME32_2 +# define PRIME32_3 XXH_PRIME32_3 +# define PRIME32_4 XXH_PRIME32_4 +# define PRIME32_5 XXH_PRIME32_5 +#endif + +/*! + * @internal + * @brief Normal stripe processing routine. + * + * This shuffles the bits so that any bit from @p input impacts several bits in + * @p acc. + * + * @param acc The accumulator lane. + * @param input The stripe of input to mix. + * @return The mixed accumulator lane. + */ +static xxh_u32 XXH32_round(xxh_u32 acc, xxh_u32 input) +{ + acc += input * XXH_PRIME32_2; + acc = XXH_rotl32(acc, 13); + acc *= XXH_PRIME32_1; +#if (defined(__SSE4_1__) || defined(__aarch64__)) && !defined(XXH_ENABLE_AUTOVECTORIZE) + /* + * UGLY HACK: + * A compiler fence is the only thing that prevents GCC and Clang from + * autovectorizing the XXH32 loop (pragmas and attributes don't work for some + * reason) without globally disabling SSE4.1. + * + * The reason we want to avoid vectorization is because despite working on + * 4 integers at a time, there are multiple factors slowing XXH32 down on + * SSE4: + * - There's a ridiculous amount of lag from pmulld (10 cycles of latency on + * newer chips!) making it slightly slower to multiply four integers at + * once compared to four integers independently. Even when pmulld was + * fastest, Sandy/Ivy Bridge, it is still not worth it to go into SSE + * just to multiply unless doing a long operation. + * + * - Four instructions are required to rotate, + * movqda tmp, v // not required with VEX encoding + * pslld tmp, 13 // tmp <<= 13 + * psrld v, 19 // x >>= 19 + * por v, tmp // x |= tmp + * compared to one for scalar: + * roll v, 13 // reliably fast across the board + * shldl v, v, 13 // Sandy Bridge and later prefer this for some reason + * + * - Instruction level parallelism is actually more beneficial here because + * the SIMD actually serializes this operation: While v1 is rotating, v2 + * can load data, while v3 can multiply. SSE forces them to operate + * together. + * + * This is also enabled on AArch64, as Clang autovectorizes it incorrectly + * and it is pointless writing a NEON implementation that is basically the + * same speed as scalar for XXH32. + */ + XXH_COMPILER_GUARD(acc); +#endif + return acc; +} + +/*! + * @internal + * @brief Mixes all bits to finalize the hash. + * + * The final mix ensures that all input bits have a chance to impact any bit in + * the output digest, resulting in an unbiased distribution. + * + * @param h32 The hash to avalanche. + * @return The avalanched hash. + */ +static xxh_u32 XXH32_avalanche(xxh_u32 h32) +{ + h32 ^= h32 >> 15; + h32 *= XXH_PRIME32_2; + h32 ^= h32 >> 13; + h32 *= XXH_PRIME32_3; + h32 ^= h32 >> 16; + return(h32); +} + +#define XXH_get32bits(p) XXH_readLE32_align(p, align) + +/*! + * @internal + * @brief Processes the last 0-15 bytes of @p ptr. + * + * There may be up to 15 bytes remaining to consume from the input. + * This final stage will digest them to ensure that all input bytes are present + * in the final mix. + * + * @param h32 The hash to finalize. + * @param ptr The pointer to the remaining input. + * @param len The remaining length, modulo 16. + * @param align Whether @p ptr is aligned. + * @return The finalized hash. + */ +static xxh_u32 +XXH32_finalize(xxh_u32 h32, const xxh_u8* ptr, size_t len, XXH_alignment align) +{ +#define XXH_PROCESS1 do { \ + h32 += (*ptr++) * XXH_PRIME32_5; \ + h32 = XXH_rotl32(h32, 11) * XXH_PRIME32_1; \ +} while (0) + +#define XXH_PROCESS4 do { \ + h32 += XXH_get32bits(ptr) * XXH_PRIME32_3; \ + ptr += 4; \ + h32 = XXH_rotl32(h32, 17) * XXH_PRIME32_4; \ +} while (0) + + if (ptr==NULL) XXH_ASSERT(len == 0); + + /* Compact rerolled version; generally faster */ + if (!XXH32_ENDJMP) { + len &= 15; + while (len >= 4) { + XXH_PROCESS4; + len -= 4; + } + while (len > 0) { + XXH_PROCESS1; + --len; + } + return XXH32_avalanche(h32); + } else { + switch(len&15) /* or switch(bEnd - p) */ { + case 12: XXH_PROCESS4; + XXH_FALLTHROUGH; + case 8: XXH_PROCESS4; + XXH_FALLTHROUGH; + case 4: XXH_PROCESS4; + return XXH32_avalanche(h32); + + case 13: XXH_PROCESS4; + XXH_FALLTHROUGH; + case 9: XXH_PROCESS4; + XXH_FALLTHROUGH; + case 5: XXH_PROCESS4; + XXH_PROCESS1; + return XXH32_avalanche(h32); + + case 14: XXH_PROCESS4; + XXH_FALLTHROUGH; + case 10: XXH_PROCESS4; + XXH_FALLTHROUGH; + case 6: XXH_PROCESS4; + XXH_PROCESS1; + XXH_PROCESS1; + return XXH32_avalanche(h32); + + case 15: XXH_PROCESS4; + XXH_FALLTHROUGH; + case 11: XXH_PROCESS4; + XXH_FALLTHROUGH; + case 7: XXH_PROCESS4; + XXH_FALLTHROUGH; + case 3: XXH_PROCESS1; + XXH_FALLTHROUGH; + case 2: XXH_PROCESS1; + XXH_FALLTHROUGH; + case 1: XXH_PROCESS1; + XXH_FALLTHROUGH; + case 0: return XXH32_avalanche(h32); + } + XXH_ASSERT(0); + return h32; /* reaching this point is deemed impossible */ + } +} + +#ifdef XXH_OLD_NAMES +# define PROCESS1 XXH_PROCESS1 +# define PROCESS4 XXH_PROCESS4 +#else +# undef XXH_PROCESS1 +# undef XXH_PROCESS4 +#endif + +/*! + * @internal + * @brief The implementation for @ref XXH32(). + * + * @param input , len , seed Directly passed from @ref XXH32(). + * @param align Whether @p input is aligned. + * @return The calculated hash. + */ +XXH_FORCE_INLINE xxh_u32 +XXH32_endian_align(const xxh_u8* input, size_t len, xxh_u32 seed, XXH_alignment align) +{ + xxh_u32 h32; + + if (input==NULL) XXH_ASSERT(len == 0); + + if (len>=16) { + const xxh_u8* const bEnd = input + len; + const xxh_u8* const limit = bEnd - 15; + xxh_u32 v1 = seed + XXH_PRIME32_1 + XXH_PRIME32_2; + xxh_u32 v2 = seed + XXH_PRIME32_2; + xxh_u32 v3 = seed + 0; + xxh_u32 v4 = seed - XXH_PRIME32_1; + + do { + v1 = XXH32_round(v1, XXH_get32bits(input)); input += 4; + v2 = XXH32_round(v2, XXH_get32bits(input)); input += 4; + v3 = XXH32_round(v3, XXH_get32bits(input)); input += 4; + v4 = XXH32_round(v4, XXH_get32bits(input)); input += 4; + } while (input < limit); + + h32 = XXH_rotl32(v1, 1) + XXH_rotl32(v2, 7) + + XXH_rotl32(v3, 12) + XXH_rotl32(v4, 18); + } else { + h32 = seed + XXH_PRIME32_5; + } + + h32 += (xxh_u32)len; + + return XXH32_finalize(h32, input, len&15, align); +} + +/*! @ingroup xxh32_family */ +XXH_PUBLIC_API XXH32_hash_t XXH32 (const void* input, size_t len, XXH32_hash_t seed) +{ +#if 0 + /* Simple version, good for code maintenance, but unfortunately slow for small inputs */ + XXH32_state_t state; + XXH32_reset(&state, seed); + XXH32_update(&state, (const xxh_u8*)input, len); + return XXH32_digest(&state); +#else + if (XXH_FORCE_ALIGN_CHECK) { + if ((((size_t)input) & 3) == 0) { /* Input is 4-bytes aligned, leverage the speed benefit */ + return XXH32_endian_align((const xxh_u8*)input, len, seed, XXH_aligned); + } } + + return XXH32_endian_align((const xxh_u8*)input, len, seed, XXH_unaligned); +#endif +} + + + +/******* Hash streaming *******/ +/*! + * @ingroup xxh32_family + */ +XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void) +{ + return (XXH32_state_t*)XXH_malloc(sizeof(XXH32_state_t)); +} +/*! @ingroup xxh32_family */ +XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr) +{ + XXH_free(statePtr); + return XXH_OK; +} + +/*! @ingroup xxh32_family */ +XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* dstState, const XXH32_state_t* srcState) +{ + XXH_memcpy(dstState, srcState, sizeof(*dstState)); +} + +/*! @ingroup xxh32_family */ +XXH_PUBLIC_API XXH_errorcode XXH32_reset(XXH32_state_t* statePtr, XXH32_hash_t seed) +{ + XXH_ASSERT(statePtr != NULL); + memset(statePtr, 0, sizeof(*statePtr)); + statePtr->v[0] = seed + XXH_PRIME32_1 + XXH_PRIME32_2; + statePtr->v[1] = seed + XXH_PRIME32_2; + statePtr->v[2] = seed + 0; + statePtr->v[3] = seed - XXH_PRIME32_1; + return XXH_OK; +} + + +/*! @ingroup xxh32_family */ +XXH_PUBLIC_API XXH_errorcode +XXH32_update(XXH32_state_t* state, const void* input, size_t len) +{ + if (input==NULL) { + XXH_ASSERT(len == 0); + return XXH_OK; + } + + { const xxh_u8* p = (const xxh_u8*)input; + const xxh_u8* const bEnd = p + len; + + state->total_len_32 += (XXH32_hash_t)len; + state->large_len |= (XXH32_hash_t)((len>=16) | (state->total_len_32>=16)); + + if (state->memsize + len < 16) { /* fill in tmp buffer */ + XXH_memcpy((xxh_u8*)(state->mem32) + state->memsize, input, len); + state->memsize += (XXH32_hash_t)len; + return XXH_OK; + } + + if (state->memsize) { /* some data left from previous update */ + XXH_memcpy((xxh_u8*)(state->mem32) + state->memsize, input, 16-state->memsize); + { const xxh_u32* p32 = state->mem32; + state->v[0] = XXH32_round(state->v[0], XXH_readLE32(p32)); p32++; + state->v[1] = XXH32_round(state->v[1], XXH_readLE32(p32)); p32++; + state->v[2] = XXH32_round(state->v[2], XXH_readLE32(p32)); p32++; + state->v[3] = XXH32_round(state->v[3], XXH_readLE32(p32)); + } + p += 16-state->memsize; + state->memsize = 0; + } + + if (p <= bEnd-16) { + const xxh_u8* const limit = bEnd - 16; + + do { + state->v[0] = XXH32_round(state->v[0], XXH_readLE32(p)); p+=4; + state->v[1] = XXH32_round(state->v[1], XXH_readLE32(p)); p+=4; + state->v[2] = XXH32_round(state->v[2], XXH_readLE32(p)); p+=4; + state->v[3] = XXH32_round(state->v[3], XXH_readLE32(p)); p+=4; + } while (p<=limit); + + } + + if (p < bEnd) { + XXH_memcpy(state->mem32, p, (size_t)(bEnd-p)); + state->memsize = (unsigned)(bEnd-p); + } + } + + return XXH_OK; +} + + +/*! @ingroup xxh32_family */ +XXH_PUBLIC_API XXH32_hash_t XXH32_digest(const XXH32_state_t* state) +{ + xxh_u32 h32; + + if (state->large_len) { + h32 = XXH_rotl32(state->v[0], 1) + + XXH_rotl32(state->v[1], 7) + + XXH_rotl32(state->v[2], 12) + + XXH_rotl32(state->v[3], 18); + } else { + h32 = state->v[2] /* == seed */ + XXH_PRIME32_5; + } + + h32 += state->total_len_32; + + return XXH32_finalize(h32, (const xxh_u8*)state->mem32, state->memsize, XXH_aligned); +} + + +/******* Canonical representation *******/ + +/*! + * @ingroup xxh32_family + * The default return values from XXH functions are unsigned 32 and 64 bit + * integers. + * + * The canonical representation uses big endian convention, the same convention + * as human-readable numbers (large digits first). + * + * This way, hash values can be written into a file or buffer, remaining + * comparable across different systems. + * + * The following functions allow transformation of hash values to and from their + * canonical format. + */ +XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash) +{ + /* XXH_STATIC_ASSERT(sizeof(XXH32_canonical_t) == sizeof(XXH32_hash_t)); */ + if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap32(hash); + XXH_memcpy(dst, &hash, sizeof(*dst)); +} +/*! @ingroup xxh32_family */ +XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src) +{ + return XXH_readBE32(src); +} + + +#ifndef XXH_NO_LONG_LONG + +/* ******************************************************************* +* 64-bit hash functions +*********************************************************************/ +/*! + * @} + * @ingroup impl + * @{ + */ +/******* Memory access *******/ + +typedef XXH64_hash_t xxh_u64; + +#ifdef XXH_OLD_NAMES +# define U64 xxh_u64 +#endif + +#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3)) +/* + * Manual byteshift. Best for old compilers which don't inline memcpy. + * We actually directly use XXH_readLE64 and XXH_readBE64. + */ +#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2)) + +/* Force direct memory access. Only works on CPU which support unaligned memory access in hardware */ +static xxh_u64 XXH_read64(const void* memPtr) +{ + return *(const xxh_u64*) memPtr; +} + +#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1)) + +/* + * __pack instructions are safer, but compiler specific, hence potentially + * problematic for some compilers. + * + * Currently only defined for GCC and ICC. + */ +#ifdef XXH_OLD_NAMES +typedef union { xxh_u32 u32; xxh_u64 u64; } __attribute__((packed)) unalign64; +#endif +static xxh_u64 XXH_read64(const void* ptr) +{ + typedef union { xxh_u32 u32; xxh_u64 u64; } __attribute__((packed)) xxh_unalign64; + return ((const xxh_unalign64*)ptr)->u64; +} + +#else + +/* + * Portable and safe solution. Generally efficient. + * see: http://fastcompression.blogspot.com/2015/08/accessing-unaligned-memory.html + */ +static xxh_u64 XXH_read64(const void* memPtr) +{ + xxh_u64 val; + XXH_memcpy(&val, memPtr, sizeof(val)); + return val; +} + +#endif /* XXH_FORCE_DIRECT_MEMORY_ACCESS */ + +#if defined(_MSC_VER) /* Visual Studio */ +# define XXH_swap64 _byteswap_uint64 +#elif XXH_GCC_VERSION >= 403 +# define XXH_swap64 __builtin_bswap64 +#else +static xxh_u64 XXH_swap64(xxh_u64 x) +{ + return ((x << 56) & 0xff00000000000000ULL) | + ((x << 40) & 0x00ff000000000000ULL) | + ((x << 24) & 0x0000ff0000000000ULL) | + ((x << 8) & 0x000000ff00000000ULL) | + ((x >> 8) & 0x00000000ff000000ULL) | + ((x >> 24) & 0x0000000000ff0000ULL) | + ((x >> 40) & 0x000000000000ff00ULL) | + ((x >> 56) & 0x00000000000000ffULL); +} +#endif + + +/* XXH_FORCE_MEMORY_ACCESS==3 is an endian-independent byteshift load. */ +#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3)) + +XXH_FORCE_INLINE xxh_u64 XXH_readLE64(const void* memPtr) +{ + const xxh_u8* bytePtr = (const xxh_u8 *)memPtr; + return bytePtr[0] + | ((xxh_u64)bytePtr[1] << 8) + | ((xxh_u64)bytePtr[2] << 16) + | ((xxh_u64)bytePtr[3] << 24) + | ((xxh_u64)bytePtr[4] << 32) + | ((xxh_u64)bytePtr[5] << 40) + | ((xxh_u64)bytePtr[6] << 48) + | ((xxh_u64)bytePtr[7] << 56); +} + +XXH_FORCE_INLINE xxh_u64 XXH_readBE64(const void* memPtr) +{ + const xxh_u8* bytePtr = (const xxh_u8 *)memPtr; + return bytePtr[7] + | ((xxh_u64)bytePtr[6] << 8) + | ((xxh_u64)bytePtr[5] << 16) + | ((xxh_u64)bytePtr[4] << 24) + | ((xxh_u64)bytePtr[3] << 32) + | ((xxh_u64)bytePtr[2] << 40) + | ((xxh_u64)bytePtr[1] << 48) + | ((xxh_u64)bytePtr[0] << 56); +} + +#else +XXH_FORCE_INLINE xxh_u64 XXH_readLE64(const void* ptr) +{ + return XXH_CPU_LITTLE_ENDIAN ? XXH_read64(ptr) : XXH_swap64(XXH_read64(ptr)); +} + +static xxh_u64 XXH_readBE64(const void* ptr) +{ + return XXH_CPU_LITTLE_ENDIAN ? XXH_swap64(XXH_read64(ptr)) : XXH_read64(ptr); +} +#endif + +XXH_FORCE_INLINE xxh_u64 +XXH_readLE64_align(const void* ptr, XXH_alignment align) +{ + if (align==XXH_unaligned) + return XXH_readLE64(ptr); + else + return XXH_CPU_LITTLE_ENDIAN ? *(const xxh_u64*)ptr : XXH_swap64(*(const xxh_u64*)ptr); +} + + +/******* xxh64 *******/ +/*! + * @} + * @defgroup xxh64_impl XXH64 implementation + * @ingroup impl + * @{ + */ +/* #define rather that static const, to be used as initializers */ +#define XXH_PRIME64_1 0x9E3779B185EBCA87ULL /*!< 0b1001111000110111011110011011000110000101111010111100101010000111 */ +#define XXH_PRIME64_2 0xC2B2AE3D27D4EB4FULL /*!< 0b1100001010110010101011100011110100100111110101001110101101001111 */ +#define XXH_PRIME64_3 0x165667B19E3779F9ULL /*!< 0b0001011001010110011001111011000110011110001101110111100111111001 */ +#define XXH_PRIME64_4 0x85EBCA77C2B2AE63ULL /*!< 0b1000010111101011110010100111011111000010101100101010111001100011 */ +#define XXH_PRIME64_5 0x27D4EB2F165667C5ULL /*!< 0b0010011111010100111010110010111100010110010101100110011111000101 */ + +#ifdef XXH_OLD_NAMES +# define PRIME64_1 XXH_PRIME64_1 +# define PRIME64_2 XXH_PRIME64_2 +# define PRIME64_3 XXH_PRIME64_3 +# define PRIME64_4 XXH_PRIME64_4 +# define PRIME64_5 XXH_PRIME64_5 +#endif + +static xxh_u64 XXH64_round(xxh_u64 acc, xxh_u64 input) +{ + acc += input * XXH_PRIME64_2; + acc = XXH_rotl64(acc, 31); + acc *= XXH_PRIME64_1; + return acc; +} + +static xxh_u64 XXH64_mergeRound(xxh_u64 acc, xxh_u64 val) +{ + val = XXH64_round(0, val); + acc ^= val; + acc = acc * XXH_PRIME64_1 + XXH_PRIME64_4; + return acc; +} + +static xxh_u64 XXH64_avalanche(xxh_u64 h64) +{ + h64 ^= h64 >> 33; + h64 *= XXH_PRIME64_2; + h64 ^= h64 >> 29; + h64 *= XXH_PRIME64_3; + h64 ^= h64 >> 32; + return h64; +} + + +#define XXH_get64bits(p) XXH_readLE64_align(p, align) + +static xxh_u64 +XXH64_finalize(xxh_u64 h64, const xxh_u8* ptr, size_t len, XXH_alignment align) +{ + if (ptr==NULL) XXH_ASSERT(len == 0); + len &= 31; + while (len >= 8) { + xxh_u64 const k1 = XXH64_round(0, XXH_get64bits(ptr)); + ptr += 8; + h64 ^= k1; + h64 = XXH_rotl64(h64,27) * XXH_PRIME64_1 + XXH_PRIME64_4; + len -= 8; + } + if (len >= 4) { + h64 ^= (xxh_u64)(XXH_get32bits(ptr)) * XXH_PRIME64_1; + ptr += 4; + h64 = XXH_rotl64(h64, 23) * XXH_PRIME64_2 + XXH_PRIME64_3; + len -= 4; + } + while (len > 0) { + h64 ^= (*ptr++) * XXH_PRIME64_5; + h64 = XXH_rotl64(h64, 11) * XXH_PRIME64_1; + --len; + } + return XXH64_avalanche(h64); +} + +#ifdef XXH_OLD_NAMES +# define PROCESS1_64 XXH_PROCESS1_64 +# define PROCESS4_64 XXH_PROCESS4_64 +# define PROCESS8_64 XXH_PROCESS8_64 +#else +# undef XXH_PROCESS1_64 +# undef XXH_PROCESS4_64 +# undef XXH_PROCESS8_64 +#endif + +XXH_FORCE_INLINE xxh_u64 +XXH64_endian_align(const xxh_u8* input, size_t len, xxh_u64 seed, XXH_alignment align) +{ + xxh_u64 h64; + if (input==NULL) XXH_ASSERT(len == 0); + + if (len>=32) { + const xxh_u8* const bEnd = input + len; + const xxh_u8* const limit = bEnd - 31; + xxh_u64 v1 = seed + XXH_PRIME64_1 + XXH_PRIME64_2; + xxh_u64 v2 = seed + XXH_PRIME64_2; + xxh_u64 v3 = seed + 0; + xxh_u64 v4 = seed - XXH_PRIME64_1; + + do { + v1 = XXH64_round(v1, XXH_get64bits(input)); input+=8; + v2 = XXH64_round(v2, XXH_get64bits(input)); input+=8; + v3 = XXH64_round(v3, XXH_get64bits(input)); input+=8; + v4 = XXH64_round(v4, XXH_get64bits(input)); input+=8; + } while (inputv[0] = seed + XXH_PRIME64_1 + XXH_PRIME64_2; + statePtr->v[1] = seed + XXH_PRIME64_2; + statePtr->v[2] = seed + 0; + statePtr->v[3] = seed - XXH_PRIME64_1; + return XXH_OK; +} + +/*! @ingroup xxh64_family */ +XXH_PUBLIC_API XXH_errorcode +XXH64_update (XXH64_state_t* state, const void* input, size_t len) +{ + if (input==NULL) { + XXH_ASSERT(len == 0); + return XXH_OK; + } + + { const xxh_u8* p = (const xxh_u8*)input; + const xxh_u8* const bEnd = p + len; + + state->total_len += len; + + if (state->memsize + len < 32) { /* fill in tmp buffer */ + XXH_memcpy(((xxh_u8*)state->mem64) + state->memsize, input, len); + state->memsize += (xxh_u32)len; + return XXH_OK; + } + + if (state->memsize) { /* tmp buffer is full */ + XXH_memcpy(((xxh_u8*)state->mem64) + state->memsize, input, 32-state->memsize); + state->v[0] = XXH64_round(state->v[0], XXH_readLE64(state->mem64+0)); + state->v[1] = XXH64_round(state->v[1], XXH_readLE64(state->mem64+1)); + state->v[2] = XXH64_round(state->v[2], XXH_readLE64(state->mem64+2)); + state->v[3] = XXH64_round(state->v[3], XXH_readLE64(state->mem64+3)); + p += 32 - state->memsize; + state->memsize = 0; + } + + if (p+32 <= bEnd) { + const xxh_u8* const limit = bEnd - 32; + + do { + state->v[0] = XXH64_round(state->v[0], XXH_readLE64(p)); p+=8; + state->v[1] = XXH64_round(state->v[1], XXH_readLE64(p)); p+=8; + state->v[2] = XXH64_round(state->v[2], XXH_readLE64(p)); p+=8; + state->v[3] = XXH64_round(state->v[3], XXH_readLE64(p)); p+=8; + } while (p<=limit); + + } + + if (p < bEnd) { + XXH_memcpy(state->mem64, p, (size_t)(bEnd-p)); + state->memsize = (unsigned)(bEnd-p); + } + } + + return XXH_OK; +} + + +/*! @ingroup xxh64_family */ +XXH_PUBLIC_API XXH64_hash_t XXH64_digest(const XXH64_state_t* state) +{ + xxh_u64 h64; + + if (state->total_len >= 32) { + h64 = XXH_rotl64(state->v[0], 1) + XXH_rotl64(state->v[1], 7) + XXH_rotl64(state->v[2], 12) + XXH_rotl64(state->v[3], 18); + h64 = XXH64_mergeRound(h64, state->v[0]); + h64 = XXH64_mergeRound(h64, state->v[1]); + h64 = XXH64_mergeRound(h64, state->v[2]); + h64 = XXH64_mergeRound(h64, state->v[3]); + } else { + h64 = state->v[2] /*seed*/ + XXH_PRIME64_5; + } + + h64 += (xxh_u64) state->total_len; + + return XXH64_finalize(h64, (const xxh_u8*)state->mem64, (size_t)state->total_len, XXH_aligned); +} + + +/******* Canonical representation *******/ + +/*! @ingroup xxh64_family */ +XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t* dst, XXH64_hash_t hash) +{ + /* XXH_STATIC_ASSERT(sizeof(XXH64_canonical_t) == sizeof(XXH64_hash_t)); */ + if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap64(hash); + XXH_memcpy(dst, &hash, sizeof(*dst)); +} + +/*! @ingroup xxh64_family */ +XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src) +{ + return XXH_readBE64(src); +} + +#ifndef XXH_NO_XXH3 + +/* ********************************************************************* +* XXH3 +* New generation hash designed for speed on small keys and vectorization +************************************************************************ */ +/*! + * @} + * @defgroup xxh3_impl XXH3 implementation + * @ingroup impl + * @{ + */ + +/* === Compiler specifics === */ + +#if ((defined(sun) || defined(__sun)) && __cplusplus) /* Solaris includes __STDC_VERSION__ with C++. Tested with GCC 5.5 */ +# define XXH_RESTRICT /* disable */ +#elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* >= C99 */ +# define XXH_RESTRICT restrict +#else +/* Note: it might be useful to define __restrict or __restrict__ for some C++ compilers */ +# define XXH_RESTRICT /* disable */ +#endif + +#if (defined(__GNUC__) && (__GNUC__ >= 3)) \ + || (defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 800)) \ + || defined(__clang__) +# define XXH_likely(x) __builtin_expect(x, 1) +# define XXH_unlikely(x) __builtin_expect(x, 0) +#else +# define XXH_likely(x) (x) +# define XXH_unlikely(x) (x) +#endif + +#if defined(__GNUC__) || defined(__clang__) +# if defined(__ARM_NEON__) || defined(__ARM_NEON) \ + || defined(__aarch64__) || defined(_M_ARM) \ + || defined(_M_ARM64) || defined(_M_ARM64EC) +# define inline __inline__ /* circumvent a clang bug */ +# include +# undef inline +# elif defined(__AVX2__) +# include +# elif defined(__SSE2__) +# include +# endif +#endif + +#if defined(_MSC_VER) +# include +#endif + +/* + * One goal of XXH3 is to make it fast on both 32-bit and 64-bit, while + * remaining a true 64-bit/128-bit hash function. + * + * This is done by prioritizing a subset of 64-bit operations that can be + * emulated without too many steps on the average 32-bit machine. + * + * For example, these two lines seem similar, and run equally fast on 64-bit: + * + * xxh_u64 x; + * x ^= (x >> 47); // good + * x ^= (x >> 13); // bad + * + * However, to a 32-bit machine, there is a major difference. + * + * x ^= (x >> 47) looks like this: + * + * x.lo ^= (x.hi >> (47 - 32)); + * + * while x ^= (x >> 13) looks like this: + * + * // note: funnel shifts are not usually cheap. + * x.lo ^= (x.lo >> 13) | (x.hi << (32 - 13)); + * x.hi ^= (x.hi >> 13); + * + * The first one is significantly faster than the second, simply because the + * shift is larger than 32. This means: + * - All the bits we need are in the upper 32 bits, so we can ignore the lower + * 32 bits in the shift. + * - The shift result will always fit in the lower 32 bits, and therefore, + * we can ignore the upper 32 bits in the xor. + * + * Thanks to this optimization, XXH3 only requires these features to be efficient: + * + * - Usable unaligned access + * - A 32-bit or 64-bit ALU + * - If 32-bit, a decent ADC instruction + * - A 32 or 64-bit multiply with a 64-bit result + * - For the 128-bit variant, a decent byteswap helps short inputs. + * + * The first two are already required by XXH32, and almost all 32-bit and 64-bit + * platforms which can run XXH32 can run XXH3 efficiently. + * + * Thumb-1, the classic 16-bit only subset of ARM's instruction set, is one + * notable exception. + * + * First of all, Thumb-1 lacks support for the UMULL instruction which + * performs the important long multiply. This means numerous __aeabi_lmul + * calls. + * + * Second of all, the 8 functional registers are just not enough. + * Setup for __aeabi_lmul, byteshift loads, pointers, and all arithmetic need + * Lo registers, and this shuffling results in thousands more MOVs than A32. + * + * A32 and T32 don't have this limitation. They can access all 14 registers, + * do a 32->64 multiply with UMULL, and the flexible operand allowing free + * shifts is helpful, too. + * + * Therefore, we do a quick sanity check. + * + * If compiling Thumb-1 for a target which supports ARM instructions, we will + * emit a warning, as it is not a "sane" platform to compile for. + * + * Usually, if this happens, it is because of an accident and you probably need + * to specify -march, as you likely meant to compile for a newer architecture. + * + * Credit: large sections of the vectorial and asm source code paths + * have been contributed by @easyaspi314 + */ +#if defined(__thumb__) && !defined(__thumb2__) && defined(__ARM_ARCH_ISA_ARM) +# warning "XXH3 is highly inefficient without ARM or Thumb-2." +#endif + +/* ========================================== + * Vectorization detection + * ========================================== */ + +#ifdef XXH_DOXYGEN +/*! + * @ingroup tuning + * @brief Overrides the vectorization implementation chosen for XXH3. + * + * Can be defined to 0 to disable SIMD or any of the values mentioned in + * @ref XXH_VECTOR_TYPE. + * + * If this is not defined, it uses predefined macros to determine the best + * implementation. + */ +# define XXH_VECTOR XXH_SCALAR +/*! + * @ingroup tuning + * @brief Possible values for @ref XXH_VECTOR. + * + * Note that these are actually implemented as macros. + * + * If this is not defined, it is detected automatically. + * @ref XXH_X86DISPATCH overrides this. + */ +enum XXH_VECTOR_TYPE /* fake enum */ { + XXH_SCALAR = 0, /*!< Portable scalar version */ + XXH_SSE2 = 1, /*!< + * SSE2 for Pentium 4, Opteron, all x86_64. + * + * @note SSE2 is also guaranteed on Windows 10, macOS, and + * Android x86. + */ + XXH_AVX2 = 2, /*!< AVX2 for Haswell and Bulldozer */ + XXH_AVX512 = 3, /*!< AVX512 for Skylake and Icelake */ + XXH_NEON = 4, /*!< NEON for most ARMv7-A and all AArch64 */ + XXH_VSX = 5, /*!< VSX and ZVector for POWER8/z13 (64-bit) */ +}; +/*! + * @ingroup tuning + * @brief Selects the minimum alignment for XXH3's accumulators. + * + * When using SIMD, this should match the alignment reqired for said vector + * type, so, for example, 32 for AVX2. + * + * Default: Auto detected. + */ +# define XXH_ACC_ALIGN 8 +#endif + +/* Actual definition */ +#ifndef XXH_DOXYGEN +# define XXH_SCALAR 0 +# define XXH_SSE2 1 +# define XXH_AVX2 2 +# define XXH_AVX512 3 +# define XXH_NEON 4 +# define XXH_VSX 5 +#endif + +#ifndef XXH_VECTOR /* can be defined on command line */ +# if ( \ + defined(__ARM_NEON__) || defined(__ARM_NEON) /* gcc */ \ + || defined(_M_ARM) || defined(_M_ARM64) || defined(_M_ARM64EC) /* msvc */ \ + ) && ( \ + defined(_WIN32) || defined(__LITTLE_ENDIAN__) /* little endian only */ \ + || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) \ + ) +# define XXH_VECTOR XXH_NEON +# elif defined(__AVX512F__) +# define XXH_VECTOR XXH_AVX512 +# elif defined(__AVX2__) +# define XXH_VECTOR XXH_AVX2 +# elif defined(__SSE2__) || defined(_M_AMD64) || defined(_M_X64) || (defined(_M_IX86_FP) && (_M_IX86_FP == 2)) +# define XXH_VECTOR XXH_SSE2 +# elif (defined(__PPC64__) && defined(__POWER8_VECTOR__)) \ + || (defined(__s390x__) && defined(__VEC__)) \ + && defined(__GNUC__) /* TODO: IBM XL */ +# define XXH_VECTOR XXH_VSX +# else +# define XXH_VECTOR XXH_SCALAR +# endif +#endif + +/* + * Controls the alignment of the accumulator, + * for compatibility with aligned vector loads, which are usually faster. + */ +#ifndef XXH_ACC_ALIGN +# if defined(XXH_X86DISPATCH) +# define XXH_ACC_ALIGN 64 /* for compatibility with avx512 */ +# elif XXH_VECTOR == XXH_SCALAR /* scalar */ +# define XXH_ACC_ALIGN 8 +# elif XXH_VECTOR == XXH_SSE2 /* sse2 */ +# define XXH_ACC_ALIGN 16 +# elif XXH_VECTOR == XXH_AVX2 /* avx2 */ +# define XXH_ACC_ALIGN 32 +# elif XXH_VECTOR == XXH_NEON /* neon */ +# define XXH_ACC_ALIGN 16 +# elif XXH_VECTOR == XXH_VSX /* vsx */ +# define XXH_ACC_ALIGN 16 +# elif XXH_VECTOR == XXH_AVX512 /* avx512 */ +# define XXH_ACC_ALIGN 64 +# endif +#endif + +#if defined(XXH_X86DISPATCH) || XXH_VECTOR == XXH_SSE2 \ + || XXH_VECTOR == XXH_AVX2 || XXH_VECTOR == XXH_AVX512 +# define XXH_SEC_ALIGN XXH_ACC_ALIGN +#else +# define XXH_SEC_ALIGN 8 +#endif + +/* + * UGLY HACK: + * GCC usually generates the best code with -O3 for xxHash. + * + * However, when targeting AVX2, it is overzealous in its unrolling resulting + * in code roughly 3/4 the speed of Clang. + * + * There are other issues, such as GCC splitting _mm256_loadu_si256 into + * _mm_loadu_si128 + _mm256_inserti128_si256. This is an optimization which + * only applies to Sandy and Ivy Bridge... which don't even support AVX2. + * + * That is why when compiling the AVX2 version, it is recommended to use either + * -O2 -mavx2 -march=haswell + * or + * -O2 -mavx2 -mno-avx256-split-unaligned-load + * for decent performance, or to use Clang instead. + * + * Fortunately, we can control the first one with a pragma that forces GCC into + * -O2, but the other one we can't control without "failed to inline always + * inline function due to target mismatch" warnings. + */ +#if XXH_VECTOR == XXH_AVX2 /* AVX2 */ \ + && defined(__GNUC__) && !defined(__clang__) /* GCC, not Clang */ \ + && defined(__OPTIMIZE__) && !defined(__OPTIMIZE_SIZE__) /* respect -O0 and -Os */ +# pragma GCC push_options +# pragma GCC optimize("-O2") +#endif + + +#if XXH_VECTOR == XXH_NEON +/* + * NEON's setup for vmlal_u32 is a little more complicated than it is on + * SSE2, AVX2, and VSX. + * + * While PMULUDQ and VMULEUW both perform a mask, VMLAL.U32 performs an upcast. + * + * To do the same operation, the 128-bit 'Q' register needs to be split into + * two 64-bit 'D' registers, performing this operation:: + * + * [ a | b ] + * | '---------. .--------' | + * | x | + * | .---------' '--------. | + * [ a & 0xFFFFFFFF | b & 0xFFFFFFFF ],[ a >> 32 | b >> 32 ] + * + * Due to significant changes in aarch64, the fastest method for aarch64 is + * completely different than the fastest method for ARMv7-A. + * + * ARMv7-A treats D registers as unions overlaying Q registers, so modifying + * D11 will modify the high half of Q5. This is similar to how modifying AH + * will only affect bits 8-15 of AX on x86. + * + * VZIP takes two registers, and puts even lanes in one register and odd lanes + * in the other. + * + * On ARMv7-A, this strangely modifies both parameters in place instead of + * taking the usual 3-operand form. + * + * Therefore, if we want to do this, we can simply use a D-form VZIP.32 on the + * lower and upper halves of the Q register to end up with the high and low + * halves where we want - all in one instruction. + * + * vzip.32 d10, d11 @ d10 = { d10[0], d11[0] }; d11 = { d10[1], d11[1] } + * + * Unfortunately we need inline assembly for this: Instructions modifying two + * registers at once is not possible in GCC or Clang's IR, and they have to + * create a copy. + * + * aarch64 requires a different approach. + * + * In order to make it easier to write a decent compiler for aarch64, many + * quirks were removed, such as conditional execution. + * + * NEON was also affected by this. + * + * aarch64 cannot access the high bits of a Q-form register, and writes to a + * D-form register zero the high bits, similar to how writes to W-form scalar + * registers (or DWORD registers on x86_64) work. + * + * The formerly free vget_high intrinsics now require a vext (with a few + * exceptions) + * + * Additionally, VZIP was replaced by ZIP1 and ZIP2, which are the equivalent + * of PUNPCKL* and PUNPCKH* in SSE, respectively, in order to only modify one + * operand. + * + * The equivalent of the VZIP.32 on the lower and upper halves would be this + * mess: + * + * ext v2.4s, v0.4s, v0.4s, #2 // v2 = { v0[2], v0[3], v0[0], v0[1] } + * zip1 v1.2s, v0.2s, v2.2s // v1 = { v0[0], v2[0] } + * zip2 v0.2s, v0.2s, v1.2s // v0 = { v0[1], v2[1] } + * + * Instead, we use a literal downcast, vmovn_u64 (XTN), and vshrn_n_u64 (SHRN): + * + * shrn v1.2s, v0.2d, #32 // v1 = (uint32x2_t)(v0 >> 32); + * xtn v0.2s, v0.2d // v0 = (uint32x2_t)(v0 & 0xFFFFFFFF); + * + * This is available on ARMv7-A, but is less efficient than a single VZIP.32. + */ + +/*! + * Function-like macro: + * void XXH_SPLIT_IN_PLACE(uint64x2_t &in, uint32x2_t &outLo, uint32x2_t &outHi) + * { + * outLo = (uint32x2_t)(in & 0xFFFFFFFF); + * outHi = (uint32x2_t)(in >> 32); + * in = UNDEFINED; + * } + */ +# if !defined(XXH_NO_VZIP_HACK) /* define to disable */ \ + && (defined(__GNUC__) || defined(__clang__)) \ + && (defined(__arm__) || defined(__thumb__) || defined(_M_ARM)) +# define XXH_SPLIT_IN_PLACE(in, outLo, outHi) \ + do { \ + /* Undocumented GCC/Clang operand modifier: %e0 = lower D half, %f0 = upper D half */ \ + /* https://github.com/gcc-mirror/gcc/blob/38cf91e5/gcc/config/arm/arm.c#L22486 */ \ + /* https://github.com/llvm-mirror/llvm/blob/2c4ca683/lib/Target/ARM/ARMAsmPrinter.cpp#L399 */ \ + __asm__("vzip.32 %e0, %f0" : "+w" (in)); \ + (outLo) = vget_low_u32 (vreinterpretq_u32_u64(in)); \ + (outHi) = vget_high_u32(vreinterpretq_u32_u64(in)); \ + } while (0) +# else +# define XXH_SPLIT_IN_PLACE(in, outLo, outHi) \ + do { \ + (outLo) = vmovn_u64 (in); \ + (outHi) = vshrn_n_u64 ((in), 32); \ + } while (0) +# endif + +/*! + * @ingroup tuning + * @brief Controls the NEON to scalar ratio for XXH3 + * + * On AArch64 when not optimizing for size, XXH3 will run 6 lanes using NEON and + * 2 lanes on scalar by default. + * + * This can be set to 2, 4, 6, or 8. ARMv7 will default to all 8 NEON lanes, as the + * emulated 64-bit arithmetic is too slow. + * + * Modern ARM CPUs are _very_ sensitive to how their pipelines are used. + * + * For example, the Cortex-A73 can dispatch 3 micro-ops per cycle, but it can't + * have more than 2 NEON (F0/F1) micro-ops. If you are only using NEON instructions, + * you are only using 2/3 of the CPU bandwidth. + * + * This is even more noticable on the more advanced cores like the A76 which + * can dispatch 8 micro-ops per cycle, but still only 2 NEON micro-ops at once. + * + * Therefore, @ref XXH3_NEON_LANES lanes will be processed using NEON, and the + * remaining lanes will use scalar instructions. This improves the bandwidth + * and also gives the integer pipelines something to do besides twiddling loop + * counters and pointers. + * + * This change benefits CPUs with large micro-op buffers without negatively affecting + * other CPUs: + * + * | Chipset | Dispatch type | NEON only | 6:2 hybrid | Diff. | + * |:----------------------|:--------------------|----------:|-----------:|------:| + * | Snapdragon 730 (A76) | 2 NEON/8 micro-ops | 8.8 GB/s | 10.1 GB/s | ~16% | + * | Snapdragon 835 (A73) | 2 NEON/3 micro-ops | 5.1 GB/s | 5.3 GB/s | ~5% | + * | Marvell PXA1928 (A53) | In-order dual-issue | 1.9 GB/s | 1.9 GB/s | 0% | + * + * It also seems to fix some bad codegen on GCC, making it almost as fast as clang. + * + * @see XXH3_accumulate_512_neon() + */ +# ifndef XXH3_NEON_LANES +# if (defined(__aarch64__) || defined(__arm64__) || defined(_M_ARM64) || defined(_M_ARM64EC)) \ + && !defined(__OPTIMIZE_SIZE__) +# define XXH3_NEON_LANES 6 +# else +# define XXH3_NEON_LANES XXH_ACC_NB +# endif +# endif +#endif /* XXH_VECTOR == XXH_NEON */ + +/* + * VSX and Z Vector helpers. + * + * This is very messy, and any pull requests to clean this up are welcome. + * + * There are a lot of problems with supporting VSX and s390x, due to + * inconsistent intrinsics, spotty coverage, and multiple endiannesses. + */ +#if XXH_VECTOR == XXH_VSX +# if defined(__s390x__) +# include +# else +/* gcc's altivec.h can have the unwanted consequence to unconditionally + * #define bool, vector, and pixel keywords, + * with bad consequences for programs already using these keywords for other purposes. + * The paragraph defining these macros is skipped when __APPLE_ALTIVEC__ is defined. + * __APPLE_ALTIVEC__ is _generally_ defined automatically by the compiler, + * but it seems that, in some cases, it isn't. + * Force the build macro to be defined, so that keywords are not altered. + */ +# if defined(__GNUC__) && !defined(__APPLE_ALTIVEC__) +# define __APPLE_ALTIVEC__ +# endif +# include +# endif + +typedef __vector unsigned long long xxh_u64x2; +typedef __vector unsigned char xxh_u8x16; +typedef __vector unsigned xxh_u32x4; + +# ifndef XXH_VSX_BE +# if defined(__BIG_ENDIAN__) \ + || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) +# define XXH_VSX_BE 1 +# elif defined(__VEC_ELEMENT_REG_ORDER__) && __VEC_ELEMENT_REG_ORDER__ == __ORDER_BIG_ENDIAN__ +# warning "-maltivec=be is not recommended. Please use native endianness." +# define XXH_VSX_BE 1 +# else +# define XXH_VSX_BE 0 +# endif +# endif /* !defined(XXH_VSX_BE) */ + +# if XXH_VSX_BE +# if defined(__POWER9_VECTOR__) || (defined(__clang__) && defined(__s390x__)) +# define XXH_vec_revb vec_revb +# else +/*! + * A polyfill for POWER9's vec_revb(). + */ +XXH_FORCE_INLINE xxh_u64x2 XXH_vec_revb(xxh_u64x2 val) +{ + xxh_u8x16 const vByteSwap = { 0x07, 0x06, 0x05, 0x04, 0x03, 0x02, 0x01, 0x00, + 0x0F, 0x0E, 0x0D, 0x0C, 0x0B, 0x0A, 0x09, 0x08 }; + return vec_perm(val, val, vByteSwap); +} +# endif +# endif /* XXH_VSX_BE */ + +/*! + * Performs an unaligned vector load and byte swaps it on big endian. + */ +XXH_FORCE_INLINE xxh_u64x2 XXH_vec_loadu(const void *ptr) +{ + xxh_u64x2 ret; + XXH_memcpy(&ret, ptr, sizeof(xxh_u64x2)); +# if XXH_VSX_BE + ret = XXH_vec_revb(ret); +# endif + return ret; +} + +/* + * vec_mulo and vec_mule are very problematic intrinsics on PowerPC + * + * These intrinsics weren't added until GCC 8, despite existing for a while, + * and they are endian dependent. Also, their meaning swap depending on version. + * */ +# if defined(__s390x__) + /* s390x is always big endian, no issue on this platform */ +# define XXH_vec_mulo vec_mulo +# define XXH_vec_mule vec_mule +# elif defined(__clang__) && XXH_HAS_BUILTIN(__builtin_altivec_vmuleuw) +/* Clang has a better way to control this, we can just use the builtin which doesn't swap. */ +# define XXH_vec_mulo __builtin_altivec_vmulouw +# define XXH_vec_mule __builtin_altivec_vmuleuw +# else +/* gcc needs inline assembly */ +/* Adapted from https://github.com/google/highwayhash/blob/master/highwayhash/hh_vsx.h. */ +XXH_FORCE_INLINE xxh_u64x2 XXH_vec_mulo(xxh_u32x4 a, xxh_u32x4 b) +{ + xxh_u64x2 result; + __asm__("vmulouw %0, %1, %2" : "=v" (result) : "v" (a), "v" (b)); + return result; +} +XXH_FORCE_INLINE xxh_u64x2 XXH_vec_mule(xxh_u32x4 a, xxh_u32x4 b) +{ + xxh_u64x2 result; + __asm__("vmuleuw %0, %1, %2" : "=v" (result) : "v" (a), "v" (b)); + return result; +} +# endif /* XXH_vec_mulo, XXH_vec_mule */ +#endif /* XXH_VECTOR == XXH_VSX */ + + +/* prefetch + * can be disabled, by declaring XXH_NO_PREFETCH build macro */ +#if defined(XXH_NO_PREFETCH) +# define XXH_PREFETCH(ptr) (void)(ptr) /* disabled */ +#else +# if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_IX86)) /* _mm_prefetch() not defined outside of x86/x64 */ +# include /* https://msdn.microsoft.com/fr-fr/library/84szxsww(v=vs.90).aspx */ +# define XXH_PREFETCH(ptr) _mm_prefetch((const char*)(ptr), _MM_HINT_T0) +# elif defined(__GNUC__) && ( (__GNUC__ >= 4) || ( (__GNUC__ == 3) && (__GNUC_MINOR__ >= 1) ) ) +# define XXH_PREFETCH(ptr) __builtin_prefetch((ptr), 0 /* rw==read */, 3 /* locality */) +# else +# define XXH_PREFETCH(ptr) (void)(ptr) /* disabled */ +# endif +#endif /* XXH_NO_PREFETCH */ + + +/* ========================================== + * XXH3 default settings + * ========================================== */ + +#define XXH_SECRET_DEFAULT_SIZE 192 /* minimum XXH3_SECRET_SIZE_MIN */ + +#if (XXH_SECRET_DEFAULT_SIZE < XXH3_SECRET_SIZE_MIN) +# error "default keyset is not large enough" +#endif + +/*! Pseudorandom secret taken directly from FARSH. */ +XXH_ALIGN(64) static const xxh_u8 XXH3_kSecret[XXH_SECRET_DEFAULT_SIZE] = { + 0xb8, 0xfe, 0x6c, 0x39, 0x23, 0xa4, 0x4b, 0xbe, 0x7c, 0x01, 0x81, 0x2c, 0xf7, 0x21, 0xad, 0x1c, + 0xde, 0xd4, 0x6d, 0xe9, 0x83, 0x90, 0x97, 0xdb, 0x72, 0x40, 0xa4, 0xa4, 0xb7, 0xb3, 0x67, 0x1f, + 0xcb, 0x79, 0xe6, 0x4e, 0xcc, 0xc0, 0xe5, 0x78, 0x82, 0x5a, 0xd0, 0x7d, 0xcc, 0xff, 0x72, 0x21, + 0xb8, 0x08, 0x46, 0x74, 0xf7, 0x43, 0x24, 0x8e, 0xe0, 0x35, 0x90, 0xe6, 0x81, 0x3a, 0x26, 0x4c, + 0x3c, 0x28, 0x52, 0xbb, 0x91, 0xc3, 0x00, 0xcb, 0x88, 0xd0, 0x65, 0x8b, 0x1b, 0x53, 0x2e, 0xa3, + 0x71, 0x64, 0x48, 0x97, 0xa2, 0x0d, 0xf9, 0x4e, 0x38, 0x19, 0xef, 0x46, 0xa9, 0xde, 0xac, 0xd8, + 0xa8, 0xfa, 0x76, 0x3f, 0xe3, 0x9c, 0x34, 0x3f, 0xf9, 0xdc, 0xbb, 0xc7, 0xc7, 0x0b, 0x4f, 0x1d, + 0x8a, 0x51, 0xe0, 0x4b, 0xcd, 0xb4, 0x59, 0x31, 0xc8, 0x9f, 0x7e, 0xc9, 0xd9, 0x78, 0x73, 0x64, + 0xea, 0xc5, 0xac, 0x83, 0x34, 0xd3, 0xeb, 0xc3, 0xc5, 0x81, 0xa0, 0xff, 0xfa, 0x13, 0x63, 0xeb, + 0x17, 0x0d, 0xdd, 0x51, 0xb7, 0xf0, 0xda, 0x49, 0xd3, 0x16, 0x55, 0x26, 0x29, 0xd4, 0x68, 0x9e, + 0x2b, 0x16, 0xbe, 0x58, 0x7d, 0x47, 0xa1, 0xfc, 0x8f, 0xf8, 0xb8, 0xd1, 0x7a, 0xd0, 0x31, 0xce, + 0x45, 0xcb, 0x3a, 0x8f, 0x95, 0x16, 0x04, 0x28, 0xaf, 0xd7, 0xfb, 0xca, 0xbb, 0x4b, 0x40, 0x7e, +}; + + +#ifdef XXH_OLD_NAMES +# define kSecret XXH3_kSecret +#endif + +#ifdef XXH_DOXYGEN +/*! + * @brief Calculates a 32-bit to 64-bit long multiply. + * + * Implemented as a macro. + * + * Wraps `__emulu` on MSVC x86 because it tends to call `__allmul` when it doesn't + * need to (but it shouldn't need to anyways, it is about 7 instructions to do + * a 64x64 multiply...). Since we know that this will _always_ emit `MULL`, we + * use that instead of the normal method. + * + * If you are compiling for platforms like Thumb-1 and don't have a better option, + * you may also want to write your own long multiply routine here. + * + * @param x, y Numbers to be multiplied + * @return 64-bit product of the low 32 bits of @p x and @p y. + */ +XXH_FORCE_INLINE xxh_u64 +XXH_mult32to64(xxh_u64 x, xxh_u64 y) +{ + return (x & 0xFFFFFFFF) * (y & 0xFFFFFFFF); +} +#elif defined(_MSC_VER) && defined(_M_IX86) +# define XXH_mult32to64(x, y) __emulu((unsigned)(x), (unsigned)(y)) +#else +/* + * Downcast + upcast is usually better than masking on older compilers like + * GCC 4.2 (especially 32-bit ones), all without affecting newer compilers. + * + * The other method, (x & 0xFFFFFFFF) * (y & 0xFFFFFFFF), will AND both operands + * and perform a full 64x64 multiply -- entirely redundant on 32-bit. + */ +# define XXH_mult32to64(x, y) ((xxh_u64)(xxh_u32)(x) * (xxh_u64)(xxh_u32)(y)) +#endif + +/*! + * @brief Calculates a 64->128-bit long multiply. + * + * Uses `__uint128_t` and `_umul128` if available, otherwise uses a scalar + * version. + * + * @param lhs , rhs The 64-bit integers to be multiplied + * @return The 128-bit result represented in an @ref XXH128_hash_t. + */ +static XXH128_hash_t +XXH_mult64to128(xxh_u64 lhs, xxh_u64 rhs) +{ + /* + * GCC/Clang __uint128_t method. + * + * On most 64-bit targets, GCC and Clang define a __uint128_t type. + * This is usually the best way as it usually uses a native long 64-bit + * multiply, such as MULQ on x86_64 or MUL + UMULH on aarch64. + * + * Usually. + * + * Despite being a 32-bit platform, Clang (and emscripten) define this type + * despite not having the arithmetic for it. This results in a laggy + * compiler builtin call which calculates a full 128-bit multiply. + * In that case it is best to use the portable one. + * https://github.com/Cyan4973/xxHash/issues/211#issuecomment-515575677 + */ +#if (defined(__GNUC__) || defined(__clang__)) && !defined(__wasm__) \ + && defined(__SIZEOF_INT128__) \ + || (defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128) + + __uint128_t const product = (__uint128_t)lhs * (__uint128_t)rhs; + XXH128_hash_t r128; + r128.low64 = (xxh_u64)(product); + r128.high64 = (xxh_u64)(product >> 64); + return r128; + + /* + * MSVC for x64's _umul128 method. + * + * xxh_u64 _umul128(xxh_u64 Multiplier, xxh_u64 Multiplicand, xxh_u64 *HighProduct); + * + * This compiles to single operand MUL on x64. + */ +#elif (defined(_M_X64) || defined(_M_IA64)) && !defined(_M_ARM64EC) + +#ifndef _MSC_VER +# pragma intrinsic(_umul128) +#endif + xxh_u64 product_high; + xxh_u64 const product_low = _umul128(lhs, rhs, &product_high); + XXH128_hash_t r128; + r128.low64 = product_low; + r128.high64 = product_high; + return r128; + + /* + * MSVC for ARM64's __umulh method. + * + * This compiles to the same MUL + UMULH as GCC/Clang's __uint128_t method. + */ +#elif defined(_M_ARM64) || defined(_M_ARM64EC) + +#ifndef _MSC_VER +# pragma intrinsic(__umulh) +#endif + XXH128_hash_t r128; + r128.low64 = lhs * rhs; + r128.high64 = __umulh(lhs, rhs); + return r128; + +#else + /* + * Portable scalar method. Optimized for 32-bit and 64-bit ALUs. + * + * This is a fast and simple grade school multiply, which is shown below + * with base 10 arithmetic instead of base 0x100000000. + * + * 9 3 // D2 lhs = 93 + * x 7 5 // D2 rhs = 75 + * ---------- + * 1 5 // D2 lo_lo = (93 % 10) * (75 % 10) = 15 + * 4 5 | // D2 hi_lo = (93 / 10) * (75 % 10) = 45 + * 2 1 | // D2 lo_hi = (93 % 10) * (75 / 10) = 21 + * + 6 3 | | // D2 hi_hi = (93 / 10) * (75 / 10) = 63 + * --------- + * 2 7 | // D2 cross = (15 / 10) + (45 % 10) + 21 = 27 + * + 6 7 | | // D2 upper = (27 / 10) + (45 / 10) + 63 = 67 + * --------- + * 6 9 7 5 // D4 res = (27 * 10) + (15 % 10) + (67 * 100) = 6975 + * + * The reasons for adding the products like this are: + * 1. It avoids manual carry tracking. Just like how + * (9 * 9) + 9 + 9 = 99, the same applies with this for UINT64_MAX. + * This avoids a lot of complexity. + * + * 2. It hints for, and on Clang, compiles to, the powerful UMAAL + * instruction available in ARM's Digital Signal Processing extension + * in 32-bit ARMv6 and later, which is shown below: + * + * void UMAAL(xxh_u32 *RdLo, xxh_u32 *RdHi, xxh_u32 Rn, xxh_u32 Rm) + * { + * xxh_u64 product = (xxh_u64)*RdLo * (xxh_u64)*RdHi + Rn + Rm; + * *RdLo = (xxh_u32)(product & 0xFFFFFFFF); + * *RdHi = (xxh_u32)(product >> 32); + * } + * + * This instruction was designed for efficient long multiplication, and + * allows this to be calculated in only 4 instructions at speeds + * comparable to some 64-bit ALUs. + * + * 3. It isn't terrible on other platforms. Usually this will be a couple + * of 32-bit ADD/ADCs. + */ + + /* First calculate all of the cross products. */ + xxh_u64 const lo_lo = XXH_mult32to64(lhs & 0xFFFFFFFF, rhs & 0xFFFFFFFF); + xxh_u64 const hi_lo = XXH_mult32to64(lhs >> 32, rhs & 0xFFFFFFFF); + xxh_u64 const lo_hi = XXH_mult32to64(lhs & 0xFFFFFFFF, rhs >> 32); + xxh_u64 const hi_hi = XXH_mult32to64(lhs >> 32, rhs >> 32); + + /* Now add the products together. These will never overflow. */ + xxh_u64 const cross = (lo_lo >> 32) + (hi_lo & 0xFFFFFFFF) + lo_hi; + xxh_u64 const upper = (hi_lo >> 32) + (cross >> 32) + hi_hi; + xxh_u64 const lower = (cross << 32) | (lo_lo & 0xFFFFFFFF); + + XXH128_hash_t r128; + r128.low64 = lower; + r128.high64 = upper; + return r128; +#endif +} + +/*! + * @brief Calculates a 64-bit to 128-bit multiply, then XOR folds it. + * + * The reason for the separate function is to prevent passing too many structs + * around by value. This will hopefully inline the multiply, but we don't force it. + * + * @param lhs , rhs The 64-bit integers to multiply + * @return The low 64 bits of the product XOR'd by the high 64 bits. + * @see XXH_mult64to128() + */ +static xxh_u64 +XXH3_mul128_fold64(xxh_u64 lhs, xxh_u64 rhs) +{ + XXH128_hash_t product = XXH_mult64to128(lhs, rhs); + return product.low64 ^ product.high64; +} + +/*! Seems to produce slightly better code on GCC for some reason. */ +XXH_FORCE_INLINE xxh_u64 XXH_xorshift64(xxh_u64 v64, int shift) +{ + XXH_ASSERT(0 <= shift && shift < 64); + return v64 ^ (v64 >> shift); +} + +/* + * This is a fast avalanche stage, + * suitable when input bits are already partially mixed + */ +static XXH64_hash_t XXH3_avalanche(xxh_u64 h64) +{ + h64 = XXH_xorshift64(h64, 37); + h64 *= 0x165667919E3779F9ULL; + h64 = XXH_xorshift64(h64, 32); + return h64; +} + +/* + * This is a stronger avalanche, + * inspired by Pelle Evensen's rrmxmx + * preferable when input has not been previously mixed + */ +static XXH64_hash_t XXH3_rrmxmx(xxh_u64 h64, xxh_u64 len) +{ + /* this mix is inspired by Pelle Evensen's rrmxmx */ + h64 ^= XXH_rotl64(h64, 49) ^ XXH_rotl64(h64, 24); + h64 *= 0x9FB21C651E98DF25ULL; + h64 ^= (h64 >> 35) + len ; + h64 *= 0x9FB21C651E98DF25ULL; + return XXH_xorshift64(h64, 28); +} + + +/* ========================================== + * Short keys + * ========================================== + * One of the shortcomings of XXH32 and XXH64 was that their performance was + * sub-optimal on short lengths. It used an iterative algorithm which strongly + * favored lengths that were a multiple of 4 or 8. + * + * Instead of iterating over individual inputs, we use a set of single shot + * functions which piece together a range of lengths and operate in constant time. + * + * Additionally, the number of multiplies has been significantly reduced. This + * reduces latency, especially when emulating 64-bit multiplies on 32-bit. + * + * Depending on the platform, this may or may not be faster than XXH32, but it + * is almost guaranteed to be faster than XXH64. + */ + +/* + * At very short lengths, there isn't enough input to fully hide secrets, or use + * the entire secret. + * + * There is also only a limited amount of mixing we can do before significantly + * impacting performance. + * + * Therefore, we use different sections of the secret and always mix two secret + * samples with an XOR. This should have no effect on performance on the + * seedless or withSeed variants because everything _should_ be constant folded + * by modern compilers. + * + * The XOR mixing hides individual parts of the secret and increases entropy. + * + * This adds an extra layer of strength for custom secrets. + */ +XXH_FORCE_INLINE XXH64_hash_t +XXH3_len_1to3_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(input != NULL); + XXH_ASSERT(1 <= len && len <= 3); + XXH_ASSERT(secret != NULL); + /* + * len = 1: combined = { input[0], 0x01, input[0], input[0] } + * len = 2: combined = { input[1], 0x02, input[0], input[1] } + * len = 3: combined = { input[2], 0x03, input[0], input[1] } + */ + { xxh_u8 const c1 = input[0]; + xxh_u8 const c2 = input[len >> 1]; + xxh_u8 const c3 = input[len - 1]; + xxh_u32 const combined = ((xxh_u32)c1 << 16) | ((xxh_u32)c2 << 24) + | ((xxh_u32)c3 << 0) | ((xxh_u32)len << 8); + xxh_u64 const bitflip = (XXH_readLE32(secret) ^ XXH_readLE32(secret+4)) + seed; + xxh_u64 const keyed = (xxh_u64)combined ^ bitflip; + return XXH64_avalanche(keyed); + } +} + +XXH_FORCE_INLINE XXH64_hash_t +XXH3_len_4to8_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(input != NULL); + XXH_ASSERT(secret != NULL); + XXH_ASSERT(4 <= len && len <= 8); + seed ^= (xxh_u64)XXH_swap32((xxh_u32)seed) << 32; + { xxh_u32 const input1 = XXH_readLE32(input); + xxh_u32 const input2 = XXH_readLE32(input + len - 4); + xxh_u64 const bitflip = (XXH_readLE64(secret+8) ^ XXH_readLE64(secret+16)) - seed; + xxh_u64 const input64 = input2 + (((xxh_u64)input1) << 32); + xxh_u64 const keyed = input64 ^ bitflip; + return XXH3_rrmxmx(keyed, len); + } +} + +XXH_FORCE_INLINE XXH64_hash_t +XXH3_len_9to16_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(input != NULL); + XXH_ASSERT(secret != NULL); + XXH_ASSERT(9 <= len && len <= 16); + { xxh_u64 const bitflip1 = (XXH_readLE64(secret+24) ^ XXH_readLE64(secret+32)) + seed; + xxh_u64 const bitflip2 = (XXH_readLE64(secret+40) ^ XXH_readLE64(secret+48)) - seed; + xxh_u64 const input_lo = XXH_readLE64(input) ^ bitflip1; + xxh_u64 const input_hi = XXH_readLE64(input + len - 8) ^ bitflip2; + xxh_u64 const acc = len + + XXH_swap64(input_lo) + input_hi + + XXH3_mul128_fold64(input_lo, input_hi); + return XXH3_avalanche(acc); + } +} + +XXH_FORCE_INLINE XXH64_hash_t +XXH3_len_0to16_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(len <= 16); + { if (XXH_likely(len > 8)) return XXH3_len_9to16_64b(input, len, secret, seed); + if (XXH_likely(len >= 4)) return XXH3_len_4to8_64b(input, len, secret, seed); + if (len) return XXH3_len_1to3_64b(input, len, secret, seed); + return XXH64_avalanche(seed ^ (XXH_readLE64(secret+56) ^ XXH_readLE64(secret+64))); + } +} + +/* + * DISCLAIMER: There are known *seed-dependent* multicollisions here due to + * multiplication by zero, affecting hashes of lengths 17 to 240. + * + * However, they are very unlikely. + * + * Keep this in mind when using the unseeded XXH3_64bits() variant: As with all + * unseeded non-cryptographic hashes, it does not attempt to defend itself + * against specially crafted inputs, only random inputs. + * + * Compared to classic UMAC where a 1 in 2^31 chance of 4 consecutive bytes + * cancelling out the secret is taken an arbitrary number of times (addressed + * in XXH3_accumulate_512), this collision is very unlikely with random inputs + * and/or proper seeding: + * + * This only has a 1 in 2^63 chance of 8 consecutive bytes cancelling out, in a + * function that is only called up to 16 times per hash with up to 240 bytes of + * input. + * + * This is not too bad for a non-cryptographic hash function, especially with + * only 64 bit outputs. + * + * The 128-bit variant (which trades some speed for strength) is NOT affected + * by this, although it is always a good idea to use a proper seed if you care + * about strength. + */ +XXH_FORCE_INLINE xxh_u64 XXH3_mix16B(const xxh_u8* XXH_RESTRICT input, + const xxh_u8* XXH_RESTRICT secret, xxh_u64 seed64) +{ +#if defined(__GNUC__) && !defined(__clang__) /* GCC, not Clang */ \ + && defined(__i386__) && defined(__SSE2__) /* x86 + SSE2 */ \ + && !defined(XXH_ENABLE_AUTOVECTORIZE) /* Define to disable like XXH32 hack */ + /* + * UGLY HACK: + * GCC for x86 tends to autovectorize the 128-bit multiply, resulting in + * slower code. + * + * By forcing seed64 into a register, we disrupt the cost model and + * cause it to scalarize. See `XXH32_round()` + * + * FIXME: Clang's output is still _much_ faster -- On an AMD Ryzen 3600, + * XXH3_64bits @ len=240 runs at 4.6 GB/s with Clang 9, but 3.3 GB/s on + * GCC 9.2, despite both emitting scalar code. + * + * GCC generates much better scalar code than Clang for the rest of XXH3, + * which is why finding a more optimal codepath is an interest. + */ + XXH_COMPILER_GUARD(seed64); +#endif + { xxh_u64 const input_lo = XXH_readLE64(input); + xxh_u64 const input_hi = XXH_readLE64(input+8); + return XXH3_mul128_fold64( + input_lo ^ (XXH_readLE64(secret) + seed64), + input_hi ^ (XXH_readLE64(secret+8) - seed64) + ); + } +} + +/* For mid range keys, XXH3 uses a Mum-hash variant. */ +XXH_FORCE_INLINE XXH64_hash_t +XXH3_len_17to128_64b(const xxh_u8* XXH_RESTRICT input, size_t len, + const xxh_u8* XXH_RESTRICT secret, size_t secretSize, + XXH64_hash_t seed) +{ + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize; + XXH_ASSERT(16 < len && len <= 128); + + { xxh_u64 acc = len * XXH_PRIME64_1; + if (len > 32) { + if (len > 64) { + if (len > 96) { + acc += XXH3_mix16B(input+48, secret+96, seed); + acc += XXH3_mix16B(input+len-64, secret+112, seed); + } + acc += XXH3_mix16B(input+32, secret+64, seed); + acc += XXH3_mix16B(input+len-48, secret+80, seed); + } + acc += XXH3_mix16B(input+16, secret+32, seed); + acc += XXH3_mix16B(input+len-32, secret+48, seed); + } + acc += XXH3_mix16B(input+0, secret+0, seed); + acc += XXH3_mix16B(input+len-16, secret+16, seed); + + return XXH3_avalanche(acc); + } +} + +#define XXH3_MIDSIZE_MAX 240 + +XXH_NO_INLINE XXH64_hash_t +XXH3_len_129to240_64b(const xxh_u8* XXH_RESTRICT input, size_t len, + const xxh_u8* XXH_RESTRICT secret, size_t secretSize, + XXH64_hash_t seed) +{ + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize; + XXH_ASSERT(128 < len && len <= XXH3_MIDSIZE_MAX); + + #define XXH3_MIDSIZE_STARTOFFSET 3 + #define XXH3_MIDSIZE_LASTOFFSET 17 + + { xxh_u64 acc = len * XXH_PRIME64_1; + int const nbRounds = (int)len / 16; + int i; + for (i=0; i<8; i++) { + acc += XXH3_mix16B(input+(16*i), secret+(16*i), seed); + } + acc = XXH3_avalanche(acc); + XXH_ASSERT(nbRounds >= 8); +#if defined(__clang__) /* Clang */ \ + && (defined(__ARM_NEON) || defined(__ARM_NEON__)) /* NEON */ \ + && !defined(XXH_ENABLE_AUTOVECTORIZE) /* Define to disable */ + /* + * UGLY HACK: + * Clang for ARMv7-A tries to vectorize this loop, similar to GCC x86. + * In everywhere else, it uses scalar code. + * + * For 64->128-bit multiplies, even if the NEON was 100% optimal, it + * would still be slower than UMAAL (see XXH_mult64to128). + * + * Unfortunately, Clang doesn't handle the long multiplies properly and + * converts them to the nonexistent "vmulq_u64" intrinsic, which is then + * scalarized into an ugly mess of VMOV.32 instructions. + * + * This mess is difficult to avoid without turning autovectorization + * off completely, but they are usually relatively minor and/or not + * worth it to fix. + * + * This loop is the easiest to fix, as unlike XXH32, this pragma + * _actually works_ because it is a loop vectorization instead of an + * SLP vectorization. + */ + #pragma clang loop vectorize(disable) +#endif + for (i=8 ; i < nbRounds; i++) { + acc += XXH3_mix16B(input+(16*i), secret+(16*(i-8)) + XXH3_MIDSIZE_STARTOFFSET, seed); + } + /* last bytes */ + acc += XXH3_mix16B(input + len - 16, secret + XXH3_SECRET_SIZE_MIN - XXH3_MIDSIZE_LASTOFFSET, seed); + return XXH3_avalanche(acc); + } +} + + +/* ======= Long Keys ======= */ + +#define XXH_STRIPE_LEN 64 +#define XXH_SECRET_CONSUME_RATE 8 /* nb of secret bytes consumed at each accumulation */ +#define XXH_ACC_NB (XXH_STRIPE_LEN / sizeof(xxh_u64)) + +#ifdef XXH_OLD_NAMES +# define STRIPE_LEN XXH_STRIPE_LEN +# define ACC_NB XXH_ACC_NB +#endif + +XXH_FORCE_INLINE void XXH_writeLE64(void* dst, xxh_u64 v64) +{ + if (!XXH_CPU_LITTLE_ENDIAN) v64 = XXH_swap64(v64); + XXH_memcpy(dst, &v64, sizeof(v64)); +} + +/* Several intrinsic functions below are supposed to accept __int64 as argument, + * as documented in https://software.intel.com/sites/landingpage/IntrinsicsGuide/ . + * However, several environments do not define __int64 type, + * requiring a workaround. + */ +#if !defined (__VMS) \ + && (defined (__cplusplus) \ + || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) + typedef int64_t xxh_i64; +#else + /* the following type must have a width of 64-bit */ + typedef long long xxh_i64; +#endif + + +/* + * XXH3_accumulate_512 is the tightest loop for long inputs, and it is the most optimized. + * + * It is a hardened version of UMAC, based off of FARSH's implementation. + * + * This was chosen because it adapts quite well to 32-bit, 64-bit, and SIMD + * implementations, and it is ridiculously fast. + * + * We harden it by mixing the original input to the accumulators as well as the product. + * + * This means that in the (relatively likely) case of a multiply by zero, the + * original input is preserved. + * + * On 128-bit inputs, we swap 64-bit pairs when we add the input to improve + * cross-pollination, as otherwise the upper and lower halves would be + * essentially independent. + * + * This doesn't matter on 64-bit hashes since they all get merged together in + * the end, so we skip the extra step. + * + * Both XXH3_64bits and XXH3_128bits use this subroutine. + */ + +#if (XXH_VECTOR == XXH_AVX512) \ + || (defined(XXH_DISPATCH_AVX512) && XXH_DISPATCH_AVX512 != 0) + +#ifndef XXH_TARGET_AVX512 +# define XXH_TARGET_AVX512 /* disable attribute target */ +#endif + +XXH_FORCE_INLINE XXH_TARGET_AVX512 void +XXH3_accumulate_512_avx512(void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + __m512i* const xacc = (__m512i *) acc; + XXH_ASSERT((((size_t)acc) & 63) == 0); + XXH_STATIC_ASSERT(XXH_STRIPE_LEN == sizeof(__m512i)); + + { + /* data_vec = input[0]; */ + __m512i const data_vec = _mm512_loadu_si512 (input); + /* key_vec = secret[0]; */ + __m512i const key_vec = _mm512_loadu_si512 (secret); + /* data_key = data_vec ^ key_vec; */ + __m512i const data_key = _mm512_xor_si512 (data_vec, key_vec); + /* data_key_lo = data_key >> 32; */ + __m512i const data_key_lo = _mm512_shuffle_epi32 (data_key, (_MM_PERM_ENUM)_MM_SHUFFLE(0, 3, 0, 1)); + /* product = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */ + __m512i const product = _mm512_mul_epu32 (data_key, data_key_lo); + /* xacc[0] += swap(data_vec); */ + __m512i const data_swap = _mm512_shuffle_epi32(data_vec, (_MM_PERM_ENUM)_MM_SHUFFLE(1, 0, 3, 2)); + __m512i const sum = _mm512_add_epi64(*xacc, data_swap); + /* xacc[0] += product; */ + *xacc = _mm512_add_epi64(product, sum); + } +} + +/* + * XXH3_scrambleAcc: Scrambles the accumulators to improve mixing. + * + * Multiplication isn't perfect, as explained by Google in HighwayHash: + * + * // Multiplication mixes/scrambles bytes 0-7 of the 64-bit result to + * // varying degrees. In descending order of goodness, bytes + * // 3 4 2 5 1 6 0 7 have quality 228 224 164 160 100 96 36 32. + * // As expected, the upper and lower bytes are much worse. + * + * Source: https://github.com/google/highwayhash/blob/0aaf66b/highwayhash/hh_avx2.h#L291 + * + * Since our algorithm uses a pseudorandom secret to add some variance into the + * mix, we don't need to (or want to) mix as often or as much as HighwayHash does. + * + * This isn't as tight as XXH3_accumulate, but still written in SIMD to avoid + * extraction. + * + * Both XXH3_64bits and XXH3_128bits use this subroutine. + */ + +XXH_FORCE_INLINE XXH_TARGET_AVX512 void +XXH3_scrambleAcc_avx512(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 63) == 0); + XXH_STATIC_ASSERT(XXH_STRIPE_LEN == sizeof(__m512i)); + { __m512i* const xacc = (__m512i*) acc; + const __m512i prime32 = _mm512_set1_epi32((int)XXH_PRIME32_1); + + /* xacc[0] ^= (xacc[0] >> 47) */ + __m512i const acc_vec = *xacc; + __m512i const shifted = _mm512_srli_epi64 (acc_vec, 47); + __m512i const data_vec = _mm512_xor_si512 (acc_vec, shifted); + /* xacc[0] ^= secret; */ + __m512i const key_vec = _mm512_loadu_si512 (secret); + __m512i const data_key = _mm512_xor_si512 (data_vec, key_vec); + + /* xacc[0] *= XXH_PRIME32_1; */ + __m512i const data_key_hi = _mm512_shuffle_epi32 (data_key, (_MM_PERM_ENUM)_MM_SHUFFLE(0, 3, 0, 1)); + __m512i const prod_lo = _mm512_mul_epu32 (data_key, prime32); + __m512i const prod_hi = _mm512_mul_epu32 (data_key_hi, prime32); + *xacc = _mm512_add_epi64(prod_lo, _mm512_slli_epi64(prod_hi, 32)); + } +} + +XXH_FORCE_INLINE XXH_TARGET_AVX512 void +XXH3_initCustomSecret_avx512(void* XXH_RESTRICT customSecret, xxh_u64 seed64) +{ + XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 63) == 0); + XXH_STATIC_ASSERT(XXH_SEC_ALIGN == 64); + XXH_ASSERT(((size_t)customSecret & 63) == 0); + (void)(&XXH_writeLE64); + { int const nbRounds = XXH_SECRET_DEFAULT_SIZE / sizeof(__m512i); + __m512i const seed = _mm512_mask_set1_epi64(_mm512_set1_epi64((xxh_i64)seed64), 0xAA, (xxh_i64)(0U - seed64)); + + const __m512i* const src = (const __m512i*) ((const void*) XXH3_kSecret); + __m512i* const dest = ( __m512i*) customSecret; + int i; + XXH_ASSERT(((size_t)src & 63) == 0); /* control alignment */ + XXH_ASSERT(((size_t)dest & 63) == 0); + for (i=0; i < nbRounds; ++i) { + /* GCC has a bug, _mm512_stream_load_si512 accepts 'void*', not 'void const*', + * this will warn "discards 'const' qualifier". */ + union { + const __m512i* cp; + void* p; + } remote_const_void; + remote_const_void.cp = src + i; + dest[i] = _mm512_add_epi64(_mm512_stream_load_si512(remote_const_void.p), seed); + } } +} + +#endif + +#if (XXH_VECTOR == XXH_AVX2) \ + || (defined(XXH_DISPATCH_AVX2) && XXH_DISPATCH_AVX2 != 0) + +#ifndef XXH_TARGET_AVX2 +# define XXH_TARGET_AVX2 /* disable attribute target */ +#endif + +XXH_FORCE_INLINE XXH_TARGET_AVX2 void +XXH3_accumulate_512_avx2( void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 31) == 0); + { __m256i* const xacc = (__m256i *) acc; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm256_loadu_si256 requires a const __m256i * pointer for some reason. */ + const __m256i* const xinput = (const __m256i *) input; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm256_loadu_si256 requires a const __m256i * pointer for some reason. */ + const __m256i* const xsecret = (const __m256i *) secret; + + size_t i; + for (i=0; i < XXH_STRIPE_LEN/sizeof(__m256i); i++) { + /* data_vec = xinput[i]; */ + __m256i const data_vec = _mm256_loadu_si256 (xinput+i); + /* key_vec = xsecret[i]; */ + __m256i const key_vec = _mm256_loadu_si256 (xsecret+i); + /* data_key = data_vec ^ key_vec; */ + __m256i const data_key = _mm256_xor_si256 (data_vec, key_vec); + /* data_key_lo = data_key >> 32; */ + __m256i const data_key_lo = _mm256_shuffle_epi32 (data_key, _MM_SHUFFLE(0, 3, 0, 1)); + /* product = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */ + __m256i const product = _mm256_mul_epu32 (data_key, data_key_lo); + /* xacc[i] += swap(data_vec); */ + __m256i const data_swap = _mm256_shuffle_epi32(data_vec, _MM_SHUFFLE(1, 0, 3, 2)); + __m256i const sum = _mm256_add_epi64(xacc[i], data_swap); + /* xacc[i] += product; */ + xacc[i] = _mm256_add_epi64(product, sum); + } } +} + +XXH_FORCE_INLINE XXH_TARGET_AVX2 void +XXH3_scrambleAcc_avx2(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 31) == 0); + { __m256i* const xacc = (__m256i*) acc; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm256_loadu_si256 requires a const __m256i * pointer for some reason. */ + const __m256i* const xsecret = (const __m256i *) secret; + const __m256i prime32 = _mm256_set1_epi32((int)XXH_PRIME32_1); + + size_t i; + for (i=0; i < XXH_STRIPE_LEN/sizeof(__m256i); i++) { + /* xacc[i] ^= (xacc[i] >> 47) */ + __m256i const acc_vec = xacc[i]; + __m256i const shifted = _mm256_srli_epi64 (acc_vec, 47); + __m256i const data_vec = _mm256_xor_si256 (acc_vec, shifted); + /* xacc[i] ^= xsecret; */ + __m256i const key_vec = _mm256_loadu_si256 (xsecret+i); + __m256i const data_key = _mm256_xor_si256 (data_vec, key_vec); + + /* xacc[i] *= XXH_PRIME32_1; */ + __m256i const data_key_hi = _mm256_shuffle_epi32 (data_key, _MM_SHUFFLE(0, 3, 0, 1)); + __m256i const prod_lo = _mm256_mul_epu32 (data_key, prime32); + __m256i const prod_hi = _mm256_mul_epu32 (data_key_hi, prime32); + xacc[i] = _mm256_add_epi64(prod_lo, _mm256_slli_epi64(prod_hi, 32)); + } + } +} + +XXH_FORCE_INLINE XXH_TARGET_AVX2 void XXH3_initCustomSecret_avx2(void* XXH_RESTRICT customSecret, xxh_u64 seed64) +{ + XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 31) == 0); + XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE / sizeof(__m256i)) == 6); + XXH_STATIC_ASSERT(XXH_SEC_ALIGN <= 64); + (void)(&XXH_writeLE64); + XXH_PREFETCH(customSecret); + { __m256i const seed = _mm256_set_epi64x((xxh_i64)(0U - seed64), (xxh_i64)seed64, (xxh_i64)(0U - seed64), (xxh_i64)seed64); + + const __m256i* const src = (const __m256i*) ((const void*) XXH3_kSecret); + __m256i* dest = ( __m256i*) customSecret; + +# if defined(__GNUC__) || defined(__clang__) + /* + * On GCC & Clang, marking 'dest' as modified will cause the compiler: + * - do not extract the secret from sse registers in the internal loop + * - use less common registers, and avoid pushing these reg into stack + */ + XXH_COMPILER_GUARD(dest); +# endif + XXH_ASSERT(((size_t)src & 31) == 0); /* control alignment */ + XXH_ASSERT(((size_t)dest & 31) == 0); + + /* GCC -O2 need unroll loop manually */ + dest[0] = _mm256_add_epi64(_mm256_stream_load_si256(src+0), seed); + dest[1] = _mm256_add_epi64(_mm256_stream_load_si256(src+1), seed); + dest[2] = _mm256_add_epi64(_mm256_stream_load_si256(src+2), seed); + dest[3] = _mm256_add_epi64(_mm256_stream_load_si256(src+3), seed); + dest[4] = _mm256_add_epi64(_mm256_stream_load_si256(src+4), seed); + dest[5] = _mm256_add_epi64(_mm256_stream_load_si256(src+5), seed); + } +} + +#endif + +/* x86dispatch always generates SSE2 */ +#if (XXH_VECTOR == XXH_SSE2) || defined(XXH_X86DISPATCH) + +#ifndef XXH_TARGET_SSE2 +# define XXH_TARGET_SSE2 /* disable attribute target */ +#endif + +XXH_FORCE_INLINE XXH_TARGET_SSE2 void +XXH3_accumulate_512_sse2( void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + /* SSE2 is just a half-scale version of the AVX2 version. */ + XXH_ASSERT((((size_t)acc) & 15) == 0); + { __m128i* const xacc = (__m128i *) acc; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm_loadu_si128 requires a const __m128i * pointer for some reason. */ + const __m128i* const xinput = (const __m128i *) input; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm_loadu_si128 requires a const __m128i * pointer for some reason. */ + const __m128i* const xsecret = (const __m128i *) secret; + + size_t i; + for (i=0; i < XXH_STRIPE_LEN/sizeof(__m128i); i++) { + /* data_vec = xinput[i]; */ + __m128i const data_vec = _mm_loadu_si128 (xinput+i); + /* key_vec = xsecret[i]; */ + __m128i const key_vec = _mm_loadu_si128 (xsecret+i); + /* data_key = data_vec ^ key_vec; */ + __m128i const data_key = _mm_xor_si128 (data_vec, key_vec); + /* data_key_lo = data_key >> 32; */ + __m128i const data_key_lo = _mm_shuffle_epi32 (data_key, _MM_SHUFFLE(0, 3, 0, 1)); + /* product = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */ + __m128i const product = _mm_mul_epu32 (data_key, data_key_lo); + /* xacc[i] += swap(data_vec); */ + __m128i const data_swap = _mm_shuffle_epi32(data_vec, _MM_SHUFFLE(1,0,3,2)); + __m128i const sum = _mm_add_epi64(xacc[i], data_swap); + /* xacc[i] += product; */ + xacc[i] = _mm_add_epi64(product, sum); + } } +} + +XXH_FORCE_INLINE XXH_TARGET_SSE2 void +XXH3_scrambleAcc_sse2(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 15) == 0); + { __m128i* const xacc = (__m128i*) acc; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm_loadu_si128 requires a const __m128i * pointer for some reason. */ + const __m128i* const xsecret = (const __m128i *) secret; + const __m128i prime32 = _mm_set1_epi32((int)XXH_PRIME32_1); + + size_t i; + for (i=0; i < XXH_STRIPE_LEN/sizeof(__m128i); i++) { + /* xacc[i] ^= (xacc[i] >> 47) */ + __m128i const acc_vec = xacc[i]; + __m128i const shifted = _mm_srli_epi64 (acc_vec, 47); + __m128i const data_vec = _mm_xor_si128 (acc_vec, shifted); + /* xacc[i] ^= xsecret[i]; */ + __m128i const key_vec = _mm_loadu_si128 (xsecret+i); + __m128i const data_key = _mm_xor_si128 (data_vec, key_vec); + + /* xacc[i] *= XXH_PRIME32_1; */ + __m128i const data_key_hi = _mm_shuffle_epi32 (data_key, _MM_SHUFFLE(0, 3, 0, 1)); + __m128i const prod_lo = _mm_mul_epu32 (data_key, prime32); + __m128i const prod_hi = _mm_mul_epu32 (data_key_hi, prime32); + xacc[i] = _mm_add_epi64(prod_lo, _mm_slli_epi64(prod_hi, 32)); + } + } +} + +XXH_FORCE_INLINE XXH_TARGET_SSE2 void XXH3_initCustomSecret_sse2(void* XXH_RESTRICT customSecret, xxh_u64 seed64) +{ + XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 15) == 0); + (void)(&XXH_writeLE64); + { int const nbRounds = XXH_SECRET_DEFAULT_SIZE / sizeof(__m128i); + +# if defined(_MSC_VER) && defined(_M_IX86) && _MSC_VER < 1900 + /* MSVC 32bit mode does not support _mm_set_epi64x before 2015 */ + XXH_ALIGN(16) const xxh_i64 seed64x2[2] = { (xxh_i64)seed64, (xxh_i64)(0U - seed64) }; + __m128i const seed = _mm_load_si128((__m128i const*)seed64x2); +# else + __m128i const seed = _mm_set_epi64x((xxh_i64)(0U - seed64), (xxh_i64)seed64); +# endif + int i; + + const void* const src16 = XXH3_kSecret; + __m128i* dst16 = (__m128i*) customSecret; +# if defined(__GNUC__) || defined(__clang__) + /* + * On GCC & Clang, marking 'dest' as modified will cause the compiler: + * - do not extract the secret from sse registers in the internal loop + * - use less common registers, and avoid pushing these reg into stack + */ + XXH_COMPILER_GUARD(dst16); +# endif + XXH_ASSERT(((size_t)src16 & 15) == 0); /* control alignment */ + XXH_ASSERT(((size_t)dst16 & 15) == 0); + + for (i=0; i < nbRounds; ++i) { + dst16[i] = _mm_add_epi64(_mm_load_si128((const __m128i *)src16+i), seed); + } } +} + +#endif + +#if (XXH_VECTOR == XXH_NEON) + +/* forward declarations for the scalar routines */ +XXH_FORCE_INLINE void +XXH3_scalarRound(void* XXH_RESTRICT acc, void const* XXH_RESTRICT input, + void const* XXH_RESTRICT secret, size_t lane); + +XXH_FORCE_INLINE void +XXH3_scalarScrambleRound(void* XXH_RESTRICT acc, + void const* XXH_RESTRICT secret, size_t lane); + +/*! + * @internal + * @brief The bulk processing loop for NEON. + * + * The NEON code path is actually partially scalar when running on AArch64. This + * is to optimize the pipelining and can have up to 15% speedup depending on the + * CPU, and it also mitigates some GCC codegen issues. + * + * @see XXH3_NEON_LANES for configuring this and details about this optimization. + */ +XXH_FORCE_INLINE void +XXH3_accumulate_512_neon( void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 15) == 0); + XXH_STATIC_ASSERT(XXH3_NEON_LANES > 0 && XXH3_NEON_LANES <= XXH_ACC_NB && XXH3_NEON_LANES % 2 == 0); + { + uint64x2_t* const xacc = (uint64x2_t *) acc; + /* We don't use a uint32x4_t pointer because it causes bus errors on ARMv7. */ + uint8_t const* const xinput = (const uint8_t *) input; + uint8_t const* const xsecret = (const uint8_t *) secret; + + size_t i; + /* NEON for the first few lanes (these loops are normally interleaved) */ + for (i=0; i < XXH3_NEON_LANES / 2; i++) { + /* data_vec = xinput[i]; */ + uint8x16_t data_vec = vld1q_u8(xinput + (i * 16)); + /* key_vec = xsecret[i]; */ + uint8x16_t key_vec = vld1q_u8(xsecret + (i * 16)); + uint64x2_t data_key; + uint32x2_t data_key_lo, data_key_hi; + /* xacc[i] += swap(data_vec); */ + uint64x2_t const data64 = vreinterpretq_u64_u8(data_vec); + uint64x2_t const swapped = vextq_u64(data64, data64, 1); + xacc[i] = vaddq_u64 (xacc[i], swapped); + /* data_key = data_vec ^ key_vec; */ + data_key = vreinterpretq_u64_u8(veorq_u8(data_vec, key_vec)); + /* data_key_lo = (uint32x2_t) (data_key & 0xFFFFFFFF); + * data_key_hi = (uint32x2_t) (data_key >> 32); + * data_key = UNDEFINED; */ + XXH_SPLIT_IN_PLACE(data_key, data_key_lo, data_key_hi); + /* xacc[i] += (uint64x2_t) data_key_lo * (uint64x2_t) data_key_hi; */ + xacc[i] = vmlal_u32 (xacc[i], data_key_lo, data_key_hi); + + } + /* Scalar for the remainder. This may be a zero iteration loop. */ + for (i = XXH3_NEON_LANES; i < XXH_ACC_NB; i++) { + XXH3_scalarRound(acc, input, secret, i); + } + } +} + +XXH_FORCE_INLINE void +XXH3_scrambleAcc_neon(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 15) == 0); + + { uint64x2_t* xacc = (uint64x2_t*) acc; + uint8_t const* xsecret = (uint8_t const*) secret; + uint32x2_t prime = vdup_n_u32 (XXH_PRIME32_1); + + size_t i; + /* NEON for the first few lanes (these loops are normally interleaved) */ + for (i=0; i < XXH3_NEON_LANES / 2; i++) { + /* xacc[i] ^= (xacc[i] >> 47); */ + uint64x2_t acc_vec = xacc[i]; + uint64x2_t shifted = vshrq_n_u64 (acc_vec, 47); + uint64x2_t data_vec = veorq_u64 (acc_vec, shifted); + + /* xacc[i] ^= xsecret[i]; */ + uint8x16_t key_vec = vld1q_u8 (xsecret + (i * 16)); + uint64x2_t data_key = veorq_u64 (data_vec, vreinterpretq_u64_u8(key_vec)); + + /* xacc[i] *= XXH_PRIME32_1 */ + uint32x2_t data_key_lo, data_key_hi; + /* data_key_lo = (uint32x2_t) (xacc[i] & 0xFFFFFFFF); + * data_key_hi = (uint32x2_t) (xacc[i] >> 32); + * xacc[i] = UNDEFINED; */ + XXH_SPLIT_IN_PLACE(data_key, data_key_lo, data_key_hi); + { /* + * prod_hi = (data_key >> 32) * XXH_PRIME32_1; + * + * Avoid vmul_u32 + vshll_n_u32 since Clang 6 and 7 will + * incorrectly "optimize" this: + * tmp = vmul_u32(vmovn_u64(a), vmovn_u64(b)); + * shifted = vshll_n_u32(tmp, 32); + * to this: + * tmp = "vmulq_u64"(a, b); // no such thing! + * shifted = vshlq_n_u64(tmp, 32); + * + * However, unlike SSE, Clang lacks a 64-bit multiply routine + * for NEON, and it scalarizes two 64-bit multiplies instead. + * + * vmull_u32 has the same timing as vmul_u32, and it avoids + * this bug completely. + * See https://bugs.llvm.org/show_bug.cgi?id=39967 + */ + uint64x2_t prod_hi = vmull_u32 (data_key_hi, prime); + /* xacc[i] = prod_hi << 32; */ + xacc[i] = vshlq_n_u64(prod_hi, 32); + /* xacc[i] += (prod_hi & 0xFFFFFFFF) * XXH_PRIME32_1; */ + xacc[i] = vmlal_u32(xacc[i], data_key_lo, prime); + } + } + /* Scalar for the remainder. This may be a zero iteration loop. */ + for (i = XXH3_NEON_LANES; i < XXH_ACC_NB; i++) { + XXH3_scalarScrambleRound(acc, secret, i); + } + } +} + +#endif + +#if (XXH_VECTOR == XXH_VSX) + +XXH_FORCE_INLINE void +XXH3_accumulate_512_vsx( void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + /* presumed aligned */ + unsigned int* const xacc = (unsigned int*) acc; + xxh_u64x2 const* const xinput = (xxh_u64x2 const*) input; /* no alignment restriction */ + xxh_u64x2 const* const xsecret = (xxh_u64x2 const*) secret; /* no alignment restriction */ + xxh_u64x2 const v32 = { 32, 32 }; + size_t i; + for (i = 0; i < XXH_STRIPE_LEN / sizeof(xxh_u64x2); i++) { + /* data_vec = xinput[i]; */ + xxh_u64x2 const data_vec = XXH_vec_loadu(xinput + i); + /* key_vec = xsecret[i]; */ + xxh_u64x2 const key_vec = XXH_vec_loadu(xsecret + i); + xxh_u64x2 const data_key = data_vec ^ key_vec; + /* shuffled = (data_key << 32) | (data_key >> 32); */ + xxh_u32x4 const shuffled = (xxh_u32x4)vec_rl(data_key, v32); + /* product = ((xxh_u64x2)data_key & 0xFFFFFFFF) * ((xxh_u64x2)shuffled & 0xFFFFFFFF); */ + xxh_u64x2 const product = XXH_vec_mulo((xxh_u32x4)data_key, shuffled); + /* acc_vec = xacc[i]; */ + xxh_u64x2 acc_vec = (xxh_u64x2)vec_xl(0, xacc + 4 * i); + acc_vec += product; + + /* swap high and low halves */ +#ifdef __s390x__ + acc_vec += vec_permi(data_vec, data_vec, 2); +#else + acc_vec += vec_xxpermdi(data_vec, data_vec, 2); +#endif + /* xacc[i] = acc_vec; */ + vec_xst((xxh_u32x4)acc_vec, 0, xacc + 4 * i); + } +} + +XXH_FORCE_INLINE void +XXH3_scrambleAcc_vsx(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 15) == 0); + + { xxh_u64x2* const xacc = (xxh_u64x2*) acc; + const xxh_u64x2* const xsecret = (const xxh_u64x2*) secret; + /* constants */ + xxh_u64x2 const v32 = { 32, 32 }; + xxh_u64x2 const v47 = { 47, 47 }; + xxh_u32x4 const prime = { XXH_PRIME32_1, XXH_PRIME32_1, XXH_PRIME32_1, XXH_PRIME32_1 }; + size_t i; + for (i = 0; i < XXH_STRIPE_LEN / sizeof(xxh_u64x2); i++) { + /* xacc[i] ^= (xacc[i] >> 47); */ + xxh_u64x2 const acc_vec = xacc[i]; + xxh_u64x2 const data_vec = acc_vec ^ (acc_vec >> v47); + + /* xacc[i] ^= xsecret[i]; */ + xxh_u64x2 const key_vec = XXH_vec_loadu(xsecret + i); + xxh_u64x2 const data_key = data_vec ^ key_vec; + + /* xacc[i] *= XXH_PRIME32_1 */ + /* prod_lo = ((xxh_u64x2)data_key & 0xFFFFFFFF) * ((xxh_u64x2)prime & 0xFFFFFFFF); */ + xxh_u64x2 const prod_even = XXH_vec_mule((xxh_u32x4)data_key, prime); + /* prod_hi = ((xxh_u64x2)data_key >> 32) * ((xxh_u64x2)prime >> 32); */ + xxh_u64x2 const prod_odd = XXH_vec_mulo((xxh_u32x4)data_key, prime); + xacc[i] = prod_odd + (prod_even << v32); + } } +} + +#endif + +/* scalar variants - universal */ + +/*! + * @internal + * @brief Scalar round for @ref XXH3_accumulate_512_scalar(). + * + * This is extracted to its own function because the NEON path uses a combination + * of NEON and scalar. + */ +XXH_FORCE_INLINE void +XXH3_scalarRound(void* XXH_RESTRICT acc, + void const* XXH_RESTRICT input, + void const* XXH_RESTRICT secret, + size_t lane) +{ + xxh_u64* xacc = (xxh_u64*) acc; + xxh_u8 const* xinput = (xxh_u8 const*) input; + xxh_u8 const* xsecret = (xxh_u8 const*) secret; + XXH_ASSERT(lane < XXH_ACC_NB); + XXH_ASSERT(((size_t)acc & (XXH_ACC_ALIGN-1)) == 0); + { + xxh_u64 const data_val = XXH_readLE64(xinput + lane * 8); + xxh_u64 const data_key = data_val ^ XXH_readLE64(xsecret + lane * 8); + xacc[lane ^ 1] += data_val; /* swap adjacent lanes */ + xacc[lane] += XXH_mult32to64(data_key & 0xFFFFFFFF, data_key >> 32); + } +} + +/*! + * @internal + * @brief Processes a 64 byte block of data using the scalar path. + */ +XXH_FORCE_INLINE void +XXH3_accumulate_512_scalar(void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + size_t i; + for (i=0; i < XXH_ACC_NB; i++) { + XXH3_scalarRound(acc, input, secret, i); + } +} + +/*! + * @internal + * @brief Scalar scramble step for @ref XXH3_scrambleAcc_scalar(). + * + * This is extracted to its own function because the NEON path uses a combination + * of NEON and scalar. + */ +XXH_FORCE_INLINE void +XXH3_scalarScrambleRound(void* XXH_RESTRICT acc, + void const* XXH_RESTRICT secret, + size_t lane) +{ + xxh_u64* const xacc = (xxh_u64*) acc; /* presumed aligned */ + const xxh_u8* const xsecret = (const xxh_u8*) secret; /* no alignment restriction */ + XXH_ASSERT((((size_t)acc) & (XXH_ACC_ALIGN-1)) == 0); + XXH_ASSERT(lane < XXH_ACC_NB); + { + xxh_u64 const key64 = XXH_readLE64(xsecret + lane * 8); + xxh_u64 acc64 = xacc[lane]; + acc64 = XXH_xorshift64(acc64, 47); + acc64 ^= key64; + acc64 *= XXH_PRIME32_1; + xacc[lane] = acc64; + } +} + +/*! + * @internal + * @brief Scrambles the accumulators after a large chunk has been read + */ +XXH_FORCE_INLINE void +XXH3_scrambleAcc_scalar(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) +{ + size_t i; + for (i=0; i < XXH_ACC_NB; i++) { + XXH3_scalarScrambleRound(acc, secret, i); + } +} + +XXH_FORCE_INLINE void +XXH3_initCustomSecret_scalar(void* XXH_RESTRICT customSecret, xxh_u64 seed64) +{ + /* + * We need a separate pointer for the hack below, + * which requires a non-const pointer. + * Any decent compiler will optimize this out otherwise. + */ + const xxh_u8* kSecretPtr = XXH3_kSecret; + XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 15) == 0); + +#if defined(__clang__) && defined(__aarch64__) + /* + * UGLY HACK: + * Clang generates a bunch of MOV/MOVK pairs for aarch64, and they are + * placed sequentially, in order, at the top of the unrolled loop. + * + * While MOVK is great for generating constants (2 cycles for a 64-bit + * constant compared to 4 cycles for LDR), it fights for bandwidth with + * the arithmetic instructions. + * + * I L S + * MOVK + * MOVK + * MOVK + * MOVK + * ADD + * SUB STR + * STR + * By forcing loads from memory (as the asm line causes Clang to assume + * that XXH3_kSecretPtr has been changed), the pipelines are used more + * efficiently: + * I L S + * LDR + * ADD LDR + * SUB STR + * STR + * + * See XXH3_NEON_LANES for details on the pipsline. + * + * XXH3_64bits_withSeed, len == 256, Snapdragon 835 + * without hack: 2654.4 MB/s + * with hack: 3202.9 MB/s + */ + XXH_COMPILER_GUARD(kSecretPtr); +#endif + /* + * Note: in debug mode, this overrides the asm optimization + * and Clang will emit MOVK chains again. + */ + XXH_ASSERT(kSecretPtr == XXH3_kSecret); + + { int const nbRounds = XXH_SECRET_DEFAULT_SIZE / 16; + int i; + for (i=0; i < nbRounds; i++) { + /* + * The asm hack causes Clang to assume that kSecretPtr aliases with + * customSecret, and on aarch64, this prevented LDP from merging two + * loads together for free. Putting the loads together before the stores + * properly generates LDP. + */ + xxh_u64 lo = XXH_readLE64(kSecretPtr + 16*i) + seed64; + xxh_u64 hi = XXH_readLE64(kSecretPtr + 16*i + 8) - seed64; + XXH_writeLE64((xxh_u8*)customSecret + 16*i, lo); + XXH_writeLE64((xxh_u8*)customSecret + 16*i + 8, hi); + } } +} + + +typedef void (*XXH3_f_accumulate_512)(void* XXH_RESTRICT, const void*, const void*); +typedef void (*XXH3_f_scrambleAcc)(void* XXH_RESTRICT, const void*); +typedef void (*XXH3_f_initCustomSecret)(void* XXH_RESTRICT, xxh_u64); + + +#if (XXH_VECTOR == XXH_AVX512) + +#define XXH3_accumulate_512 XXH3_accumulate_512_avx512 +#define XXH3_scrambleAcc XXH3_scrambleAcc_avx512 +#define XXH3_initCustomSecret XXH3_initCustomSecret_avx512 + +#elif (XXH_VECTOR == XXH_AVX2) + +#define XXH3_accumulate_512 XXH3_accumulate_512_avx2 +#define XXH3_scrambleAcc XXH3_scrambleAcc_avx2 +#define XXH3_initCustomSecret XXH3_initCustomSecret_avx2 + +#elif (XXH_VECTOR == XXH_SSE2) + +#define XXH3_accumulate_512 XXH3_accumulate_512_sse2 +#define XXH3_scrambleAcc XXH3_scrambleAcc_sse2 +#define XXH3_initCustomSecret XXH3_initCustomSecret_sse2 + +#elif (XXH_VECTOR == XXH_NEON) + +#define XXH3_accumulate_512 XXH3_accumulate_512_neon +#define XXH3_scrambleAcc XXH3_scrambleAcc_neon +#define XXH3_initCustomSecret XXH3_initCustomSecret_scalar + +#elif (XXH_VECTOR == XXH_VSX) + +#define XXH3_accumulate_512 XXH3_accumulate_512_vsx +#define XXH3_scrambleAcc XXH3_scrambleAcc_vsx +#define XXH3_initCustomSecret XXH3_initCustomSecret_scalar + +#else /* scalar */ + +#define XXH3_accumulate_512 XXH3_accumulate_512_scalar +#define XXH3_scrambleAcc XXH3_scrambleAcc_scalar +#define XXH3_initCustomSecret XXH3_initCustomSecret_scalar + +#endif + + + +#ifndef XXH_PREFETCH_DIST +# ifdef __clang__ +# define XXH_PREFETCH_DIST 320 +# else +# if (XXH_VECTOR == XXH_AVX512) +# define XXH_PREFETCH_DIST 512 +# else +# define XXH_PREFETCH_DIST 384 +# endif +# endif /* __clang__ */ +#endif /* XXH_PREFETCH_DIST */ + +/* + * XXH3_accumulate() + * Loops over XXH3_accumulate_512(). + * Assumption: nbStripes will not overflow the secret size + */ +XXH_FORCE_INLINE void +XXH3_accumulate( xxh_u64* XXH_RESTRICT acc, + const xxh_u8* XXH_RESTRICT input, + const xxh_u8* XXH_RESTRICT secret, + size_t nbStripes, + XXH3_f_accumulate_512 f_acc512) +{ + size_t n; + for (n = 0; n < nbStripes; n++ ) { + const xxh_u8* const in = input + n*XXH_STRIPE_LEN; + XXH_PREFETCH(in + XXH_PREFETCH_DIST); + f_acc512(acc, + in, + secret + n*XXH_SECRET_CONSUME_RATE); + } +} + +XXH_FORCE_INLINE void +XXH3_hashLong_internal_loop(xxh_u64* XXH_RESTRICT acc, + const xxh_u8* XXH_RESTRICT input, size_t len, + const xxh_u8* XXH_RESTRICT secret, size_t secretSize, + XXH3_f_accumulate_512 f_acc512, + XXH3_f_scrambleAcc f_scramble) +{ + size_t const nbStripesPerBlock = (secretSize - XXH_STRIPE_LEN) / XXH_SECRET_CONSUME_RATE; + size_t const block_len = XXH_STRIPE_LEN * nbStripesPerBlock; + size_t const nb_blocks = (len - 1) / block_len; + + size_t n; + + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); + + for (n = 0; n < nb_blocks; n++) { + XXH3_accumulate(acc, input + n*block_len, secret, nbStripesPerBlock, f_acc512); + f_scramble(acc, secret + secretSize - XXH_STRIPE_LEN); + } + + /* last partial block */ + XXH_ASSERT(len > XXH_STRIPE_LEN); + { size_t const nbStripes = ((len - 1) - (block_len * nb_blocks)) / XXH_STRIPE_LEN; + XXH_ASSERT(nbStripes <= (secretSize / XXH_SECRET_CONSUME_RATE)); + XXH3_accumulate(acc, input + nb_blocks*block_len, secret, nbStripes, f_acc512); + + /* last stripe */ + { const xxh_u8* const p = input + len - XXH_STRIPE_LEN; +#define XXH_SECRET_LASTACC_START 7 /* not aligned on 8, last secret is different from acc & scrambler */ + f_acc512(acc, p, secret + secretSize - XXH_STRIPE_LEN - XXH_SECRET_LASTACC_START); + } } +} + +XXH_FORCE_INLINE xxh_u64 +XXH3_mix2Accs(const xxh_u64* XXH_RESTRICT acc, const xxh_u8* XXH_RESTRICT secret) +{ + return XXH3_mul128_fold64( + acc[0] ^ XXH_readLE64(secret), + acc[1] ^ XXH_readLE64(secret+8) ); +} + +static XXH64_hash_t +XXH3_mergeAccs(const xxh_u64* XXH_RESTRICT acc, const xxh_u8* XXH_RESTRICT secret, xxh_u64 start) +{ + xxh_u64 result64 = start; + size_t i = 0; + + for (i = 0; i < 4; i++) { + result64 += XXH3_mix2Accs(acc+2*i, secret + 16*i); +#if defined(__clang__) /* Clang */ \ + && (defined(__arm__) || defined(__thumb__)) /* ARMv7 */ \ + && (defined(__ARM_NEON) || defined(__ARM_NEON__)) /* NEON */ \ + && !defined(XXH_ENABLE_AUTOVECTORIZE) /* Define to disable */ + /* + * UGLY HACK: + * Prevent autovectorization on Clang ARMv7-a. Exact same problem as + * the one in XXH3_len_129to240_64b. Speeds up shorter keys > 240b. + * XXH3_64bits, len == 256, Snapdragon 835: + * without hack: 2063.7 MB/s + * with hack: 2560.7 MB/s + */ + XXH_COMPILER_GUARD(result64); +#endif + } + + return XXH3_avalanche(result64); +} + +#define XXH3_INIT_ACC { XXH_PRIME32_3, XXH_PRIME64_1, XXH_PRIME64_2, XXH_PRIME64_3, \ + XXH_PRIME64_4, XXH_PRIME32_2, XXH_PRIME64_5, XXH_PRIME32_1 } + +XXH_FORCE_INLINE XXH64_hash_t +XXH3_hashLong_64b_internal(const void* XXH_RESTRICT input, size_t len, + const void* XXH_RESTRICT secret, size_t secretSize, + XXH3_f_accumulate_512 f_acc512, + XXH3_f_scrambleAcc f_scramble) +{ + XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64 acc[XXH_ACC_NB] = XXH3_INIT_ACC; + + XXH3_hashLong_internal_loop(acc, (const xxh_u8*)input, len, (const xxh_u8*)secret, secretSize, f_acc512, f_scramble); + + /* converge into final hash */ + XXH_STATIC_ASSERT(sizeof(acc) == 64); + /* do not align on 8, so that the secret is different from the accumulator */ +#define XXH_SECRET_MERGEACCS_START 11 + XXH_ASSERT(secretSize >= sizeof(acc) + XXH_SECRET_MERGEACCS_START); + return XXH3_mergeAccs(acc, (const xxh_u8*)secret + XXH_SECRET_MERGEACCS_START, (xxh_u64)len * XXH_PRIME64_1); +} + +/* + * It's important for performance to transmit secret's size (when it's static) + * so that the compiler can properly optimize the vectorized loop. + * This makes a big performance difference for "medium" keys (<1 KB) when using AVX instruction set. + */ +XXH_FORCE_INLINE XXH64_hash_t +XXH3_hashLong_64b_withSecret(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, const xxh_u8* XXH_RESTRICT secret, size_t secretLen) +{ + (void)seed64; + return XXH3_hashLong_64b_internal(input, len, secret, secretLen, XXH3_accumulate_512, XXH3_scrambleAcc); +} + +/* + * It's preferable for performance that XXH3_hashLong is not inlined, + * as it results in a smaller function for small data, easier to the instruction cache. + * Note that inside this no_inline function, we do inline the internal loop, + * and provide a statically defined secret size to allow optimization of vector loop. + */ +XXH_NO_INLINE XXH64_hash_t +XXH3_hashLong_64b_default(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, const xxh_u8* XXH_RESTRICT secret, size_t secretLen) +{ + (void)seed64; (void)secret; (void)secretLen; + return XXH3_hashLong_64b_internal(input, len, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_accumulate_512, XXH3_scrambleAcc); +} + +/* + * XXH3_hashLong_64b_withSeed(): + * Generate a custom key based on alteration of default XXH3_kSecret with the seed, + * and then use this key for long mode hashing. + * + * This operation is decently fast but nonetheless costs a little bit of time. + * Try to avoid it whenever possible (typically when seed==0). + * + * It's important for performance that XXH3_hashLong is not inlined. Not sure + * why (uop cache maybe?), but the difference is large and easily measurable. + */ +XXH_FORCE_INLINE XXH64_hash_t +XXH3_hashLong_64b_withSeed_internal(const void* input, size_t len, + XXH64_hash_t seed, + XXH3_f_accumulate_512 f_acc512, + XXH3_f_scrambleAcc f_scramble, + XXH3_f_initCustomSecret f_initSec) +{ + if (seed == 0) + return XXH3_hashLong_64b_internal(input, len, + XXH3_kSecret, sizeof(XXH3_kSecret), + f_acc512, f_scramble); + { XXH_ALIGN(XXH_SEC_ALIGN) xxh_u8 secret[XXH_SECRET_DEFAULT_SIZE]; + f_initSec(secret, seed); + return XXH3_hashLong_64b_internal(input, len, secret, sizeof(secret), + f_acc512, f_scramble); + } +} + +/* + * It's important for performance that XXH3_hashLong is not inlined. + */ +XXH_NO_INLINE XXH64_hash_t +XXH3_hashLong_64b_withSeed(const void* input, size_t len, + XXH64_hash_t seed, const xxh_u8* secret, size_t secretLen) +{ + (void)secret; (void)secretLen; + return XXH3_hashLong_64b_withSeed_internal(input, len, seed, + XXH3_accumulate_512, XXH3_scrambleAcc, XXH3_initCustomSecret); +} + + +typedef XXH64_hash_t (*XXH3_hashLong64_f)(const void* XXH_RESTRICT, size_t, + XXH64_hash_t, const xxh_u8* XXH_RESTRICT, size_t); + +XXH_FORCE_INLINE XXH64_hash_t +XXH3_64bits_internal(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, const void* XXH_RESTRICT secret, size_t secretLen, + XXH3_hashLong64_f f_hashLong) +{ + XXH_ASSERT(secretLen >= XXH3_SECRET_SIZE_MIN); + /* + * If an action is to be taken if `secretLen` condition is not respected, + * it should be done here. + * For now, it's a contract pre-condition. + * Adding a check and a branch here would cost performance at every hash. + * Also, note that function signature doesn't offer room to return an error. + */ + if (len <= 16) + return XXH3_len_0to16_64b((const xxh_u8*)input, len, (const xxh_u8*)secret, seed64); + if (len <= 128) + return XXH3_len_17to128_64b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64); + if (len <= XXH3_MIDSIZE_MAX) + return XXH3_len_129to240_64b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64); + return f_hashLong(input, len, seed64, (const xxh_u8*)secret, secretLen); +} + + +/* === Public entry point === */ + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH64_hash_t XXH3_64bits(const void* input, size_t len) +{ + return XXH3_64bits_internal(input, len, 0, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_hashLong_64b_default); +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH64_hash_t +XXH3_64bits_withSecret(const void* input, size_t len, const void* secret, size_t secretSize) +{ + return XXH3_64bits_internal(input, len, 0, secret, secretSize, XXH3_hashLong_64b_withSecret); +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH64_hash_t +XXH3_64bits_withSeed(const void* input, size_t len, XXH64_hash_t seed) +{ + return XXH3_64bits_internal(input, len, seed, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_hashLong_64b_withSeed); +} + +XXH_PUBLIC_API XXH64_hash_t +XXH3_64bits_withSecretandSeed(const void* input, size_t len, const void* secret, size_t secretSize, XXH64_hash_t seed) +{ + if (len <= XXH3_MIDSIZE_MAX) + return XXH3_64bits_internal(input, len, seed, XXH3_kSecret, sizeof(XXH3_kSecret), NULL); + return XXH3_hashLong_64b_withSecret(input, len, seed, (const xxh_u8*)secret, secretSize); +} + + +/* === XXH3 streaming === */ + +/* + * Malloc's a pointer that is always aligned to align. + * + * This must be freed with `XXH_alignedFree()`. + * + * malloc typically guarantees 16 byte alignment on 64-bit systems and 8 byte + * alignment on 32-bit. This isn't enough for the 32 byte aligned loads in AVX2 + * or on 32-bit, the 16 byte aligned loads in SSE2 and NEON. + * + * This underalignment previously caused a rather obvious crash which went + * completely unnoticed due to XXH3_createState() not actually being tested. + * Credit to RedSpah for noticing this bug. + * + * The alignment is done manually: Functions like posix_memalign or _mm_malloc + * are avoided: To maintain portability, we would have to write a fallback + * like this anyways, and besides, testing for the existence of library + * functions without relying on external build tools is impossible. + * + * The method is simple: Overallocate, manually align, and store the offset + * to the original behind the returned pointer. + * + * Align must be a power of 2 and 8 <= align <= 128. + */ +static void* XXH_alignedMalloc(size_t s, size_t align) +{ + XXH_ASSERT(align <= 128 && align >= 8); /* range check */ + XXH_ASSERT((align & (align-1)) == 0); /* power of 2 */ + XXH_ASSERT(s != 0 && s < (s + align)); /* empty/overflow */ + { /* Overallocate to make room for manual realignment and an offset byte */ + xxh_u8* base = (xxh_u8*)XXH_malloc(s + align); + if (base != NULL) { + /* + * Get the offset needed to align this pointer. + * + * Even if the returned pointer is aligned, there will always be + * at least one byte to store the offset to the original pointer. + */ + size_t offset = align - ((size_t)base & (align - 1)); /* base % align */ + /* Add the offset for the now-aligned pointer */ + xxh_u8* ptr = base + offset; + + XXH_ASSERT((size_t)ptr % align == 0); + + /* Store the offset immediately before the returned pointer. */ + ptr[-1] = (xxh_u8)offset; + return ptr; + } + return NULL; + } +} +/* + * Frees an aligned pointer allocated by XXH_alignedMalloc(). Don't pass + * normal malloc'd pointers, XXH_alignedMalloc has a specific data layout. + */ +static void XXH_alignedFree(void* p) +{ + if (p != NULL) { + xxh_u8* ptr = (xxh_u8*)p; + /* Get the offset byte we added in XXH_malloc. */ + xxh_u8 offset = ptr[-1]; + /* Free the original malloc'd pointer */ + xxh_u8* base = ptr - offset; + XXH_free(base); + } +} +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH3_state_t* XXH3_createState(void) +{ + XXH3_state_t* const state = (XXH3_state_t*)XXH_alignedMalloc(sizeof(XXH3_state_t), 64); + if (state==NULL) return NULL; + XXH3_INITSTATE(state); + return state; +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH_errorcode XXH3_freeState(XXH3_state_t* statePtr) +{ + XXH_alignedFree(statePtr); + return XXH_OK; +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API void +XXH3_copyState(XXH3_state_t* dst_state, const XXH3_state_t* src_state) +{ + XXH_memcpy(dst_state, src_state, sizeof(*dst_state)); +} + +static void +XXH3_reset_internal(XXH3_state_t* statePtr, + XXH64_hash_t seed, + const void* secret, size_t secretSize) +{ + size_t const initStart = offsetof(XXH3_state_t, bufferedSize); + size_t const initLength = offsetof(XXH3_state_t, nbStripesPerBlock) - initStart; + XXH_ASSERT(offsetof(XXH3_state_t, nbStripesPerBlock) > initStart); + XXH_ASSERT(statePtr != NULL); + /* set members from bufferedSize to nbStripesPerBlock (excluded) to 0 */ + memset((char*)statePtr + initStart, 0, initLength); + statePtr->acc[0] = XXH_PRIME32_3; + statePtr->acc[1] = XXH_PRIME64_1; + statePtr->acc[2] = XXH_PRIME64_2; + statePtr->acc[3] = XXH_PRIME64_3; + statePtr->acc[4] = XXH_PRIME64_4; + statePtr->acc[5] = XXH_PRIME32_2; + statePtr->acc[6] = XXH_PRIME64_5; + statePtr->acc[7] = XXH_PRIME32_1; + statePtr->seed = seed; + statePtr->useSeed = (seed != 0); + statePtr->extSecret = (const unsigned char*)secret; + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); + statePtr->secretLimit = secretSize - XXH_STRIPE_LEN; + statePtr->nbStripesPerBlock = statePtr->secretLimit / XXH_SECRET_CONSUME_RATE; +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_reset(XXH3_state_t* statePtr) +{ + if (statePtr == NULL) return XXH_ERROR; + XXH3_reset_internal(statePtr, 0, XXH3_kSecret, XXH_SECRET_DEFAULT_SIZE); + return XXH_OK; +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_reset_withSecret(XXH3_state_t* statePtr, const void* secret, size_t secretSize) +{ + if (statePtr == NULL) return XXH_ERROR; + XXH3_reset_internal(statePtr, 0, secret, secretSize); + if (secret == NULL) return XXH_ERROR; + if (secretSize < XXH3_SECRET_SIZE_MIN) return XXH_ERROR; + return XXH_OK; +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_reset_withSeed(XXH3_state_t* statePtr, XXH64_hash_t seed) +{ + if (statePtr == NULL) return XXH_ERROR; + if (seed==0) return XXH3_64bits_reset(statePtr); + if ((seed != statePtr->seed) || (statePtr->extSecret != NULL)) + XXH3_initCustomSecret(statePtr->customSecret, seed); + XXH3_reset_internal(statePtr, seed, NULL, XXH_SECRET_DEFAULT_SIZE); + return XXH_OK; +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_reset_withSecretandSeed(XXH3_state_t* statePtr, const void* secret, size_t secretSize, XXH64_hash_t seed64) +{ + if (statePtr == NULL) return XXH_ERROR; + if (secret == NULL) return XXH_ERROR; + if (secretSize < XXH3_SECRET_SIZE_MIN) return XXH_ERROR; + XXH3_reset_internal(statePtr, seed64, secret, secretSize); + statePtr->useSeed = 1; /* always, even if seed64==0 */ + return XXH_OK; +} + +/* Note : when XXH3_consumeStripes() is invoked, + * there must be a guarantee that at least one more byte must be consumed from input + * so that the function can blindly consume all stripes using the "normal" secret segment */ +XXH_FORCE_INLINE void +XXH3_consumeStripes(xxh_u64* XXH_RESTRICT acc, + size_t* XXH_RESTRICT nbStripesSoFarPtr, size_t nbStripesPerBlock, + const xxh_u8* XXH_RESTRICT input, size_t nbStripes, + const xxh_u8* XXH_RESTRICT secret, size_t secretLimit, + XXH3_f_accumulate_512 f_acc512, + XXH3_f_scrambleAcc f_scramble) +{ + XXH_ASSERT(nbStripes <= nbStripesPerBlock); /* can handle max 1 scramble per invocation */ + XXH_ASSERT(*nbStripesSoFarPtr < nbStripesPerBlock); + if (nbStripesPerBlock - *nbStripesSoFarPtr <= nbStripes) { + /* need a scrambling operation */ + size_t const nbStripesToEndofBlock = nbStripesPerBlock - *nbStripesSoFarPtr; + size_t const nbStripesAfterBlock = nbStripes - nbStripesToEndofBlock; + XXH3_accumulate(acc, input, secret + nbStripesSoFarPtr[0] * XXH_SECRET_CONSUME_RATE, nbStripesToEndofBlock, f_acc512); + f_scramble(acc, secret + secretLimit); + XXH3_accumulate(acc, input + nbStripesToEndofBlock * XXH_STRIPE_LEN, secret, nbStripesAfterBlock, f_acc512); + *nbStripesSoFarPtr = nbStripesAfterBlock; + } else { + XXH3_accumulate(acc, input, secret + nbStripesSoFarPtr[0] * XXH_SECRET_CONSUME_RATE, nbStripes, f_acc512); + *nbStripesSoFarPtr += nbStripes; + } +} + +#ifndef XXH3_STREAM_USE_STACK +# ifndef __clang__ /* clang doesn't need additional stack space */ +# define XXH3_STREAM_USE_STACK 1 +# endif +#endif +/* + * Both XXH3_64bits_update and XXH3_128bits_update use this routine. + */ +XXH_FORCE_INLINE XXH_errorcode +XXH3_update(XXH3_state_t* XXH_RESTRICT const state, + const xxh_u8* XXH_RESTRICT input, size_t len, + XXH3_f_accumulate_512 f_acc512, + XXH3_f_scrambleAcc f_scramble) +{ + if (input==NULL) { + XXH_ASSERT(len == 0); + return XXH_OK; + } + + XXH_ASSERT(state != NULL); + { const xxh_u8* const bEnd = input + len; + const unsigned char* const secret = (state->extSecret == NULL) ? state->customSecret : state->extSecret; +#if defined(XXH3_STREAM_USE_STACK) && XXH3_STREAM_USE_STACK >= 1 + /* For some reason, gcc and MSVC seem to suffer greatly + * when operating accumulators directly into state. + * Operating into stack space seems to enable proper optimization. + * clang, on the other hand, doesn't seem to need this trick */ + XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64 acc[8]; memcpy(acc, state->acc, sizeof(acc)); +#else + xxh_u64* XXH_RESTRICT const acc = state->acc; +#endif + state->totalLen += len; + XXH_ASSERT(state->bufferedSize <= XXH3_INTERNALBUFFER_SIZE); + + /* small input : just fill in tmp buffer */ + if (state->bufferedSize + len <= XXH3_INTERNALBUFFER_SIZE) { + XXH_memcpy(state->buffer + state->bufferedSize, input, len); + state->bufferedSize += (XXH32_hash_t)len; + return XXH_OK; + } + + /* total input is now > XXH3_INTERNALBUFFER_SIZE */ + #define XXH3_INTERNALBUFFER_STRIPES (XXH3_INTERNALBUFFER_SIZE / XXH_STRIPE_LEN) + XXH_STATIC_ASSERT(XXH3_INTERNALBUFFER_SIZE % XXH_STRIPE_LEN == 0); /* clean multiple */ + + /* + * Internal buffer is partially filled (always, except at beginning) + * Complete it, then consume it. + */ + if (state->bufferedSize) { + size_t const loadSize = XXH3_INTERNALBUFFER_SIZE - state->bufferedSize; + XXH_memcpy(state->buffer + state->bufferedSize, input, loadSize); + input += loadSize; + XXH3_consumeStripes(acc, + &state->nbStripesSoFar, state->nbStripesPerBlock, + state->buffer, XXH3_INTERNALBUFFER_STRIPES, + secret, state->secretLimit, + f_acc512, f_scramble); + state->bufferedSize = 0; + } + XXH_ASSERT(input < bEnd); + + /* large input to consume : ingest per full block */ + if ((size_t)(bEnd - input) > state->nbStripesPerBlock * XXH_STRIPE_LEN) { + size_t nbStripes = (size_t)(bEnd - 1 - input) / XXH_STRIPE_LEN; + XXH_ASSERT(state->nbStripesPerBlock >= state->nbStripesSoFar); + /* join to current block's end */ + { size_t const nbStripesToEnd = state->nbStripesPerBlock - state->nbStripesSoFar; + XXH_ASSERT(nbStripesToEnd <= nbStripes); + XXH3_accumulate(acc, input, secret + state->nbStripesSoFar * XXH_SECRET_CONSUME_RATE, nbStripesToEnd, f_acc512); + f_scramble(acc, secret + state->secretLimit); + state->nbStripesSoFar = 0; + input += nbStripesToEnd * XXH_STRIPE_LEN; + nbStripes -= nbStripesToEnd; + } + /* consume per entire blocks */ + while(nbStripes >= state->nbStripesPerBlock) { + XXH3_accumulate(acc, input, secret, state->nbStripesPerBlock, f_acc512); + f_scramble(acc, secret + state->secretLimit); + input += state->nbStripesPerBlock * XXH_STRIPE_LEN; + nbStripes -= state->nbStripesPerBlock; + } + /* consume last partial block */ + XXH3_accumulate(acc, input, secret, nbStripes, f_acc512); + input += nbStripes * XXH_STRIPE_LEN; + XXH_ASSERT(input < bEnd); /* at least some bytes left */ + state->nbStripesSoFar = nbStripes; + /* buffer predecessor of last partial stripe */ + XXH_memcpy(state->buffer + sizeof(state->buffer) - XXH_STRIPE_LEN, input - XXH_STRIPE_LEN, XXH_STRIPE_LEN); + XXH_ASSERT(bEnd - input <= XXH_STRIPE_LEN); + } else { + /* content to consume <= block size */ + /* Consume input by a multiple of internal buffer size */ + if (bEnd - input > XXH3_INTERNALBUFFER_SIZE) { + const xxh_u8* const limit = bEnd - XXH3_INTERNALBUFFER_SIZE; + do { + XXH3_consumeStripes(acc, + &state->nbStripesSoFar, state->nbStripesPerBlock, + input, XXH3_INTERNALBUFFER_STRIPES, + secret, state->secretLimit, + f_acc512, f_scramble); + input += XXH3_INTERNALBUFFER_SIZE; + } while (inputbuffer + sizeof(state->buffer) - XXH_STRIPE_LEN, input - XXH_STRIPE_LEN, XXH_STRIPE_LEN); + } + } + + /* Some remaining input (always) : buffer it */ + XXH_ASSERT(input < bEnd); + XXH_ASSERT(bEnd - input <= XXH3_INTERNALBUFFER_SIZE); + XXH_ASSERT(state->bufferedSize == 0); + XXH_memcpy(state->buffer, input, (size_t)(bEnd-input)); + state->bufferedSize = (XXH32_hash_t)(bEnd-input); +#if defined(XXH3_STREAM_USE_STACK) && XXH3_STREAM_USE_STACK >= 1 + /* save stack accumulators into state */ + memcpy(state->acc, acc, sizeof(acc)); +#endif + } + + return XXH_OK; +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_update(XXH3_state_t* state, const void* input, size_t len) +{ + return XXH3_update(state, (const xxh_u8*)input, len, + XXH3_accumulate_512, XXH3_scrambleAcc); +} + + +XXH_FORCE_INLINE void +XXH3_digest_long (XXH64_hash_t* acc, + const XXH3_state_t* state, + const unsigned char* secret) +{ + /* + * Digest on a local copy. This way, the state remains unaltered, and it can + * continue ingesting more input afterwards. + */ + XXH_memcpy(acc, state->acc, sizeof(state->acc)); + if (state->bufferedSize >= XXH_STRIPE_LEN) { + size_t const nbStripes = (state->bufferedSize - 1) / XXH_STRIPE_LEN; + size_t nbStripesSoFar = state->nbStripesSoFar; + XXH3_consumeStripes(acc, + &nbStripesSoFar, state->nbStripesPerBlock, + state->buffer, nbStripes, + secret, state->secretLimit, + XXH3_accumulate_512, XXH3_scrambleAcc); + /* last stripe */ + XXH3_accumulate_512(acc, + state->buffer + state->bufferedSize - XXH_STRIPE_LEN, + secret + state->secretLimit - XXH_SECRET_LASTACC_START); + } else { /* bufferedSize < XXH_STRIPE_LEN */ + xxh_u8 lastStripe[XXH_STRIPE_LEN]; + size_t const catchupSize = XXH_STRIPE_LEN - state->bufferedSize; + XXH_ASSERT(state->bufferedSize > 0); /* there is always some input buffered */ + XXH_memcpy(lastStripe, state->buffer + sizeof(state->buffer) - catchupSize, catchupSize); + XXH_memcpy(lastStripe + catchupSize, state->buffer, state->bufferedSize); + XXH3_accumulate_512(acc, + lastStripe, + secret + state->secretLimit - XXH_SECRET_LASTACC_START); + } +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_digest (const XXH3_state_t* state) +{ + const unsigned char* const secret = (state->extSecret == NULL) ? state->customSecret : state->extSecret; + if (state->totalLen > XXH3_MIDSIZE_MAX) { + XXH_ALIGN(XXH_ACC_ALIGN) XXH64_hash_t acc[XXH_ACC_NB]; + XXH3_digest_long(acc, state, secret); + return XXH3_mergeAccs(acc, + secret + XXH_SECRET_MERGEACCS_START, + (xxh_u64)state->totalLen * XXH_PRIME64_1); + } + /* totalLen <= XXH3_MIDSIZE_MAX: digesting a short input */ + if (state->useSeed) + return XXH3_64bits_withSeed(state->buffer, (size_t)state->totalLen, state->seed); + return XXH3_64bits_withSecret(state->buffer, (size_t)(state->totalLen), + secret, state->secretLimit + XXH_STRIPE_LEN); +} + + + +/* ========================================== + * XXH3 128 bits (a.k.a XXH128) + * ========================================== + * XXH3's 128-bit variant has better mixing and strength than the 64-bit variant, + * even without counting the significantly larger output size. + * + * For example, extra steps are taken to avoid the seed-dependent collisions + * in 17-240 byte inputs (See XXH3_mix16B and XXH128_mix32B). + * + * This strength naturally comes at the cost of some speed, especially on short + * lengths. Note that longer hashes are about as fast as the 64-bit version + * due to it using only a slight modification of the 64-bit loop. + * + * XXH128 is also more oriented towards 64-bit machines. It is still extremely + * fast for a _128-bit_ hash on 32-bit (it usually clears XXH64). + */ + +XXH_FORCE_INLINE XXH128_hash_t +XXH3_len_1to3_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + /* A doubled version of 1to3_64b with different constants. */ + XXH_ASSERT(input != NULL); + XXH_ASSERT(1 <= len && len <= 3); + XXH_ASSERT(secret != NULL); + /* + * len = 1: combinedl = { input[0], 0x01, input[0], input[0] } + * len = 2: combinedl = { input[1], 0x02, input[0], input[1] } + * len = 3: combinedl = { input[2], 0x03, input[0], input[1] } + */ + { xxh_u8 const c1 = input[0]; + xxh_u8 const c2 = input[len >> 1]; + xxh_u8 const c3 = input[len - 1]; + xxh_u32 const combinedl = ((xxh_u32)c1 <<16) | ((xxh_u32)c2 << 24) + | ((xxh_u32)c3 << 0) | ((xxh_u32)len << 8); + xxh_u32 const combinedh = XXH_rotl32(XXH_swap32(combinedl), 13); + xxh_u64 const bitflipl = (XXH_readLE32(secret) ^ XXH_readLE32(secret+4)) + seed; + xxh_u64 const bitfliph = (XXH_readLE32(secret+8) ^ XXH_readLE32(secret+12)) - seed; + xxh_u64 const keyed_lo = (xxh_u64)combinedl ^ bitflipl; + xxh_u64 const keyed_hi = (xxh_u64)combinedh ^ bitfliph; + XXH128_hash_t h128; + h128.low64 = XXH64_avalanche(keyed_lo); + h128.high64 = XXH64_avalanche(keyed_hi); + return h128; + } +} + +XXH_FORCE_INLINE XXH128_hash_t +XXH3_len_4to8_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(input != NULL); + XXH_ASSERT(secret != NULL); + XXH_ASSERT(4 <= len && len <= 8); + seed ^= (xxh_u64)XXH_swap32((xxh_u32)seed) << 32; + { xxh_u32 const input_lo = XXH_readLE32(input); + xxh_u32 const input_hi = XXH_readLE32(input + len - 4); + xxh_u64 const input_64 = input_lo + ((xxh_u64)input_hi << 32); + xxh_u64 const bitflip = (XXH_readLE64(secret+16) ^ XXH_readLE64(secret+24)) + seed; + xxh_u64 const keyed = input_64 ^ bitflip; + + /* Shift len to the left to ensure it is even, this avoids even multiplies. */ + XXH128_hash_t m128 = XXH_mult64to128(keyed, XXH_PRIME64_1 + (len << 2)); + + m128.high64 += (m128.low64 << 1); + m128.low64 ^= (m128.high64 >> 3); + + m128.low64 = XXH_xorshift64(m128.low64, 35); + m128.low64 *= 0x9FB21C651E98DF25ULL; + m128.low64 = XXH_xorshift64(m128.low64, 28); + m128.high64 = XXH3_avalanche(m128.high64); + return m128; + } +} + +XXH_FORCE_INLINE XXH128_hash_t +XXH3_len_9to16_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(input != NULL); + XXH_ASSERT(secret != NULL); + XXH_ASSERT(9 <= len && len <= 16); + { xxh_u64 const bitflipl = (XXH_readLE64(secret+32) ^ XXH_readLE64(secret+40)) - seed; + xxh_u64 const bitfliph = (XXH_readLE64(secret+48) ^ XXH_readLE64(secret+56)) + seed; + xxh_u64 const input_lo = XXH_readLE64(input); + xxh_u64 input_hi = XXH_readLE64(input + len - 8); + XXH128_hash_t m128 = XXH_mult64to128(input_lo ^ input_hi ^ bitflipl, XXH_PRIME64_1); + /* + * Put len in the middle of m128 to ensure that the length gets mixed to + * both the low and high bits in the 128x64 multiply below. + */ + m128.low64 += (xxh_u64)(len - 1) << 54; + input_hi ^= bitfliph; + /* + * Add the high 32 bits of input_hi to the high 32 bits of m128, then + * add the long product of the low 32 bits of input_hi and XXH_PRIME32_2 to + * the high 64 bits of m128. + * + * The best approach to this operation is different on 32-bit and 64-bit. + */ + if (sizeof(void *) < sizeof(xxh_u64)) { /* 32-bit */ + /* + * 32-bit optimized version, which is more readable. + * + * On 32-bit, it removes an ADC and delays a dependency between the two + * halves of m128.high64, but it generates an extra mask on 64-bit. + */ + m128.high64 += (input_hi & 0xFFFFFFFF00000000ULL) + XXH_mult32to64((xxh_u32)input_hi, XXH_PRIME32_2); + } else { + /* + * 64-bit optimized (albeit more confusing) version. + * + * Uses some properties of addition and multiplication to remove the mask: + * + * Let: + * a = input_hi.lo = (input_hi & 0x00000000FFFFFFFF) + * b = input_hi.hi = (input_hi & 0xFFFFFFFF00000000) + * c = XXH_PRIME32_2 + * + * a + (b * c) + * Inverse Property: x + y - x == y + * a + (b * (1 + c - 1)) + * Distributive Property: x * (y + z) == (x * y) + (x * z) + * a + (b * 1) + (b * (c - 1)) + * Identity Property: x * 1 == x + * a + b + (b * (c - 1)) + * + * Substitute a, b, and c: + * input_hi.hi + input_hi.lo + ((xxh_u64)input_hi.lo * (XXH_PRIME32_2 - 1)) + * + * Since input_hi.hi + input_hi.lo == input_hi, we get this: + * input_hi + ((xxh_u64)input_hi.lo * (XXH_PRIME32_2 - 1)) + */ + m128.high64 += input_hi + XXH_mult32to64((xxh_u32)input_hi, XXH_PRIME32_2 - 1); + } + /* m128 ^= XXH_swap64(m128 >> 64); */ + m128.low64 ^= XXH_swap64(m128.high64); + + { /* 128x64 multiply: h128 = m128 * XXH_PRIME64_2; */ + XXH128_hash_t h128 = XXH_mult64to128(m128.low64, XXH_PRIME64_2); + h128.high64 += m128.high64 * XXH_PRIME64_2; + + h128.low64 = XXH3_avalanche(h128.low64); + h128.high64 = XXH3_avalanche(h128.high64); + return h128; + } } +} + +/* + * Assumption: `secret` size is >= XXH3_SECRET_SIZE_MIN + */ +XXH_FORCE_INLINE XXH128_hash_t +XXH3_len_0to16_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(len <= 16); + { if (len > 8) return XXH3_len_9to16_128b(input, len, secret, seed); + if (len >= 4) return XXH3_len_4to8_128b(input, len, secret, seed); + if (len) return XXH3_len_1to3_128b(input, len, secret, seed); + { XXH128_hash_t h128; + xxh_u64 const bitflipl = XXH_readLE64(secret+64) ^ XXH_readLE64(secret+72); + xxh_u64 const bitfliph = XXH_readLE64(secret+80) ^ XXH_readLE64(secret+88); + h128.low64 = XXH64_avalanche(seed ^ bitflipl); + h128.high64 = XXH64_avalanche( seed ^ bitfliph); + return h128; + } } +} + +/* + * A bit slower than XXH3_mix16B, but handles multiply by zero better. + */ +XXH_FORCE_INLINE XXH128_hash_t +XXH128_mix32B(XXH128_hash_t acc, const xxh_u8* input_1, const xxh_u8* input_2, + const xxh_u8* secret, XXH64_hash_t seed) +{ + acc.low64 += XXH3_mix16B (input_1, secret+0, seed); + acc.low64 ^= XXH_readLE64(input_2) + XXH_readLE64(input_2 + 8); + acc.high64 += XXH3_mix16B (input_2, secret+16, seed); + acc.high64 ^= XXH_readLE64(input_1) + XXH_readLE64(input_1 + 8); + return acc; +} + + +XXH_FORCE_INLINE XXH128_hash_t +XXH3_len_17to128_128b(const xxh_u8* XXH_RESTRICT input, size_t len, + const xxh_u8* XXH_RESTRICT secret, size_t secretSize, + XXH64_hash_t seed) +{ + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize; + XXH_ASSERT(16 < len && len <= 128); + + { XXH128_hash_t acc; + acc.low64 = len * XXH_PRIME64_1; + acc.high64 = 0; + if (len > 32) { + if (len > 64) { + if (len > 96) { + acc = XXH128_mix32B(acc, input+48, input+len-64, secret+96, seed); + } + acc = XXH128_mix32B(acc, input+32, input+len-48, secret+64, seed); + } + acc = XXH128_mix32B(acc, input+16, input+len-32, secret+32, seed); + } + acc = XXH128_mix32B(acc, input, input+len-16, secret, seed); + { XXH128_hash_t h128; + h128.low64 = acc.low64 + acc.high64; + h128.high64 = (acc.low64 * XXH_PRIME64_1) + + (acc.high64 * XXH_PRIME64_4) + + ((len - seed) * XXH_PRIME64_2); + h128.low64 = XXH3_avalanche(h128.low64); + h128.high64 = (XXH64_hash_t)0 - XXH3_avalanche(h128.high64); + return h128; + } + } +} + +XXH_NO_INLINE XXH128_hash_t +XXH3_len_129to240_128b(const xxh_u8* XXH_RESTRICT input, size_t len, + const xxh_u8* XXH_RESTRICT secret, size_t secretSize, + XXH64_hash_t seed) +{ + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize; + XXH_ASSERT(128 < len && len <= XXH3_MIDSIZE_MAX); + + { XXH128_hash_t acc; + int const nbRounds = (int)len / 32; + int i; + acc.low64 = len * XXH_PRIME64_1; + acc.high64 = 0; + for (i=0; i<4; i++) { + acc = XXH128_mix32B(acc, + input + (32 * i), + input + (32 * i) + 16, + secret + (32 * i), + seed); + } + acc.low64 = XXH3_avalanche(acc.low64); + acc.high64 = XXH3_avalanche(acc.high64); + XXH_ASSERT(nbRounds >= 4); + for (i=4 ; i < nbRounds; i++) { + acc = XXH128_mix32B(acc, + input + (32 * i), + input + (32 * i) + 16, + secret + XXH3_MIDSIZE_STARTOFFSET + (32 * (i - 4)), + seed); + } + /* last bytes */ + acc = XXH128_mix32B(acc, + input + len - 16, + input + len - 32, + secret + XXH3_SECRET_SIZE_MIN - XXH3_MIDSIZE_LASTOFFSET - 16, + 0ULL - seed); + + { XXH128_hash_t h128; + h128.low64 = acc.low64 + acc.high64; + h128.high64 = (acc.low64 * XXH_PRIME64_1) + + (acc.high64 * XXH_PRIME64_4) + + ((len - seed) * XXH_PRIME64_2); + h128.low64 = XXH3_avalanche(h128.low64); + h128.high64 = (XXH64_hash_t)0 - XXH3_avalanche(h128.high64); + return h128; + } + } +} + +XXH_FORCE_INLINE XXH128_hash_t +XXH3_hashLong_128b_internal(const void* XXH_RESTRICT input, size_t len, + const xxh_u8* XXH_RESTRICT secret, size_t secretSize, + XXH3_f_accumulate_512 f_acc512, + XXH3_f_scrambleAcc f_scramble) +{ + XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64 acc[XXH_ACC_NB] = XXH3_INIT_ACC; + + XXH3_hashLong_internal_loop(acc, (const xxh_u8*)input, len, secret, secretSize, f_acc512, f_scramble); + + /* converge into final hash */ + XXH_STATIC_ASSERT(sizeof(acc) == 64); + XXH_ASSERT(secretSize >= sizeof(acc) + XXH_SECRET_MERGEACCS_START); + { XXH128_hash_t h128; + h128.low64 = XXH3_mergeAccs(acc, + secret + XXH_SECRET_MERGEACCS_START, + (xxh_u64)len * XXH_PRIME64_1); + h128.high64 = XXH3_mergeAccs(acc, + secret + secretSize + - sizeof(acc) - XXH_SECRET_MERGEACCS_START, + ~((xxh_u64)len * XXH_PRIME64_2)); + return h128; + } +} + +/* + * It's important for performance that XXH3_hashLong is not inlined. + */ +XXH_NO_INLINE XXH128_hash_t +XXH3_hashLong_128b_default(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, + const void* XXH_RESTRICT secret, size_t secretLen) +{ + (void)seed64; (void)secret; (void)secretLen; + return XXH3_hashLong_128b_internal(input, len, XXH3_kSecret, sizeof(XXH3_kSecret), + XXH3_accumulate_512, XXH3_scrambleAcc); +} + +/* + * It's important for performance to pass @secretLen (when it's static) + * to the compiler, so that it can properly optimize the vectorized loop. + */ +XXH_FORCE_INLINE XXH128_hash_t +XXH3_hashLong_128b_withSecret(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, + const void* XXH_RESTRICT secret, size_t secretLen) +{ + (void)seed64; + return XXH3_hashLong_128b_internal(input, len, (const xxh_u8*)secret, secretLen, + XXH3_accumulate_512, XXH3_scrambleAcc); +} + +XXH_FORCE_INLINE XXH128_hash_t +XXH3_hashLong_128b_withSeed_internal(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, + XXH3_f_accumulate_512 f_acc512, + XXH3_f_scrambleAcc f_scramble, + XXH3_f_initCustomSecret f_initSec) +{ + if (seed64 == 0) + return XXH3_hashLong_128b_internal(input, len, + XXH3_kSecret, sizeof(XXH3_kSecret), + f_acc512, f_scramble); + { XXH_ALIGN(XXH_SEC_ALIGN) xxh_u8 secret[XXH_SECRET_DEFAULT_SIZE]; + f_initSec(secret, seed64); + return XXH3_hashLong_128b_internal(input, len, (const xxh_u8*)secret, sizeof(secret), + f_acc512, f_scramble); + } +} + +/* + * It's important for performance that XXH3_hashLong is not inlined. + */ +XXH_NO_INLINE XXH128_hash_t +XXH3_hashLong_128b_withSeed(const void* input, size_t len, + XXH64_hash_t seed64, const void* XXH_RESTRICT secret, size_t secretLen) +{ + (void)secret; (void)secretLen; + return XXH3_hashLong_128b_withSeed_internal(input, len, seed64, + XXH3_accumulate_512, XXH3_scrambleAcc, XXH3_initCustomSecret); +} + +typedef XXH128_hash_t (*XXH3_hashLong128_f)(const void* XXH_RESTRICT, size_t, + XXH64_hash_t, const void* XXH_RESTRICT, size_t); + +XXH_FORCE_INLINE XXH128_hash_t +XXH3_128bits_internal(const void* input, size_t len, + XXH64_hash_t seed64, const void* XXH_RESTRICT secret, size_t secretLen, + XXH3_hashLong128_f f_hl128) +{ + XXH_ASSERT(secretLen >= XXH3_SECRET_SIZE_MIN); + /* + * If an action is to be taken if `secret` conditions are not respected, + * it should be done here. + * For now, it's a contract pre-condition. + * Adding a check and a branch here would cost performance at every hash. + */ + if (len <= 16) + return XXH3_len_0to16_128b((const xxh_u8*)input, len, (const xxh_u8*)secret, seed64); + if (len <= 128) + return XXH3_len_17to128_128b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64); + if (len <= XXH3_MIDSIZE_MAX) + return XXH3_len_129to240_128b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64); + return f_hl128(input, len, seed64, secret, secretLen); +} + + +/* === Public XXH128 API === */ + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH128_hash_t XXH3_128bits(const void* input, size_t len) +{ + return XXH3_128bits_internal(input, len, 0, + XXH3_kSecret, sizeof(XXH3_kSecret), + XXH3_hashLong_128b_default); +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH128_hash_t +XXH3_128bits_withSecret(const void* input, size_t len, const void* secret, size_t secretSize) +{ + return XXH3_128bits_internal(input, len, 0, + (const xxh_u8*)secret, secretSize, + XXH3_hashLong_128b_withSecret); +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH128_hash_t +XXH3_128bits_withSeed(const void* input, size_t len, XXH64_hash_t seed) +{ + return XXH3_128bits_internal(input, len, seed, + XXH3_kSecret, sizeof(XXH3_kSecret), + XXH3_hashLong_128b_withSeed); +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH128_hash_t +XXH3_128bits_withSecretandSeed(const void* input, size_t len, const void* secret, size_t secretSize, XXH64_hash_t seed) +{ + if (len <= XXH3_MIDSIZE_MAX) + return XXH3_128bits_internal(input, len, seed, XXH3_kSecret, sizeof(XXH3_kSecret), NULL); + return XXH3_hashLong_128b_withSecret(input, len, seed, secret, secretSize); +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH128_hash_t +XXH128(const void* input, size_t len, XXH64_hash_t seed) +{ + return XXH3_128bits_withSeed(input, len, seed); +} + + +/* === XXH3 128-bit streaming === */ + +/* + * All initialization and update functions are identical to 64-bit streaming variant. + * The only difference is the finalization routine. + */ + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_128bits_reset(XXH3_state_t* statePtr) +{ + return XXH3_64bits_reset(statePtr); +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_128bits_reset_withSecret(XXH3_state_t* statePtr, const void* secret, size_t secretSize) +{ + return XXH3_64bits_reset_withSecret(statePtr, secret, secretSize); +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_128bits_reset_withSeed(XXH3_state_t* statePtr, XXH64_hash_t seed) +{ + return XXH3_64bits_reset_withSeed(statePtr, seed); +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_128bits_reset_withSecretandSeed(XXH3_state_t* statePtr, const void* secret, size_t secretSize, XXH64_hash_t seed) +{ + return XXH3_64bits_reset_withSecretandSeed(statePtr, secret, secretSize, seed); +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_128bits_update(XXH3_state_t* state, const void* input, size_t len) +{ + return XXH3_update(state, (const xxh_u8*)input, len, + XXH3_accumulate_512, XXH3_scrambleAcc); +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_digest (const XXH3_state_t* state) +{ + const unsigned char* const secret = (state->extSecret == NULL) ? state->customSecret : state->extSecret; + if (state->totalLen > XXH3_MIDSIZE_MAX) { + XXH_ALIGN(XXH_ACC_ALIGN) XXH64_hash_t acc[XXH_ACC_NB]; + XXH3_digest_long(acc, state, secret); + XXH_ASSERT(state->secretLimit + XXH_STRIPE_LEN >= sizeof(acc) + XXH_SECRET_MERGEACCS_START); + { XXH128_hash_t h128; + h128.low64 = XXH3_mergeAccs(acc, + secret + XXH_SECRET_MERGEACCS_START, + (xxh_u64)state->totalLen * XXH_PRIME64_1); + h128.high64 = XXH3_mergeAccs(acc, + secret + state->secretLimit + XXH_STRIPE_LEN + - sizeof(acc) - XXH_SECRET_MERGEACCS_START, + ~((xxh_u64)state->totalLen * XXH_PRIME64_2)); + return h128; + } + } + /* len <= XXH3_MIDSIZE_MAX : short code */ + if (state->seed) + return XXH3_128bits_withSeed(state->buffer, (size_t)state->totalLen, state->seed); + return XXH3_128bits_withSecret(state->buffer, (size_t)(state->totalLen), + secret, state->secretLimit + XXH_STRIPE_LEN); +} + +/* 128-bit utility functions */ + +#include /* memcmp, memcpy */ + +/* return : 1 is equal, 0 if different */ +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API int XXH128_isEqual(XXH128_hash_t h1, XXH128_hash_t h2) +{ + /* note : XXH128_hash_t is compact, it has no padding byte */ + return !(memcmp(&h1, &h2, sizeof(h1))); +} + +/* This prototype is compatible with stdlib's qsort(). + * return : >0 if *h128_1 > *h128_2 + * <0 if *h128_1 < *h128_2 + * =0 if *h128_1 == *h128_2 */ +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API int XXH128_cmp(const void* h128_1, const void* h128_2) +{ + XXH128_hash_t const h1 = *(const XXH128_hash_t*)h128_1; + XXH128_hash_t const h2 = *(const XXH128_hash_t*)h128_2; + int const hcmp = (h1.high64 > h2.high64) - (h2.high64 > h1.high64); + /* note : bets that, in most cases, hash values are different */ + if (hcmp) return hcmp; + return (h1.low64 > h2.low64) - (h2.low64 > h1.low64); +} + + +/*====== Canonical representation ======*/ +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API void +XXH128_canonicalFromHash(XXH128_canonical_t* dst, XXH128_hash_t hash) +{ + XXH_STATIC_ASSERT(sizeof(XXH128_canonical_t) == sizeof(XXH128_hash_t)); + if (XXH_CPU_LITTLE_ENDIAN) { + hash.high64 = XXH_swap64(hash.high64); + hash.low64 = XXH_swap64(hash.low64); + } + XXH_memcpy(dst, &hash.high64, sizeof(hash.high64)); + XXH_memcpy((char*)dst + sizeof(hash.high64), &hash.low64, sizeof(hash.low64)); +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH128_hash_t +XXH128_hashFromCanonical(const XXH128_canonical_t* src) +{ + XXH128_hash_t h; + h.high64 = XXH_readBE64(src); + h.low64 = XXH_readBE64(src->digest + 8); + return h; +} + + + +/* ========================================== + * Secret generators + * ========================================== + */ +#define XXH_MIN(x, y) (((x) > (y)) ? (y) : (x)) + +XXH_FORCE_INLINE void XXH3_combine16(void* dst, XXH128_hash_t h128) +{ + XXH_writeLE64( dst, XXH_readLE64(dst) ^ h128.low64 ); + XXH_writeLE64( (char*)dst+8, XXH_readLE64((char*)dst+8) ^ h128.high64 ); +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_generateSecret(void* secretBuffer, size_t secretSize, const void* customSeed, size_t customSeedSize) +{ +#if (XXH_DEBUGLEVEL >= 1) + XXH_ASSERT(secretBuffer != NULL); + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); +#else + /* production mode, assert() are disabled */ + if (secretBuffer == NULL) return XXH_ERROR; + if (secretSize < XXH3_SECRET_SIZE_MIN) return XXH_ERROR; +#endif + + if (customSeedSize == 0) { + customSeed = XXH3_kSecret; + customSeedSize = XXH_SECRET_DEFAULT_SIZE; + } +#if (XXH_DEBUGLEVEL >= 1) + XXH_ASSERT(customSeed != NULL); +#else + if (customSeed == NULL) return XXH_ERROR; +#endif + + /* Fill secretBuffer with a copy of customSeed - repeat as needed */ + { size_t pos = 0; + while (pos < secretSize) { + size_t const toCopy = XXH_MIN((secretSize - pos), customSeedSize); + memcpy((char*)secretBuffer + pos, customSeed, toCopy); + pos += toCopy; + } } + + { size_t const nbSeg16 = secretSize / 16; + size_t n; + XXH128_canonical_t scrambler; + XXH128_canonicalFromHash(&scrambler, XXH128(customSeed, customSeedSize, 0)); + for (n=0; n +#include +#include + +#if defined(__GNUC__) && __GNUC__ >= 4 +# define ZSTD_memcpy(d,s,l) __builtin_memcpy((d),(s),(l)) +# define ZSTD_memmove(d,s,l) __builtin_memmove((d),(s),(l)) +# define ZSTD_memset(p,v,l) __builtin_memset((p),(v),(l)) +#else +# define ZSTD_memcpy(d,s,l) memcpy((d),(s),(l)) +# define ZSTD_memmove(d,s,l) memmove((d),(s),(l)) +# define ZSTD_memset(p,v,l) memset((p),(v),(l)) +#endif + +#endif /* ZSTD_DEPS_COMMON */ + +/* Need: + * ZSTD_malloc() + * ZSTD_free() + * ZSTD_calloc() + */ +#ifdef ZSTD_DEPS_NEED_MALLOC +#ifndef ZSTD_DEPS_MALLOC +#define ZSTD_DEPS_MALLOC + +#include + +#define ZSTD_malloc(s) malloc(s) +#define ZSTD_calloc(n,s) calloc((n), (s)) +#define ZSTD_free(p) free((p)) + +#endif /* ZSTD_DEPS_MALLOC */ +#endif /* ZSTD_DEPS_NEED_MALLOC */ + +/* + * Provides 64-bit math support. + * Need: + * U64 ZSTD_div64(U64 dividend, U32 divisor) + */ +#ifdef ZSTD_DEPS_NEED_MATH64 +#ifndef ZSTD_DEPS_MATH64 +#define ZSTD_DEPS_MATH64 + +#define ZSTD_div64(dividend, divisor) ((dividend) / (divisor)) + +#endif /* ZSTD_DEPS_MATH64 */ +#endif /* ZSTD_DEPS_NEED_MATH64 */ + +/* Need: + * assert() + */ +#ifdef ZSTD_DEPS_NEED_ASSERT +#ifndef ZSTD_DEPS_ASSERT +#define ZSTD_DEPS_ASSERT + +#include + +#endif /* ZSTD_DEPS_ASSERT */ +#endif /* ZSTD_DEPS_NEED_ASSERT */ + +/* Need: + * ZSTD_DEBUG_PRINT() + */ +#ifdef ZSTD_DEPS_NEED_IO +#ifndef ZSTD_DEPS_IO +#define ZSTD_DEPS_IO + +#include +#define ZSTD_DEBUG_PRINT(...) fprintf(stderr, __VA_ARGS__) + +#endif /* ZSTD_DEPS_IO */ +#endif /* ZSTD_DEPS_NEED_IO */ + +/* Only requested when is known to be present. + * Need: + * intptr_t + */ +#ifdef ZSTD_DEPS_NEED_STDINT +#ifndef ZSTD_DEPS_STDINT +#define ZSTD_DEPS_STDINT + +#include + +#endif /* ZSTD_DEPS_STDINT */ +#endif /* ZSTD_DEPS_NEED_STDINT */ diff --git a/src/blosc2/internal-complibs/zstd-1.5.2/common/zstd_internal.h b/src/blosc2/internal-complibs/zstd-1.5.2/common/zstd_internal.h new file mode 100644 index 0000000..e4d36ce --- /dev/null +++ b/src/blosc2/internal-complibs/zstd-1.5.2/common/zstd_internal.h @@ -0,0 +1,493 @@ +/* + * Copyright (c) Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_CCOMMON_H_MODULE +#define ZSTD_CCOMMON_H_MODULE + +/* this module contains definitions which must be identical + * across compression, decompression and dictBuilder. + * It also contains a few functions useful to at least 2 of them + * and which benefit from being inlined */ + +/*-************************************* +* Dependencies +***************************************/ +#include "compiler.h" +#include "cpu.h" +#include "mem.h" +#include "debug.h" /* assert, DEBUGLOG, RAWLOG, g_debuglevel */ +#include "error_private.h" +#define ZSTD_STATIC_LINKING_ONLY +#include "../zstd.h" +#define FSE_STATIC_LINKING_ONLY +#include "fse.h" +#define HUF_STATIC_LINKING_ONLY +#include "huf.h" +#ifndef XXH_STATIC_LINKING_ONLY +# define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */ +#endif +#include "xxhash.h" /* XXH_reset, update, digest */ +#ifndef ZSTD_NO_TRACE +# include "zstd_trace.h" +#else +# define ZSTD_TRACE 0 +#endif + +#if defined (__cplusplus) +extern "C" { +#endif + +/* ---- static assert (debug) --- */ +#define ZSTD_STATIC_ASSERT(c) DEBUG_STATIC_ASSERT(c) +#define ZSTD_isError ERR_isError /* for inlining */ +#define FSE_isError ERR_isError +#define HUF_isError ERR_isError + + +/*-************************************* +* shared macros +***************************************/ +#undef MIN +#undef MAX +#define MIN(a,b) ((a)<(b) ? (a) : (b)) +#define MAX(a,b) ((a)>(b) ? (a) : (b)) +#define BOUNDED(min,val,max) (MAX(min,MIN(val,max))) + + +/*-************************************* +* Common constants +***************************************/ +#define ZSTD_OPT_NUM (1<<12) + +#define ZSTD_REP_NUM 3 /* number of repcodes */ +static UNUSED_ATTR const U32 repStartValue[ZSTD_REP_NUM] = { 1, 4, 8 }; + +#define KB *(1 <<10) +#define MB *(1 <<20) +#define GB *(1U<<30) + +#define BIT7 128 +#define BIT6 64 +#define BIT5 32 +#define BIT4 16 +#define BIT1 2 +#define BIT0 1 + +#define ZSTD_WINDOWLOG_ABSOLUTEMIN 10 +static UNUSED_ATTR const size_t ZSTD_fcs_fieldSize[4] = { 0, 2, 4, 8 }; +static UNUSED_ATTR const size_t ZSTD_did_fieldSize[4] = { 0, 1, 2, 4 }; + +#define ZSTD_FRAMEIDSIZE 4 /* magic number size */ + +#define ZSTD_BLOCKHEADERSIZE 3 /* C standard doesn't allow `static const` variable to be init using another `static const` variable */ +static UNUSED_ATTR const size_t ZSTD_blockHeaderSize = ZSTD_BLOCKHEADERSIZE; +typedef enum { bt_raw, bt_rle, bt_compressed, bt_reserved } blockType_e; + +#define ZSTD_FRAMECHECKSUMSIZE 4 + +#define MIN_SEQUENCES_SIZE 1 /* nbSeq==0 */ +#define MIN_CBLOCK_SIZE (1 /*litCSize*/ + 1 /* RLE or RAW */ + MIN_SEQUENCES_SIZE /* nbSeq==0 */) /* for a non-null block */ + +#define HufLog 12 +typedef enum { set_basic, set_rle, set_compressed, set_repeat } symbolEncodingType_e; + +#define LONGNBSEQ 0x7F00 + +#define MINMATCH 3 + +#define Litbits 8 +#define MaxLit ((1<= WILDCOPY_VECLEN || diff <= -WILDCOPY_VECLEN); + /* Separate out the first COPY16() call because the copy length is + * almost certain to be short, so the branches have different + * probabilities. Since it is almost certain to be short, only do + * one COPY16() in the first call. Then, do two calls per loop since + * at that point it is more likely to have a high trip count. + */ +#ifdef __aarch64__ + do { + COPY16(op, ip); + } + while (op < oend); +#else + ZSTD_copy16(op, ip); + if (16 >= length) return; + op += 16; + ip += 16; + do { + COPY16(op, ip); + COPY16(op, ip); + } + while (op < oend); +#endif + } +} + +MEM_STATIC size_t ZSTD_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ + size_t const length = MIN(dstCapacity, srcSize); + if (length > 0) { + ZSTD_memcpy(dst, src, length); + } + return length; +} + +/* define "workspace is too large" as this number of times larger than needed */ +#define ZSTD_WORKSPACETOOLARGE_FACTOR 3 + +/* when workspace is continuously too large + * during at least this number of times, + * context's memory usage is considered wasteful, + * because it's sized to handle a worst case scenario which rarely happens. + * In which case, resize it down to free some memory */ +#define ZSTD_WORKSPACETOOLARGE_MAXDURATION 128 + +/* Controls whether the input/output buffer is buffered or stable. */ +typedef enum { + ZSTD_bm_buffered = 0, /* Buffer the input/output */ + ZSTD_bm_stable = 1 /* ZSTD_inBuffer/ZSTD_outBuffer is stable */ +} ZSTD_bufferMode_e; + + +/*-******************************************* +* Private declarations +*********************************************/ +typedef struct seqDef_s { + U32 offBase; /* offBase == Offset + ZSTD_REP_NUM, or repcode 1,2,3 */ + U16 litLength; + U16 mlBase; /* mlBase == matchLength - MINMATCH */ +} seqDef; + +/* Controls whether seqStore has a single "long" litLength or matchLength. See seqStore_t. */ +typedef enum { + ZSTD_llt_none = 0, /* no longLengthType */ + ZSTD_llt_literalLength = 1, /* represents a long literal */ + ZSTD_llt_matchLength = 2 /* represents a long match */ +} ZSTD_longLengthType_e; + +typedef struct { + seqDef* sequencesStart; + seqDef* sequences; /* ptr to end of sequences */ + BYTE* litStart; + BYTE* lit; /* ptr to end of literals */ + BYTE* llCode; + BYTE* mlCode; + BYTE* ofCode; + size_t maxNbSeq; + size_t maxNbLit; + + /* longLengthPos and longLengthType to allow us to represent either a single litLength or matchLength + * in the seqStore that has a value larger than U16 (if it exists). To do so, we increment + * the existing value of the litLength or matchLength by 0x10000. + */ + ZSTD_longLengthType_e longLengthType; + U32 longLengthPos; /* Index of the sequence to apply long length modification to */ +} seqStore_t; + +typedef struct { + U32 litLength; + U32 matchLength; +} ZSTD_sequenceLength; + +/** + * Returns the ZSTD_sequenceLength for the given sequences. It handles the decoding of long sequences + * indicated by longLengthPos and longLengthType, and adds MINMATCH back to matchLength. + */ +MEM_STATIC ZSTD_sequenceLength ZSTD_getSequenceLength(seqStore_t const* seqStore, seqDef const* seq) +{ + ZSTD_sequenceLength seqLen; + seqLen.litLength = seq->litLength; + seqLen.matchLength = seq->mlBase + MINMATCH; + if (seqStore->longLengthPos == (U32)(seq - seqStore->sequencesStart)) { + if (seqStore->longLengthType == ZSTD_llt_literalLength) { + seqLen.litLength += 0xFFFF; + } + if (seqStore->longLengthType == ZSTD_llt_matchLength) { + seqLen.matchLength += 0xFFFF; + } + } + return seqLen; +} + +/** + * Contains the compressed frame size and an upper-bound for the decompressed frame size. + * Note: before using `compressedSize`, check for errors using ZSTD_isError(). + * similarly, before using `decompressedBound`, check for errors using: + * `decompressedBound != ZSTD_CONTENTSIZE_ERROR` + */ +typedef struct { + size_t compressedSize; + unsigned long long decompressedBound; +} ZSTD_frameSizeInfo; /* decompress & legacy */ + +const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx); /* compress & dictBuilder */ +void ZSTD_seqToCodes(const seqStore_t* seqStorePtr); /* compress, dictBuilder, decodeCorpus (shouldn't get its definition from here) */ + +/* custom memory allocation functions */ +void* ZSTD_customMalloc(size_t size, ZSTD_customMem customMem); +void* ZSTD_customCalloc(size_t size, ZSTD_customMem customMem); +void ZSTD_customFree(void* ptr, ZSTD_customMem customMem); + + +MEM_STATIC U32 ZSTD_highbit32(U32 val) /* compress, dictBuilder, decodeCorpus */ +{ + assert(val != 0); + { +# if defined(_MSC_VER) /* Visual */ +# if STATIC_BMI2 == 1 + return _lzcnt_u32(val)^31; +# else + if (val != 0) { + unsigned long r; + _BitScanReverse(&r, val); + return (unsigned)r; + } else { + /* Should not reach this code path */ + __assume(0); + } +# endif +# elif defined(__GNUC__) && (__GNUC__ >= 3) /* GCC Intrinsic */ + return __builtin_clz (val) ^ 31; +# elif defined(__ICCARM__) /* IAR Intrinsic */ + return 31 - __CLZ(val); +# else /* Software version */ + static const U32 DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 }; + U32 v = val; + v |= v >> 1; + v |= v >> 2; + v |= v >> 4; + v |= v >> 8; + v |= v >> 16; + return DeBruijnClz[(v * 0x07C4ACDDU) >> 27]; +# endif + } +} + +/** + * Counts the number of trailing zeros of a `size_t`. + * Most compilers should support CTZ as a builtin. A backup + * implementation is provided if the builtin isn't supported, but + * it may not be terribly efficient. + */ +MEM_STATIC unsigned ZSTD_countTrailingZeros(size_t val) +{ + if (MEM_64bits()) { +# if defined(_MSC_VER) && defined(_WIN64) +# if STATIC_BMI2 + return _tzcnt_u64(val); +# else + if (val != 0) { + unsigned long r; + _BitScanForward64(&r, (U64)val); + return (unsigned)r; + } else { + /* Should not reach this code path */ + __assume(0); + } +# endif +# elif defined(__GNUC__) && (__GNUC__ >= 4) + return __builtin_ctzll((U64)val); +# else + static const int DeBruijnBytePos[64] = { 0, 1, 2, 7, 3, 13, 8, 19, + 4, 25, 14, 28, 9, 34, 20, 56, + 5, 17, 26, 54, 15, 41, 29, 43, + 10, 31, 38, 35, 21, 45, 49, 57, + 63, 6, 12, 18, 24, 27, 33, 55, + 16, 53, 40, 42, 30, 37, 44, 48, + 62, 11, 23, 32, 52, 39, 36, 47, + 61, 22, 51, 46, 60, 50, 59, 58 }; + return DeBruijnBytePos[((U64)((val & -(long long)val) * 0x0218A392CDABBD3FULL)) >> 58]; +# endif + } else { /* 32 bits */ +# if defined(_MSC_VER) + if (val != 0) { + unsigned long r; + _BitScanForward(&r, (U32)val); + return (unsigned)r; + } else { + /* Should not reach this code path */ + __assume(0); + } +# elif defined(__GNUC__) && (__GNUC__ >= 3) + return __builtin_ctz((U32)val); +# else + static const int DeBruijnBytePos[32] = { 0, 1, 28, 2, 29, 14, 24, 3, + 30, 22, 20, 15, 25, 17, 4, 8, + 31, 27, 13, 23, 21, 19, 16, 7, + 26, 12, 18, 6, 11, 5, 10, 9 }; + return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27]; +# endif + } +} + + +/* ZSTD_invalidateRepCodes() : + * ensures next compression will not use repcodes from previous block. + * Note : only works with regular variant; + * do not use with extDict variant ! */ +void ZSTD_invalidateRepCodes(ZSTD_CCtx* cctx); /* zstdmt, adaptive_compression (shouldn't get this definition from here) */ + + +typedef struct { + blockType_e blockType; + U32 lastBlock; + U32 origSize; +} blockProperties_t; /* declared here for decompress and fullbench */ + +/*! ZSTD_getcBlockSize() : + * Provides the size of compressed block from block header `src` */ +/* Used by: decompress, fullbench (does not get its definition from here) */ +size_t ZSTD_getcBlockSize(const void* src, size_t srcSize, + blockProperties_t* bpPtr); + +/*! ZSTD_decodeSeqHeaders() : + * decode sequence header from src */ +/* Used by: decompress, fullbench (does not get its definition from here) */ +size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr, + const void* src, size_t srcSize); + +/** + * @returns true iff the CPU supports dynamic BMI2 dispatch. + */ +MEM_STATIC int ZSTD_cpuSupportsBmi2(void) +{ + ZSTD_cpuid_t cpuid = ZSTD_cpuid(); + return ZSTD_cpuid_bmi1(cpuid) && ZSTD_cpuid_bmi2(cpuid); +} + +#if defined (__cplusplus) +} +#endif + +#endif /* ZSTD_CCOMMON_H_MODULE */ diff --git a/src/blosc2/internal-complibs/zstd-1.5.2/common/zstd_trace.h b/src/blosc2/internal-complibs/zstd-1.5.2/common/zstd_trace.h new file mode 100644 index 0000000..f9121f7 --- /dev/null +++ b/src/blosc2/internal-complibs/zstd-1.5.2/common/zstd_trace.h @@ -0,0 +1,163 @@ +/* + * Copyright (c) Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_TRACE_H +#define ZSTD_TRACE_H + +#if defined (__cplusplus) +extern "C" { +#endif + +#include + +/* weak symbol support + * For now, enable conservatively: + * - Only GNUC + * - Only ELF + * - Only x86-64 and i386 + * Also, explicitly disable on platforms known not to work so they aren't + * forgotten in the future. + */ +#if !defined(ZSTD_HAVE_WEAK_SYMBOLS) && \ + defined(__GNUC__) && defined(__ELF__) && \ + (defined(__x86_64__) || defined(_M_X64) || defined(__i386__) || defined(_M_IX86)) && \ + !defined(__APPLE__) && !defined(_WIN32) && !defined(__MINGW32__) && \ + !defined(__CYGWIN__) && !defined(_AIX) +# define ZSTD_HAVE_WEAK_SYMBOLS 1 +#else +# define ZSTD_HAVE_WEAK_SYMBOLS 0 +#endif +#if ZSTD_HAVE_WEAK_SYMBOLS +# define ZSTD_WEAK_ATTR __attribute__((__weak__)) +#else +# define ZSTD_WEAK_ATTR +#endif + +/* Only enable tracing when weak symbols are available. */ +#ifndef ZSTD_TRACE +# define ZSTD_TRACE ZSTD_HAVE_WEAK_SYMBOLS +#endif + +#if ZSTD_TRACE + +struct ZSTD_CCtx_s; +struct ZSTD_DCtx_s; +struct ZSTD_CCtx_params_s; + +typedef struct { + /** + * ZSTD_VERSION_NUMBER + * + * This is guaranteed to be the first member of ZSTD_trace. + * Otherwise, this struct is not stable between versions. If + * the version number does not match your expectation, you + * should not interpret the rest of the struct. + */ + unsigned version; + /** + * Non-zero if streaming (de)compression is used. + */ + unsigned streaming; + /** + * The dictionary ID. + */ + unsigned dictionaryID; + /** + * Is the dictionary cold? + * Only set on decompression. + */ + unsigned dictionaryIsCold; + /** + * The dictionary size or zero if no dictionary. + */ + size_t dictionarySize; + /** + * The uncompressed size of the data. + */ + size_t uncompressedSize; + /** + * The compressed size of the data. + */ + size_t compressedSize; + /** + * The fully resolved CCtx parameters (NULL on decompression). + */ + struct ZSTD_CCtx_params_s const* params; + /** + * The ZSTD_CCtx pointer (NULL on decompression). + */ + struct ZSTD_CCtx_s const* cctx; + /** + * The ZSTD_DCtx pointer (NULL on compression). + */ + struct ZSTD_DCtx_s const* dctx; +} ZSTD_Trace; + +/** + * A tracing context. It must be 0 when tracing is disabled. + * Otherwise, any non-zero value returned by a tracing begin() + * function is presented to any subsequent calls to end(). + * + * Any non-zero value is treated as tracing is enabled and not + * interpreted by the library. + * + * Two possible uses are: + * * A timestamp for when the begin() function was called. + * * A unique key identifying the (de)compression, like the + * address of the [dc]ctx pointer if you need to track + * more information than just a timestamp. + */ +typedef unsigned long long ZSTD_TraceCtx; + +/** + * Trace the beginning of a compression call. + * @param cctx The dctx pointer for the compression. + * It can be used as a key to map begin() to end(). + * @returns Non-zero if tracing is enabled. The return value is + * passed to ZSTD_trace_compress_end(). + */ +ZSTD_WEAK_ATTR ZSTD_TraceCtx ZSTD_trace_compress_begin( + struct ZSTD_CCtx_s const* cctx); + +/** + * Trace the end of a compression call. + * @param ctx The return value of ZSTD_trace_compress_begin(). + * @param trace The zstd tracing info. + */ +ZSTD_WEAK_ATTR void ZSTD_trace_compress_end( + ZSTD_TraceCtx ctx, + ZSTD_Trace const* trace); + +/** + * Trace the beginning of a decompression call. + * @param dctx The dctx pointer for the decompression. + * It can be used as a key to map begin() to end(). + * @returns Non-zero if tracing is enabled. The return value is + * passed to ZSTD_trace_compress_end(). + */ +ZSTD_WEAK_ATTR ZSTD_TraceCtx ZSTD_trace_decompress_begin( + struct ZSTD_DCtx_s const* dctx); + +/** + * Trace the end of a decompression call. + * @param ctx The return value of ZSTD_trace_decompress_begin(). + * @param trace The zstd tracing info. + */ +ZSTD_WEAK_ATTR void ZSTD_trace_decompress_end( + ZSTD_TraceCtx ctx, + ZSTD_Trace const* trace); + +#endif /* ZSTD_TRACE */ + +#if defined (__cplusplus) +} +#endif + +#endif /* ZSTD_TRACE_H */ diff --git a/src/blosc2/internal-complibs/zstd-1.5.2/compress/clevels.h b/src/blosc2/internal-complibs/zstd-1.5.2/compress/clevels.h new file mode 100644 index 0000000..7ed2e00 --- /dev/null +++ b/src/blosc2/internal-complibs/zstd-1.5.2/compress/clevels.h @@ -0,0 +1,134 @@ +/* + * Copyright (c) Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_CLEVELS_H +#define ZSTD_CLEVELS_H + +#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressionParameters */ +#include "../zstd.h" + +/*-===== Pre-defined compression levels =====-*/ + +#define ZSTD_MAX_CLEVEL 22 + +#ifdef __GNUC__ +__attribute__((__unused__)) +#endif + +static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEVEL+1] = { +{ /* "default" - for any srcSize > 256 KB */ + /* W, C, H, S, L, TL, strat */ + { 19, 12, 13, 1, 6, 1, ZSTD_fast }, /* base for negative levels */ + { 19, 13, 14, 1, 7, 0, ZSTD_fast }, /* level 1 */ + { 20, 15, 16, 1, 6, 0, ZSTD_fast }, /* level 2 */ + { 21, 16, 17, 1, 5, 0, ZSTD_dfast }, /* level 3 */ + { 21, 18, 18, 1, 5, 0, ZSTD_dfast }, /* level 4 */ + { 21, 18, 19, 3, 5, 2, ZSTD_greedy }, /* level 5 */ + { 21, 18, 19, 3, 5, 4, ZSTD_lazy }, /* level 6 */ + { 21, 19, 20, 4, 5, 8, ZSTD_lazy }, /* level 7 */ + { 21, 19, 20, 4, 5, 16, ZSTD_lazy2 }, /* level 8 */ + { 22, 20, 21, 4, 5, 16, ZSTD_lazy2 }, /* level 9 */ + { 22, 21, 22, 5, 5, 16, ZSTD_lazy2 }, /* level 10 */ + { 22, 21, 22, 6, 5, 16, ZSTD_lazy2 }, /* level 11 */ + { 22, 22, 23, 6, 5, 32, ZSTD_lazy2 }, /* level 12 */ + { 22, 22, 22, 4, 5, 32, ZSTD_btlazy2 }, /* level 13 */ + { 22, 22, 23, 5, 5, 32, ZSTD_btlazy2 }, /* level 14 */ + { 22, 23, 23, 6, 5, 32, ZSTD_btlazy2 }, /* level 15 */ + { 22, 22, 22, 5, 5, 48, ZSTD_btopt }, /* level 16 */ + { 23, 23, 22, 5, 4, 64, ZSTD_btopt }, /* level 17 */ + { 23, 23, 22, 6, 3, 64, ZSTD_btultra }, /* level 18 */ + { 23, 24, 22, 7, 3,256, ZSTD_btultra2}, /* level 19 */ + { 25, 25, 23, 7, 3,256, ZSTD_btultra2}, /* level 20 */ + { 26, 26, 24, 7, 3,512, ZSTD_btultra2}, /* level 21 */ + { 27, 27, 25, 9, 3,999, ZSTD_btultra2}, /* level 22 */ +}, +{ /* for srcSize <= 256 KB */ + /* W, C, H, S, L, T, strat */ + { 18, 12, 13, 1, 5, 1, ZSTD_fast }, /* base for negative levels */ + { 18, 13, 14, 1, 6, 0, ZSTD_fast }, /* level 1 */ + { 18, 14, 14, 1, 5, 0, ZSTD_dfast }, /* level 2 */ + { 18, 16, 16, 1, 4, 0, ZSTD_dfast }, /* level 3 */ + { 18, 16, 17, 3, 5, 2, ZSTD_greedy }, /* level 4.*/ + { 18, 17, 18, 5, 5, 2, ZSTD_greedy }, /* level 5.*/ + { 18, 18, 19, 3, 5, 4, ZSTD_lazy }, /* level 6.*/ + { 18, 18, 19, 4, 4, 4, ZSTD_lazy }, /* level 7 */ + { 18, 18, 19, 4, 4, 8, ZSTD_lazy2 }, /* level 8 */ + { 18, 18, 19, 5, 4, 8, ZSTD_lazy2 }, /* level 9 */ + { 18, 18, 19, 6, 4, 8, ZSTD_lazy2 }, /* level 10 */ + { 18, 18, 19, 5, 4, 12, ZSTD_btlazy2 }, /* level 11.*/ + { 18, 19, 19, 7, 4, 12, ZSTD_btlazy2 }, /* level 12.*/ + { 18, 18, 19, 4, 4, 16, ZSTD_btopt }, /* level 13 */ + { 18, 18, 19, 4, 3, 32, ZSTD_btopt }, /* level 14.*/ + { 18, 18, 19, 6, 3,128, ZSTD_btopt }, /* level 15.*/ + { 18, 19, 19, 6, 3,128, ZSTD_btultra }, /* level 16.*/ + { 18, 19, 19, 8, 3,256, ZSTD_btultra }, /* level 17.*/ + { 18, 19, 19, 6, 3,128, ZSTD_btultra2}, /* level 18.*/ + { 18, 19, 19, 8, 3,256, ZSTD_btultra2}, /* level 19.*/ + { 18, 19, 19, 10, 3,512, ZSTD_btultra2}, /* level 20.*/ + { 18, 19, 19, 12, 3,512, ZSTD_btultra2}, /* level 21.*/ + { 18, 19, 19, 13, 3,999, ZSTD_btultra2}, /* level 22.*/ +}, +{ /* for srcSize <= 128 KB */ + /* W, C, H, S, L, T, strat */ + { 17, 12, 12, 1, 5, 1, ZSTD_fast }, /* base for negative levels */ + { 17, 12, 13, 1, 6, 0, ZSTD_fast }, /* level 1 */ + { 17, 13, 15, 1, 5, 0, ZSTD_fast }, /* level 2 */ + { 17, 15, 16, 2, 5, 0, ZSTD_dfast }, /* level 3 */ + { 17, 17, 17, 2, 4, 0, ZSTD_dfast }, /* level 4 */ + { 17, 16, 17, 3, 4, 2, ZSTD_greedy }, /* level 5 */ + { 17, 16, 17, 3, 4, 4, ZSTD_lazy }, /* level 6 */ + { 17, 16, 17, 3, 4, 8, ZSTD_lazy2 }, /* level 7 */ + { 17, 16, 17, 4, 4, 8, ZSTD_lazy2 }, /* level 8 */ + { 17, 16, 17, 5, 4, 8, ZSTD_lazy2 }, /* level 9 */ + { 17, 16, 17, 6, 4, 8, ZSTD_lazy2 }, /* level 10 */ + { 17, 17, 17, 5, 4, 8, ZSTD_btlazy2 }, /* level 11 */ + { 17, 18, 17, 7, 4, 12, ZSTD_btlazy2 }, /* level 12 */ + { 17, 18, 17, 3, 4, 12, ZSTD_btopt }, /* level 13.*/ + { 17, 18, 17, 4, 3, 32, ZSTD_btopt }, /* level 14.*/ + { 17, 18, 17, 6, 3,256, ZSTD_btopt }, /* level 15.*/ + { 17, 18, 17, 6, 3,128, ZSTD_btultra }, /* level 16.*/ + { 17, 18, 17, 8, 3,256, ZSTD_btultra }, /* level 17.*/ + { 17, 18, 17, 10, 3,512, ZSTD_btultra }, /* level 18.*/ + { 17, 18, 17, 5, 3,256, ZSTD_btultra2}, /* level 19.*/ + { 17, 18, 17, 7, 3,512, ZSTD_btultra2}, /* level 20.*/ + { 17, 18, 17, 9, 3,512, ZSTD_btultra2}, /* level 21.*/ + { 17, 18, 17, 11, 3,999, ZSTD_btultra2}, /* level 22.*/ +}, +{ /* for srcSize <= 16 KB */ + /* W, C, H, S, L, T, strat */ + { 14, 12, 13, 1, 5, 1, ZSTD_fast }, /* base for negative levels */ + { 14, 14, 15, 1, 5, 0, ZSTD_fast }, /* level 1 */ + { 14, 14, 15, 1, 4, 0, ZSTD_fast }, /* level 2 */ + { 14, 14, 15, 2, 4, 0, ZSTD_dfast }, /* level 3 */ + { 14, 14, 14, 4, 4, 2, ZSTD_greedy }, /* level 4 */ + { 14, 14, 14, 3, 4, 4, ZSTD_lazy }, /* level 5.*/ + { 14, 14, 14, 4, 4, 8, ZSTD_lazy2 }, /* level 6 */ + { 14, 14, 14, 6, 4, 8, ZSTD_lazy2 }, /* level 7 */ + { 14, 14, 14, 8, 4, 8, ZSTD_lazy2 }, /* level 8.*/ + { 14, 15, 14, 5, 4, 8, ZSTD_btlazy2 }, /* level 9.*/ + { 14, 15, 14, 9, 4, 8, ZSTD_btlazy2 }, /* level 10.*/ + { 14, 15, 14, 3, 4, 12, ZSTD_btopt }, /* level 11.*/ + { 14, 15, 14, 4, 3, 24, ZSTD_btopt }, /* level 12.*/ + { 14, 15, 14, 5, 3, 32, ZSTD_btultra }, /* level 13.*/ + { 14, 15, 15, 6, 3, 64, ZSTD_btultra }, /* level 14.*/ + { 14, 15, 15, 7, 3,256, ZSTD_btultra }, /* level 15.*/ + { 14, 15, 15, 5, 3, 48, ZSTD_btultra2}, /* level 16.*/ + { 14, 15, 15, 6, 3,128, ZSTD_btultra2}, /* level 17.*/ + { 14, 15, 15, 7, 3,256, ZSTD_btultra2}, /* level 18.*/ + { 14, 15, 15, 8, 3,256, ZSTD_btultra2}, /* level 19.*/ + { 14, 15, 15, 8, 3,512, ZSTD_btultra2}, /* level 20.*/ + { 14, 15, 15, 9, 3,512, ZSTD_btultra2}, /* level 21.*/ + { 14, 15, 15, 10, 3,999, ZSTD_btultra2}, /* level 22.*/ +}, +}; + + + +#endif /* ZSTD_CLEVELS_H */ diff --git a/src/blosc2/internal-complibs/zstd-1.5.2/compress/fse_compress.c b/src/blosc2/internal-complibs/zstd-1.5.2/compress/fse_compress.c new file mode 100644 index 0000000..5547b4a --- /dev/null +++ b/src/blosc2/internal-complibs/zstd-1.5.2/compress/fse_compress.c @@ -0,0 +1,741 @@ +/* ****************************************************************** + * FSE : Finite State Entropy encoder + * Copyright (c) Yann Collet, Facebook, Inc. + * + * You can contact the author at : + * - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy + * - Public forum : https://groups.google.com/forum/#!forum/lz4c + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. +****************************************************************** */ + +/* ************************************************************** +* Includes +****************************************************************/ +#include "../common/compiler.h" +#include "../common/mem.h" /* U32, U16, etc. */ +#include "../common/debug.h" /* assert, DEBUGLOG */ +#include "hist.h" /* HIST_count_wksp */ +#include "../common/bitstream.h" +#define FSE_STATIC_LINKING_ONLY +#include "../common/fse.h" +#include "../common/error_private.h" +#define ZSTD_DEPS_NEED_MALLOC +#define ZSTD_DEPS_NEED_MATH64 +#include "../common/zstd_deps.h" /* ZSTD_malloc, ZSTD_free, ZSTD_memcpy, ZSTD_memset */ + + +/* ************************************************************** +* Error Management +****************************************************************/ +#define FSE_isError ERR_isError + + +/* ************************************************************** +* Templates +****************************************************************/ +/* + designed to be included + for type-specific functions (template emulation in C) + Objective is to write these functions only once, for improved maintenance +*/ + +/* safety checks */ +#ifndef FSE_FUNCTION_EXTENSION +# error "FSE_FUNCTION_EXTENSION must be defined" +#endif +#ifndef FSE_FUNCTION_TYPE +# error "FSE_FUNCTION_TYPE must be defined" +#endif + +/* Function names */ +#define FSE_CAT(X,Y) X##Y +#define FSE_FUNCTION_NAME(X,Y) FSE_CAT(X,Y) +#define FSE_TYPE_NAME(X,Y) FSE_CAT(X,Y) + + +/* Function templates */ + +/* FSE_buildCTable_wksp() : + * Same as FSE_buildCTable(), but using an externally allocated scratch buffer (`workSpace`). + * wkspSize should be sized to handle worst case situation, which is `1<>1 : 1) ; + FSE_symbolCompressionTransform* const symbolTT = (FSE_symbolCompressionTransform*) (FSCT); + U32 const step = FSE_TABLESTEP(tableSize); + U32 const maxSV1 = maxSymbolValue+1; + + U16* cumul = (U16*)workSpace; /* size = maxSV1 */ + FSE_FUNCTION_TYPE* const tableSymbol = (FSE_FUNCTION_TYPE*)(cumul + (maxSV1+1)); /* size = tableSize */ + + U32 highThreshold = tableSize-1; + + assert(((size_t)workSpace & 1) == 0); /* Must be 2 bytes-aligned */ + if (FSE_BUILD_CTABLE_WORKSPACE_SIZE(maxSymbolValue, tableLog) > wkspSize) return ERROR(tableLog_tooLarge); + /* CTable header */ + tableU16[-2] = (U16) tableLog; + tableU16[-1] = (U16) maxSymbolValue; + assert(tableLog < 16); /* required for threshold strategy to work */ + + /* For explanations on how to distribute symbol values over the table : + * http://fastcompression.blogspot.fr/2014/02/fse-distributing-symbol-values.html */ + + #ifdef __clang_analyzer__ + ZSTD_memset(tableSymbol, 0, sizeof(*tableSymbol) * tableSize); /* useless initialization, just to keep scan-build happy */ + #endif + + /* symbol start positions */ + { U32 u; + cumul[0] = 0; + for (u=1; u <= maxSV1; u++) { + if (normalizedCounter[u-1]==-1) { /* Low proba symbol */ + cumul[u] = cumul[u-1] + 1; + tableSymbol[highThreshold--] = (FSE_FUNCTION_TYPE)(u-1); + } else { + assert(normalizedCounter[u-1] >= 0); + cumul[u] = cumul[u-1] + (U16)normalizedCounter[u-1]; + assert(cumul[u] >= cumul[u-1]); /* no overflow */ + } } + cumul[maxSV1] = (U16)(tableSize+1); + } + + /* Spread symbols */ + if (highThreshold == tableSize - 1) { + /* Case for no low prob count symbols. Lay down 8 bytes at a time + * to reduce branch misses since we are operating on a small block + */ + BYTE* const spread = tableSymbol + tableSize; /* size = tableSize + 8 (may write beyond tableSize) */ + { U64 const add = 0x0101010101010101ull; + size_t pos = 0; + U64 sv = 0; + U32 s; + for (s=0; s=0); + pos += (size_t)n; + } + } + /* Spread symbols across the table. Lack of lowprob symbols means that + * we don't need variable sized inner loop, so we can unroll the loop and + * reduce branch misses. + */ + { size_t position = 0; + size_t s; + size_t const unroll = 2; /* Experimentally determined optimal unroll */ + assert(tableSize % unroll == 0); /* FSE_MIN_TABLELOG is 5 */ + for (s = 0; s < (size_t)tableSize; s += unroll) { + size_t u; + for (u = 0; u < unroll; ++u) { + size_t const uPosition = (position + (u * step)) & tableMask; + tableSymbol[uPosition] = spread[s + u]; + } + position = (position + (unroll * step)) & tableMask; + } + assert(position == 0); /* Must have initialized all positions */ + } + } else { + U32 position = 0; + U32 symbol; + for (symbol=0; symbol highThreshold) + position = (position + step) & tableMask; /* Low proba area */ + } } + assert(position==0); /* Must have initialized all positions */ + } + + /* Build table */ + { U32 u; for (u=0; u 1); + { U32 const maxBitsOut = tableLog - BIT_highbit32 ((U32)normalizedCounter[s]-1); + U32 const minStatePlus = (U32)normalizedCounter[s] << maxBitsOut; + symbolTT[s].deltaNbBits = (maxBitsOut << 16) - minStatePlus; + symbolTT[s].deltaFindState = (int)(total - (unsigned)normalizedCounter[s]); + total += (unsigned)normalizedCounter[s]; + } } } } + +#if 0 /* debug : symbol costs */ + DEBUGLOG(5, "\n --- table statistics : "); + { U32 symbol; + for (symbol=0; symbol<=maxSymbolValue; symbol++) { + DEBUGLOG(5, "%3u: w=%3i, maxBits=%u, fracBits=%.2f", + symbol, normalizedCounter[symbol], + FSE_getMaxNbBits(symbolTT, symbol), + (double)FSE_bitCost(symbolTT, tableLog, symbol, 8) / 256); + } } +#endif + + return 0; +} + + + +#ifndef FSE_COMMONDEFS_ONLY + +/*-************************************************************** +* FSE NCount encoding +****************************************************************/ +size_t FSE_NCountWriteBound(unsigned maxSymbolValue, unsigned tableLog) +{ + size_t const maxHeaderSize = (((maxSymbolValue+1) * tableLog + + 4 /* bitCount initialized at 4 */ + + 2 /* first two symbols may use one additional bit each */) / 8) + + 1 /* round up to whole nb bytes */ + + 2 /* additional two bytes for bitstream flush */; + return maxSymbolValue ? maxHeaderSize : FSE_NCOUNTBOUND; /* maxSymbolValue==0 ? use default */ +} + +static size_t +FSE_writeNCount_generic (void* header, size_t headerBufferSize, + const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, + unsigned writeIsSafe) +{ + BYTE* const ostart = (BYTE*) header; + BYTE* out = ostart; + BYTE* const oend = ostart + headerBufferSize; + int nbBits; + const int tableSize = 1 << tableLog; + int remaining; + int threshold; + U32 bitStream = 0; + int bitCount = 0; + unsigned symbol = 0; + unsigned const alphabetSize = maxSymbolValue + 1; + int previousIs0 = 0; + + /* Table Size */ + bitStream += (tableLog-FSE_MIN_TABLELOG) << bitCount; + bitCount += 4; + + /* Init */ + remaining = tableSize+1; /* +1 for extra accuracy */ + threshold = tableSize; + nbBits = tableLog+1; + + while ((symbol < alphabetSize) && (remaining>1)) { /* stops at 1 */ + if (previousIs0) { + unsigned start = symbol; + while ((symbol < alphabetSize) && !normalizedCounter[symbol]) symbol++; + if (symbol == alphabetSize) break; /* incorrect distribution */ + while (symbol >= start+24) { + start+=24; + bitStream += 0xFFFFU << bitCount; + if ((!writeIsSafe) && (out > oend-2)) + return ERROR(dstSize_tooSmall); /* Buffer overflow */ + out[0] = (BYTE) bitStream; + out[1] = (BYTE)(bitStream>>8); + out+=2; + bitStream>>=16; + } + while (symbol >= start+3) { + start+=3; + bitStream += 3 << bitCount; + bitCount += 2; + } + bitStream += (symbol-start) << bitCount; + bitCount += 2; + if (bitCount>16) { + if ((!writeIsSafe) && (out > oend - 2)) + return ERROR(dstSize_tooSmall); /* Buffer overflow */ + out[0] = (BYTE)bitStream; + out[1] = (BYTE)(bitStream>>8); + out += 2; + bitStream >>= 16; + bitCount -= 16; + } } + { int count = normalizedCounter[symbol++]; + int const max = (2*threshold-1) - remaining; + remaining -= count < 0 ? -count : count; + count++; /* +1 for extra accuracy */ + if (count>=threshold) + count += max; /* [0..max[ [max..threshold[ (...) [threshold+max 2*threshold[ */ + bitStream += count << bitCount; + bitCount += nbBits; + bitCount -= (count>=1; } + } + if (bitCount>16) { + if ((!writeIsSafe) && (out > oend - 2)) + return ERROR(dstSize_tooSmall); /* Buffer overflow */ + out[0] = (BYTE)bitStream; + out[1] = (BYTE)(bitStream>>8); + out += 2; + bitStream >>= 16; + bitCount -= 16; + } } + + if (remaining != 1) + return ERROR(GENERIC); /* incorrect normalized distribution */ + assert(symbol <= alphabetSize); + + /* flush remaining bitStream */ + if ((!writeIsSafe) && (out > oend - 2)) + return ERROR(dstSize_tooSmall); /* Buffer overflow */ + out[0] = (BYTE)bitStream; + out[1] = (BYTE)(bitStream>>8); + out+= (bitCount+7) /8; + + return (out-ostart); +} + + +size_t FSE_writeNCount (void* buffer, size_t bufferSize, + const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog) +{ + if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge); /* Unsupported */ + if (tableLog < FSE_MIN_TABLELOG) return ERROR(GENERIC); /* Unsupported */ + + if (bufferSize < FSE_NCountWriteBound(maxSymbolValue, tableLog)) + return FSE_writeNCount_generic(buffer, bufferSize, normalizedCounter, maxSymbolValue, tableLog, 0); + + return FSE_writeNCount_generic(buffer, bufferSize, normalizedCounter, maxSymbolValue, tableLog, 1 /* write in buffer is safe */); +} + + +/*-************************************************************** +* FSE Compression Code +****************************************************************/ + +FSE_CTable* FSE_createCTable (unsigned maxSymbolValue, unsigned tableLog) +{ + size_t size; + if (tableLog > FSE_TABLELOG_ABSOLUTE_MAX) tableLog = FSE_TABLELOG_ABSOLUTE_MAX; + size = FSE_CTABLE_SIZE_U32 (tableLog, maxSymbolValue) * sizeof(U32); + return (FSE_CTable*)ZSTD_malloc(size); +} + +void FSE_freeCTable (FSE_CTable* ct) { ZSTD_free(ct); } + +/* provides the minimum logSize to safely represent a distribution */ +static unsigned FSE_minTableLog(size_t srcSize, unsigned maxSymbolValue) +{ + U32 minBitsSrc = BIT_highbit32((U32)(srcSize)) + 1; + U32 minBitsSymbols = BIT_highbit32(maxSymbolValue) + 2; + U32 minBits = minBitsSrc < minBitsSymbols ? minBitsSrc : minBitsSymbols; + assert(srcSize > 1); /* Not supported, RLE should be used instead */ + return minBits; +} + +unsigned FSE_optimalTableLog_internal(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, unsigned minus) +{ + U32 maxBitsSrc = BIT_highbit32((U32)(srcSize - 1)) - minus; + U32 tableLog = maxTableLog; + U32 minBits = FSE_minTableLog(srcSize, maxSymbolValue); + assert(srcSize > 1); /* Not supported, RLE should be used instead */ + if (tableLog==0) tableLog = FSE_DEFAULT_TABLELOG; + if (maxBitsSrc < tableLog) tableLog = maxBitsSrc; /* Accuracy can be reduced */ + if (minBits > tableLog) tableLog = minBits; /* Need a minimum to safely represent all symbol values */ + if (tableLog < FSE_MIN_TABLELOG) tableLog = FSE_MIN_TABLELOG; + if (tableLog > FSE_MAX_TABLELOG) tableLog = FSE_MAX_TABLELOG; + return tableLog; +} + +unsigned FSE_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue) +{ + return FSE_optimalTableLog_internal(maxTableLog, srcSize, maxSymbolValue, 2); +} + +/* Secondary normalization method. + To be used when primary method fails. */ + +static size_t FSE_normalizeM2(short* norm, U32 tableLog, const unsigned* count, size_t total, U32 maxSymbolValue, short lowProbCount) +{ + short const NOT_YET_ASSIGNED = -2; + U32 s; + U32 distributed = 0; + U32 ToDistribute; + + /* Init */ + U32 const lowThreshold = (U32)(total >> tableLog); + U32 lowOne = (U32)((total * 3) >> (tableLog + 1)); + + for (s=0; s<=maxSymbolValue; s++) { + if (count[s] == 0) { + norm[s]=0; + continue; + } + if (count[s] <= lowThreshold) { + norm[s] = lowProbCount; + distributed++; + total -= count[s]; + continue; + } + if (count[s] <= lowOne) { + norm[s] = 1; + distributed++; + total -= count[s]; + continue; + } + + norm[s]=NOT_YET_ASSIGNED; + } + ToDistribute = (1 << tableLog) - distributed; + + if (ToDistribute == 0) + return 0; + + if ((total / ToDistribute) > lowOne) { + /* risk of rounding to zero */ + lowOne = (U32)((total * 3) / (ToDistribute * 2)); + for (s=0; s<=maxSymbolValue; s++) { + if ((norm[s] == NOT_YET_ASSIGNED) && (count[s] <= lowOne)) { + norm[s] = 1; + distributed++; + total -= count[s]; + continue; + } } + ToDistribute = (1 << tableLog) - distributed; + } + + if (distributed == maxSymbolValue+1) { + /* all values are pretty poor; + probably incompressible data (should have already been detected); + find max, then give all remaining points to max */ + U32 maxV = 0, maxC = 0; + for (s=0; s<=maxSymbolValue; s++) + if (count[s] > maxC) { maxV=s; maxC=count[s]; } + norm[maxV] += (short)ToDistribute; + return 0; + } + + if (total == 0) { + /* all of the symbols were low enough for the lowOne or lowThreshold */ + for (s=0; ToDistribute > 0; s = (s+1)%(maxSymbolValue+1)) + if (norm[s] > 0) { ToDistribute--; norm[s]++; } + return 0; + } + + { U64 const vStepLog = 62 - tableLog; + U64 const mid = (1ULL << (vStepLog-1)) - 1; + U64 const rStep = ZSTD_div64((((U64)1<> vStepLog); + U32 const sEnd = (U32)(end >> vStepLog); + U32 const weight = sEnd - sStart; + if (weight < 1) + return ERROR(GENERIC); + norm[s] = (short)weight; + tmpTotal = end; + } } } + + return 0; +} + +size_t FSE_normalizeCount (short* normalizedCounter, unsigned tableLog, + const unsigned* count, size_t total, + unsigned maxSymbolValue, unsigned useLowProbCount) +{ + /* Sanity checks */ + if (tableLog==0) tableLog = FSE_DEFAULT_TABLELOG; + if (tableLog < FSE_MIN_TABLELOG) return ERROR(GENERIC); /* Unsupported size */ + if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge); /* Unsupported size */ + if (tableLog < FSE_minTableLog(total, maxSymbolValue)) return ERROR(GENERIC); /* Too small tableLog, compression potentially impossible */ + + { static U32 const rtbTable[] = { 0, 473195, 504333, 520860, 550000, 700000, 750000, 830000 }; + short const lowProbCount = useLowProbCount ? -1 : 1; + U64 const scale = 62 - tableLog; + U64 const step = ZSTD_div64((U64)1<<62, (U32)total); /* <== here, one division ! */ + U64 const vStep = 1ULL<<(scale-20); + int stillToDistribute = 1<> tableLog); + + for (s=0; s<=maxSymbolValue; s++) { + if (count[s] == total) return 0; /* rle special case */ + if (count[s] == 0) { normalizedCounter[s]=0; continue; } + if (count[s] <= lowThreshold) { + normalizedCounter[s] = lowProbCount; + stillToDistribute--; + } else { + short proba = (short)((count[s]*step) >> scale); + if (proba<8) { + U64 restToBeat = vStep * rtbTable[proba]; + proba += (count[s]*step) - ((U64)proba< restToBeat; + } + if (proba > largestP) { largestP=proba; largest=s; } + normalizedCounter[s] = proba; + stillToDistribute -= proba; + } } + if (-stillToDistribute >= (normalizedCounter[largest] >> 1)) { + /* corner case, need another normalization method */ + size_t const errorCode = FSE_normalizeM2(normalizedCounter, tableLog, count, total, maxSymbolValue, lowProbCount); + if (FSE_isError(errorCode)) return errorCode; + } + else normalizedCounter[largest] += (short)stillToDistribute; + } + +#if 0 + { /* Print Table (debug) */ + U32 s; + U32 nTotal = 0; + for (s=0; s<=maxSymbolValue; s++) + RAWLOG(2, "%3i: %4i \n", s, normalizedCounter[s]); + for (s=0; s<=maxSymbolValue; s++) + nTotal += abs(normalizedCounter[s]); + if (nTotal != (1U<>1); /* assumption : tableLog >= 1 */ + FSE_symbolCompressionTransform* const symbolTT = (FSE_symbolCompressionTransform*) (FSCT); + unsigned s; + + /* Sanity checks */ + if (nbBits < 1) return ERROR(GENERIC); /* min size */ + + /* header */ + tableU16[-2] = (U16) nbBits; + tableU16[-1] = (U16) maxSymbolValue; + + /* Build table */ + for (s=0; s FSE_MAX_TABLELOG*4+7 ) && (srcSize & 2)) { /* test bit 2 */ + FSE_encodeSymbol(&bitC, &CState2, *--ip); + FSE_encodeSymbol(&bitC, &CState1, *--ip); + FSE_FLUSHBITS(&bitC); + } + + /* 2 or 4 encoding per loop */ + while ( ip>istart ) { + + FSE_encodeSymbol(&bitC, &CState2, *--ip); + + if (sizeof(bitC.bitContainer)*8 < FSE_MAX_TABLELOG*2+7 ) /* this test must be static */ + FSE_FLUSHBITS(&bitC); + + FSE_encodeSymbol(&bitC, &CState1, *--ip); + + if (sizeof(bitC.bitContainer)*8 > FSE_MAX_TABLELOG*4+7 ) { /* this test must be static */ + FSE_encodeSymbol(&bitC, &CState2, *--ip); + FSE_encodeSymbol(&bitC, &CState1, *--ip); + } + + FSE_FLUSHBITS(&bitC); + } + + FSE_flushCState(&bitC, &CState2); + FSE_flushCState(&bitC, &CState1); + return BIT_closeCStream(&bitC); +} + +size_t FSE_compress_usingCTable (void* dst, size_t dstSize, + const void* src, size_t srcSize, + const FSE_CTable* ct) +{ + unsigned const fast = (dstSize >= FSE_BLOCKBOUND(srcSize)); + + if (fast) + return FSE_compress_usingCTable_generic(dst, dstSize, src, srcSize, ct, 1); + else + return FSE_compress_usingCTable_generic(dst, dstSize, src, srcSize, ct, 0); +} + + +size_t FSE_compressBound(size_t size) { return FSE_COMPRESSBOUND(size); } + +#ifndef ZSTD_NO_UNUSED_FUNCTIONS +/* FSE_compress_wksp() : + * Same as FSE_compress2(), but using an externally allocated scratch buffer (`workSpace`). + * `wkspSize` size must be `(1< not compressible */ + if (maxCount < (srcSize >> 7)) return 0; /* Heuristic : not compressible enough */ + } + + tableLog = FSE_optimalTableLog(tableLog, srcSize, maxSymbolValue); + CHECK_F( FSE_normalizeCount(norm, tableLog, count, srcSize, maxSymbolValue, /* useLowProbCount */ srcSize >= 2048) ); + + /* Write table description header */ + { CHECK_V_F(nc_err, FSE_writeNCount(op, oend-op, norm, maxSymbolValue, tableLog) ); + op += nc_err; + } + + /* Compress */ + CHECK_F( FSE_buildCTable_wksp(CTable, norm, maxSymbolValue, tableLog, scratchBuffer, scratchBufferSize) ); + { CHECK_V_F(cSize, FSE_compress_usingCTable(op, oend - op, src, srcSize, CTable) ); + if (cSize == 0) return 0; /* not enough space for compressed data */ + op += cSize; + } + + /* check compressibility */ + if ( (size_t)(op-ostart) >= srcSize-1 ) return 0; + + return op-ostart; +} + +typedef struct { + FSE_CTable CTable_max[FSE_CTABLE_SIZE_U32(FSE_MAX_TABLELOG, FSE_MAX_SYMBOL_VALUE)]; + union { + U32 hist_wksp[HIST_WKSP_SIZE_U32]; + BYTE scratchBuffer[1 << FSE_MAX_TABLELOG]; + } workspace; +} fseWkspMax_t; + +size_t FSE_compress2 (void* dst, size_t dstCapacity, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog) +{ + fseWkspMax_t scratchBuffer; + DEBUG_STATIC_ASSERT(sizeof(scratchBuffer) >= FSE_COMPRESS_WKSP_SIZE_U32(FSE_MAX_TABLELOG, FSE_MAX_SYMBOL_VALUE)); /* compilation failures here means scratchBuffer is not large enough */ + if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge); + return FSE_compress_wksp(dst, dstCapacity, src, srcSize, maxSymbolValue, tableLog, &scratchBuffer, sizeof(scratchBuffer)); +} + +size_t FSE_compress (void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ + return FSE_compress2(dst, dstCapacity, src, srcSize, FSE_MAX_SYMBOL_VALUE, FSE_DEFAULT_TABLELOG); +} +#endif + +#endif /* FSE_COMMONDEFS_ONLY */ diff --git a/src/blosc2/internal-complibs/zstd-1.5.2/compress/hist.c b/src/blosc2/internal-complibs/zstd-1.5.2/compress/hist.c new file mode 100644 index 0000000..073c57e --- /dev/null +++ b/src/blosc2/internal-complibs/zstd-1.5.2/compress/hist.c @@ -0,0 +1,181 @@ +/* ****************************************************************** + * hist : Histogram functions + * part of Finite State Entropy project + * Copyright (c) Yann Collet, Facebook, Inc. + * + * You can contact the author at : + * - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy + * - Public forum : https://groups.google.com/forum/#!forum/lz4c + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. +****************************************************************** */ + +/* --- dependencies --- */ +#include "../common/mem.h" /* U32, BYTE, etc. */ +#include "../common/debug.h" /* assert, DEBUGLOG */ +#include "../common/error_private.h" /* ERROR */ +#include "hist.h" + + +/* --- Error management --- */ +unsigned HIST_isError(size_t code) { return ERR_isError(code); } + +/*-************************************************************** + * Histogram functions + ****************************************************************/ +unsigned HIST_count_simple(unsigned* count, unsigned* maxSymbolValuePtr, + const void* src, size_t srcSize) +{ + const BYTE* ip = (const BYTE*)src; + const BYTE* const end = ip + srcSize; + unsigned maxSymbolValue = *maxSymbolValuePtr; + unsigned largestCount=0; + + ZSTD_memset(count, 0, (maxSymbolValue+1) * sizeof(*count)); + if (srcSize==0) { *maxSymbolValuePtr = 0; return 0; } + + while (ip largestCount) largestCount = count[s]; + } + + return largestCount; +} + +typedef enum { trustInput, checkMaxSymbolValue } HIST_checkInput_e; + +/* HIST_count_parallel_wksp() : + * store histogram into 4 intermediate tables, recombined at the end. + * this design makes better use of OoO cpus, + * and is noticeably faster when some values are heavily repeated. + * But it needs some additional workspace for intermediate tables. + * `workSpace` must be a U32 table of size >= HIST_WKSP_SIZE_U32. + * @return : largest histogram frequency, + * or an error code (notably when histogram's alphabet is larger than *maxSymbolValuePtr) */ +static size_t HIST_count_parallel_wksp( + unsigned* count, unsigned* maxSymbolValuePtr, + const void* source, size_t sourceSize, + HIST_checkInput_e check, + U32* const workSpace) +{ + const BYTE* ip = (const BYTE*)source; + const BYTE* const iend = ip+sourceSize; + size_t const countSize = (*maxSymbolValuePtr + 1) * sizeof(*count); + unsigned max=0; + U32* const Counting1 = workSpace; + U32* const Counting2 = Counting1 + 256; + U32* const Counting3 = Counting2 + 256; + U32* const Counting4 = Counting3 + 256; + + /* safety checks */ + assert(*maxSymbolValuePtr <= 255); + if (!sourceSize) { + ZSTD_memset(count, 0, countSize); + *maxSymbolValuePtr = 0; + return 0; + } + ZSTD_memset(workSpace, 0, 4*256*sizeof(unsigned)); + + /* by stripes of 16 bytes */ + { U32 cached = MEM_read32(ip); ip += 4; + while (ip < iend-15) { + U32 c = cached; cached = MEM_read32(ip); ip += 4; + Counting1[(BYTE) c ]++; + Counting2[(BYTE)(c>>8) ]++; + Counting3[(BYTE)(c>>16)]++; + Counting4[ c>>24 ]++; + c = cached; cached = MEM_read32(ip); ip += 4; + Counting1[(BYTE) c ]++; + Counting2[(BYTE)(c>>8) ]++; + Counting3[(BYTE)(c>>16)]++; + Counting4[ c>>24 ]++; + c = cached; cached = MEM_read32(ip); ip += 4; + Counting1[(BYTE) c ]++; + Counting2[(BYTE)(c>>8) ]++; + Counting3[(BYTE)(c>>16)]++; + Counting4[ c>>24 ]++; + c = cached; cached = MEM_read32(ip); ip += 4; + Counting1[(BYTE) c ]++; + Counting2[(BYTE)(c>>8) ]++; + Counting3[(BYTE)(c>>16)]++; + Counting4[ c>>24 ]++; + } + ip-=4; + } + + /* finish last symbols */ + while (ip max) max = Counting1[s]; + } } + + { unsigned maxSymbolValue = 255; + while (!Counting1[maxSymbolValue]) maxSymbolValue--; + if (check && maxSymbolValue > *maxSymbolValuePtr) return ERROR(maxSymbolValue_tooSmall); + *maxSymbolValuePtr = maxSymbolValue; + ZSTD_memmove(count, Counting1, countSize); /* in case count & Counting1 are overlapping */ + } + return (size_t)max; +} + +/* HIST_countFast_wksp() : + * Same as HIST_countFast(), but using an externally provided scratch buffer. + * `workSpace` is a writable buffer which must be 4-bytes aligned, + * `workSpaceSize` must be >= HIST_WKSP_SIZE + */ +size_t HIST_countFast_wksp(unsigned* count, unsigned* maxSymbolValuePtr, + const void* source, size_t sourceSize, + void* workSpace, size_t workSpaceSize) +{ + if (sourceSize < 1500) /* heuristic threshold */ + return HIST_count_simple(count, maxSymbolValuePtr, source, sourceSize); + if ((size_t)workSpace & 3) return ERROR(GENERIC); /* must be aligned on 4-bytes boundaries */ + if (workSpaceSize < HIST_WKSP_SIZE) return ERROR(workSpace_tooSmall); + return HIST_count_parallel_wksp(count, maxSymbolValuePtr, source, sourceSize, trustInput, (U32*)workSpace); +} + +/* HIST_count_wksp() : + * Same as HIST_count(), but using an externally provided scratch buffer. + * `workSpace` size must be table of >= HIST_WKSP_SIZE_U32 unsigned */ +size_t HIST_count_wksp(unsigned* count, unsigned* maxSymbolValuePtr, + const void* source, size_t sourceSize, + void* workSpace, size_t workSpaceSize) +{ + if ((size_t)workSpace & 3) return ERROR(GENERIC); /* must be aligned on 4-bytes boundaries */ + if (workSpaceSize < HIST_WKSP_SIZE) return ERROR(workSpace_tooSmall); + if (*maxSymbolValuePtr < 255) + return HIST_count_parallel_wksp(count, maxSymbolValuePtr, source, sourceSize, checkMaxSymbolValue, (U32*)workSpace); + *maxSymbolValuePtr = 255; + return HIST_countFast_wksp(count, maxSymbolValuePtr, source, sourceSize, workSpace, workSpaceSize); +} + +#ifndef ZSTD_NO_UNUSED_FUNCTIONS +/* fast variant (unsafe : won't check if src contains values beyond count[] limit) */ +size_t HIST_countFast(unsigned* count, unsigned* maxSymbolValuePtr, + const void* source, size_t sourceSize) +{ + unsigned tmpCounters[HIST_WKSP_SIZE_U32]; + return HIST_countFast_wksp(count, maxSymbolValuePtr, source, sourceSize, tmpCounters, sizeof(tmpCounters)); +} + +size_t HIST_count(unsigned* count, unsigned* maxSymbolValuePtr, + const void* src, size_t srcSize) +{ + unsigned tmpCounters[HIST_WKSP_SIZE_U32]; + return HIST_count_wksp(count, maxSymbolValuePtr, src, srcSize, tmpCounters, sizeof(tmpCounters)); +} +#endif diff --git a/src/blosc2/internal-complibs/zstd-1.5.2/compress/hist.h b/src/blosc2/internal-complibs/zstd-1.5.2/compress/hist.h new file mode 100644 index 0000000..228ed48 --- /dev/null +++ b/src/blosc2/internal-complibs/zstd-1.5.2/compress/hist.h @@ -0,0 +1,75 @@ +/* ****************************************************************** + * hist : Histogram functions + * part of Finite State Entropy project + * Copyright (c) Yann Collet, Facebook, Inc. + * + * You can contact the author at : + * - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy + * - Public forum : https://groups.google.com/forum/#!forum/lz4c + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. +****************************************************************** */ + +/* --- dependencies --- */ +#include "../common/zstd_deps.h" /* size_t */ + + +/* --- simple histogram functions --- */ + +/*! HIST_count(): + * Provides the precise count of each byte within a table 'count'. + * 'count' is a table of unsigned int, of minimum size (*maxSymbolValuePtr+1). + * Updates *maxSymbolValuePtr with actual largest symbol value detected. + * @return : count of the most frequent symbol (which isn't identified). + * or an error code, which can be tested using HIST_isError(). + * note : if return == srcSize, there is only one symbol. + */ +size_t HIST_count(unsigned* count, unsigned* maxSymbolValuePtr, + const void* src, size_t srcSize); + +unsigned HIST_isError(size_t code); /**< tells if a return value is an error code */ + + +/* --- advanced histogram functions --- */ + +#define HIST_WKSP_SIZE_U32 1024 +#define HIST_WKSP_SIZE (HIST_WKSP_SIZE_U32 * sizeof(unsigned)) +/** HIST_count_wksp() : + * Same as HIST_count(), but using an externally provided scratch buffer. + * Benefit is this function will use very little stack space. + * `workSpace` is a writable buffer which must be 4-bytes aligned, + * `workSpaceSize` must be >= HIST_WKSP_SIZE + */ +size_t HIST_count_wksp(unsigned* count, unsigned* maxSymbolValuePtr, + const void* src, size_t srcSize, + void* workSpace, size_t workSpaceSize); + +/** HIST_countFast() : + * same as HIST_count(), but blindly trusts that all byte values within src are <= *maxSymbolValuePtr. + * This function is unsafe, and will segfault if any value within `src` is `> *maxSymbolValuePtr` + */ +size_t HIST_countFast(unsigned* count, unsigned* maxSymbolValuePtr, + const void* src, size_t srcSize); + +/** HIST_countFast_wksp() : + * Same as HIST_countFast(), but using an externally provided scratch buffer. + * `workSpace` is a writable buffer which must be 4-bytes aligned, + * `workSpaceSize` must be >= HIST_WKSP_SIZE + */ +size_t HIST_countFast_wksp(unsigned* count, unsigned* maxSymbolValuePtr, + const void* src, size_t srcSize, + void* workSpace, size_t workSpaceSize); + +/*! HIST_count_simple() : + * Same as HIST_countFast(), this function is unsafe, + * and will segfault if any value within `src` is `> *maxSymbolValuePtr`. + * It is also a bit slower for large inputs. + * However, it does not need any additional memory (not even on stack). + * @return : count of the most frequent symbol. + * Note this function doesn't produce any error (i.e. it must succeed). + */ +unsigned HIST_count_simple(unsigned* count, unsigned* maxSymbolValuePtr, + const void* src, size_t srcSize); diff --git a/src/blosc2/internal-complibs/zstd-1.5.2/compress/huf_compress.c b/src/blosc2/internal-complibs/zstd-1.5.2/compress/huf_compress.c new file mode 100644 index 0000000..2b3d6ad --- /dev/null +++ b/src/blosc2/internal-complibs/zstd-1.5.2/compress/huf_compress.c @@ -0,0 +1,1370 @@ +/* ****************************************************************** + * Huffman encoder, part of New Generation Entropy library + * Copyright (c) Yann Collet, Facebook, Inc. + * + * You can contact the author at : + * - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy + * - Public forum : https://groups.google.com/forum/#!forum/lz4c + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. +****************************************************************** */ + +/* ************************************************************** +* Compiler specifics +****************************************************************/ +#ifdef _MSC_VER /* Visual Studio */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +#endif + + +/* ************************************************************** +* Includes +****************************************************************/ +#include "../common/zstd_deps.h" /* ZSTD_memcpy, ZSTD_memset */ +#include "../common/compiler.h" +#include "../common/bitstream.h" +#include "hist.h" +#define FSE_STATIC_LINKING_ONLY /* FSE_optimalTableLog_internal */ +#include "../common/fse.h" /* header compression */ +#define HUF_STATIC_LINKING_ONLY +#include "../common/huf.h" +#include "../common/error_private.h" + + +/* ************************************************************** +* Error Management +****************************************************************/ +#define HUF_isError ERR_isError +#define HUF_STATIC_ASSERT(c) DEBUG_STATIC_ASSERT(c) /* use only *after* variable declarations */ + + +/* ************************************************************** +* Utils +****************************************************************/ +unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue) +{ + return FSE_optimalTableLog_internal(maxTableLog, srcSize, maxSymbolValue, 1); +} + + +/* ******************************************************* +* HUF : Huffman block compression +*********************************************************/ +#define HUF_WORKSPACE_MAX_ALIGNMENT 8 + +static void* HUF_alignUpWorkspace(void* workspace, size_t* workspaceSizePtr, size_t align) +{ + size_t const mask = align - 1; + size_t const rem = (size_t)workspace & mask; + size_t const add = (align - rem) & mask; + BYTE* const aligned = (BYTE*)workspace + add; + assert((align & (align - 1)) == 0); /* pow 2 */ + assert(align <= HUF_WORKSPACE_MAX_ALIGNMENT); + if (*workspaceSizePtr >= add) { + assert(add < align); + assert(((size_t)aligned & mask) == 0); + *workspaceSizePtr -= add; + return aligned; + } else { + *workspaceSizePtr = 0; + return NULL; + } +} + + +/* HUF_compressWeights() : + * Same as FSE_compress(), but dedicated to huff0's weights compression. + * The use case needs much less stack memory. + * Note : all elements within weightTable are supposed to be <= HUF_TABLELOG_MAX. + */ +#define MAX_FSE_TABLELOG_FOR_HUFF_HEADER 6 + +typedef struct { + FSE_CTable CTable[FSE_CTABLE_SIZE_U32(MAX_FSE_TABLELOG_FOR_HUFF_HEADER, HUF_TABLELOG_MAX)]; + U32 scratchBuffer[FSE_BUILD_CTABLE_WORKSPACE_SIZE_U32(HUF_TABLELOG_MAX, MAX_FSE_TABLELOG_FOR_HUFF_HEADER)]; + unsigned count[HUF_TABLELOG_MAX+1]; + S16 norm[HUF_TABLELOG_MAX+1]; +} HUF_CompressWeightsWksp; + +static size_t HUF_compressWeights(void* dst, size_t dstSize, const void* weightTable, size_t wtSize, void* workspace, size_t workspaceSize) +{ + BYTE* const ostart = (BYTE*) dst; + BYTE* op = ostart; + BYTE* const oend = ostart + dstSize; + + unsigned maxSymbolValue = HUF_TABLELOG_MAX; + U32 tableLog = MAX_FSE_TABLELOG_FOR_HUFF_HEADER; + HUF_CompressWeightsWksp* wksp = (HUF_CompressWeightsWksp*)HUF_alignUpWorkspace(workspace, &workspaceSize, ZSTD_ALIGNOF(U32)); + + if (workspaceSize < sizeof(HUF_CompressWeightsWksp)) return ERROR(GENERIC); + + /* init conditions */ + if (wtSize <= 1) return 0; /* Not compressible */ + + /* Scan input and build symbol stats */ + { unsigned const maxCount = HIST_count_simple(wksp->count, &maxSymbolValue, weightTable, wtSize); /* never fails */ + if (maxCount == wtSize) return 1; /* only a single symbol in src : rle */ + if (maxCount == 1) return 0; /* each symbol present maximum once => not compressible */ + } + + tableLog = FSE_optimalTableLog(tableLog, wtSize, maxSymbolValue); + CHECK_F( FSE_normalizeCount(wksp->norm, tableLog, wksp->count, wtSize, maxSymbolValue, /* useLowProbCount */ 0) ); + + /* Write table description header */ + { CHECK_V_F(hSize, FSE_writeNCount(op, (size_t)(oend-op), wksp->norm, maxSymbolValue, tableLog) ); + op += hSize; + } + + /* Compress */ + CHECK_F( FSE_buildCTable_wksp(wksp->CTable, wksp->norm, maxSymbolValue, tableLog, wksp->scratchBuffer, sizeof(wksp->scratchBuffer)) ); + { CHECK_V_F(cSize, FSE_compress_usingCTable(op, (size_t)(oend - op), weightTable, wtSize, wksp->CTable) ); + if (cSize == 0) return 0; /* not enough space for compressed data */ + op += cSize; + } + + return (size_t)(op-ostart); +} + +static size_t HUF_getNbBits(HUF_CElt elt) +{ + return elt & 0xFF; +} + +static size_t HUF_getNbBitsFast(HUF_CElt elt) +{ + return elt; +} + +static size_t HUF_getValue(HUF_CElt elt) +{ + return elt & ~0xFF; +} + +static size_t HUF_getValueFast(HUF_CElt elt) +{ + return elt; +} + +static void HUF_setNbBits(HUF_CElt* elt, size_t nbBits) +{ + assert(nbBits <= HUF_TABLELOG_ABSOLUTEMAX); + *elt = nbBits; +} + +static void HUF_setValue(HUF_CElt* elt, size_t value) +{ + size_t const nbBits = HUF_getNbBits(*elt); + if (nbBits > 0) { + assert((value >> nbBits) == 0); + *elt |= value << (sizeof(HUF_CElt) * 8 - nbBits); + } +} + +typedef struct { + HUF_CompressWeightsWksp wksp; + BYTE bitsToWeight[HUF_TABLELOG_MAX + 1]; /* precomputed conversion table */ + BYTE huffWeight[HUF_SYMBOLVALUE_MAX]; +} HUF_WriteCTableWksp; + +size_t HUF_writeCTable_wksp(void* dst, size_t maxDstSize, + const HUF_CElt* CTable, unsigned maxSymbolValue, unsigned huffLog, + void* workspace, size_t workspaceSize) +{ + HUF_CElt const* const ct = CTable + 1; + BYTE* op = (BYTE*)dst; + U32 n; + HUF_WriteCTableWksp* wksp = (HUF_WriteCTableWksp*)HUF_alignUpWorkspace(workspace, &workspaceSize, ZSTD_ALIGNOF(U32)); + + /* check conditions */ + if (workspaceSize < sizeof(HUF_WriteCTableWksp)) return ERROR(GENERIC); + if (maxSymbolValue > HUF_SYMBOLVALUE_MAX) return ERROR(maxSymbolValue_tooLarge); + + /* convert to weight */ + wksp->bitsToWeight[0] = 0; + for (n=1; nbitsToWeight[n] = (BYTE)(huffLog + 1 - n); + for (n=0; nhuffWeight[n] = wksp->bitsToWeight[HUF_getNbBits(ct[n])]; + + /* attempt weights compression by FSE */ + if (maxDstSize < 1) return ERROR(dstSize_tooSmall); + { CHECK_V_F(hSize, HUF_compressWeights(op+1, maxDstSize-1, wksp->huffWeight, maxSymbolValue, &wksp->wksp, sizeof(wksp->wksp)) ); + if ((hSize>1) & (hSize < maxSymbolValue/2)) { /* FSE compressed */ + op[0] = (BYTE)hSize; + return hSize+1; + } } + + /* write raw values as 4-bits (max : 15) */ + if (maxSymbolValue > (256-128)) return ERROR(GENERIC); /* should not happen : likely means source cannot be compressed */ + if (((maxSymbolValue+1)/2) + 1 > maxDstSize) return ERROR(dstSize_tooSmall); /* not enough space within dst buffer */ + op[0] = (BYTE)(128 /*special case*/ + (maxSymbolValue-1)); + wksp->huffWeight[maxSymbolValue] = 0; /* to be sure it doesn't cause msan issue in final combination */ + for (n=0; nhuffWeight[n] << 4) + wksp->huffWeight[n+1]); + return ((maxSymbolValue+1)/2) + 1; +} + +/*! HUF_writeCTable() : + `CTable` : Huffman tree to save, using huf representation. + @return : size of saved CTable */ +size_t HUF_writeCTable (void* dst, size_t maxDstSize, + const HUF_CElt* CTable, unsigned maxSymbolValue, unsigned huffLog) +{ + HUF_WriteCTableWksp wksp; + return HUF_writeCTable_wksp(dst, maxDstSize, CTable, maxSymbolValue, huffLog, &wksp, sizeof(wksp)); +} + + +size_t HUF_readCTable (HUF_CElt* CTable, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize, unsigned* hasZeroWeights) +{ + BYTE huffWeight[HUF_SYMBOLVALUE_MAX + 1]; /* init not required, even though some static analyzer may complain */ + U32 rankVal[HUF_TABLELOG_ABSOLUTEMAX + 1]; /* large enough for values from 0 to 16 */ + U32 tableLog = 0; + U32 nbSymbols = 0; + HUF_CElt* const ct = CTable + 1; + + /* get symbol weights */ + CHECK_V_F(readSize, HUF_readStats(huffWeight, HUF_SYMBOLVALUE_MAX+1, rankVal, &nbSymbols, &tableLog, src, srcSize)); + *hasZeroWeights = (rankVal[0] > 0); + + /* check result */ + if (tableLog > HUF_TABLELOG_MAX) return ERROR(tableLog_tooLarge); + if (nbSymbols > *maxSymbolValuePtr+1) return ERROR(maxSymbolValue_tooSmall); + + CTable[0] = tableLog; + + /* Prepare base value per rank */ + { U32 n, nextRankStart = 0; + for (n=1; n<=tableLog; n++) { + U32 curr = nextRankStart; + nextRankStart += (rankVal[n] << (n-1)); + rankVal[n] = curr; + } } + + /* fill nbBits */ + { U32 n; for (n=0; nn=tableLog+1 */ + U16 valPerRank[HUF_TABLELOG_MAX+2] = {0}; + { U32 n; for (n=0; n0; n--) { /* start at n=tablelog <-> w=1 */ + valPerRank[n] = min; /* get starting value within each rank */ + min += nbPerRank[n]; + min >>= 1; + } } + /* assign value within rank, symbol order */ + { U32 n; for (n=0; n maxNbBits to be maxNbBits. Then it adjusts + * the tree to so that it is a valid canonical Huffman tree. + * + * @pre The sum of the ranks of each symbol == 2^largestBits, + * where largestBits == huffNode[lastNonNull].nbBits. + * @post The sum of the ranks of each symbol == 2^largestBits, + * where largestBits is the return value <= maxNbBits. + * + * @param huffNode The Huffman tree modified in place to enforce maxNbBits. + * @param lastNonNull The symbol with the lowest count in the Huffman tree. + * @param maxNbBits The maximum allowed number of bits, which the Huffman tree + * may not respect. After this function the Huffman tree will + * respect maxNbBits. + * @return The maximum number of bits of the Huffman tree after adjustment, + * necessarily no more than maxNbBits. + */ +static U32 HUF_setMaxHeight(nodeElt* huffNode, U32 lastNonNull, U32 maxNbBits) +{ + const U32 largestBits = huffNode[lastNonNull].nbBits; + /* early exit : no elt > maxNbBits, so the tree is already valid. */ + if (largestBits <= maxNbBits) return largestBits; + + /* there are several too large elements (at least >= 2) */ + { int totalCost = 0; + const U32 baseCost = 1 << (largestBits - maxNbBits); + int n = (int)lastNonNull; + + /* Adjust any ranks > maxNbBits to maxNbBits. + * Compute totalCost, which is how far the sum of the ranks is + * we are over 2^largestBits after adjust the offending ranks. + */ + while (huffNode[n].nbBits > maxNbBits) { + totalCost += baseCost - (1 << (largestBits - huffNode[n].nbBits)); + huffNode[n].nbBits = (BYTE)maxNbBits; + n--; + } + /* n stops at huffNode[n].nbBits <= maxNbBits */ + assert(huffNode[n].nbBits <= maxNbBits); + /* n end at index of smallest symbol using < maxNbBits */ + while (huffNode[n].nbBits == maxNbBits) --n; + + /* renorm totalCost from 2^largestBits to 2^maxNbBits + * note : totalCost is necessarily a multiple of baseCost */ + assert((totalCost & (baseCost - 1)) == 0); + totalCost >>= (largestBits - maxNbBits); + assert(totalCost > 0); + + /* repay normalized cost */ + { U32 const noSymbol = 0xF0F0F0F0; + U32 rankLast[HUF_TABLELOG_MAX+2]; + + /* Get pos of last (smallest = lowest cum. count) symbol per rank */ + ZSTD_memset(rankLast, 0xF0, sizeof(rankLast)); + { U32 currentNbBits = maxNbBits; + int pos; + for (pos=n ; pos >= 0; pos--) { + if (huffNode[pos].nbBits >= currentNbBits) continue; + currentNbBits = huffNode[pos].nbBits; /* < maxNbBits */ + rankLast[maxNbBits-currentNbBits] = (U32)pos; + } } + + while (totalCost > 0) { + /* Try to reduce the next power of 2 above totalCost because we + * gain back half the rank. + */ + U32 nBitsToDecrease = BIT_highbit32((U32)totalCost) + 1; + for ( ; nBitsToDecrease > 1; nBitsToDecrease--) { + U32 const highPos = rankLast[nBitsToDecrease]; + U32 const lowPos = rankLast[nBitsToDecrease-1]; + if (highPos == noSymbol) continue; + /* Decrease highPos if no symbols of lowPos or if it is + * not cheaper to remove 2 lowPos than highPos. + */ + if (lowPos == noSymbol) break; + { U32 const highTotal = huffNode[highPos].count; + U32 const lowTotal = 2 * huffNode[lowPos].count; + if (highTotal <= lowTotal) break; + } } + /* only triggered when no more rank 1 symbol left => find closest one (note : there is necessarily at least one !) */ + assert(rankLast[nBitsToDecrease] != noSymbol || nBitsToDecrease == 1); + /* HUF_MAX_TABLELOG test just to please gcc 5+; but it should not be necessary */ + while ((nBitsToDecrease<=HUF_TABLELOG_MAX) && (rankLast[nBitsToDecrease] == noSymbol)) + nBitsToDecrease++; + assert(rankLast[nBitsToDecrease] != noSymbol); + /* Increase the number of bits to gain back half the rank cost. */ + totalCost -= 1 << (nBitsToDecrease-1); + huffNode[rankLast[nBitsToDecrease]].nbBits++; + + /* Fix up the new rank. + * If the new rank was empty, this symbol is now its smallest. + * Otherwise, this symbol will be the largest in the new rank so no adjustment. + */ + if (rankLast[nBitsToDecrease-1] == noSymbol) + rankLast[nBitsToDecrease-1] = rankLast[nBitsToDecrease]; + /* Fix up the old rank. + * If the symbol was at position 0, meaning it was the highest weight symbol in the tree, + * it must be the only symbol in its rank, so the old rank now has no symbols. + * Otherwise, since the Huffman nodes are sorted by count, the previous position is now + * the smallest node in the rank. If the previous position belongs to a different rank, + * then the rank is now empty. + */ + if (rankLast[nBitsToDecrease] == 0) /* special case, reached largest symbol */ + rankLast[nBitsToDecrease] = noSymbol; + else { + rankLast[nBitsToDecrease]--; + if (huffNode[rankLast[nBitsToDecrease]].nbBits != maxNbBits-nBitsToDecrease) + rankLast[nBitsToDecrease] = noSymbol; /* this rank is now empty */ + } + } /* while (totalCost > 0) */ + + /* If we've removed too much weight, then we have to add it back. + * To avoid overshooting again, we only adjust the smallest rank. + * We take the largest nodes from the lowest rank 0 and move them + * to rank 1. There's guaranteed to be enough rank 0 symbols because + * TODO. + */ + while (totalCost < 0) { /* Sometimes, cost correction overshoot */ + /* special case : no rank 1 symbol (using maxNbBits-1); + * let's create one from largest rank 0 (using maxNbBits). + */ + if (rankLast[1] == noSymbol) { + while (huffNode[n].nbBits == maxNbBits) n--; + huffNode[n+1].nbBits--; + assert(n >= 0); + rankLast[1] = (U32)(n+1); + totalCost++; + continue; + } + huffNode[ rankLast[1] + 1 ].nbBits--; + rankLast[1]++; + totalCost ++; + } + } /* repay normalized cost */ + } /* there are several too large elements (at least >= 2) */ + + return maxNbBits; +} + +typedef struct { + U16 base; + U16 curr; +} rankPos; + +typedef nodeElt huffNodeTable[HUF_CTABLE_WORKSPACE_SIZE_U32]; + +/* Number of buckets available for HUF_sort() */ +#define RANK_POSITION_TABLE_SIZE 192 + +typedef struct { + huffNodeTable huffNodeTbl; + rankPos rankPosition[RANK_POSITION_TABLE_SIZE]; +} HUF_buildCTable_wksp_tables; + +/* RANK_POSITION_DISTINCT_COUNT_CUTOFF == Cutoff point in HUF_sort() buckets for which we use log2 bucketing. + * Strategy is to use as many buckets as possible for representing distinct + * counts while using the remainder to represent all "large" counts. + * + * To satisfy this requirement for 192 buckets, we can do the following: + * Let buckets 0-166 represent distinct counts of [0, 166] + * Let buckets 166 to 192 represent all remaining counts up to RANK_POSITION_MAX_COUNT_LOG using log2 bucketing. + */ +#define RANK_POSITION_MAX_COUNT_LOG 32 +#define RANK_POSITION_LOG_BUCKETS_BEGIN (RANK_POSITION_TABLE_SIZE - 1) - RANK_POSITION_MAX_COUNT_LOG - 1 /* == 158 */ +#define RANK_POSITION_DISTINCT_COUNT_CUTOFF RANK_POSITION_LOG_BUCKETS_BEGIN + BIT_highbit32(RANK_POSITION_LOG_BUCKETS_BEGIN) /* == 166 */ + +/* Return the appropriate bucket index for a given count. See definition of + * RANK_POSITION_DISTINCT_COUNT_CUTOFF for explanation of bucketing strategy. + */ +static U32 HUF_getIndex(U32 const count) { + return (count < RANK_POSITION_DISTINCT_COUNT_CUTOFF) + ? count + : BIT_highbit32(count) + RANK_POSITION_LOG_BUCKETS_BEGIN; +} + +/* Helper swap function for HUF_quickSortPartition() */ +static void HUF_swapNodes(nodeElt* a, nodeElt* b) { + nodeElt tmp = *a; + *a = *b; + *b = tmp; +} + +/* Returns 0 if the huffNode array is not sorted by descending count */ +MEM_STATIC int HUF_isSorted(nodeElt huffNode[], U32 const maxSymbolValue1) { + U32 i; + for (i = 1; i < maxSymbolValue1; ++i) { + if (huffNode[i].count > huffNode[i-1].count) { + return 0; + } + } + return 1; +} + +/* Insertion sort by descending order */ +HINT_INLINE void HUF_insertionSort(nodeElt huffNode[], int const low, int const high) { + int i; + int const size = high-low+1; + huffNode += low; + for (i = 1; i < size; ++i) { + nodeElt const key = huffNode[i]; + int j = i - 1; + while (j >= 0 && huffNode[j].count < key.count) { + huffNode[j + 1] = huffNode[j]; + j--; + } + huffNode[j + 1] = key; + } +} + +/* Pivot helper function for quicksort. */ +static int HUF_quickSortPartition(nodeElt arr[], int const low, int const high) { + /* Simply select rightmost element as pivot. "Better" selectors like + * median-of-three don't experimentally appear to have any benefit. + */ + U32 const pivot = arr[high].count; + int i = low - 1; + int j = low; + for ( ; j < high; j++) { + if (arr[j].count > pivot) { + i++; + HUF_swapNodes(&arr[i], &arr[j]); + } + } + HUF_swapNodes(&arr[i + 1], &arr[high]); + return i + 1; +} + +/* Classic quicksort by descending with partially iterative calls + * to reduce worst case callstack size. + */ +static void HUF_simpleQuickSort(nodeElt arr[], int low, int high) { + int const kInsertionSortThreshold = 8; + if (high - low < kInsertionSortThreshold) { + HUF_insertionSort(arr, low, high); + return; + } + while (low < high) { + int const idx = HUF_quickSortPartition(arr, low, high); + if (idx - low < high - idx) { + HUF_simpleQuickSort(arr, low, idx - 1); + low = idx + 1; + } else { + HUF_simpleQuickSort(arr, idx + 1, high); + high = idx - 1; + } + } +} + +/** + * HUF_sort(): + * Sorts the symbols [0, maxSymbolValue] by count[symbol] in decreasing order. + * This is a typical bucket sorting strategy that uses either quicksort or insertion sort to sort each bucket. + * + * @param[out] huffNode Sorted symbols by decreasing count. Only members `.count` and `.byte` are filled. + * Must have (maxSymbolValue + 1) entries. + * @param[in] count Histogram of the symbols. + * @param[in] maxSymbolValue Maximum symbol value. + * @param rankPosition This is a scratch workspace. Must have RANK_POSITION_TABLE_SIZE entries. + */ +static void HUF_sort(nodeElt huffNode[], const unsigned count[], U32 const maxSymbolValue, rankPos rankPosition[]) { + U32 n; + U32 const maxSymbolValue1 = maxSymbolValue+1; + + /* Compute base and set curr to base. + * For symbol s let lowerRank = HUF_getIndex(count[n]) and rank = lowerRank + 1. + * See HUF_getIndex to see bucketing strategy. + * We attribute each symbol to lowerRank's base value, because we want to know where + * each rank begins in the output, so for rank R we want to count ranks R+1 and above. + */ + ZSTD_memset(rankPosition, 0, sizeof(*rankPosition) * RANK_POSITION_TABLE_SIZE); + for (n = 0; n < maxSymbolValue1; ++n) { + U32 lowerRank = HUF_getIndex(count[n]); + assert(lowerRank < RANK_POSITION_TABLE_SIZE - 1); + rankPosition[lowerRank].base++; + } + + assert(rankPosition[RANK_POSITION_TABLE_SIZE - 1].base == 0); + /* Set up the rankPosition table */ + for (n = RANK_POSITION_TABLE_SIZE - 1; n > 0; --n) { + rankPosition[n-1].base += rankPosition[n].base; + rankPosition[n-1].curr = rankPosition[n-1].base; + } + + /* Insert each symbol into their appropriate bucket, setting up rankPosition table. */ + for (n = 0; n < maxSymbolValue1; ++n) { + U32 const c = count[n]; + U32 const r = HUF_getIndex(c) + 1; + U32 const pos = rankPosition[r].curr++; + assert(pos < maxSymbolValue1); + huffNode[pos].count = c; + huffNode[pos].byte = (BYTE)n; + } + + /* Sort each bucket. */ + for (n = RANK_POSITION_DISTINCT_COUNT_CUTOFF; n < RANK_POSITION_TABLE_SIZE - 1; ++n) { + U32 const bucketSize = rankPosition[n].curr-rankPosition[n].base; + U32 const bucketStartIdx = rankPosition[n].base; + if (bucketSize > 1) { + assert(bucketStartIdx < maxSymbolValue1); + HUF_simpleQuickSort(huffNode + bucketStartIdx, 0, bucketSize-1); + } + } + + assert(HUF_isSorted(huffNode, maxSymbolValue1)); +} + +/** HUF_buildCTable_wksp() : + * Same as HUF_buildCTable(), but using externally allocated scratch buffer. + * `workSpace` must be aligned on 4-bytes boundaries, and be at least as large as sizeof(HUF_buildCTable_wksp_tables). + */ +#define STARTNODE (HUF_SYMBOLVALUE_MAX+1) + +/* HUF_buildTree(): + * Takes the huffNode array sorted by HUF_sort() and builds an unlimited-depth Huffman tree. + * + * @param huffNode The array sorted by HUF_sort(). Builds the Huffman tree in this array. + * @param maxSymbolValue The maximum symbol value. + * @return The smallest node in the Huffman tree (by count). + */ +static int HUF_buildTree(nodeElt* huffNode, U32 maxSymbolValue) +{ + nodeElt* const huffNode0 = huffNode - 1; + int nonNullRank; + int lowS, lowN; + int nodeNb = STARTNODE; + int n, nodeRoot; + /* init for parents */ + nonNullRank = (int)maxSymbolValue; + while(huffNode[nonNullRank].count == 0) nonNullRank--; + lowS = nonNullRank; nodeRoot = nodeNb + lowS - 1; lowN = nodeNb; + huffNode[nodeNb].count = huffNode[lowS].count + huffNode[lowS-1].count; + huffNode[lowS].parent = huffNode[lowS-1].parent = (U16)nodeNb; + nodeNb++; lowS-=2; + for (n=nodeNb; n<=nodeRoot; n++) huffNode[n].count = (U32)(1U<<30); + huffNode0[0].count = (U32)(1U<<31); /* fake entry, strong barrier */ + + /* create parents */ + while (nodeNb <= nodeRoot) { + int const n1 = (huffNode[lowS].count < huffNode[lowN].count) ? lowS-- : lowN++; + int const n2 = (huffNode[lowS].count < huffNode[lowN].count) ? lowS-- : lowN++; + huffNode[nodeNb].count = huffNode[n1].count + huffNode[n2].count; + huffNode[n1].parent = huffNode[n2].parent = (U16)nodeNb; + nodeNb++; + } + + /* distribute weights (unlimited tree height) */ + huffNode[nodeRoot].nbBits = 0; + for (n=nodeRoot-1; n>=STARTNODE; n--) + huffNode[n].nbBits = huffNode[ huffNode[n].parent ].nbBits + 1; + for (n=0; n<=nonNullRank; n++) + huffNode[n].nbBits = huffNode[ huffNode[n].parent ].nbBits + 1; + + return nonNullRank; +} + +/** + * HUF_buildCTableFromTree(): + * Build the CTable given the Huffman tree in huffNode. + * + * @param[out] CTable The output Huffman CTable. + * @param huffNode The Huffman tree. + * @param nonNullRank The last and smallest node in the Huffman tree. + * @param maxSymbolValue The maximum symbol value. + * @param maxNbBits The exact maximum number of bits used in the Huffman tree. + */ +static void HUF_buildCTableFromTree(HUF_CElt* CTable, nodeElt const* huffNode, int nonNullRank, U32 maxSymbolValue, U32 maxNbBits) +{ + HUF_CElt* const ct = CTable + 1; + /* fill result into ctable (val, nbBits) */ + int n; + U16 nbPerRank[HUF_TABLELOG_MAX+1] = {0}; + U16 valPerRank[HUF_TABLELOG_MAX+1] = {0}; + int const alphabetSize = (int)(maxSymbolValue + 1); + for (n=0; n<=nonNullRank; n++) + nbPerRank[huffNode[n].nbBits]++; + /* determine starting value per rank */ + { U16 min = 0; + for (n=(int)maxNbBits; n>0; n--) { + valPerRank[n] = min; /* get starting value within each rank */ + min += nbPerRank[n]; + min >>= 1; + } } + for (n=0; nhuffNodeTbl; + nodeElt* const huffNode = huffNode0+1; + int nonNullRank; + + /* safety checks */ + if (wkspSize < sizeof(HUF_buildCTable_wksp_tables)) + return ERROR(workSpace_tooSmall); + if (maxNbBits == 0) maxNbBits = HUF_TABLELOG_DEFAULT; + if (maxSymbolValue > HUF_SYMBOLVALUE_MAX) + return ERROR(maxSymbolValue_tooLarge); + ZSTD_memset(huffNode0, 0, sizeof(huffNodeTable)); + + /* sort, decreasing order */ + HUF_sort(huffNode, count, maxSymbolValue, wksp_tables->rankPosition); + + /* build tree */ + nonNullRank = HUF_buildTree(huffNode, maxSymbolValue); + + /* enforce maxTableLog */ + maxNbBits = HUF_setMaxHeight(huffNode, (U32)nonNullRank, maxNbBits); + if (maxNbBits > HUF_TABLELOG_MAX) return ERROR(GENERIC); /* check fit into table */ + + HUF_buildCTableFromTree(CTable, huffNode, nonNullRank, maxSymbolValue, maxNbBits); + + return maxNbBits; +} + +size_t HUF_estimateCompressedSize(const HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue) +{ + HUF_CElt const* ct = CTable + 1; + size_t nbBits = 0; + int s; + for (s = 0; s <= (int)maxSymbolValue; ++s) { + nbBits += HUF_getNbBits(ct[s]) * count[s]; + } + return nbBits >> 3; +} + +int HUF_validateCTable(const HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue) { + HUF_CElt const* ct = CTable + 1; + int bad = 0; + int s; + for (s = 0; s <= (int)maxSymbolValue; ++s) { + bad |= (count[s] != 0) & (HUF_getNbBits(ct[s]) == 0); + } + return !bad; +} + +size_t HUF_compressBound(size_t size) { return HUF_COMPRESSBOUND(size); } + +/** HUF_CStream_t: + * Huffman uses its own BIT_CStream_t implementation. + * There are three major differences from BIT_CStream_t: + * 1. HUF_addBits() takes a HUF_CElt (size_t) which is + * the pair (nbBits, value) in the format: + * format: + * - Bits [0, 4) = nbBits + * - Bits [4, 64 - nbBits) = 0 + * - Bits [64 - nbBits, 64) = value + * 2. The bitContainer is built from the upper bits and + * right shifted. E.g. to add a new value of N bits + * you right shift the bitContainer by N, then or in + * the new value into the N upper bits. + * 3. The bitstream has two bit containers. You can add + * bits to the second container and merge them into + * the first container. + */ + +#define HUF_BITS_IN_CONTAINER (sizeof(size_t) * 8) + +typedef struct { + size_t bitContainer[2]; + size_t bitPos[2]; + + BYTE* startPtr; + BYTE* ptr; + BYTE* endPtr; +} HUF_CStream_t; + +/**! HUF_initCStream(): + * Initializes the bitstream. + * @returns 0 or an error code. + */ +static size_t HUF_initCStream(HUF_CStream_t* bitC, + void* startPtr, size_t dstCapacity) +{ + ZSTD_memset(bitC, 0, sizeof(*bitC)); + bitC->startPtr = (BYTE*)startPtr; + bitC->ptr = bitC->startPtr; + bitC->endPtr = bitC->startPtr + dstCapacity - sizeof(bitC->bitContainer[0]); + if (dstCapacity <= sizeof(bitC->bitContainer[0])) return ERROR(dstSize_tooSmall); + return 0; +} + +/*! HUF_addBits(): + * Adds the symbol stored in HUF_CElt elt to the bitstream. + * + * @param elt The element we're adding. This is a (nbBits, value) pair. + * See the HUF_CStream_t docs for the format. + * @param idx Insert into the bitstream at this idx. + * @param kFast This is a template parameter. If the bitstream is guaranteed + * to have at least 4 unused bits after this call it may be 1, + * otherwise it must be 0. HUF_addBits() is faster when fast is set. + */ +FORCE_INLINE_TEMPLATE void HUF_addBits(HUF_CStream_t* bitC, HUF_CElt elt, int idx, int kFast) +{ + assert(idx <= 1); + assert(HUF_getNbBits(elt) <= HUF_TABLELOG_ABSOLUTEMAX); + /* This is efficient on x86-64 with BMI2 because shrx + * only reads the low 6 bits of the register. The compiler + * knows this and elides the mask. When fast is set, + * every operation can use the same value loaded from elt. + */ + bitC->bitContainer[idx] >>= HUF_getNbBits(elt); + bitC->bitContainer[idx] |= kFast ? HUF_getValueFast(elt) : HUF_getValue(elt); + /* We only read the low 8 bits of bitC->bitPos[idx] so it + * doesn't matter that the high bits have noise from the value. + */ + bitC->bitPos[idx] += HUF_getNbBitsFast(elt); + assert((bitC->bitPos[idx] & 0xFF) <= HUF_BITS_IN_CONTAINER); + /* The last 4-bits of elt are dirty if fast is set, + * so we must not be overwriting bits that have already been + * inserted into the bit container. + */ +#if DEBUGLEVEL >= 1 + { + size_t const nbBits = HUF_getNbBits(elt); + size_t const dirtyBits = nbBits == 0 ? 0 : BIT_highbit32((U32)nbBits) + 1; + (void)dirtyBits; + /* Middle bits are 0. */ + assert(((elt >> dirtyBits) << (dirtyBits + nbBits)) == 0); + /* We didn't overwrite any bits in the bit container. */ + assert(!kFast || (bitC->bitPos[idx] & 0xFF) <= HUF_BITS_IN_CONTAINER); + (void)dirtyBits; + } +#endif +} + +FORCE_INLINE_TEMPLATE void HUF_zeroIndex1(HUF_CStream_t* bitC) +{ + bitC->bitContainer[1] = 0; + bitC->bitPos[1] = 0; +} + +/*! HUF_mergeIndex1() : + * Merges the bit container @ index 1 into the bit container @ index 0 + * and zeros the bit container @ index 1. + */ +FORCE_INLINE_TEMPLATE void HUF_mergeIndex1(HUF_CStream_t* bitC) +{ + assert((bitC->bitPos[1] & 0xFF) < HUF_BITS_IN_CONTAINER); + bitC->bitContainer[0] >>= (bitC->bitPos[1] & 0xFF); + bitC->bitContainer[0] |= bitC->bitContainer[1]; + bitC->bitPos[0] += bitC->bitPos[1]; + assert((bitC->bitPos[0] & 0xFF) <= HUF_BITS_IN_CONTAINER); +} + +/*! HUF_flushBits() : +* Flushes the bits in the bit container @ index 0. +* +* @post bitPos will be < 8. +* @param kFast If kFast is set then we must know a-priori that +* the bit container will not overflow. +*/ +FORCE_INLINE_TEMPLATE void HUF_flushBits(HUF_CStream_t* bitC, int kFast) +{ + /* The upper bits of bitPos are noisy, so we must mask by 0xFF. */ + size_t const nbBits = bitC->bitPos[0] & 0xFF; + size_t const nbBytes = nbBits >> 3; + /* The top nbBits bits of bitContainer are the ones we need. */ + size_t const bitContainer = bitC->bitContainer[0] >> (HUF_BITS_IN_CONTAINER - nbBits); + /* Mask bitPos to account for the bytes we consumed. */ + bitC->bitPos[0] &= 7; + assert(nbBits > 0); + assert(nbBits <= sizeof(bitC->bitContainer[0]) * 8); + assert(bitC->ptr <= bitC->endPtr); + MEM_writeLEST(bitC->ptr, bitContainer); + bitC->ptr += nbBytes; + assert(!kFast || bitC->ptr <= bitC->endPtr); + if (!kFast && bitC->ptr > bitC->endPtr) bitC->ptr = bitC->endPtr; + /* bitContainer doesn't need to be modified because the leftover + * bits are already the top bitPos bits. And we don't care about + * noise in the lower values. + */ +} + +/*! HUF_endMark() + * @returns The Huffman stream end mark: A 1-bit value = 1. + */ +static HUF_CElt HUF_endMark(void) +{ + HUF_CElt endMark; + HUF_setNbBits(&endMark, 1); + HUF_setValue(&endMark, 1); + return endMark; +} + +/*! HUF_closeCStream() : + * @return Size of CStream, in bytes, + * or 0 if it could not fit into dstBuffer */ +static size_t HUF_closeCStream(HUF_CStream_t* bitC) +{ + HUF_addBits(bitC, HUF_endMark(), /* idx */ 0, /* kFast */ 0); + HUF_flushBits(bitC, /* kFast */ 0); + { + size_t const nbBits = bitC->bitPos[0] & 0xFF; + if (bitC->ptr >= bitC->endPtr) return 0; /* overflow detected */ + return (bitC->ptr - bitC->startPtr) + (nbBits > 0); + } +} + +FORCE_INLINE_TEMPLATE void +HUF_encodeSymbol(HUF_CStream_t* bitCPtr, U32 symbol, const HUF_CElt* CTable, int idx, int fast) +{ + HUF_addBits(bitCPtr, CTable[symbol], idx, fast); +} + +FORCE_INLINE_TEMPLATE void +HUF_compress1X_usingCTable_internal_body_loop(HUF_CStream_t* bitC, + const BYTE* ip, size_t srcSize, + const HUF_CElt* ct, + int kUnroll, int kFastFlush, int kLastFast) +{ + /* Join to kUnroll */ + int n = (int)srcSize; + int rem = n % kUnroll; + if (rem > 0) { + for (; rem > 0; --rem) { + HUF_encodeSymbol(bitC, ip[--n], ct, 0, /* fast */ 0); + } + HUF_flushBits(bitC, kFastFlush); + } + assert(n % kUnroll == 0); + + /* Join to 2 * kUnroll */ + if (n % (2 * kUnroll)) { + int u; + for (u = 1; u < kUnroll; ++u) { + HUF_encodeSymbol(bitC, ip[n - u], ct, 0, 1); + } + HUF_encodeSymbol(bitC, ip[n - kUnroll], ct, 0, kLastFast); + HUF_flushBits(bitC, kFastFlush); + n -= kUnroll; + } + assert(n % (2 * kUnroll) == 0); + + for (; n>0; n-= 2 * kUnroll) { + /* Encode kUnroll symbols into the bitstream @ index 0. */ + int u; + for (u = 1; u < kUnroll; ++u) { + HUF_encodeSymbol(bitC, ip[n - u], ct, /* idx */ 0, /* fast */ 1); + } + HUF_encodeSymbol(bitC, ip[n - kUnroll], ct, /* idx */ 0, /* fast */ kLastFast); + HUF_flushBits(bitC, kFastFlush); + /* Encode kUnroll symbols into the bitstream @ index 1. + * This allows us to start filling the bit container + * without any data dependencies. + */ + HUF_zeroIndex1(bitC); + for (u = 1; u < kUnroll; ++u) { + HUF_encodeSymbol(bitC, ip[n - kUnroll - u], ct, /* idx */ 1, /* fast */ 1); + } + HUF_encodeSymbol(bitC, ip[n - kUnroll - kUnroll], ct, /* idx */ 1, /* fast */ kLastFast); + /* Merge bitstream @ index 1 into the bitstream @ index 0 */ + HUF_mergeIndex1(bitC); + HUF_flushBits(bitC, kFastFlush); + } + assert(n == 0); + +} + +/** + * Returns a tight upper bound on the output space needed by Huffman + * with 8 bytes buffer to handle over-writes. If the output is at least + * this large we don't need to do bounds checks during Huffman encoding. + */ +static size_t HUF_tightCompressBound(size_t srcSize, size_t tableLog) +{ + return ((srcSize * tableLog) >> 3) + 8; +} + + +FORCE_INLINE_TEMPLATE size_t +HUF_compress1X_usingCTable_internal_body(void* dst, size_t dstSize, + const void* src, size_t srcSize, + const HUF_CElt* CTable) +{ + U32 const tableLog = (U32)CTable[0]; + HUF_CElt const* ct = CTable + 1; + const BYTE* ip = (const BYTE*) src; + BYTE* const ostart = (BYTE*)dst; + BYTE* const oend = ostart + dstSize; + BYTE* op = ostart; + HUF_CStream_t bitC; + + /* init */ + if (dstSize < 8) return 0; /* not enough space to compress */ + { size_t const initErr = HUF_initCStream(&bitC, op, (size_t)(oend-op)); + if (HUF_isError(initErr)) return 0; } + + if (dstSize < HUF_tightCompressBound(srcSize, (size_t)tableLog) || tableLog > 11) + HUF_compress1X_usingCTable_internal_body_loop(&bitC, ip, srcSize, ct, /* kUnroll */ MEM_32bits() ? 2 : 4, /* kFast */ 0, /* kLastFast */ 0); + else { + if (MEM_32bits()) { + switch (tableLog) { + case 11: + HUF_compress1X_usingCTable_internal_body_loop(&bitC, ip, srcSize, ct, /* kUnroll */ 2, /* kFastFlush */ 1, /* kLastFast */ 0); + break; + case 10: ZSTD_FALLTHROUGH; + case 9: ZSTD_FALLTHROUGH; + case 8: + HUF_compress1X_usingCTable_internal_body_loop(&bitC, ip, srcSize, ct, /* kUnroll */ 2, /* kFastFlush */ 1, /* kLastFast */ 1); + break; + case 7: ZSTD_FALLTHROUGH; + default: + HUF_compress1X_usingCTable_internal_body_loop(&bitC, ip, srcSize, ct, /* kUnroll */ 3, /* kFastFlush */ 1, /* kLastFast */ 1); + break; + } + } else { + switch (tableLog) { + case 11: + HUF_compress1X_usingCTable_internal_body_loop(&bitC, ip, srcSize, ct, /* kUnroll */ 5, /* kFastFlush */ 1, /* kLastFast */ 0); + break; + case 10: + HUF_compress1X_usingCTable_internal_body_loop(&bitC, ip, srcSize, ct, /* kUnroll */ 5, /* kFastFlush */ 1, /* kLastFast */ 1); + break; + case 9: + HUF_compress1X_usingCTable_internal_body_loop(&bitC, ip, srcSize, ct, /* kUnroll */ 6, /* kFastFlush */ 1, /* kLastFast */ 0); + break; + case 8: + HUF_compress1X_usingCTable_internal_body_loop(&bitC, ip, srcSize, ct, /* kUnroll */ 7, /* kFastFlush */ 1, /* kLastFast */ 0); + break; + case 7: + HUF_compress1X_usingCTable_internal_body_loop(&bitC, ip, srcSize, ct, /* kUnroll */ 8, /* kFastFlush */ 1, /* kLastFast */ 0); + break; + case 6: ZSTD_FALLTHROUGH; + default: + HUF_compress1X_usingCTable_internal_body_loop(&bitC, ip, srcSize, ct, /* kUnroll */ 9, /* kFastFlush */ 1, /* kLastFast */ 1); + break; + } + } + } + assert(bitC.ptr <= bitC.endPtr); + + return HUF_closeCStream(&bitC); +} + +#if DYNAMIC_BMI2 + +static BMI2_TARGET_ATTRIBUTE size_t +HUF_compress1X_usingCTable_internal_bmi2(void* dst, size_t dstSize, + const void* src, size_t srcSize, + const HUF_CElt* CTable) +{ + return HUF_compress1X_usingCTable_internal_body(dst, dstSize, src, srcSize, CTable); +} + +static size_t +HUF_compress1X_usingCTable_internal_default(void* dst, size_t dstSize, + const void* src, size_t srcSize, + const HUF_CElt* CTable) +{ + return HUF_compress1X_usingCTable_internal_body(dst, dstSize, src, srcSize, CTable); +} + +static size_t +HUF_compress1X_usingCTable_internal(void* dst, size_t dstSize, + const void* src, size_t srcSize, + const HUF_CElt* CTable, const int bmi2) +{ + if (bmi2) { + return HUF_compress1X_usingCTable_internal_bmi2(dst, dstSize, src, srcSize, CTable); + } + return HUF_compress1X_usingCTable_internal_default(dst, dstSize, src, srcSize, CTable); +} + +#else + +static size_t +HUF_compress1X_usingCTable_internal(void* dst, size_t dstSize, + const void* src, size_t srcSize, + const HUF_CElt* CTable, const int bmi2) +{ + (void)bmi2; + return HUF_compress1X_usingCTable_internal_body(dst, dstSize, src, srcSize, CTable); +} + +#endif + +size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable) +{ + return HUF_compress1X_usingCTable_bmi2(dst, dstSize, src, srcSize, CTable, /* bmi2 */ 0); +} + +size_t HUF_compress1X_usingCTable_bmi2(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable, int bmi2) +{ + return HUF_compress1X_usingCTable_internal(dst, dstSize, src, srcSize, CTable, bmi2); +} + +static size_t +HUF_compress4X_usingCTable_internal(void* dst, size_t dstSize, + const void* src, size_t srcSize, + const HUF_CElt* CTable, int bmi2) +{ + size_t const segmentSize = (srcSize+3)/4; /* first 3 segments */ + const BYTE* ip = (const BYTE*) src; + const BYTE* const iend = ip + srcSize; + BYTE* const ostart = (BYTE*) dst; + BYTE* const oend = ostart + dstSize; + BYTE* op = ostart; + + if (dstSize < 6 + 1 + 1 + 1 + 8) return 0; /* minimum space to compress successfully */ + if (srcSize < 12) return 0; /* no saving possible : too small input */ + op += 6; /* jumpTable */ + + assert(op <= oend); + { CHECK_V_F(cSize, HUF_compress1X_usingCTable_internal(op, (size_t)(oend-op), ip, segmentSize, CTable, bmi2) ); + if (cSize == 0 || cSize > 65535) return 0; + MEM_writeLE16(ostart, (U16)cSize); + op += cSize; + } + + ip += segmentSize; + assert(op <= oend); + { CHECK_V_F(cSize, HUF_compress1X_usingCTable_internal(op, (size_t)(oend-op), ip, segmentSize, CTable, bmi2) ); + if (cSize == 0 || cSize > 65535) return 0; + MEM_writeLE16(ostart+2, (U16)cSize); + op += cSize; + } + + ip += segmentSize; + assert(op <= oend); + { CHECK_V_F(cSize, HUF_compress1X_usingCTable_internal(op, (size_t)(oend-op), ip, segmentSize, CTable, bmi2) ); + if (cSize == 0 || cSize > 65535) return 0; + MEM_writeLE16(ostart+4, (U16)cSize); + op += cSize; + } + + ip += segmentSize; + assert(op <= oend); + assert(ip <= iend); + { CHECK_V_F(cSize, HUF_compress1X_usingCTable_internal(op, (size_t)(oend-op), ip, (size_t)(iend-ip), CTable, bmi2) ); + if (cSize == 0 || cSize > 65535) return 0; + op += cSize; + } + + return (size_t)(op-ostart); +} + +size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable) +{ + return HUF_compress4X_usingCTable_bmi2(dst, dstSize, src, srcSize, CTable, /* bmi2 */ 0); +} + +size_t HUF_compress4X_usingCTable_bmi2(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable, int bmi2) +{ + return HUF_compress4X_usingCTable_internal(dst, dstSize, src, srcSize, CTable, bmi2); +} + +typedef enum { HUF_singleStream, HUF_fourStreams } HUF_nbStreams_e; + +static size_t HUF_compressCTable_internal( + BYTE* const ostart, BYTE* op, BYTE* const oend, + const void* src, size_t srcSize, + HUF_nbStreams_e nbStreams, const HUF_CElt* CTable, const int bmi2) +{ + size_t const cSize = (nbStreams==HUF_singleStream) ? + HUF_compress1X_usingCTable_internal(op, (size_t)(oend - op), src, srcSize, CTable, bmi2) : + HUF_compress4X_usingCTable_internal(op, (size_t)(oend - op), src, srcSize, CTable, bmi2); + if (HUF_isError(cSize)) { return cSize; } + if (cSize==0) { return 0; } /* uncompressible */ + op += cSize; + /* check compressibility */ + assert(op >= ostart); + if ((size_t)(op-ostart) >= srcSize-1) { return 0; } + return (size_t)(op-ostart); +} + +typedef struct { + unsigned count[HUF_SYMBOLVALUE_MAX + 1]; + HUF_CElt CTable[HUF_CTABLE_SIZE_ST(HUF_SYMBOLVALUE_MAX)]; + union { + HUF_buildCTable_wksp_tables buildCTable_wksp; + HUF_WriteCTableWksp writeCTable_wksp; + U32 hist_wksp[HIST_WKSP_SIZE_U32]; + } wksps; +} HUF_compress_tables_t; + +#define SUSPECT_INCOMPRESSIBLE_SAMPLE_SIZE 4096 +#define SUSPECT_INCOMPRESSIBLE_SAMPLE_RATIO 10 /* Must be >= 2 */ + +/* HUF_compress_internal() : + * `workSpace_align4` must be aligned on 4-bytes boundaries, + * and occupies the same space as a table of HUF_WORKSPACE_SIZE_U64 unsigned */ +static size_t +HUF_compress_internal (void* dst, size_t dstSize, + const void* src, size_t srcSize, + unsigned maxSymbolValue, unsigned huffLog, + HUF_nbStreams_e nbStreams, + void* workSpace, size_t wkspSize, + HUF_CElt* oldHufTable, HUF_repeat* repeat, int preferRepeat, + const int bmi2, unsigned suspectUncompressible) +{ + HUF_compress_tables_t* const table = (HUF_compress_tables_t*)HUF_alignUpWorkspace(workSpace, &wkspSize, ZSTD_ALIGNOF(size_t)); + BYTE* const ostart = (BYTE*)dst; + BYTE* const oend = ostart + dstSize; + BYTE* op = ostart; + + HUF_STATIC_ASSERT(sizeof(*table) + HUF_WORKSPACE_MAX_ALIGNMENT <= HUF_WORKSPACE_SIZE); + + /* checks & inits */ + if (wkspSize < sizeof(*table)) return ERROR(workSpace_tooSmall); + if (!srcSize) return 0; /* Uncompressed */ + if (!dstSize) return 0; /* cannot fit anything within dst budget */ + if (srcSize > HUF_BLOCKSIZE_MAX) return ERROR(srcSize_wrong); /* current block size limit */ + if (huffLog > HUF_TABLELOG_MAX) return ERROR(tableLog_tooLarge); + if (maxSymbolValue > HUF_SYMBOLVALUE_MAX) return ERROR(maxSymbolValue_tooLarge); + if (!maxSymbolValue) maxSymbolValue = HUF_SYMBOLVALUE_MAX; + if (!huffLog) huffLog = HUF_TABLELOG_DEFAULT; + + /* Heuristic : If old table is valid, use it for small inputs */ + if (preferRepeat && repeat && *repeat == HUF_repeat_valid) { + return HUF_compressCTable_internal(ostart, op, oend, + src, srcSize, + nbStreams, oldHufTable, bmi2); + } + + /* If uncompressible data is suspected, do a smaller sampling first */ + DEBUG_STATIC_ASSERT(SUSPECT_INCOMPRESSIBLE_SAMPLE_RATIO >= 2); + if (suspectUncompressible && srcSize >= (SUSPECT_INCOMPRESSIBLE_SAMPLE_SIZE * SUSPECT_INCOMPRESSIBLE_SAMPLE_RATIO)) { + size_t largestTotal = 0; + { unsigned maxSymbolValueBegin = maxSymbolValue; + CHECK_V_F(largestBegin, HIST_count_simple (table->count, &maxSymbolValueBegin, (const BYTE*)src, SUSPECT_INCOMPRESSIBLE_SAMPLE_SIZE) ); + largestTotal += largestBegin; + } + { unsigned maxSymbolValueEnd = maxSymbolValue; + CHECK_V_F(largestEnd, HIST_count_simple (table->count, &maxSymbolValueEnd, (const BYTE*)src + srcSize - SUSPECT_INCOMPRESSIBLE_SAMPLE_SIZE, SUSPECT_INCOMPRESSIBLE_SAMPLE_SIZE) ); + largestTotal += largestEnd; + } + if (largestTotal <= ((2 * SUSPECT_INCOMPRESSIBLE_SAMPLE_SIZE) >> 7)+4) return 0; /* heuristic : probably not compressible enough */ + } + + /* Scan input and build symbol stats */ + { CHECK_V_F(largest, HIST_count_wksp (table->count, &maxSymbolValue, (const BYTE*)src, srcSize, table->wksps.hist_wksp, sizeof(table->wksps.hist_wksp)) ); + if (largest == srcSize) { *ostart = ((const BYTE*)src)[0]; return 1; } /* single symbol, rle */ + if (largest <= (srcSize >> 7)+4) return 0; /* heuristic : probably not compressible enough */ + } + + /* Check validity of previous table */ + if ( repeat + && *repeat == HUF_repeat_check + && !HUF_validateCTable(oldHufTable, table->count, maxSymbolValue)) { + *repeat = HUF_repeat_none; + } + /* Heuristic : use existing table for small inputs */ + if (preferRepeat && repeat && *repeat != HUF_repeat_none) { + return HUF_compressCTable_internal(ostart, op, oend, + src, srcSize, + nbStreams, oldHufTable, bmi2); + } + + /* Build Huffman Tree */ + huffLog = HUF_optimalTableLog(huffLog, srcSize, maxSymbolValue); + { size_t const maxBits = HUF_buildCTable_wksp(table->CTable, table->count, + maxSymbolValue, huffLog, + &table->wksps.buildCTable_wksp, sizeof(table->wksps.buildCTable_wksp)); + CHECK_F(maxBits); + huffLog = (U32)maxBits; + } + /* Zero unused symbols in CTable, so we can check it for validity */ + { + size_t const ctableSize = HUF_CTABLE_SIZE_ST(maxSymbolValue); + size_t const unusedSize = sizeof(table->CTable) - ctableSize * sizeof(HUF_CElt); + ZSTD_memset(table->CTable + ctableSize, 0, unusedSize); + } + + /* Write table description header */ + { CHECK_V_F(hSize, HUF_writeCTable_wksp(op, dstSize, table->CTable, maxSymbolValue, huffLog, + &table->wksps.writeCTable_wksp, sizeof(table->wksps.writeCTable_wksp)) ); + /* Check if using previous huffman table is beneficial */ + if (repeat && *repeat != HUF_repeat_none) { + size_t const oldSize = HUF_estimateCompressedSize(oldHufTable, table->count, maxSymbolValue); + size_t const newSize = HUF_estimateCompressedSize(table->CTable, table->count, maxSymbolValue); + if (oldSize <= hSize + newSize || hSize + 12 >= srcSize) { + return HUF_compressCTable_internal(ostart, op, oend, + src, srcSize, + nbStreams, oldHufTable, bmi2); + } } + + /* Use the new huffman table */ + if (hSize + 12ul >= srcSize) { return 0; } + op += hSize; + if (repeat) { *repeat = HUF_repeat_none; } + if (oldHufTable) + ZSTD_memcpy(oldHufTable, table->CTable, sizeof(table->CTable)); /* Save new table */ + } + return HUF_compressCTable_internal(ostart, op, oend, + src, srcSize, + nbStreams, table->CTable, bmi2); +} + + +size_t HUF_compress1X_wksp (void* dst, size_t dstSize, + const void* src, size_t srcSize, + unsigned maxSymbolValue, unsigned huffLog, + void* workSpace, size_t wkspSize) +{ + return HUF_compress_internal(dst, dstSize, src, srcSize, + maxSymbolValue, huffLog, HUF_singleStream, + workSpace, wkspSize, + NULL, NULL, 0, 0 /*bmi2*/, 0); +} + +size_t HUF_compress1X_repeat (void* dst, size_t dstSize, + const void* src, size_t srcSize, + unsigned maxSymbolValue, unsigned huffLog, + void* workSpace, size_t wkspSize, + HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat, + int bmi2, unsigned suspectUncompressible) +{ + return HUF_compress_internal(dst, dstSize, src, srcSize, + maxSymbolValue, huffLog, HUF_singleStream, + workSpace, wkspSize, hufTable, + repeat, preferRepeat, bmi2, suspectUncompressible); +} + +/* HUF_compress4X_repeat(): + * compress input using 4 streams. + * provide workspace to generate compression tables */ +size_t HUF_compress4X_wksp (void* dst, size_t dstSize, + const void* src, size_t srcSize, + unsigned maxSymbolValue, unsigned huffLog, + void* workSpace, size_t wkspSize) +{ + return HUF_compress_internal(dst, dstSize, src, srcSize, + maxSymbolValue, huffLog, HUF_fourStreams, + workSpace, wkspSize, + NULL, NULL, 0, 0 /*bmi2*/, 0); +} + +/* HUF_compress4X_repeat(): + * compress input using 4 streams. + * consider skipping quickly + * re-use an existing huffman compression table */ +size_t HUF_compress4X_repeat (void* dst, size_t dstSize, + const void* src, size_t srcSize, + unsigned maxSymbolValue, unsigned huffLog, + void* workSpace, size_t wkspSize, + HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat, int bmi2, unsigned suspectUncompressible) +{ + return HUF_compress_internal(dst, dstSize, src, srcSize, + maxSymbolValue, huffLog, HUF_fourStreams, + workSpace, wkspSize, + hufTable, repeat, preferRepeat, bmi2, suspectUncompressible); +} + +#ifndef ZSTD_NO_UNUSED_FUNCTIONS +/** HUF_buildCTable() : + * @return : maxNbBits + * Note : count is used before tree is written, so they can safely overlap + */ +size_t HUF_buildCTable (HUF_CElt* tree, const unsigned* count, unsigned maxSymbolValue, unsigned maxNbBits) +{ + HUF_buildCTable_wksp_tables workspace; + return HUF_buildCTable_wksp(tree, count, maxSymbolValue, maxNbBits, &workspace, sizeof(workspace)); +} + +size_t HUF_compress1X (void* dst, size_t dstSize, + const void* src, size_t srcSize, + unsigned maxSymbolValue, unsigned huffLog) +{ + U64 workSpace[HUF_WORKSPACE_SIZE_U64]; + return HUF_compress1X_wksp(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, workSpace, sizeof(workSpace)); +} + +size_t HUF_compress2 (void* dst, size_t dstSize, + const void* src, size_t srcSize, + unsigned maxSymbolValue, unsigned huffLog) +{ + U64 workSpace[HUF_WORKSPACE_SIZE_U64]; + return HUF_compress4X_wksp(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, workSpace, sizeof(workSpace)); +} + +size_t HUF_compress (void* dst, size_t maxDstSize, const void* src, size_t srcSize) +{ + return HUF_compress2(dst, maxDstSize, src, srcSize, 255, HUF_TABLELOG_DEFAULT); +} +#endif diff --git a/src/blosc2/internal-complibs/zstd-1.5.2/compress/zstd_compress.c b/src/blosc2/internal-complibs/zstd-1.5.2/compress/zstd_compress.c new file mode 100644 index 0000000..f06456a --- /dev/null +++ b/src/blosc2/internal-complibs/zstd-1.5.2/compress/zstd_compress.c @@ -0,0 +1,6327 @@ +/* + * Copyright (c) Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/*-************************************* +* Dependencies +***************************************/ +#include "../common/zstd_deps.h" /* INT_MAX, ZSTD_memset, ZSTD_memcpy */ +#include "../common/mem.h" +#include "hist.h" /* HIST_countFast_wksp */ +#define FSE_STATIC_LINKING_ONLY /* FSE_encodeSymbol */ +#include "../common/fse.h" +#define HUF_STATIC_LINKING_ONLY +#include "../common/huf.h" +#include "zstd_compress_internal.h" +#include "zstd_compress_sequences.h" +#include "zstd_compress_literals.h" +#include "zstd_fast.h" +#include "zstd_double_fast.h" +#include "zstd_lazy.h" +#include "zstd_opt.h" +#include "zstd_ldm.h" +#include "zstd_compress_superblock.h" + +/* *************************************************************** +* Tuning parameters +*****************************************************************/ +/*! + * COMPRESS_HEAPMODE : + * Select how default decompression function ZSTD_compress() allocates its context, + * on stack (0, default), or into heap (1). + * Note that functions with explicit context such as ZSTD_compressCCtx() are unaffected. + */ +#ifndef ZSTD_COMPRESS_HEAPMODE +# define ZSTD_COMPRESS_HEAPMODE 0 +#endif + +/*! + * ZSTD_HASHLOG3_MAX : + * Maximum size of the hash table dedicated to find 3-bytes matches, + * in log format, aka 17 => 1 << 17 == 128Ki positions. + * This structure is only used in zstd_opt. + * Since allocation is centralized for all strategies, it has to be known here. + * The actual (selected) size of the hash table is then stored in ZSTD_matchState_t.hashLog3, + * so that zstd_opt.c doesn't need to know about this constant. + */ +#ifndef ZSTD_HASHLOG3_MAX +# define ZSTD_HASHLOG3_MAX 17 +#endif + +/*-************************************* +* Helper functions +***************************************/ +/* ZSTD_compressBound() + * Note that the result from this function is only compatible with the "normal" + * full-block strategy. + * When there are a lot of small blocks due to frequent flush in streaming mode + * the overhead of headers can make the compressed data to be larger than the + * return value of ZSTD_compressBound(). + */ +size_t ZSTD_compressBound(size_t srcSize) { + return ZSTD_COMPRESSBOUND(srcSize); +} + + +/*-************************************* +* Context memory management +***************************************/ +struct ZSTD_CDict_s { + const void* dictContent; + size_t dictContentSize; + ZSTD_dictContentType_e dictContentType; /* The dictContentType the CDict was created with */ + U32* entropyWorkspace; /* entropy workspace of HUF_WORKSPACE_SIZE bytes */ + ZSTD_cwksp workspace; + ZSTD_matchState_t matchState; + ZSTD_compressedBlockState_t cBlockState; + ZSTD_customMem customMem; + U32 dictID; + int compressionLevel; /* 0 indicates that advanced API was used to select CDict params */ + ZSTD_paramSwitch_e useRowMatchFinder; /* Indicates whether the CDict was created with params that would use + * row-based matchfinder. Unless the cdict is reloaded, we will use + * the same greedy/lazy matchfinder at compression time. + */ +}; /* typedef'd to ZSTD_CDict within "zstd.h" */ + +ZSTD_CCtx* ZSTD_createCCtx(void) +{ + return ZSTD_createCCtx_advanced(ZSTD_defaultCMem); +} + +static void ZSTD_initCCtx(ZSTD_CCtx* cctx, ZSTD_customMem memManager) +{ + assert(cctx != NULL); + ZSTD_memset(cctx, 0, sizeof(*cctx)); + cctx->customMem = memManager; + cctx->bmi2 = ZSTD_cpuSupportsBmi2(); + { size_t const err = ZSTD_CCtx_reset(cctx, ZSTD_reset_parameters); + assert(!ZSTD_isError(err)); + (void)err; + } +} + +ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem) +{ + ZSTD_STATIC_ASSERT(zcss_init==0); + ZSTD_STATIC_ASSERT(ZSTD_CONTENTSIZE_UNKNOWN==(0ULL - 1)); + if ((!customMem.customAlloc) ^ (!customMem.customFree)) return NULL; + { ZSTD_CCtx* const cctx = (ZSTD_CCtx*)ZSTD_customMalloc(sizeof(ZSTD_CCtx), customMem); + if (!cctx) return NULL; + ZSTD_initCCtx(cctx, customMem); + return cctx; + } +} + +ZSTD_CCtx* ZSTD_initStaticCCtx(void* workspace, size_t workspaceSize) +{ + ZSTD_cwksp ws; + ZSTD_CCtx* cctx; + if (workspaceSize <= sizeof(ZSTD_CCtx)) return NULL; /* minimum size */ + if ((size_t)workspace & 7) return NULL; /* must be 8-aligned */ + ZSTD_cwksp_init(&ws, workspace, workspaceSize, ZSTD_cwksp_static_alloc); + + cctx = (ZSTD_CCtx*)ZSTD_cwksp_reserve_object(&ws, sizeof(ZSTD_CCtx)); + if (cctx == NULL) return NULL; + + ZSTD_memset(cctx, 0, sizeof(ZSTD_CCtx)); + ZSTD_cwksp_move(&cctx->workspace, &ws); + cctx->staticSize = workspaceSize; + + /* statically sized space. entropyWorkspace never moves (but prev/next block swap places) */ + if (!ZSTD_cwksp_check_available(&cctx->workspace, ENTROPY_WORKSPACE_SIZE + 2 * sizeof(ZSTD_compressedBlockState_t))) return NULL; + cctx->blockState.prevCBlock = (ZSTD_compressedBlockState_t*)ZSTD_cwksp_reserve_object(&cctx->workspace, sizeof(ZSTD_compressedBlockState_t)); + cctx->blockState.nextCBlock = (ZSTD_compressedBlockState_t*)ZSTD_cwksp_reserve_object(&cctx->workspace, sizeof(ZSTD_compressedBlockState_t)); + cctx->entropyWorkspace = (U32*)ZSTD_cwksp_reserve_object(&cctx->workspace, ENTROPY_WORKSPACE_SIZE); + cctx->bmi2 = ZSTD_cpuid_bmi2(ZSTD_cpuid()); + return cctx; +} + +/** + * Clears and frees all of the dictionaries in the CCtx. + */ +static void ZSTD_clearAllDicts(ZSTD_CCtx* cctx) +{ + ZSTD_customFree(cctx->localDict.dictBuffer, cctx->customMem); + ZSTD_freeCDict(cctx->localDict.cdict); + ZSTD_memset(&cctx->localDict, 0, sizeof(cctx->localDict)); + ZSTD_memset(&cctx->prefixDict, 0, sizeof(cctx->prefixDict)); + cctx->cdict = NULL; +} + +static size_t ZSTD_sizeof_localDict(ZSTD_localDict dict) +{ + size_t const bufferSize = dict.dictBuffer != NULL ? dict.dictSize : 0; + size_t const cdictSize = ZSTD_sizeof_CDict(dict.cdict); + return bufferSize + cdictSize; +} + +static void ZSTD_freeCCtxContent(ZSTD_CCtx* cctx) +{ + assert(cctx != NULL); + assert(cctx->staticSize == 0); + ZSTD_clearAllDicts(cctx); +#ifdef ZSTD_MULTITHREAD + ZSTDMT_freeCCtx(cctx->mtctx); cctx->mtctx = NULL; +#endif + ZSTD_cwksp_free(&cctx->workspace, cctx->customMem); +} + +size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx) +{ + if (cctx==NULL) return 0; /* support free on NULL */ + RETURN_ERROR_IF(cctx->staticSize, memory_allocation, + "not compatible with static CCtx"); + { + int cctxInWorkspace = ZSTD_cwksp_owns_buffer(&cctx->workspace, cctx); + ZSTD_freeCCtxContent(cctx); + if (!cctxInWorkspace) { + ZSTD_customFree(cctx, cctx->customMem); + } + } + return 0; +} + + +static size_t ZSTD_sizeof_mtctx(const ZSTD_CCtx* cctx) +{ +#ifdef ZSTD_MULTITHREAD + return ZSTDMT_sizeof_CCtx(cctx->mtctx); +#else + (void)cctx; + return 0; +#endif +} + + +size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx) +{ + if (cctx==NULL) return 0; /* support sizeof on NULL */ + /* cctx may be in the workspace */ + return (cctx->workspace.workspace == cctx ? 0 : sizeof(*cctx)) + + ZSTD_cwksp_sizeof(&cctx->workspace) + + ZSTD_sizeof_localDict(cctx->localDict) + + ZSTD_sizeof_mtctx(cctx); +} + +size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs) +{ + return ZSTD_sizeof_CCtx(zcs); /* same object */ +} + +/* private API call, for dictBuilder only */ +const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx) { return &(ctx->seqStore); } + +/* Returns true if the strategy supports using a row based matchfinder */ +static int ZSTD_rowMatchFinderSupported(const ZSTD_strategy strategy) { + return (strategy >= ZSTD_greedy && strategy <= ZSTD_lazy2); +} + +/* Returns true if the strategy and useRowMatchFinder mode indicate that we will use the row based matchfinder + * for this compression. + */ +static int ZSTD_rowMatchFinderUsed(const ZSTD_strategy strategy, const ZSTD_paramSwitch_e mode) { + assert(mode != ZSTD_ps_auto); + return ZSTD_rowMatchFinderSupported(strategy) && (mode == ZSTD_ps_enable); +} + +/* Returns row matchfinder usage given an initial mode and cParams */ +static ZSTD_paramSwitch_e ZSTD_resolveRowMatchFinderMode(ZSTD_paramSwitch_e mode, + const ZSTD_compressionParameters* const cParams) { +#if defined(ZSTD_ARCH_X86_SSE2) || defined(ZSTD_ARCH_ARM_NEON) + int const kHasSIMD128 = 1; +#else + int const kHasSIMD128 = 0; +#endif + if (mode != ZSTD_ps_auto) return mode; /* if requested enabled, but no SIMD, we still will use row matchfinder */ + mode = ZSTD_ps_disable; + if (!ZSTD_rowMatchFinderSupported(cParams->strategy)) return mode; + if (kHasSIMD128) { + if (cParams->windowLog > 14) mode = ZSTD_ps_enable; + } else { + if (cParams->windowLog > 17) mode = ZSTD_ps_enable; + } + return mode; +} + +/* Returns block splitter usage (generally speaking, when using slower/stronger compression modes) */ +static ZSTD_paramSwitch_e ZSTD_resolveBlockSplitterMode(ZSTD_paramSwitch_e mode, + const ZSTD_compressionParameters* const cParams) { + if (mode != ZSTD_ps_auto) return mode; + return (cParams->strategy >= ZSTD_btopt && cParams->windowLog >= 17) ? ZSTD_ps_enable : ZSTD_ps_disable; +} + +/* Returns 1 if the arguments indicate that we should allocate a chainTable, 0 otherwise */ +static int ZSTD_allocateChainTable(const ZSTD_strategy strategy, + const ZSTD_paramSwitch_e useRowMatchFinder, + const U32 forDDSDict) { + assert(useRowMatchFinder != ZSTD_ps_auto); + /* We always should allocate a chaintable if we are allocating a matchstate for a DDS dictionary matchstate. + * We do not allocate a chaintable if we are using ZSTD_fast, or are using the row-based matchfinder. + */ + return forDDSDict || ((strategy != ZSTD_fast) && !ZSTD_rowMatchFinderUsed(strategy, useRowMatchFinder)); +} + +/* Returns 1 if compression parameters are such that we should + * enable long distance matching (wlog >= 27, strategy >= btopt). + * Returns 0 otherwise. + */ +static ZSTD_paramSwitch_e ZSTD_resolveEnableLdm(ZSTD_paramSwitch_e mode, + const ZSTD_compressionParameters* const cParams) { + if (mode != ZSTD_ps_auto) return mode; + return (cParams->strategy >= ZSTD_btopt && cParams->windowLog >= 27) ? ZSTD_ps_enable : ZSTD_ps_disable; +} + +static ZSTD_CCtx_params ZSTD_makeCCtxParamsFromCParams( + ZSTD_compressionParameters cParams) +{ + ZSTD_CCtx_params cctxParams; + /* should not matter, as all cParams are presumed properly defined */ + ZSTD_CCtxParams_init(&cctxParams, ZSTD_CLEVEL_DEFAULT); + cctxParams.cParams = cParams; + + /* Adjust advanced params according to cParams */ + cctxParams.ldmParams.enableLdm = ZSTD_resolveEnableLdm(cctxParams.ldmParams.enableLdm, &cParams); + if (cctxParams.ldmParams.enableLdm == ZSTD_ps_enable) { + ZSTD_ldm_adjustParameters(&cctxParams.ldmParams, &cParams); + assert(cctxParams.ldmParams.hashLog >= cctxParams.ldmParams.bucketSizeLog); + assert(cctxParams.ldmParams.hashRateLog < 32); + } + cctxParams.useBlockSplitter = ZSTD_resolveBlockSplitterMode(cctxParams.useBlockSplitter, &cParams); + cctxParams.useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(cctxParams.useRowMatchFinder, &cParams); + assert(!ZSTD_checkCParams(cParams)); + return cctxParams; +} + +static ZSTD_CCtx_params* ZSTD_createCCtxParams_advanced( + ZSTD_customMem customMem) +{ + ZSTD_CCtx_params* params; + if ((!customMem.customAlloc) ^ (!customMem.customFree)) return NULL; + params = (ZSTD_CCtx_params*)ZSTD_customCalloc( + sizeof(ZSTD_CCtx_params), customMem); + if (!params) { return NULL; } + ZSTD_CCtxParams_init(params, ZSTD_CLEVEL_DEFAULT); + params->customMem = customMem; + return params; +} + +ZSTD_CCtx_params* ZSTD_createCCtxParams(void) +{ + return ZSTD_createCCtxParams_advanced(ZSTD_defaultCMem); +} + +size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params) +{ + if (params == NULL) { return 0; } + ZSTD_customFree(params, params->customMem); + return 0; +} + +size_t ZSTD_CCtxParams_reset(ZSTD_CCtx_params* params) +{ + return ZSTD_CCtxParams_init(params, ZSTD_CLEVEL_DEFAULT); +} + +size_t ZSTD_CCtxParams_init(ZSTD_CCtx_params* cctxParams, int compressionLevel) { + RETURN_ERROR_IF(!cctxParams, GENERIC, "NULL pointer!"); + ZSTD_memset(cctxParams, 0, sizeof(*cctxParams)); + cctxParams->compressionLevel = compressionLevel; + cctxParams->fParams.contentSizeFlag = 1; + return 0; +} + +#define ZSTD_NO_CLEVEL 0 + +/** + * Initializes the cctxParams from params and compressionLevel. + * @param compressionLevel If params are derived from a compression level then that compression level, otherwise ZSTD_NO_CLEVEL. + */ +static void ZSTD_CCtxParams_init_internal(ZSTD_CCtx_params* cctxParams, ZSTD_parameters const* params, int compressionLevel) +{ + assert(!ZSTD_checkCParams(params->cParams)); + ZSTD_memset(cctxParams, 0, sizeof(*cctxParams)); + cctxParams->cParams = params->cParams; + cctxParams->fParams = params->fParams; + /* Should not matter, as all cParams are presumed properly defined. + * But, set it for tracing anyway. + */ + cctxParams->compressionLevel = compressionLevel; + cctxParams->useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(cctxParams->useRowMatchFinder, ¶ms->cParams); + cctxParams->useBlockSplitter = ZSTD_resolveBlockSplitterMode(cctxParams->useBlockSplitter, ¶ms->cParams); + cctxParams->ldmParams.enableLdm = ZSTD_resolveEnableLdm(cctxParams->ldmParams.enableLdm, ¶ms->cParams); + DEBUGLOG(4, "ZSTD_CCtxParams_init_internal: useRowMatchFinder=%d, useBlockSplitter=%d ldm=%d", + cctxParams->useRowMatchFinder, cctxParams->useBlockSplitter, cctxParams->ldmParams.enableLdm); +} + +size_t ZSTD_CCtxParams_init_advanced(ZSTD_CCtx_params* cctxParams, ZSTD_parameters params) +{ + RETURN_ERROR_IF(!cctxParams, GENERIC, "NULL pointer!"); + FORWARD_IF_ERROR( ZSTD_checkCParams(params.cParams) , ""); + ZSTD_CCtxParams_init_internal(cctxParams, ¶ms, ZSTD_NO_CLEVEL); + return 0; +} + +/** + * Sets cctxParams' cParams and fParams from params, but otherwise leaves them alone. + * @param param Validated zstd parameters. + */ +static void ZSTD_CCtxParams_setZstdParams( + ZSTD_CCtx_params* cctxParams, const ZSTD_parameters* params) +{ + assert(!ZSTD_checkCParams(params->cParams)); + cctxParams->cParams = params->cParams; + cctxParams->fParams = params->fParams; + /* Should not matter, as all cParams are presumed properly defined. + * But, set it for tracing anyway. + */ + cctxParams->compressionLevel = ZSTD_NO_CLEVEL; +} + +ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter param) +{ + ZSTD_bounds bounds = { 0, 0, 0 }; + + switch(param) + { + case ZSTD_c_compressionLevel: + bounds.lowerBound = ZSTD_minCLevel(); + bounds.upperBound = ZSTD_maxCLevel(); + return bounds; + + case ZSTD_c_windowLog: + bounds.lowerBound = ZSTD_WINDOWLOG_MIN; + bounds.upperBound = ZSTD_WINDOWLOG_MAX; + return bounds; + + case ZSTD_c_hashLog: + bounds.lowerBound = ZSTD_HASHLOG_MIN; + bounds.upperBound = ZSTD_HASHLOG_MAX; + return bounds; + + case ZSTD_c_chainLog: + bounds.lowerBound = ZSTD_CHAINLOG_MIN; + bounds.upperBound = ZSTD_CHAINLOG_MAX; + return bounds; + + case ZSTD_c_searchLog: + bounds.lowerBound = ZSTD_SEARCHLOG_MIN; + bounds.upperBound = ZSTD_SEARCHLOG_MAX; + return bounds; + + case ZSTD_c_minMatch: + bounds.lowerBound = ZSTD_MINMATCH_MIN; + bounds.upperBound = ZSTD_MINMATCH_MAX; + return bounds; + + case ZSTD_c_targetLength: + bounds.lowerBound = ZSTD_TARGETLENGTH_MIN; + bounds.upperBound = ZSTD_TARGETLENGTH_MAX; + return bounds; + + case ZSTD_c_strategy: + bounds.lowerBound = ZSTD_STRATEGY_MIN; + bounds.upperBound = ZSTD_STRATEGY_MAX; + return bounds; + + case ZSTD_c_contentSizeFlag: + bounds.lowerBound = 0; + bounds.upperBound = 1; + return bounds; + + case ZSTD_c_checksumFlag: + bounds.lowerBound = 0; + bounds.upperBound = 1; + return bounds; + + case ZSTD_c_dictIDFlag: + bounds.lowerBound = 0; + bounds.upperBound = 1; + return bounds; + + case ZSTD_c_nbWorkers: + bounds.lowerBound = 0; +#ifdef ZSTD_MULTITHREAD + bounds.upperBound = ZSTDMT_NBWORKERS_MAX; +#else + bounds.upperBound = 0; +#endif + return bounds; + + case ZSTD_c_jobSize: + bounds.lowerBound = 0; +#ifdef ZSTD_MULTITHREAD + bounds.upperBound = ZSTDMT_JOBSIZE_MAX; +#else + bounds.upperBound = 0; +#endif + return bounds; + + case ZSTD_c_overlapLog: +#ifdef ZSTD_MULTITHREAD + bounds.lowerBound = ZSTD_OVERLAPLOG_MIN; + bounds.upperBound = ZSTD_OVERLAPLOG_MAX; +#else + bounds.lowerBound = 0; + bounds.upperBound = 0; +#endif + return bounds; + + case ZSTD_c_enableDedicatedDictSearch: + bounds.lowerBound = 0; + bounds.upperBound = 1; + return bounds; + + case ZSTD_c_enableLongDistanceMatching: + bounds.lowerBound = 0; + bounds.upperBound = 1; + return bounds; + + case ZSTD_c_ldmHashLog: + bounds.lowerBound = ZSTD_LDM_HASHLOG_MIN; + bounds.upperBound = ZSTD_LDM_HASHLOG_MAX; + return bounds; + + case ZSTD_c_ldmMinMatch: + bounds.lowerBound = ZSTD_LDM_MINMATCH_MIN; + bounds.upperBound = ZSTD_LDM_MINMATCH_MAX; + return bounds; + + case ZSTD_c_ldmBucketSizeLog: + bounds.lowerBound = ZSTD_LDM_BUCKETSIZELOG_MIN; + bounds.upperBound = ZSTD_LDM_BUCKETSIZELOG_MAX; + return bounds; + + case ZSTD_c_ldmHashRateLog: + bounds.lowerBound = ZSTD_LDM_HASHRATELOG_MIN; + bounds.upperBound = ZSTD_LDM_HASHRATELOG_MAX; + return bounds; + + /* experimental parameters */ + case ZSTD_c_rsyncable: + bounds.lowerBound = 0; + bounds.upperBound = 1; + return bounds; + + case ZSTD_c_forceMaxWindow : + bounds.lowerBound = 0; + bounds.upperBound = 1; + return bounds; + + case ZSTD_c_format: + ZSTD_STATIC_ASSERT(ZSTD_f_zstd1 < ZSTD_f_zstd1_magicless); + bounds.lowerBound = ZSTD_f_zstd1; + bounds.upperBound = ZSTD_f_zstd1_magicless; /* note : how to ensure at compile time that this is the highest value enum ? */ + return bounds; + + case ZSTD_c_forceAttachDict: + ZSTD_STATIC_ASSERT(ZSTD_dictDefaultAttach < ZSTD_dictForceLoad); + bounds.lowerBound = ZSTD_dictDefaultAttach; + bounds.upperBound = ZSTD_dictForceLoad; /* note : how to ensure at compile time that this is the highest value enum ? */ + return bounds; + + case ZSTD_c_literalCompressionMode: + ZSTD_STATIC_ASSERT(ZSTD_ps_auto < ZSTD_ps_enable && ZSTD_ps_enable < ZSTD_ps_disable); + bounds.lowerBound = (int)ZSTD_ps_auto; + bounds.upperBound = (int)ZSTD_ps_disable; + return bounds; + + case ZSTD_c_targetCBlockSize: + bounds.lowerBound = ZSTD_TARGETCBLOCKSIZE_MIN; + bounds.upperBound = ZSTD_TARGETCBLOCKSIZE_MAX; + return bounds; + + case ZSTD_c_srcSizeHint: + bounds.lowerBound = ZSTD_SRCSIZEHINT_MIN; + bounds.upperBound = ZSTD_SRCSIZEHINT_MAX; + return bounds; + + case ZSTD_c_stableInBuffer: + case ZSTD_c_stableOutBuffer: + bounds.lowerBound = (int)ZSTD_bm_buffered; + bounds.upperBound = (int)ZSTD_bm_stable; + return bounds; + + case ZSTD_c_blockDelimiters: + bounds.lowerBound = (int)ZSTD_sf_noBlockDelimiters; + bounds.upperBound = (int)ZSTD_sf_explicitBlockDelimiters; + return bounds; + + case ZSTD_c_validateSequences: + bounds.lowerBound = 0; + bounds.upperBound = 1; + return bounds; + + case ZSTD_c_useBlockSplitter: + bounds.lowerBound = (int)ZSTD_ps_auto; + bounds.upperBound = (int)ZSTD_ps_disable; + return bounds; + + case ZSTD_c_useRowMatchFinder: + bounds.lowerBound = (int)ZSTD_ps_auto; + bounds.upperBound = (int)ZSTD_ps_disable; + return bounds; + + case ZSTD_c_deterministicRefPrefix: + bounds.lowerBound = 0; + bounds.upperBound = 1; + return bounds; + + default: + bounds.error = ERROR(parameter_unsupported); + return bounds; + } +} + +/* ZSTD_cParam_clampBounds: + * Clamps the value into the bounded range. + */ +static size_t ZSTD_cParam_clampBounds(ZSTD_cParameter cParam, int* value) +{ + ZSTD_bounds const bounds = ZSTD_cParam_getBounds(cParam); + if (ZSTD_isError(bounds.error)) return bounds.error; + if (*value < bounds.lowerBound) *value = bounds.lowerBound; + if (*value > bounds.upperBound) *value = bounds.upperBound; + return 0; +} + +#define BOUNDCHECK(cParam, val) { \ + RETURN_ERROR_IF(!ZSTD_cParam_withinBounds(cParam,val), \ + parameter_outOfBound, "Param out of bounds"); \ +} + + +static int ZSTD_isUpdateAuthorized(ZSTD_cParameter param) +{ + switch(param) + { + case ZSTD_c_compressionLevel: + case ZSTD_c_hashLog: + case ZSTD_c_chainLog: + case ZSTD_c_searchLog: + case ZSTD_c_minMatch: + case ZSTD_c_targetLength: + case ZSTD_c_strategy: + return 1; + + case ZSTD_c_format: + case ZSTD_c_windowLog: + case ZSTD_c_contentSizeFlag: + case ZSTD_c_checksumFlag: + case ZSTD_c_dictIDFlag: + case ZSTD_c_forceMaxWindow : + case ZSTD_c_nbWorkers: + case ZSTD_c_jobSize: + case ZSTD_c_overlapLog: + case ZSTD_c_rsyncable: + case ZSTD_c_enableDedicatedDictSearch: + case ZSTD_c_enableLongDistanceMatching: + case ZSTD_c_ldmHashLog: + case ZSTD_c_ldmMinMatch: + case ZSTD_c_ldmBucketSizeLog: + case ZSTD_c_ldmHashRateLog: + case ZSTD_c_forceAttachDict: + case ZSTD_c_literalCompressionMode: + case ZSTD_c_targetCBlockSize: + case ZSTD_c_srcSizeHint: + case ZSTD_c_stableInBuffer: + case ZSTD_c_stableOutBuffer: + case ZSTD_c_blockDelimiters: + case ZSTD_c_validateSequences: + case ZSTD_c_useBlockSplitter: + case ZSTD_c_useRowMatchFinder: + case ZSTD_c_deterministicRefPrefix: + default: + return 0; + } +} + +size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int value) +{ + DEBUGLOG(4, "ZSTD_CCtx_setParameter (%i, %i)", (int)param, value); + if (cctx->streamStage != zcss_init) { + if (ZSTD_isUpdateAuthorized(param)) { + cctx->cParamsChanged = 1; + } else { + RETURN_ERROR(stage_wrong, "can only set params in ctx init stage"); + } } + + switch(param) + { + case ZSTD_c_nbWorkers: + RETURN_ERROR_IF((value!=0) && cctx->staticSize, parameter_unsupported, + "MT not compatible with static alloc"); + break; + + case ZSTD_c_compressionLevel: + case ZSTD_c_windowLog: + case ZSTD_c_hashLog: + case ZSTD_c_chainLog: + case ZSTD_c_searchLog: + case ZSTD_c_minMatch: + case ZSTD_c_targetLength: + case ZSTD_c_strategy: + case ZSTD_c_ldmHashRateLog: + case ZSTD_c_format: + case ZSTD_c_contentSizeFlag: + case ZSTD_c_checksumFlag: + case ZSTD_c_dictIDFlag: + case ZSTD_c_forceMaxWindow: + case ZSTD_c_forceAttachDict: + case ZSTD_c_literalCompressionMode: + case ZSTD_c_jobSize: + case ZSTD_c_overlapLog: + case ZSTD_c_rsyncable: + case ZSTD_c_enableDedicatedDictSearch: + case ZSTD_c_enableLongDistanceMatching: + case ZSTD_c_ldmHashLog: + case ZSTD_c_ldmMinMatch: + case ZSTD_c_ldmBucketSizeLog: + case ZSTD_c_targetCBlockSize: + case ZSTD_c_srcSizeHint: + case ZSTD_c_stableInBuffer: + case ZSTD_c_stableOutBuffer: + case ZSTD_c_blockDelimiters: + case ZSTD_c_validateSequences: + case ZSTD_c_useBlockSplitter: + case ZSTD_c_useRowMatchFinder: + case ZSTD_c_deterministicRefPrefix: + break; + + default: RETURN_ERROR(parameter_unsupported, "unknown parameter"); + } + return ZSTD_CCtxParams_setParameter(&cctx->requestedParams, param, value); +} + +size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* CCtxParams, + ZSTD_cParameter param, int value) +{ + DEBUGLOG(4, "ZSTD_CCtxParams_setParameter (%i, %i)", (int)param, value); + switch(param) + { + case ZSTD_c_format : + BOUNDCHECK(ZSTD_c_format, value); + CCtxParams->format = (ZSTD_format_e)value; + return (size_t)CCtxParams->format; + + case ZSTD_c_compressionLevel : { + FORWARD_IF_ERROR(ZSTD_cParam_clampBounds(param, &value), ""); + if (value == 0) + CCtxParams->compressionLevel = ZSTD_CLEVEL_DEFAULT; /* 0 == default */ + else + CCtxParams->compressionLevel = value; + if (CCtxParams->compressionLevel >= 0) return (size_t)CCtxParams->compressionLevel; + return 0; /* return type (size_t) cannot represent negative values */ + } + + case ZSTD_c_windowLog : + if (value!=0) /* 0 => use default */ + BOUNDCHECK(ZSTD_c_windowLog, value); + CCtxParams->cParams.windowLog = (U32)value; + return CCtxParams->cParams.windowLog; + + case ZSTD_c_hashLog : + if (value!=0) /* 0 => use default */ + BOUNDCHECK(ZSTD_c_hashLog, value); + CCtxParams->cParams.hashLog = (U32)value; + return CCtxParams->cParams.hashLog; + + case ZSTD_c_chainLog : + if (value!=0) /* 0 => use default */ + BOUNDCHECK(ZSTD_c_chainLog, value); + CCtxParams->cParams.chainLog = (U32)value; + return CCtxParams->cParams.chainLog; + + case ZSTD_c_searchLog : + if (value!=0) /* 0 => use default */ + BOUNDCHECK(ZSTD_c_searchLog, value); + CCtxParams->cParams.searchLog = (U32)value; + return (size_t)value; + + case ZSTD_c_minMatch : + if (value!=0) /* 0 => use default */ + BOUNDCHECK(ZSTD_c_minMatch, value); + CCtxParams->cParams.minMatch = value; + return CCtxParams->cParams.minMatch; + + case ZSTD_c_targetLength : + BOUNDCHECK(ZSTD_c_targetLength, value); + CCtxParams->cParams.targetLength = value; + return CCtxParams->cParams.targetLength; + + case ZSTD_c_strategy : + if (value!=0) /* 0 => use default */ + BOUNDCHECK(ZSTD_c_strategy, value); + CCtxParams->cParams.strategy = (ZSTD_strategy)value; + return (size_t)CCtxParams->cParams.strategy; + + case ZSTD_c_contentSizeFlag : + /* Content size written in frame header _when known_ (default:1) */ + DEBUGLOG(4, "set content size flag = %u", (value!=0)); + CCtxParams->fParams.contentSizeFlag = value != 0; + return CCtxParams->fParams.contentSizeFlag; + + case ZSTD_c_checksumFlag : + /* A 32-bits content checksum will be calculated and written at end of frame (default:0) */ + CCtxParams->fParams.checksumFlag = value != 0; + return CCtxParams->fParams.checksumFlag; + + case ZSTD_c_dictIDFlag : /* When applicable, dictionary's dictID is provided in frame header (default:1) */ + DEBUGLOG(4, "set dictIDFlag = %u", (value!=0)); + CCtxParams->fParams.noDictIDFlag = !value; + return !CCtxParams->fParams.noDictIDFlag; + + case ZSTD_c_forceMaxWindow : + CCtxParams->forceWindow = (value != 0); + return CCtxParams->forceWindow; + + case ZSTD_c_forceAttachDict : { + const ZSTD_dictAttachPref_e pref = (ZSTD_dictAttachPref_e)value; + BOUNDCHECK(ZSTD_c_forceAttachDict, pref); + CCtxParams->attachDictPref = pref; + return CCtxParams->attachDictPref; + } + + case ZSTD_c_literalCompressionMode : { + const ZSTD_paramSwitch_e lcm = (ZSTD_paramSwitch_e)value; + BOUNDCHECK(ZSTD_c_literalCompressionMode, lcm); + CCtxParams->literalCompressionMode = lcm; + return CCtxParams->literalCompressionMode; + } + + case ZSTD_c_nbWorkers : +#ifndef ZSTD_MULTITHREAD + RETURN_ERROR_IF(value!=0, parameter_unsupported, "not compiled with multithreading"); + return 0; +#else + FORWARD_IF_ERROR(ZSTD_cParam_clampBounds(param, &value), ""); + CCtxParams->nbWorkers = value; + return CCtxParams->nbWorkers; +#endif + + case ZSTD_c_jobSize : +#ifndef ZSTD_MULTITHREAD + RETURN_ERROR_IF(value!=0, parameter_unsupported, "not compiled with multithreading"); + return 0; +#else + /* Adjust to the minimum non-default value. */ + if (value != 0 && value < ZSTDMT_JOBSIZE_MIN) + value = ZSTDMT_JOBSIZE_MIN; + FORWARD_IF_ERROR(ZSTD_cParam_clampBounds(param, &value), ""); + assert(value >= 0); + CCtxParams->jobSize = value; + return CCtxParams->jobSize; +#endif + + case ZSTD_c_overlapLog : +#ifndef ZSTD_MULTITHREAD + RETURN_ERROR_IF(value!=0, parameter_unsupported, "not compiled with multithreading"); + return 0; +#else + FORWARD_IF_ERROR(ZSTD_cParam_clampBounds(ZSTD_c_overlapLog, &value), ""); + CCtxParams->overlapLog = value; + return CCtxParams->overlapLog; +#endif + + case ZSTD_c_rsyncable : +#ifndef ZSTD_MULTITHREAD + RETURN_ERROR_IF(value!=0, parameter_unsupported, "not compiled with multithreading"); + return 0; +#else + FORWARD_IF_ERROR(ZSTD_cParam_clampBounds(ZSTD_c_overlapLog, &value), ""); + CCtxParams->rsyncable = value; + return CCtxParams->rsyncable; +#endif + + case ZSTD_c_enableDedicatedDictSearch : + CCtxParams->enableDedicatedDictSearch = (value!=0); + return CCtxParams->enableDedicatedDictSearch; + + case ZSTD_c_enableLongDistanceMatching : + CCtxParams->ldmParams.enableLdm = (ZSTD_paramSwitch_e)value; + return CCtxParams->ldmParams.enableLdm; + + case ZSTD_c_ldmHashLog : + if (value!=0) /* 0 ==> auto */ + BOUNDCHECK(ZSTD_c_ldmHashLog, value); + CCtxParams->ldmParams.hashLog = value; + return CCtxParams->ldmParams.hashLog; + + case ZSTD_c_ldmMinMatch : + if (value!=0) /* 0 ==> default */ + BOUNDCHECK(ZSTD_c_ldmMinMatch, value); + CCtxParams->ldmParams.minMatchLength = value; + return CCtxParams->ldmParams.minMatchLength; + + case ZSTD_c_ldmBucketSizeLog : + if (value!=0) /* 0 ==> default */ + BOUNDCHECK(ZSTD_c_ldmBucketSizeLog, value); + CCtxParams->ldmParams.bucketSizeLog = value; + return CCtxParams->ldmParams.bucketSizeLog; + + case ZSTD_c_ldmHashRateLog : + if (value!=0) /* 0 ==> default */ + BOUNDCHECK(ZSTD_c_ldmHashRateLog, value); + CCtxParams->ldmParams.hashRateLog = value; + return CCtxParams->ldmParams.hashRateLog; + + case ZSTD_c_targetCBlockSize : + if (value!=0) /* 0 ==> default */ + BOUNDCHECK(ZSTD_c_targetCBlockSize, value); + CCtxParams->targetCBlockSize = value; + return CCtxParams->targetCBlockSize; + + case ZSTD_c_srcSizeHint : + if (value!=0) /* 0 ==> default */ + BOUNDCHECK(ZSTD_c_srcSizeHint, value); + CCtxParams->srcSizeHint = value; + return CCtxParams->srcSizeHint; + + case ZSTD_c_stableInBuffer: + BOUNDCHECK(ZSTD_c_stableInBuffer, value); + CCtxParams->inBufferMode = (ZSTD_bufferMode_e)value; + return CCtxParams->inBufferMode; + + case ZSTD_c_stableOutBuffer: + BOUNDCHECK(ZSTD_c_stableOutBuffer, value); + CCtxParams->outBufferMode = (ZSTD_bufferMode_e)value; + return CCtxParams->outBufferMode; + + case ZSTD_c_blockDelimiters: + BOUNDCHECK(ZSTD_c_blockDelimiters, value); + CCtxParams->blockDelimiters = (ZSTD_sequenceFormat_e)value; + return CCtxParams->blockDelimiters; + + case ZSTD_c_validateSequences: + BOUNDCHECK(ZSTD_c_validateSequences, value); + CCtxParams->validateSequences = value; + return CCtxParams->validateSequences; + + case ZSTD_c_useBlockSplitter: + BOUNDCHECK(ZSTD_c_useBlockSplitter, value); + CCtxParams->useBlockSplitter = (ZSTD_paramSwitch_e)value; + return CCtxParams->useBlockSplitter; + + case ZSTD_c_useRowMatchFinder: + BOUNDCHECK(ZSTD_c_useRowMatchFinder, value); + CCtxParams->useRowMatchFinder = (ZSTD_paramSwitch_e)value; + return CCtxParams->useRowMatchFinder; + + case ZSTD_c_deterministicRefPrefix: + BOUNDCHECK(ZSTD_c_deterministicRefPrefix, value); + CCtxParams->deterministicRefPrefix = !!value; + return CCtxParams->deterministicRefPrefix; + + default: RETURN_ERROR(parameter_unsupported, "unknown parameter"); + } +} + +size_t ZSTD_CCtx_getParameter(ZSTD_CCtx const* cctx, ZSTD_cParameter param, int* value) +{ + return ZSTD_CCtxParams_getParameter(&cctx->requestedParams, param, value); +} + +size_t ZSTD_CCtxParams_getParameter( + ZSTD_CCtx_params const* CCtxParams, ZSTD_cParameter param, int* value) +{ + switch(param) + { + case ZSTD_c_format : + *value = CCtxParams->format; + break; + case ZSTD_c_compressionLevel : + *value = CCtxParams->compressionLevel; + break; + case ZSTD_c_windowLog : + *value = (int)CCtxParams->cParams.windowLog; + break; + case ZSTD_c_hashLog : + *value = (int)CCtxParams->cParams.hashLog; + break; + case ZSTD_c_chainLog : + *value = (int)CCtxParams->cParams.chainLog; + break; + case ZSTD_c_searchLog : + *value = CCtxParams->cParams.searchLog; + break; + case ZSTD_c_minMatch : + *value = CCtxParams->cParams.minMatch; + break; + case ZSTD_c_targetLength : + *value = CCtxParams->cParams.targetLength; + break; + case ZSTD_c_strategy : + *value = (unsigned)CCtxParams->cParams.strategy; + break; + case ZSTD_c_contentSizeFlag : + *value = CCtxParams->fParams.contentSizeFlag; + break; + case ZSTD_c_checksumFlag : + *value = CCtxParams->fParams.checksumFlag; + break; + case ZSTD_c_dictIDFlag : + *value = !CCtxParams->fParams.noDictIDFlag; + break; + case ZSTD_c_forceMaxWindow : + *value = CCtxParams->forceWindow; + break; + case ZSTD_c_forceAttachDict : + *value = CCtxParams->attachDictPref; + break; + case ZSTD_c_literalCompressionMode : + *value = CCtxParams->literalCompressionMode; + break; + case ZSTD_c_nbWorkers : +#ifndef ZSTD_MULTITHREAD + assert(CCtxParams->nbWorkers == 0); +#endif + *value = CCtxParams->nbWorkers; + break; + case ZSTD_c_jobSize : +#ifndef ZSTD_MULTITHREAD + RETURN_ERROR(parameter_unsupported, "not compiled with multithreading"); +#else + assert(CCtxParams->jobSize <= INT_MAX); + *value = (int)CCtxParams->jobSize; + break; +#endif + case ZSTD_c_overlapLog : +#ifndef ZSTD_MULTITHREAD + RETURN_ERROR(parameter_unsupported, "not compiled with multithreading"); +#else + *value = CCtxParams->overlapLog; + break; +#endif + case ZSTD_c_rsyncable : +#ifndef ZSTD_MULTITHREAD + RETURN_ERROR(parameter_unsupported, "not compiled with multithreading"); +#else + *value = CCtxParams->rsyncable; + break; +#endif + case ZSTD_c_enableDedicatedDictSearch : + *value = CCtxParams->enableDedicatedDictSearch; + break; + case ZSTD_c_enableLongDistanceMatching : + *value = CCtxParams->ldmParams.enableLdm; + break; + case ZSTD_c_ldmHashLog : + *value = CCtxParams->ldmParams.hashLog; + break; + case ZSTD_c_ldmMinMatch : + *value = CCtxParams->ldmParams.minMatchLength; + break; + case ZSTD_c_ldmBucketSizeLog : + *value = CCtxParams->ldmParams.bucketSizeLog; + break; + case ZSTD_c_ldmHashRateLog : + *value = CCtxParams->ldmParams.hashRateLog; + break; + case ZSTD_c_targetCBlockSize : + *value = (int)CCtxParams->targetCBlockSize; + break; + case ZSTD_c_srcSizeHint : + *value = (int)CCtxParams->srcSizeHint; + break; + case ZSTD_c_stableInBuffer : + *value = (int)CCtxParams->inBufferMode; + break; + case ZSTD_c_stableOutBuffer : + *value = (int)CCtxParams->outBufferMode; + break; + case ZSTD_c_blockDelimiters : + *value = (int)CCtxParams->blockDelimiters; + break; + case ZSTD_c_validateSequences : + *value = (int)CCtxParams->validateSequences; + break; + case ZSTD_c_useBlockSplitter : + *value = (int)CCtxParams->useBlockSplitter; + break; + case ZSTD_c_useRowMatchFinder : + *value = (int)CCtxParams->useRowMatchFinder; + break; + case ZSTD_c_deterministicRefPrefix: + *value = (int)CCtxParams->deterministicRefPrefix; + break; + default: RETURN_ERROR(parameter_unsupported, "unknown parameter"); + } + return 0; +} + +/** ZSTD_CCtx_setParametersUsingCCtxParams() : + * just applies `params` into `cctx` + * no action is performed, parameters are merely stored. + * If ZSTDMT is enabled, parameters are pushed to cctx->mtctx. + * This is possible even if a compression is ongoing. + * In which case, new parameters will be applied on the fly, starting with next compression job. + */ +size_t ZSTD_CCtx_setParametersUsingCCtxParams( + ZSTD_CCtx* cctx, const ZSTD_CCtx_params* params) +{ + DEBUGLOG(4, "ZSTD_CCtx_setParametersUsingCCtxParams"); + RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong, + "The context is in the wrong stage!"); + RETURN_ERROR_IF(cctx->cdict, stage_wrong, + "Can't override parameters with cdict attached (some must " + "be inherited from the cdict)."); + + cctx->requestedParams = *params; + return 0; +} + +size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long long pledgedSrcSize) +{ + DEBUGLOG(4, "ZSTD_CCtx_setPledgedSrcSize to %u bytes", (U32)pledgedSrcSize); + RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong, + "Can't set pledgedSrcSize when not in init stage."); + cctx->pledgedSrcSizePlusOne = pledgedSrcSize+1; + return 0; +} + +static ZSTD_compressionParameters ZSTD_dedicatedDictSearch_getCParams( + int const compressionLevel, + size_t const dictSize); +static int ZSTD_dedicatedDictSearch_isSupported( + const ZSTD_compressionParameters* cParams); +static void ZSTD_dedicatedDictSearch_revertCParams( + ZSTD_compressionParameters* cParams); + +/** + * Initializes the local dict using the requested parameters. + * NOTE: This does not use the pledged src size, because it may be used for more + * than one compression. + */ +static size_t ZSTD_initLocalDict(ZSTD_CCtx* cctx) +{ + ZSTD_localDict* const dl = &cctx->localDict; + if (dl->dict == NULL) { + /* No local dictionary. */ + assert(dl->dictBuffer == NULL); + assert(dl->cdict == NULL); + assert(dl->dictSize == 0); + return 0; + } + if (dl->cdict != NULL) { + assert(cctx->cdict == dl->cdict); + /* Local dictionary already initialized. */ + return 0; + } + assert(dl->dictSize > 0); + assert(cctx->cdict == NULL); + assert(cctx->prefixDict.dict == NULL); + + dl->cdict = ZSTD_createCDict_advanced2( + dl->dict, + dl->dictSize, + ZSTD_dlm_byRef, + dl->dictContentType, + &cctx->requestedParams, + cctx->customMem); + RETURN_ERROR_IF(!dl->cdict, memory_allocation, "ZSTD_createCDict_advanced failed"); + cctx->cdict = dl->cdict; + return 0; +} + +size_t ZSTD_CCtx_loadDictionary_advanced( + ZSTD_CCtx* cctx, const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType) +{ + RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong, + "Can't load a dictionary when ctx is not in init stage."); + DEBUGLOG(4, "ZSTD_CCtx_loadDictionary_advanced (size: %u)", (U32)dictSize); + ZSTD_clearAllDicts(cctx); /* in case one already exists */ + if (dict == NULL || dictSize == 0) /* no dictionary mode */ + return 0; + if (dictLoadMethod == ZSTD_dlm_byRef) { + cctx->localDict.dict = dict; + } else { + void* dictBuffer; + RETURN_ERROR_IF(cctx->staticSize, memory_allocation, + "no malloc for static CCtx"); + dictBuffer = ZSTD_customMalloc(dictSize, cctx->customMem); + RETURN_ERROR_IF(!dictBuffer, memory_allocation, "NULL pointer!"); + ZSTD_memcpy(dictBuffer, dict, dictSize); + cctx->localDict.dictBuffer = dictBuffer; + cctx->localDict.dict = dictBuffer; + } + cctx->localDict.dictSize = dictSize; + cctx->localDict.dictContentType = dictContentType; + return 0; +} + +size_t ZSTD_CCtx_loadDictionary_byReference( + ZSTD_CCtx* cctx, const void* dict, size_t dictSize) +{ + return ZSTD_CCtx_loadDictionary_advanced( + cctx, dict, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto); +} + +size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize) +{ + return ZSTD_CCtx_loadDictionary_advanced( + cctx, dict, dictSize, ZSTD_dlm_byCopy, ZSTD_dct_auto); +} + + +size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict) +{ + RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong, + "Can't ref a dict when ctx not in init stage."); + /* Free the existing local cdict (if any) to save memory. */ + ZSTD_clearAllDicts(cctx); + cctx->cdict = cdict; + return 0; +} + +size_t ZSTD_CCtx_refThreadPool(ZSTD_CCtx* cctx, ZSTD_threadPool* pool) +{ + RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong, + "Can't ref a pool when ctx not in init stage."); + cctx->pool = pool; + return 0; +} + +size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize) +{ + return ZSTD_CCtx_refPrefix_advanced(cctx, prefix, prefixSize, ZSTD_dct_rawContent); +} + +size_t ZSTD_CCtx_refPrefix_advanced( + ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType) +{ + RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong, + "Can't ref a prefix when ctx not in init stage."); + ZSTD_clearAllDicts(cctx); + if (prefix != NULL && prefixSize > 0) { + cctx->prefixDict.dict = prefix; + cctx->prefixDict.dictSize = prefixSize; + cctx->prefixDict.dictContentType = dictContentType; + } + return 0; +} + +/*! ZSTD_CCtx_reset() : + * Also dumps dictionary */ +size_t ZSTD_CCtx_reset(ZSTD_CCtx* cctx, ZSTD_ResetDirective reset) +{ + if ( (reset == ZSTD_reset_session_only) + || (reset == ZSTD_reset_session_and_parameters) ) { + cctx->streamStage = zcss_init; + cctx->pledgedSrcSizePlusOne = 0; + } + if ( (reset == ZSTD_reset_parameters) + || (reset == ZSTD_reset_session_and_parameters) ) { + RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong, + "Can't reset parameters only when not in init stage."); + ZSTD_clearAllDicts(cctx); + return ZSTD_CCtxParams_reset(&cctx->requestedParams); + } + return 0; +} + + +/** ZSTD_checkCParams() : + control CParam values remain within authorized range. + @return : 0, or an error code if one value is beyond authorized range */ +size_t ZSTD_checkCParams(ZSTD_compressionParameters cParams) +{ + BOUNDCHECK(ZSTD_c_windowLog, (int)cParams.windowLog); + BOUNDCHECK(ZSTD_c_chainLog, (int)cParams.chainLog); + BOUNDCHECK(ZSTD_c_hashLog, (int)cParams.hashLog); + BOUNDCHECK(ZSTD_c_searchLog, (int)cParams.searchLog); + BOUNDCHECK(ZSTD_c_minMatch, (int)cParams.minMatch); + BOUNDCHECK(ZSTD_c_targetLength,(int)cParams.targetLength); + BOUNDCHECK(ZSTD_c_strategy, cParams.strategy); + return 0; +} + +/** ZSTD_clampCParams() : + * make CParam values within valid range. + * @return : valid CParams */ +static ZSTD_compressionParameters +ZSTD_clampCParams(ZSTD_compressionParameters cParams) +{ +# define CLAMP_TYPE(cParam, val, type) { \ + ZSTD_bounds const bounds = ZSTD_cParam_getBounds(cParam); \ + if ((int)valbounds.upperBound) val=(type)bounds.upperBound; \ + } +# define CLAMP(cParam, val) CLAMP_TYPE(cParam, val, unsigned) + CLAMP(ZSTD_c_windowLog, cParams.windowLog); + CLAMP(ZSTD_c_chainLog, cParams.chainLog); + CLAMP(ZSTD_c_hashLog, cParams.hashLog); + CLAMP(ZSTD_c_searchLog, cParams.searchLog); + CLAMP(ZSTD_c_minMatch, cParams.minMatch); + CLAMP(ZSTD_c_targetLength,cParams.targetLength); + CLAMP_TYPE(ZSTD_c_strategy,cParams.strategy, ZSTD_strategy); + return cParams; +} + +/** ZSTD_cycleLog() : + * condition for correct operation : hashLog > 1 */ +U32 ZSTD_cycleLog(U32 hashLog, ZSTD_strategy strat) +{ + U32 const btScale = ((U32)strat >= (U32)ZSTD_btlazy2); + return hashLog - btScale; +} + +/** ZSTD_dictAndWindowLog() : + * Returns an adjusted window log that is large enough to fit the source and the dictionary. + * The zstd format says that the entire dictionary is valid if one byte of the dictionary + * is within the window. So the hashLog and chainLog should be large enough to reference both + * the dictionary and the window. So we must use this adjusted dictAndWindowLog when downsizing + * the hashLog and windowLog. + * NOTE: srcSize must not be ZSTD_CONTENTSIZE_UNKNOWN. + */ +static U32 ZSTD_dictAndWindowLog(U32 windowLog, U64 srcSize, U64 dictSize) +{ + const U64 maxWindowSize = 1ULL << ZSTD_WINDOWLOG_MAX; + /* No dictionary ==> No change */ + if (dictSize == 0) { + return windowLog; + } + assert(windowLog <= ZSTD_WINDOWLOG_MAX); + assert(srcSize != ZSTD_CONTENTSIZE_UNKNOWN); /* Handled in ZSTD_adjustCParams_internal() */ + { + U64 const windowSize = 1ULL << windowLog; + U64 const dictAndWindowSize = dictSize + windowSize; + /* If the window size is already large enough to fit both the source and the dictionary + * then just use the window size. Otherwise adjust so that it fits the dictionary and + * the window. + */ + if (windowSize >= dictSize + srcSize) { + return windowLog; /* Window size large enough already */ + } else if (dictAndWindowSize >= maxWindowSize) { + return ZSTD_WINDOWLOG_MAX; /* Larger than max window log */ + } else { + return ZSTD_highbit32((U32)dictAndWindowSize - 1) + 1; + } + } +} + +/** ZSTD_adjustCParams_internal() : + * optimize `cPar` for a specified input (`srcSize` and `dictSize`). + * mostly downsize to reduce memory consumption and initialization latency. + * `srcSize` can be ZSTD_CONTENTSIZE_UNKNOWN when not known. + * `mode` is the mode for parameter adjustment. See docs for `ZSTD_cParamMode_e`. + * note : `srcSize==0` means 0! + * condition : cPar is presumed validated (can be checked using ZSTD_checkCParams()). */ +static ZSTD_compressionParameters +ZSTD_adjustCParams_internal(ZSTD_compressionParameters cPar, + unsigned long long srcSize, + size_t dictSize, + ZSTD_cParamMode_e mode) +{ + const U64 minSrcSize = 513; /* (1<<9) + 1 */ + const U64 maxWindowResize = 1ULL << (ZSTD_WINDOWLOG_MAX-1); + assert(ZSTD_checkCParams(cPar)==0); + + switch (mode) { + case ZSTD_cpm_unknown: + case ZSTD_cpm_noAttachDict: + /* If we don't know the source size, don't make any + * assumptions about it. We will already have selected + * smaller parameters if a dictionary is in use. + */ + break; + case ZSTD_cpm_createCDict: + /* Assume a small source size when creating a dictionary + * with an unknown source size. + */ + if (dictSize && srcSize == ZSTD_CONTENTSIZE_UNKNOWN) + srcSize = minSrcSize; + break; + case ZSTD_cpm_attachDict: + /* Dictionary has its own dedicated parameters which have + * already been selected. We are selecting parameters + * for only the source. + */ + dictSize = 0; + break; + default: + assert(0); + break; + } + + /* resize windowLog if input is small enough, to use less memory */ + if ( (srcSize < maxWindowResize) + && (dictSize < maxWindowResize) ) { + U32 const tSize = (U32)(srcSize + dictSize); + static U32 const hashSizeMin = 1 << ZSTD_HASHLOG_MIN; + U32 const srcLog = (tSize < hashSizeMin) ? ZSTD_HASHLOG_MIN : + ZSTD_highbit32(tSize-1) + 1; + if (cPar.windowLog > srcLog) cPar.windowLog = srcLog; + } + if (srcSize != ZSTD_CONTENTSIZE_UNKNOWN) { + U32 const dictAndWindowLog = ZSTD_dictAndWindowLog(cPar.windowLog, (U64)srcSize, (U64)dictSize); + U32 const cycleLog = ZSTD_cycleLog(cPar.chainLog, cPar.strategy); + if (cPar.hashLog > dictAndWindowLog+1) cPar.hashLog = dictAndWindowLog+1; + if (cycleLog > dictAndWindowLog) + cPar.chainLog -= (cycleLog - dictAndWindowLog); + } + + if (cPar.windowLog < ZSTD_WINDOWLOG_ABSOLUTEMIN) + cPar.windowLog = ZSTD_WINDOWLOG_ABSOLUTEMIN; /* minimum wlog required for valid frame header */ + + return cPar; +} + +ZSTD_compressionParameters +ZSTD_adjustCParams(ZSTD_compressionParameters cPar, + unsigned long long srcSize, + size_t dictSize) +{ + cPar = ZSTD_clampCParams(cPar); /* resulting cPar is necessarily valid (all parameters within range) */ + if (srcSize == 0) srcSize = ZSTD_CONTENTSIZE_UNKNOWN; + return ZSTD_adjustCParams_internal(cPar, srcSize, dictSize, ZSTD_cpm_unknown); +} + +static ZSTD_compressionParameters ZSTD_getCParams_internal(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize, ZSTD_cParamMode_e mode); +static ZSTD_parameters ZSTD_getParams_internal(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize, ZSTD_cParamMode_e mode); + +static void ZSTD_overrideCParams( + ZSTD_compressionParameters* cParams, + const ZSTD_compressionParameters* overrides) +{ + if (overrides->windowLog) cParams->windowLog = overrides->windowLog; + if (overrides->hashLog) cParams->hashLog = overrides->hashLog; + if (overrides->chainLog) cParams->chainLog = overrides->chainLog; + if (overrides->searchLog) cParams->searchLog = overrides->searchLog; + if (overrides->minMatch) cParams->minMatch = overrides->minMatch; + if (overrides->targetLength) cParams->targetLength = overrides->targetLength; + if (overrides->strategy) cParams->strategy = overrides->strategy; +} + +ZSTD_compressionParameters ZSTD_getCParamsFromCCtxParams( + const ZSTD_CCtx_params* CCtxParams, U64 srcSizeHint, size_t dictSize, ZSTD_cParamMode_e mode) +{ + ZSTD_compressionParameters cParams; + if (srcSizeHint == ZSTD_CONTENTSIZE_UNKNOWN && CCtxParams->srcSizeHint > 0) { + srcSizeHint = CCtxParams->srcSizeHint; + } + cParams = ZSTD_getCParams_internal(CCtxParams->compressionLevel, srcSizeHint, dictSize, mode); + if (CCtxParams->ldmParams.enableLdm == ZSTD_ps_enable) cParams.windowLog = ZSTD_LDM_DEFAULT_WINDOW_LOG; + ZSTD_overrideCParams(&cParams, &CCtxParams->cParams); + assert(!ZSTD_checkCParams(cParams)); + /* srcSizeHint == 0 means 0 */ + return ZSTD_adjustCParams_internal(cParams, srcSizeHint, dictSize, mode); +} + +static size_t +ZSTD_sizeof_matchState(const ZSTD_compressionParameters* const cParams, + const ZSTD_paramSwitch_e useRowMatchFinder, + const U32 enableDedicatedDictSearch, + const U32 forCCtx) +{ + /* chain table size should be 0 for fast or row-hash strategies */ + size_t const chainSize = ZSTD_allocateChainTable(cParams->strategy, useRowMatchFinder, enableDedicatedDictSearch && !forCCtx) + ? ((size_t)1 << cParams->chainLog) + : 0; + size_t const hSize = ((size_t)1) << cParams->hashLog; + U32 const hashLog3 = (forCCtx && cParams->minMatch==3) ? MIN(ZSTD_HASHLOG3_MAX, cParams->windowLog) : 0; + size_t const h3Size = hashLog3 ? ((size_t)1) << hashLog3 : 0; + /* We don't use ZSTD_cwksp_alloc_size() here because the tables aren't + * surrounded by redzones in ASAN. */ + size_t const tableSpace = chainSize * sizeof(U32) + + hSize * sizeof(U32) + + h3Size * sizeof(U32); + size_t const optPotentialSpace = + ZSTD_cwksp_aligned_alloc_size((MaxML+1) * sizeof(U32)) + + ZSTD_cwksp_aligned_alloc_size((MaxLL+1) * sizeof(U32)) + + ZSTD_cwksp_aligned_alloc_size((MaxOff+1) * sizeof(U32)) + + ZSTD_cwksp_aligned_alloc_size((1<strategy, useRowMatchFinder) + ? ZSTD_cwksp_aligned_alloc_size(hSize*sizeof(U16)) + : 0; + size_t const optSpace = (forCCtx && (cParams->strategy >= ZSTD_btopt)) + ? optPotentialSpace + : 0; + size_t const slackSpace = ZSTD_cwksp_slack_space_required(); + + /* tables are guaranteed to be sized in multiples of 64 bytes (or 16 uint32_t) */ + ZSTD_STATIC_ASSERT(ZSTD_HASHLOG_MIN >= 4 && ZSTD_WINDOWLOG_MIN >= 4 && ZSTD_CHAINLOG_MIN >= 4); + assert(useRowMatchFinder != ZSTD_ps_auto); + + DEBUGLOG(4, "chainSize: %u - hSize: %u - h3Size: %u", + (U32)chainSize, (U32)hSize, (U32)h3Size); + return tableSpace + optSpace + slackSpace + lazyAdditionalSpace; +} + +static size_t ZSTD_estimateCCtxSize_usingCCtxParams_internal( + const ZSTD_compressionParameters* cParams, + const ldmParams_t* ldmParams, + const int isStatic, + const ZSTD_paramSwitch_e useRowMatchFinder, + const size_t buffInSize, + const size_t buffOutSize, + const U64 pledgedSrcSize) +{ + size_t const windowSize = (size_t) BOUNDED(1ULL, 1ULL << cParams->windowLog, pledgedSrcSize); + size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, windowSize); + U32 const divider = (cParams->minMatch==3) ? 3 : 4; + size_t const maxNbSeq = blockSize / divider; + size_t const tokenSpace = ZSTD_cwksp_alloc_size(WILDCOPY_OVERLENGTH + blockSize) + + ZSTD_cwksp_aligned_alloc_size(maxNbSeq * sizeof(seqDef)) + + 3 * ZSTD_cwksp_alloc_size(maxNbSeq * sizeof(BYTE)); + size_t const entropySpace = ZSTD_cwksp_alloc_size(ENTROPY_WORKSPACE_SIZE); + size_t const blockStateSpace = 2 * ZSTD_cwksp_alloc_size(sizeof(ZSTD_compressedBlockState_t)); + size_t const matchStateSize = ZSTD_sizeof_matchState(cParams, useRowMatchFinder, /* enableDedicatedDictSearch */ 0, /* forCCtx */ 1); + + size_t const ldmSpace = ZSTD_ldm_getTableSize(*ldmParams); + size_t const maxNbLdmSeq = ZSTD_ldm_getMaxNbSeq(*ldmParams, blockSize); + size_t const ldmSeqSpace = ldmParams->enableLdm == ZSTD_ps_enable ? + ZSTD_cwksp_aligned_alloc_size(maxNbLdmSeq * sizeof(rawSeq)) : 0; + + + size_t const bufferSpace = ZSTD_cwksp_alloc_size(buffInSize) + + ZSTD_cwksp_alloc_size(buffOutSize); + + size_t const cctxSpace = isStatic ? ZSTD_cwksp_alloc_size(sizeof(ZSTD_CCtx)) : 0; + + size_t const neededSpace = + cctxSpace + + entropySpace + + blockStateSpace + + ldmSpace + + ldmSeqSpace + + matchStateSize + + tokenSpace + + bufferSpace; + + DEBUGLOG(5, "estimate workspace : %u", (U32)neededSpace); + return neededSpace; +} + +size_t ZSTD_estimateCCtxSize_usingCCtxParams(const ZSTD_CCtx_params* params) +{ + ZSTD_compressionParameters const cParams = + ZSTD_getCParamsFromCCtxParams(params, ZSTD_CONTENTSIZE_UNKNOWN, 0, ZSTD_cpm_noAttachDict); + ZSTD_paramSwitch_e const useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(params->useRowMatchFinder, + &cParams); + + RETURN_ERROR_IF(params->nbWorkers > 0, GENERIC, "Estimate CCtx size is supported for single-threaded compression only."); + /* estimateCCtxSize is for one-shot compression. So no buffers should + * be needed. However, we still allocate two 0-sized buffers, which can + * take space under ASAN. */ + return ZSTD_estimateCCtxSize_usingCCtxParams_internal( + &cParams, ¶ms->ldmParams, 1, useRowMatchFinder, 0, 0, ZSTD_CONTENTSIZE_UNKNOWN); +} + +size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams) +{ + ZSTD_CCtx_params initialParams = ZSTD_makeCCtxParamsFromCParams(cParams); + if (ZSTD_rowMatchFinderSupported(cParams.strategy)) { + /* Pick bigger of not using and using row-based matchfinder for greedy and lazy strategies */ + size_t noRowCCtxSize; + size_t rowCCtxSize; + initialParams.useRowMatchFinder = ZSTD_ps_disable; + noRowCCtxSize = ZSTD_estimateCCtxSize_usingCCtxParams(&initialParams); + initialParams.useRowMatchFinder = ZSTD_ps_enable; + rowCCtxSize = ZSTD_estimateCCtxSize_usingCCtxParams(&initialParams); + return MAX(noRowCCtxSize, rowCCtxSize); + } else { + return ZSTD_estimateCCtxSize_usingCCtxParams(&initialParams); + } +} + +static size_t ZSTD_estimateCCtxSize_internal(int compressionLevel) +{ + int tier = 0; + size_t largestSize = 0; + static const unsigned long long srcSizeTiers[4] = {16 KB, 128 KB, 256 KB, ZSTD_CONTENTSIZE_UNKNOWN}; + for (; tier < 4; ++tier) { + /* Choose the set of cParams for a given level across all srcSizes that give the largest cctxSize */ + ZSTD_compressionParameters const cParams = ZSTD_getCParams_internal(compressionLevel, srcSizeTiers[tier], 0, ZSTD_cpm_noAttachDict); + largestSize = MAX(ZSTD_estimateCCtxSize_usingCParams(cParams), largestSize); + } + return largestSize; +} + +size_t ZSTD_estimateCCtxSize(int compressionLevel) +{ + int level; + size_t memBudget = 0; + for (level=MIN(compressionLevel, 1); level<=compressionLevel; level++) { + /* Ensure monotonically increasing memory usage as compression level increases */ + size_t const newMB = ZSTD_estimateCCtxSize_internal(level); + if (newMB > memBudget) memBudget = newMB; + } + return memBudget; +} + +size_t ZSTD_estimateCStreamSize_usingCCtxParams(const ZSTD_CCtx_params* params) +{ + RETURN_ERROR_IF(params->nbWorkers > 0, GENERIC, "Estimate CCtx size is supported for single-threaded compression only."); + { ZSTD_compressionParameters const cParams = + ZSTD_getCParamsFromCCtxParams(params, ZSTD_CONTENTSIZE_UNKNOWN, 0, ZSTD_cpm_noAttachDict); + size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, (size_t)1 << cParams.windowLog); + size_t const inBuffSize = (params->inBufferMode == ZSTD_bm_buffered) + ? ((size_t)1 << cParams.windowLog) + blockSize + : 0; + size_t const outBuffSize = (params->outBufferMode == ZSTD_bm_buffered) + ? ZSTD_compressBound(blockSize) + 1 + : 0; + ZSTD_paramSwitch_e const useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(params->useRowMatchFinder, ¶ms->cParams); + + return ZSTD_estimateCCtxSize_usingCCtxParams_internal( + &cParams, ¶ms->ldmParams, 1, useRowMatchFinder, inBuffSize, outBuffSize, + ZSTD_CONTENTSIZE_UNKNOWN); + } +} + +size_t ZSTD_estimateCStreamSize_usingCParams(ZSTD_compressionParameters cParams) +{ + ZSTD_CCtx_params initialParams = ZSTD_makeCCtxParamsFromCParams(cParams); + if (ZSTD_rowMatchFinderSupported(cParams.strategy)) { + /* Pick bigger of not using and using row-based matchfinder for greedy and lazy strategies */ + size_t noRowCCtxSize; + size_t rowCCtxSize; + initialParams.useRowMatchFinder = ZSTD_ps_disable; + noRowCCtxSize = ZSTD_estimateCStreamSize_usingCCtxParams(&initialParams); + initialParams.useRowMatchFinder = ZSTD_ps_enable; + rowCCtxSize = ZSTD_estimateCStreamSize_usingCCtxParams(&initialParams); + return MAX(noRowCCtxSize, rowCCtxSize); + } else { + return ZSTD_estimateCStreamSize_usingCCtxParams(&initialParams); + } +} + +static size_t ZSTD_estimateCStreamSize_internal(int compressionLevel) +{ + ZSTD_compressionParameters const cParams = ZSTD_getCParams_internal(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, 0, ZSTD_cpm_noAttachDict); + return ZSTD_estimateCStreamSize_usingCParams(cParams); +} + +size_t ZSTD_estimateCStreamSize(int compressionLevel) +{ + int level; + size_t memBudget = 0; + for (level=MIN(compressionLevel, 1); level<=compressionLevel; level++) { + size_t const newMB = ZSTD_estimateCStreamSize_internal(level); + if (newMB > memBudget) memBudget = newMB; + } + return memBudget; +} + +/* ZSTD_getFrameProgression(): + * tells how much data has been consumed (input) and produced (output) for current frame. + * able to count progression inside worker threads (non-blocking mode). + */ +ZSTD_frameProgression ZSTD_getFrameProgression(const ZSTD_CCtx* cctx) +{ +#ifdef ZSTD_MULTITHREAD + if (cctx->appliedParams.nbWorkers > 0) { + return ZSTDMT_getFrameProgression(cctx->mtctx); + } +#endif + { ZSTD_frameProgression fp; + size_t const buffered = (cctx->inBuff == NULL) ? 0 : + cctx->inBuffPos - cctx->inToCompress; + if (buffered) assert(cctx->inBuffPos >= cctx->inToCompress); + assert(buffered <= ZSTD_BLOCKSIZE_MAX); + fp.ingested = cctx->consumedSrcSize + buffered; + fp.consumed = cctx->consumedSrcSize; + fp.produced = cctx->producedCSize; + fp.flushed = cctx->producedCSize; /* simplified; some data might still be left within streaming output buffer */ + fp.currentJobID = 0; + fp.nbActiveWorkers = 0; + return fp; +} } + +/*! ZSTD_toFlushNow() + * Only useful for multithreading scenarios currently (nbWorkers >= 1). + */ +size_t ZSTD_toFlushNow(ZSTD_CCtx* cctx) +{ +#ifdef ZSTD_MULTITHREAD + if (cctx->appliedParams.nbWorkers > 0) { + return ZSTDMT_toFlushNow(cctx->mtctx); + } +#endif + (void)cctx; + return 0; /* over-simplification; could also check if context is currently running in streaming mode, and in which case, report how many bytes are left to be flushed within output buffer */ +} + +static void ZSTD_assertEqualCParams(ZSTD_compressionParameters cParams1, + ZSTD_compressionParameters cParams2) +{ + (void)cParams1; + (void)cParams2; + assert(cParams1.windowLog == cParams2.windowLog); + assert(cParams1.chainLog == cParams2.chainLog); + assert(cParams1.hashLog == cParams2.hashLog); + assert(cParams1.searchLog == cParams2.searchLog); + assert(cParams1.minMatch == cParams2.minMatch); + assert(cParams1.targetLength == cParams2.targetLength); + assert(cParams1.strategy == cParams2.strategy); +} + +void ZSTD_reset_compressedBlockState(ZSTD_compressedBlockState_t* bs) +{ + int i; + for (i = 0; i < ZSTD_REP_NUM; ++i) + bs->rep[i] = repStartValue[i]; + bs->entropy.huf.repeatMode = HUF_repeat_none; + bs->entropy.fse.offcode_repeatMode = FSE_repeat_none; + bs->entropy.fse.matchlength_repeatMode = FSE_repeat_none; + bs->entropy.fse.litlength_repeatMode = FSE_repeat_none; +} + +/*! ZSTD_invalidateMatchState() + * Invalidate all the matches in the match finder tables. + * Requires nextSrc and base to be set (can be NULL). + */ +static void ZSTD_invalidateMatchState(ZSTD_matchState_t* ms) +{ + ZSTD_window_clear(&ms->window); + + ms->nextToUpdate = ms->window.dictLimit; + ms->loadedDictEnd = 0; + ms->opt.litLengthSum = 0; /* force reset of btopt stats */ + ms->dictMatchState = NULL; +} + +/** + * Controls, for this matchState reset, whether the tables need to be cleared / + * prepared for the coming compression (ZSTDcrp_makeClean), or whether the + * tables can be left unclean (ZSTDcrp_leaveDirty), because we know that a + * subsequent operation will overwrite the table space anyways (e.g., copying + * the matchState contents in from a CDict). + */ +typedef enum { + ZSTDcrp_makeClean, + ZSTDcrp_leaveDirty +} ZSTD_compResetPolicy_e; + +/** + * Controls, for this matchState reset, whether indexing can continue where it + * left off (ZSTDirp_continue), or whether it needs to be restarted from zero + * (ZSTDirp_reset). + */ +typedef enum { + ZSTDirp_continue, + ZSTDirp_reset +} ZSTD_indexResetPolicy_e; + +typedef enum { + ZSTD_resetTarget_CDict, + ZSTD_resetTarget_CCtx +} ZSTD_resetTarget_e; + + +static size_t +ZSTD_reset_matchState(ZSTD_matchState_t* ms, + ZSTD_cwksp* ws, + const ZSTD_compressionParameters* cParams, + const ZSTD_paramSwitch_e useRowMatchFinder, + const ZSTD_compResetPolicy_e crp, + const ZSTD_indexResetPolicy_e forceResetIndex, + const ZSTD_resetTarget_e forWho) +{ + /* disable chain table allocation for fast or row-based strategies */ + size_t const chainSize = ZSTD_allocateChainTable(cParams->strategy, useRowMatchFinder, + ms->dedicatedDictSearch && (forWho == ZSTD_resetTarget_CDict)) + ? ((size_t)1 << cParams->chainLog) + : 0; + size_t const hSize = ((size_t)1) << cParams->hashLog; + U32 const hashLog3 = ((forWho == ZSTD_resetTarget_CCtx) && cParams->minMatch==3) ? MIN(ZSTD_HASHLOG3_MAX, cParams->windowLog) : 0; + size_t const h3Size = hashLog3 ? ((size_t)1) << hashLog3 : 0; + + DEBUGLOG(4, "reset indices : %u", forceResetIndex == ZSTDirp_reset); + assert(useRowMatchFinder != ZSTD_ps_auto); + if (forceResetIndex == ZSTDirp_reset) { + ZSTD_window_init(&ms->window); + ZSTD_cwksp_mark_tables_dirty(ws); + } + + ms->hashLog3 = hashLog3; + + ZSTD_invalidateMatchState(ms); + + assert(!ZSTD_cwksp_reserve_failed(ws)); /* check that allocation hasn't already failed */ + + ZSTD_cwksp_clear_tables(ws); + + DEBUGLOG(5, "reserving table space"); + /* table Space */ + ms->hashTable = (U32*)ZSTD_cwksp_reserve_table(ws, hSize * sizeof(U32)); + ms->chainTable = (U32*)ZSTD_cwksp_reserve_table(ws, chainSize * sizeof(U32)); + ms->hashTable3 = (U32*)ZSTD_cwksp_reserve_table(ws, h3Size * sizeof(U32)); + RETURN_ERROR_IF(ZSTD_cwksp_reserve_failed(ws), memory_allocation, + "failed a workspace allocation in ZSTD_reset_matchState"); + + DEBUGLOG(4, "reset table : %u", crp!=ZSTDcrp_leaveDirty); + if (crp!=ZSTDcrp_leaveDirty) { + /* reset tables only */ + ZSTD_cwksp_clean_tables(ws); + } + + /* opt parser space */ + if ((forWho == ZSTD_resetTarget_CCtx) && (cParams->strategy >= ZSTD_btopt)) { + DEBUGLOG(4, "reserving optimal parser space"); + ms->opt.litFreq = (unsigned*)ZSTD_cwksp_reserve_aligned(ws, (1<opt.litLengthFreq = (unsigned*)ZSTD_cwksp_reserve_aligned(ws, (MaxLL+1) * sizeof(unsigned)); + ms->opt.matchLengthFreq = (unsigned*)ZSTD_cwksp_reserve_aligned(ws, (MaxML+1) * sizeof(unsigned)); + ms->opt.offCodeFreq = (unsigned*)ZSTD_cwksp_reserve_aligned(ws, (MaxOff+1) * sizeof(unsigned)); + ms->opt.matchTable = (ZSTD_match_t*)ZSTD_cwksp_reserve_aligned(ws, (ZSTD_OPT_NUM+1) * sizeof(ZSTD_match_t)); + ms->opt.priceTable = (ZSTD_optimal_t*)ZSTD_cwksp_reserve_aligned(ws, (ZSTD_OPT_NUM+1) * sizeof(ZSTD_optimal_t)); + } + + if (ZSTD_rowMatchFinderUsed(cParams->strategy, useRowMatchFinder)) { + { /* Row match finder needs an additional table of hashes ("tags") */ + size_t const tagTableSize = hSize*sizeof(U16); + ms->tagTable = (U16*)ZSTD_cwksp_reserve_aligned(ws, tagTableSize); + if (ms->tagTable) ZSTD_memset(ms->tagTable, 0, tagTableSize); + } + { /* Switch to 32-entry rows if searchLog is 5 (or more) */ + U32 const rowLog = BOUNDED(4, cParams->searchLog, 6); + assert(cParams->hashLog >= rowLog); + ms->rowHashLog = cParams->hashLog - rowLog; + } + } + + ms->cParams = *cParams; + + RETURN_ERROR_IF(ZSTD_cwksp_reserve_failed(ws), memory_allocation, + "failed a workspace allocation in ZSTD_reset_matchState"); + return 0; +} + +/* ZSTD_indexTooCloseToMax() : + * minor optimization : prefer memset() rather than reduceIndex() + * which is measurably slow in some circumstances (reported for Visual Studio). + * Works when re-using a context for a lot of smallish inputs : + * if all inputs are smaller than ZSTD_INDEXOVERFLOW_MARGIN, + * memset() will be triggered before reduceIndex(). + */ +#define ZSTD_INDEXOVERFLOW_MARGIN (16 MB) +static int ZSTD_indexTooCloseToMax(ZSTD_window_t w) +{ + return (size_t)(w.nextSrc - w.base) > (ZSTD_CURRENT_MAX - ZSTD_INDEXOVERFLOW_MARGIN); +} + +/** ZSTD_dictTooBig(): + * When dictionaries are larger than ZSTD_CHUNKSIZE_MAX they can't be loaded in + * one go generically. So we ensure that in that case we reset the tables to zero, + * so that we can load as much of the dictionary as possible. + */ +static int ZSTD_dictTooBig(size_t const loadedDictSize) +{ + return loadedDictSize > ZSTD_CHUNKSIZE_MAX; +} + +/*! ZSTD_resetCCtx_internal() : + * @param loadedDictSize The size of the dictionary to be loaded + * into the context, if any. If no dictionary is used, or the + * dictionary is being attached / copied, then pass 0. + * note : `params` are assumed fully validated at this stage. + */ +static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, + ZSTD_CCtx_params const* params, + U64 const pledgedSrcSize, + size_t const loadedDictSize, + ZSTD_compResetPolicy_e const crp, + ZSTD_buffered_policy_e const zbuff) +{ + ZSTD_cwksp* const ws = &zc->workspace; + DEBUGLOG(4, "ZSTD_resetCCtx_internal: pledgedSrcSize=%u, wlog=%u, useRowMatchFinder=%d useBlockSplitter=%d", + (U32)pledgedSrcSize, params->cParams.windowLog, (int)params->useRowMatchFinder, (int)params->useBlockSplitter); + assert(!ZSTD_isError(ZSTD_checkCParams(params->cParams))); + + zc->isFirstBlock = 1; + + /* Set applied params early so we can modify them for LDM, + * and point params at the applied params. + */ + zc->appliedParams = *params; + params = &zc->appliedParams; + + assert(params->useRowMatchFinder != ZSTD_ps_auto); + assert(params->useBlockSplitter != ZSTD_ps_auto); + assert(params->ldmParams.enableLdm != ZSTD_ps_auto); + if (params->ldmParams.enableLdm == ZSTD_ps_enable) { + /* Adjust long distance matching parameters */ + ZSTD_ldm_adjustParameters(&zc->appliedParams.ldmParams, ¶ms->cParams); + assert(params->ldmParams.hashLog >= params->ldmParams.bucketSizeLog); + assert(params->ldmParams.hashRateLog < 32); + } + + { size_t const windowSize = MAX(1, (size_t)MIN(((U64)1 << params->cParams.windowLog), pledgedSrcSize)); + size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, windowSize); + U32 const divider = (params->cParams.minMatch==3) ? 3 : 4; + size_t const maxNbSeq = blockSize / divider; + size_t const buffOutSize = (zbuff == ZSTDb_buffered && params->outBufferMode == ZSTD_bm_buffered) + ? ZSTD_compressBound(blockSize) + 1 + : 0; + size_t const buffInSize = (zbuff == ZSTDb_buffered && params->inBufferMode == ZSTD_bm_buffered) + ? windowSize + blockSize + : 0; + size_t const maxNbLdmSeq = ZSTD_ldm_getMaxNbSeq(params->ldmParams, blockSize); + + int const indexTooClose = ZSTD_indexTooCloseToMax(zc->blockState.matchState.window); + int const dictTooBig = ZSTD_dictTooBig(loadedDictSize); + ZSTD_indexResetPolicy_e needsIndexReset = + (indexTooClose || dictTooBig || !zc->initialized) ? ZSTDirp_reset : ZSTDirp_continue; + + size_t const neededSpace = + ZSTD_estimateCCtxSize_usingCCtxParams_internal( + ¶ms->cParams, ¶ms->ldmParams, zc->staticSize != 0, params->useRowMatchFinder, + buffInSize, buffOutSize, pledgedSrcSize); + int resizeWorkspace; + + FORWARD_IF_ERROR(neededSpace, "cctx size estimate failed!"); + + if (!zc->staticSize) ZSTD_cwksp_bump_oversized_duration(ws, 0); + + { /* Check if workspace is large enough, alloc a new one if needed */ + int const workspaceTooSmall = ZSTD_cwksp_sizeof(ws) < neededSpace; + int const workspaceWasteful = ZSTD_cwksp_check_wasteful(ws, neededSpace); + resizeWorkspace = workspaceTooSmall || workspaceWasteful; + DEBUGLOG(4, "Need %zu B workspace", neededSpace); + DEBUGLOG(4, "windowSize: %zu - blockSize: %zu", windowSize, blockSize); + + if (resizeWorkspace) { + DEBUGLOG(4, "Resize workspaceSize from %zuKB to %zuKB", + ZSTD_cwksp_sizeof(ws) >> 10, + neededSpace >> 10); + + RETURN_ERROR_IF(zc->staticSize, memory_allocation, "static cctx : no resize"); + + needsIndexReset = ZSTDirp_reset; + + ZSTD_cwksp_free(ws, zc->customMem); + FORWARD_IF_ERROR(ZSTD_cwksp_create(ws, neededSpace, zc->customMem), ""); + + DEBUGLOG(5, "reserving object space"); + /* Statically sized space. + * entropyWorkspace never moves, + * though prev/next block swap places */ + assert(ZSTD_cwksp_check_available(ws, 2 * sizeof(ZSTD_compressedBlockState_t))); + zc->blockState.prevCBlock = (ZSTD_compressedBlockState_t*) ZSTD_cwksp_reserve_object(ws, sizeof(ZSTD_compressedBlockState_t)); + RETURN_ERROR_IF(zc->blockState.prevCBlock == NULL, memory_allocation, "couldn't allocate prevCBlock"); + zc->blockState.nextCBlock = (ZSTD_compressedBlockState_t*) ZSTD_cwksp_reserve_object(ws, sizeof(ZSTD_compressedBlockState_t)); + RETURN_ERROR_IF(zc->blockState.nextCBlock == NULL, memory_allocation, "couldn't allocate nextCBlock"); + zc->entropyWorkspace = (U32*) ZSTD_cwksp_reserve_object(ws, ENTROPY_WORKSPACE_SIZE); + RETURN_ERROR_IF(zc->entropyWorkspace == NULL, memory_allocation, "couldn't allocate entropyWorkspace"); + } } + + ZSTD_cwksp_clear(ws); + + /* init params */ + zc->blockState.matchState.cParams = params->cParams; + zc->pledgedSrcSizePlusOne = pledgedSrcSize+1; + zc->consumedSrcSize = 0; + zc->producedCSize = 0; + if (pledgedSrcSize == ZSTD_CONTENTSIZE_UNKNOWN) + zc->appliedParams.fParams.contentSizeFlag = 0; + DEBUGLOG(4, "pledged content size : %u ; flag : %u", + (unsigned)pledgedSrcSize, zc->appliedParams.fParams.contentSizeFlag); + zc->blockSize = blockSize; + + XXH64_reset(&zc->xxhState, 0); + zc->stage = ZSTDcs_init; + zc->dictID = 0; + zc->dictContentSize = 0; + + ZSTD_reset_compressedBlockState(zc->blockState.prevCBlock); + + /* ZSTD_wildcopy() is used to copy into the literals buffer, + * so we have to oversize the buffer by WILDCOPY_OVERLENGTH bytes. + */ + zc->seqStore.litStart = ZSTD_cwksp_reserve_buffer(ws, blockSize + WILDCOPY_OVERLENGTH); + zc->seqStore.maxNbLit = blockSize; + + /* buffers */ + zc->bufferedPolicy = zbuff; + zc->inBuffSize = buffInSize; + zc->inBuff = (char*)ZSTD_cwksp_reserve_buffer(ws, buffInSize); + zc->outBuffSize = buffOutSize; + zc->outBuff = (char*)ZSTD_cwksp_reserve_buffer(ws, buffOutSize); + + /* ldm bucketOffsets table */ + if (params->ldmParams.enableLdm == ZSTD_ps_enable) { + /* TODO: avoid memset? */ + size_t const numBuckets = + ((size_t)1) << (params->ldmParams.hashLog - + params->ldmParams.bucketSizeLog); + zc->ldmState.bucketOffsets = ZSTD_cwksp_reserve_buffer(ws, numBuckets); + ZSTD_memset(zc->ldmState.bucketOffsets, 0, numBuckets); + } + + /* sequences storage */ + ZSTD_referenceExternalSequences(zc, NULL, 0); + zc->seqStore.maxNbSeq = maxNbSeq; + zc->seqStore.llCode = ZSTD_cwksp_reserve_buffer(ws, maxNbSeq * sizeof(BYTE)); + zc->seqStore.mlCode = ZSTD_cwksp_reserve_buffer(ws, maxNbSeq * sizeof(BYTE)); + zc->seqStore.ofCode = ZSTD_cwksp_reserve_buffer(ws, maxNbSeq * sizeof(BYTE)); + zc->seqStore.sequencesStart = (seqDef*)ZSTD_cwksp_reserve_aligned(ws, maxNbSeq * sizeof(seqDef)); + + FORWARD_IF_ERROR(ZSTD_reset_matchState( + &zc->blockState.matchState, + ws, + ¶ms->cParams, + params->useRowMatchFinder, + crp, + needsIndexReset, + ZSTD_resetTarget_CCtx), ""); + + /* ldm hash table */ + if (params->ldmParams.enableLdm == ZSTD_ps_enable) { + /* TODO: avoid memset? */ + size_t const ldmHSize = ((size_t)1) << params->ldmParams.hashLog; + zc->ldmState.hashTable = (ldmEntry_t*)ZSTD_cwksp_reserve_aligned(ws, ldmHSize * sizeof(ldmEntry_t)); + ZSTD_memset(zc->ldmState.hashTable, 0, ldmHSize * sizeof(ldmEntry_t)); + zc->ldmSequences = (rawSeq*)ZSTD_cwksp_reserve_aligned(ws, maxNbLdmSeq * sizeof(rawSeq)); + zc->maxNbLdmSequences = maxNbLdmSeq; + + ZSTD_window_init(&zc->ldmState.window); + zc->ldmState.loadedDictEnd = 0; + } + + DEBUGLOG(3, "wksp: finished allocating, %zd bytes remain available", ZSTD_cwksp_available_space(ws)); + assert(ZSTD_cwksp_estimated_space_within_bounds(ws, neededSpace, resizeWorkspace)); + + zc->initialized = 1; + + return 0; + } +} + +/* ZSTD_invalidateRepCodes() : + * ensures next compression will not use repcodes from previous block. + * Note : only works with regular variant; + * do not use with extDict variant ! */ +void ZSTD_invalidateRepCodes(ZSTD_CCtx* cctx) { + int i; + for (i=0; iblockState.prevCBlock->rep[i] = 0; + assert(!ZSTD_window_hasExtDict(cctx->blockState.matchState.window)); +} + +/* These are the approximate sizes for each strategy past which copying the + * dictionary tables into the working context is faster than using them + * in-place. + */ +static const size_t attachDictSizeCutoffs[ZSTD_STRATEGY_MAX+1] = { + 8 KB, /* unused */ + 8 KB, /* ZSTD_fast */ + 16 KB, /* ZSTD_dfast */ + 32 KB, /* ZSTD_greedy */ + 32 KB, /* ZSTD_lazy */ + 32 KB, /* ZSTD_lazy2 */ + 32 KB, /* ZSTD_btlazy2 */ + 32 KB, /* ZSTD_btopt */ + 8 KB, /* ZSTD_btultra */ + 8 KB /* ZSTD_btultra2 */ +}; + +static int ZSTD_shouldAttachDict(const ZSTD_CDict* cdict, + const ZSTD_CCtx_params* params, + U64 pledgedSrcSize) +{ + size_t cutoff = attachDictSizeCutoffs[cdict->matchState.cParams.strategy]; + int const dedicatedDictSearch = cdict->matchState.dedicatedDictSearch; + return dedicatedDictSearch + || ( ( pledgedSrcSize <= cutoff + || pledgedSrcSize == ZSTD_CONTENTSIZE_UNKNOWN + || params->attachDictPref == ZSTD_dictForceAttach ) + && params->attachDictPref != ZSTD_dictForceCopy + && !params->forceWindow ); /* dictMatchState isn't correctly + * handled in _enforceMaxDist */ +} + +static size_t +ZSTD_resetCCtx_byAttachingCDict(ZSTD_CCtx* cctx, + const ZSTD_CDict* cdict, + ZSTD_CCtx_params params, + U64 pledgedSrcSize, + ZSTD_buffered_policy_e zbuff) +{ + DEBUGLOG(4, "ZSTD_resetCCtx_byAttachingCDict() pledgedSrcSize=%llu", + (unsigned long long)pledgedSrcSize); + { + ZSTD_compressionParameters adjusted_cdict_cParams = cdict->matchState.cParams; + unsigned const windowLog = params.cParams.windowLog; + assert(windowLog != 0); + /* Resize working context table params for input only, since the dict + * has its own tables. */ + /* pledgedSrcSize == 0 means 0! */ + + if (cdict->matchState.dedicatedDictSearch) { + ZSTD_dedicatedDictSearch_revertCParams(&adjusted_cdict_cParams); + } + + params.cParams = ZSTD_adjustCParams_internal(adjusted_cdict_cParams, pledgedSrcSize, + cdict->dictContentSize, ZSTD_cpm_attachDict); + params.cParams.windowLog = windowLog; + params.useRowMatchFinder = cdict->useRowMatchFinder; /* cdict overrides */ + FORWARD_IF_ERROR(ZSTD_resetCCtx_internal(cctx, ¶ms, pledgedSrcSize, + /* loadedDictSize */ 0, + ZSTDcrp_makeClean, zbuff), ""); + assert(cctx->appliedParams.cParams.strategy == adjusted_cdict_cParams.strategy); + } + + { const U32 cdictEnd = (U32)( cdict->matchState.window.nextSrc + - cdict->matchState.window.base); + const U32 cdictLen = cdictEnd - cdict->matchState.window.dictLimit; + if (cdictLen == 0) { + /* don't even attach dictionaries with no contents */ + DEBUGLOG(4, "skipping attaching empty dictionary"); + } else { + DEBUGLOG(4, "attaching dictionary into context"); + cctx->blockState.matchState.dictMatchState = &cdict->matchState; + + /* prep working match state so dict matches never have negative indices + * when they are translated to the working context's index space. */ + if (cctx->blockState.matchState.window.dictLimit < cdictEnd) { + cctx->blockState.matchState.window.nextSrc = + cctx->blockState.matchState.window.base + cdictEnd; + ZSTD_window_clear(&cctx->blockState.matchState.window); + } + /* loadedDictEnd is expressed within the referential of the active context */ + cctx->blockState.matchState.loadedDictEnd = cctx->blockState.matchState.window.dictLimit; + } } + + cctx->dictID = cdict->dictID; + cctx->dictContentSize = cdict->dictContentSize; + + /* copy block state */ + ZSTD_memcpy(cctx->blockState.prevCBlock, &cdict->cBlockState, sizeof(cdict->cBlockState)); + + return 0; +} + +static size_t ZSTD_resetCCtx_byCopyingCDict(ZSTD_CCtx* cctx, + const ZSTD_CDict* cdict, + ZSTD_CCtx_params params, + U64 pledgedSrcSize, + ZSTD_buffered_policy_e zbuff) +{ + const ZSTD_compressionParameters *cdict_cParams = &cdict->matchState.cParams; + + assert(!cdict->matchState.dedicatedDictSearch); + DEBUGLOG(4, "ZSTD_resetCCtx_byCopyingCDict() pledgedSrcSize=%llu", + (unsigned long long)pledgedSrcSize); + + { unsigned const windowLog = params.cParams.windowLog; + assert(windowLog != 0); + /* Copy only compression parameters related to tables. */ + params.cParams = *cdict_cParams; + params.cParams.windowLog = windowLog; + params.useRowMatchFinder = cdict->useRowMatchFinder; + FORWARD_IF_ERROR(ZSTD_resetCCtx_internal(cctx, ¶ms, pledgedSrcSize, + /* loadedDictSize */ 0, + ZSTDcrp_leaveDirty, zbuff), ""); + assert(cctx->appliedParams.cParams.strategy == cdict_cParams->strategy); + assert(cctx->appliedParams.cParams.hashLog == cdict_cParams->hashLog); + assert(cctx->appliedParams.cParams.chainLog == cdict_cParams->chainLog); + } + + ZSTD_cwksp_mark_tables_dirty(&cctx->workspace); + assert(params.useRowMatchFinder != ZSTD_ps_auto); + + /* copy tables */ + { size_t const chainSize = ZSTD_allocateChainTable(cdict_cParams->strategy, cdict->useRowMatchFinder, 0 /* DDS guaranteed disabled */) + ? ((size_t)1 << cdict_cParams->chainLog) + : 0; + size_t const hSize = (size_t)1 << cdict_cParams->hashLog; + + ZSTD_memcpy(cctx->blockState.matchState.hashTable, + cdict->matchState.hashTable, + hSize * sizeof(U32)); + /* Do not copy cdict's chainTable if cctx has parameters such that it would not use chainTable */ + if (ZSTD_allocateChainTable(cctx->appliedParams.cParams.strategy, cctx->appliedParams.useRowMatchFinder, 0 /* forDDSDict */)) { + ZSTD_memcpy(cctx->blockState.matchState.chainTable, + cdict->matchState.chainTable, + chainSize * sizeof(U32)); + } + /* copy tag table */ + if (ZSTD_rowMatchFinderUsed(cdict_cParams->strategy, cdict->useRowMatchFinder)) { + size_t const tagTableSize = hSize*sizeof(U16); + ZSTD_memcpy(cctx->blockState.matchState.tagTable, + cdict->matchState.tagTable, + tagTableSize); + } + } + + /* Zero the hashTable3, since the cdict never fills it */ + { int const h3log = cctx->blockState.matchState.hashLog3; + size_t const h3Size = h3log ? ((size_t)1 << h3log) : 0; + assert(cdict->matchState.hashLog3 == 0); + ZSTD_memset(cctx->blockState.matchState.hashTable3, 0, h3Size * sizeof(U32)); + } + + ZSTD_cwksp_mark_tables_clean(&cctx->workspace); + + /* copy dictionary offsets */ + { ZSTD_matchState_t const* srcMatchState = &cdict->matchState; + ZSTD_matchState_t* dstMatchState = &cctx->blockState.matchState; + dstMatchState->window = srcMatchState->window; + dstMatchState->nextToUpdate = srcMatchState->nextToUpdate; + dstMatchState->loadedDictEnd= srcMatchState->loadedDictEnd; + } + + cctx->dictID = cdict->dictID; + cctx->dictContentSize = cdict->dictContentSize; + + /* copy block state */ + ZSTD_memcpy(cctx->blockState.prevCBlock, &cdict->cBlockState, sizeof(cdict->cBlockState)); + + return 0; +} + +/* We have a choice between copying the dictionary context into the working + * context, or referencing the dictionary context from the working context + * in-place. We decide here which strategy to use. */ +static size_t ZSTD_resetCCtx_usingCDict(ZSTD_CCtx* cctx, + const ZSTD_CDict* cdict, + const ZSTD_CCtx_params* params, + U64 pledgedSrcSize, + ZSTD_buffered_policy_e zbuff) +{ + + DEBUGLOG(4, "ZSTD_resetCCtx_usingCDict (pledgedSrcSize=%u)", + (unsigned)pledgedSrcSize); + + if (ZSTD_shouldAttachDict(cdict, params, pledgedSrcSize)) { + return ZSTD_resetCCtx_byAttachingCDict( + cctx, cdict, *params, pledgedSrcSize, zbuff); + } else { + return ZSTD_resetCCtx_byCopyingCDict( + cctx, cdict, *params, pledgedSrcSize, zbuff); + } +} + +/*! ZSTD_copyCCtx_internal() : + * Duplicate an existing context `srcCCtx` into another one `dstCCtx`. + * Only works during stage ZSTDcs_init (i.e. after creation, but before first call to ZSTD_compressContinue()). + * The "context", in this case, refers to the hash and chain tables, + * entropy tables, and dictionary references. + * `windowLog` value is enforced if != 0, otherwise value is copied from srcCCtx. + * @return : 0, or an error code */ +static size_t ZSTD_copyCCtx_internal(ZSTD_CCtx* dstCCtx, + const ZSTD_CCtx* srcCCtx, + ZSTD_frameParameters fParams, + U64 pledgedSrcSize, + ZSTD_buffered_policy_e zbuff) +{ + RETURN_ERROR_IF(srcCCtx->stage!=ZSTDcs_init, stage_wrong, + "Can't copy a ctx that's not in init stage."); + DEBUGLOG(5, "ZSTD_copyCCtx_internal"); + ZSTD_memcpy(&dstCCtx->customMem, &srcCCtx->customMem, sizeof(ZSTD_customMem)); + { ZSTD_CCtx_params params = dstCCtx->requestedParams; + /* Copy only compression parameters related to tables. */ + params.cParams = srcCCtx->appliedParams.cParams; + assert(srcCCtx->appliedParams.useRowMatchFinder != ZSTD_ps_auto); + assert(srcCCtx->appliedParams.useBlockSplitter != ZSTD_ps_auto); + assert(srcCCtx->appliedParams.ldmParams.enableLdm != ZSTD_ps_auto); + params.useRowMatchFinder = srcCCtx->appliedParams.useRowMatchFinder; + params.useBlockSplitter = srcCCtx->appliedParams.useBlockSplitter; + params.ldmParams = srcCCtx->appliedParams.ldmParams; + params.fParams = fParams; + ZSTD_resetCCtx_internal(dstCCtx, ¶ms, pledgedSrcSize, + /* loadedDictSize */ 0, + ZSTDcrp_leaveDirty, zbuff); + assert(dstCCtx->appliedParams.cParams.windowLog == srcCCtx->appliedParams.cParams.windowLog); + assert(dstCCtx->appliedParams.cParams.strategy == srcCCtx->appliedParams.cParams.strategy); + assert(dstCCtx->appliedParams.cParams.hashLog == srcCCtx->appliedParams.cParams.hashLog); + assert(dstCCtx->appliedParams.cParams.chainLog == srcCCtx->appliedParams.cParams.chainLog); + assert(dstCCtx->blockState.matchState.hashLog3 == srcCCtx->blockState.matchState.hashLog3); + } + + ZSTD_cwksp_mark_tables_dirty(&dstCCtx->workspace); + + /* copy tables */ + { size_t const chainSize = ZSTD_allocateChainTable(srcCCtx->appliedParams.cParams.strategy, + srcCCtx->appliedParams.useRowMatchFinder, + 0 /* forDDSDict */) + ? ((size_t)1 << srcCCtx->appliedParams.cParams.chainLog) + : 0; + size_t const hSize = (size_t)1 << srcCCtx->appliedParams.cParams.hashLog; + int const h3log = srcCCtx->blockState.matchState.hashLog3; + size_t const h3Size = h3log ? ((size_t)1 << h3log) : 0; + + ZSTD_memcpy(dstCCtx->blockState.matchState.hashTable, + srcCCtx->blockState.matchState.hashTable, + hSize * sizeof(U32)); + ZSTD_memcpy(dstCCtx->blockState.matchState.chainTable, + srcCCtx->blockState.matchState.chainTable, + chainSize * sizeof(U32)); + ZSTD_memcpy(dstCCtx->blockState.matchState.hashTable3, + srcCCtx->blockState.matchState.hashTable3, + h3Size * sizeof(U32)); + } + + ZSTD_cwksp_mark_tables_clean(&dstCCtx->workspace); + + /* copy dictionary offsets */ + { + const ZSTD_matchState_t* srcMatchState = &srcCCtx->blockState.matchState; + ZSTD_matchState_t* dstMatchState = &dstCCtx->blockState.matchState; + dstMatchState->window = srcMatchState->window; + dstMatchState->nextToUpdate = srcMatchState->nextToUpdate; + dstMatchState->loadedDictEnd= srcMatchState->loadedDictEnd; + } + dstCCtx->dictID = srcCCtx->dictID; + dstCCtx->dictContentSize = srcCCtx->dictContentSize; + + /* copy block state */ + ZSTD_memcpy(dstCCtx->blockState.prevCBlock, srcCCtx->blockState.prevCBlock, sizeof(*srcCCtx->blockState.prevCBlock)); + + return 0; +} + +/*! ZSTD_copyCCtx() : + * Duplicate an existing context `srcCCtx` into another one `dstCCtx`. + * Only works during stage ZSTDcs_init (i.e. after creation, but before first call to ZSTD_compressContinue()). + * pledgedSrcSize==0 means "unknown". +* @return : 0, or an error code */ +size_t ZSTD_copyCCtx(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx, unsigned long long pledgedSrcSize) +{ + ZSTD_frameParameters fParams = { 1 /*content*/, 0 /*checksum*/, 0 /*noDictID*/ }; + ZSTD_buffered_policy_e const zbuff = srcCCtx->bufferedPolicy; + ZSTD_STATIC_ASSERT((U32)ZSTDb_buffered==1); + if (pledgedSrcSize==0) pledgedSrcSize = ZSTD_CONTENTSIZE_UNKNOWN; + fParams.contentSizeFlag = (pledgedSrcSize != ZSTD_CONTENTSIZE_UNKNOWN); + + return ZSTD_copyCCtx_internal(dstCCtx, srcCCtx, + fParams, pledgedSrcSize, + zbuff); +} + + +#define ZSTD_ROWSIZE 16 +/*! ZSTD_reduceTable() : + * reduce table indexes by `reducerValue`, or squash to zero. + * PreserveMark preserves "unsorted mark" for btlazy2 strategy. + * It must be set to a clear 0/1 value, to remove branch during inlining. + * Presume table size is a multiple of ZSTD_ROWSIZE + * to help auto-vectorization */ +FORCE_INLINE_TEMPLATE void +ZSTD_reduceTable_internal (U32* const table, U32 const size, U32 const reducerValue, int const preserveMark) +{ + int const nbRows = (int)size / ZSTD_ROWSIZE; + int cellNb = 0; + int rowNb; + /* Protect special index values < ZSTD_WINDOW_START_INDEX. */ + U32 const reducerThreshold = reducerValue + ZSTD_WINDOW_START_INDEX; + assert((size & (ZSTD_ROWSIZE-1)) == 0); /* multiple of ZSTD_ROWSIZE */ + assert(size < (1U<<31)); /* can be casted to int */ + +#if ZSTD_MEMORY_SANITIZER && !defined (ZSTD_MSAN_DONT_POISON_WORKSPACE) + /* To validate that the table re-use logic is sound, and that we don't + * access table space that we haven't cleaned, we re-"poison" the table + * space every time we mark it dirty. + * + * This function however is intended to operate on those dirty tables and + * re-clean them. So when this function is used correctly, we can unpoison + * the memory it operated on. This introduces a blind spot though, since + * if we now try to operate on __actually__ poisoned memory, we will not + * detect that. */ + __msan_unpoison(table, size * sizeof(U32)); +#endif + + for (rowNb=0 ; rowNb < nbRows ; rowNb++) { + int column; + for (column=0; columncParams.hashLog; + ZSTD_reduceTable(ms->hashTable, hSize, reducerValue); + } + + if (ZSTD_allocateChainTable(params->cParams.strategy, params->useRowMatchFinder, (U32)ms->dedicatedDictSearch)) { + U32 const chainSize = (U32)1 << params->cParams.chainLog; + if (params->cParams.strategy == ZSTD_btlazy2) + ZSTD_reduceTable_btlazy2(ms->chainTable, chainSize, reducerValue); + else + ZSTD_reduceTable(ms->chainTable, chainSize, reducerValue); + } + + if (ms->hashLog3) { + U32 const h3Size = (U32)1 << ms->hashLog3; + ZSTD_reduceTable(ms->hashTable3, h3Size, reducerValue); + } +} + + +/*-******************************************************* +* Block entropic compression +*********************************************************/ + +/* See doc/zstd_compression_format.md for detailed format description */ + +void ZSTD_seqToCodes(const seqStore_t* seqStorePtr) +{ + const seqDef* const sequences = seqStorePtr->sequencesStart; + BYTE* const llCodeTable = seqStorePtr->llCode; + BYTE* const ofCodeTable = seqStorePtr->ofCode; + BYTE* const mlCodeTable = seqStorePtr->mlCode; + U32 const nbSeq = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart); + U32 u; + assert(nbSeq <= seqStorePtr->maxNbSeq); + for (u=0; ulongLengthType==ZSTD_llt_literalLength) + llCodeTable[seqStorePtr->longLengthPos] = MaxLL; + if (seqStorePtr->longLengthType==ZSTD_llt_matchLength) + mlCodeTable[seqStorePtr->longLengthPos] = MaxML; +} + +/* ZSTD_useTargetCBlockSize(): + * Returns if target compressed block size param is being used. + * If used, compression will do best effort to make a compressed block size to be around targetCBlockSize. + * Returns 1 if true, 0 otherwise. */ +static int ZSTD_useTargetCBlockSize(const ZSTD_CCtx_params* cctxParams) +{ + DEBUGLOG(5, "ZSTD_useTargetCBlockSize (targetCBlockSize=%zu)", cctxParams->targetCBlockSize); + return (cctxParams->targetCBlockSize != 0); +} + +/* ZSTD_blockSplitterEnabled(): + * Returns if block splitting param is being used + * If used, compression will do best effort to split a block in order to improve compression ratio. + * At the time this function is called, the parameter must be finalized. + * Returns 1 if true, 0 otherwise. */ +static int ZSTD_blockSplitterEnabled(ZSTD_CCtx_params* cctxParams) +{ + DEBUGLOG(5, "ZSTD_blockSplitterEnabled (useBlockSplitter=%d)", cctxParams->useBlockSplitter); + assert(cctxParams->useBlockSplitter != ZSTD_ps_auto); + return (cctxParams->useBlockSplitter == ZSTD_ps_enable); +} + +/* Type returned by ZSTD_buildSequencesStatistics containing finalized symbol encoding types + * and size of the sequences statistics + */ +typedef struct { + U32 LLtype; + U32 Offtype; + U32 MLtype; + size_t size; + size_t lastCountSize; /* Accounts for bug in 1.3.4. More detail in ZSTD_entropyCompressSeqStore_internal() */ +} ZSTD_symbolEncodingTypeStats_t; + +/* ZSTD_buildSequencesStatistics(): + * Returns a ZSTD_symbolEncodingTypeStats_t, or a zstd error code in the `size` field. + * Modifies `nextEntropy` to have the appropriate values as a side effect. + * nbSeq must be greater than 0. + * + * entropyWkspSize must be of size at least ENTROPY_WORKSPACE_SIZE - (MaxSeq + 1)*sizeof(U32) + */ +static ZSTD_symbolEncodingTypeStats_t +ZSTD_buildSequencesStatistics(seqStore_t* seqStorePtr, size_t nbSeq, + const ZSTD_fseCTables_t* prevEntropy, ZSTD_fseCTables_t* nextEntropy, + BYTE* dst, const BYTE* const dstEnd, + ZSTD_strategy strategy, unsigned* countWorkspace, + void* entropyWorkspace, size_t entropyWkspSize) { + BYTE* const ostart = dst; + const BYTE* const oend = dstEnd; + BYTE* op = ostart; + FSE_CTable* CTable_LitLength = nextEntropy->litlengthCTable; + FSE_CTable* CTable_OffsetBits = nextEntropy->offcodeCTable; + FSE_CTable* CTable_MatchLength = nextEntropy->matchlengthCTable; + const BYTE* const ofCodeTable = seqStorePtr->ofCode; + const BYTE* const llCodeTable = seqStorePtr->llCode; + const BYTE* const mlCodeTable = seqStorePtr->mlCode; + ZSTD_symbolEncodingTypeStats_t stats; + + stats.lastCountSize = 0; + /* convert length/distances into codes */ + ZSTD_seqToCodes(seqStorePtr); + assert(op <= oend); + assert(nbSeq != 0); /* ZSTD_selectEncodingType() divides by nbSeq */ + /* build CTable for Literal Lengths */ + { unsigned max = MaxLL; + size_t const mostFrequent = HIST_countFast_wksp(countWorkspace, &max, llCodeTable, nbSeq, entropyWorkspace, entropyWkspSize); /* can't fail */ + DEBUGLOG(5, "Building LL table"); + nextEntropy->litlength_repeatMode = prevEntropy->litlength_repeatMode; + stats.LLtype = ZSTD_selectEncodingType(&nextEntropy->litlength_repeatMode, + countWorkspace, max, mostFrequent, nbSeq, + LLFSELog, prevEntropy->litlengthCTable, + LL_defaultNorm, LL_defaultNormLog, + ZSTD_defaultAllowed, strategy); + assert(set_basic < set_compressed && set_rle < set_compressed); + assert(!(stats.LLtype < set_compressed && nextEntropy->litlength_repeatMode != FSE_repeat_none)); /* We don't copy tables */ + { size_t const countSize = ZSTD_buildCTable( + op, (size_t)(oend - op), + CTable_LitLength, LLFSELog, (symbolEncodingType_e)stats.LLtype, + countWorkspace, max, llCodeTable, nbSeq, + LL_defaultNorm, LL_defaultNormLog, MaxLL, + prevEntropy->litlengthCTable, + sizeof(prevEntropy->litlengthCTable), + entropyWorkspace, entropyWkspSize); + if (ZSTD_isError(countSize)) { + DEBUGLOG(3, "ZSTD_buildCTable for LitLens failed"); + stats.size = countSize; + return stats; + } + if (stats.LLtype == set_compressed) + stats.lastCountSize = countSize; + op += countSize; + assert(op <= oend); + } } + /* build CTable for Offsets */ + { unsigned max = MaxOff; + size_t const mostFrequent = HIST_countFast_wksp( + countWorkspace, &max, ofCodeTable, nbSeq, entropyWorkspace, entropyWkspSize); /* can't fail */ + /* We can only use the basic table if max <= DefaultMaxOff, otherwise the offsets are too large */ + ZSTD_defaultPolicy_e const defaultPolicy = (max <= DefaultMaxOff) ? ZSTD_defaultAllowed : ZSTD_defaultDisallowed; + DEBUGLOG(5, "Building OF table"); + nextEntropy->offcode_repeatMode = prevEntropy->offcode_repeatMode; + stats.Offtype = ZSTD_selectEncodingType(&nextEntropy->offcode_repeatMode, + countWorkspace, max, mostFrequent, nbSeq, + OffFSELog, prevEntropy->offcodeCTable, + OF_defaultNorm, OF_defaultNormLog, + defaultPolicy, strategy); + assert(!(stats.Offtype < set_compressed && nextEntropy->offcode_repeatMode != FSE_repeat_none)); /* We don't copy tables */ + { size_t const countSize = ZSTD_buildCTable( + op, (size_t)(oend - op), + CTable_OffsetBits, OffFSELog, (symbolEncodingType_e)stats.Offtype, + countWorkspace, max, ofCodeTable, nbSeq, + OF_defaultNorm, OF_defaultNormLog, DefaultMaxOff, + prevEntropy->offcodeCTable, + sizeof(prevEntropy->offcodeCTable), + entropyWorkspace, entropyWkspSize); + if (ZSTD_isError(countSize)) { + DEBUGLOG(3, "ZSTD_buildCTable for Offsets failed"); + stats.size = countSize; + return stats; + } + if (stats.Offtype == set_compressed) + stats.lastCountSize = countSize; + op += countSize; + assert(op <= oend); + } } + /* build CTable for MatchLengths */ + { unsigned max = MaxML; + size_t const mostFrequent = HIST_countFast_wksp( + countWorkspace, &max, mlCodeTable, nbSeq, entropyWorkspace, entropyWkspSize); /* can't fail */ + DEBUGLOG(5, "Building ML table (remaining space : %i)", (int)(oend-op)); + nextEntropy->matchlength_repeatMode = prevEntropy->matchlength_repeatMode; + stats.MLtype = ZSTD_selectEncodingType(&nextEntropy->matchlength_repeatMode, + countWorkspace, max, mostFrequent, nbSeq, + MLFSELog, prevEntropy->matchlengthCTable, + ML_defaultNorm, ML_defaultNormLog, + ZSTD_defaultAllowed, strategy); + assert(!(stats.MLtype < set_compressed && nextEntropy->matchlength_repeatMode != FSE_repeat_none)); /* We don't copy tables */ + { size_t const countSize = ZSTD_buildCTable( + op, (size_t)(oend - op), + CTable_MatchLength, MLFSELog, (symbolEncodingType_e)stats.MLtype, + countWorkspace, max, mlCodeTable, nbSeq, + ML_defaultNorm, ML_defaultNormLog, MaxML, + prevEntropy->matchlengthCTable, + sizeof(prevEntropy->matchlengthCTable), + entropyWorkspace, entropyWkspSize); + if (ZSTD_isError(countSize)) { + DEBUGLOG(3, "ZSTD_buildCTable for MatchLengths failed"); + stats.size = countSize; + return stats; + } + if (stats.MLtype == set_compressed) + stats.lastCountSize = countSize; + op += countSize; + assert(op <= oend); + } } + stats.size = (size_t)(op-ostart); + return stats; +} + +/* ZSTD_entropyCompressSeqStore_internal(): + * compresses both literals and sequences + * Returns compressed size of block, or a zstd error. + */ +#define SUSPECT_UNCOMPRESSIBLE_LITERAL_RATIO 20 +MEM_STATIC size_t +ZSTD_entropyCompressSeqStore_internal(seqStore_t* seqStorePtr, + const ZSTD_entropyCTables_t* prevEntropy, + ZSTD_entropyCTables_t* nextEntropy, + const ZSTD_CCtx_params* cctxParams, + void* dst, size_t dstCapacity, + void* entropyWorkspace, size_t entropyWkspSize, + const int bmi2) +{ + const int longOffsets = cctxParams->cParams.windowLog > STREAM_ACCUMULATOR_MIN; + ZSTD_strategy const strategy = cctxParams->cParams.strategy; + unsigned* count = (unsigned*)entropyWorkspace; + FSE_CTable* CTable_LitLength = nextEntropy->fse.litlengthCTable; + FSE_CTable* CTable_OffsetBits = nextEntropy->fse.offcodeCTable; + FSE_CTable* CTable_MatchLength = nextEntropy->fse.matchlengthCTable; + const seqDef* const sequences = seqStorePtr->sequencesStart; + const size_t nbSeq = seqStorePtr->sequences - seqStorePtr->sequencesStart; + const BYTE* const ofCodeTable = seqStorePtr->ofCode; + const BYTE* const llCodeTable = seqStorePtr->llCode; + const BYTE* const mlCodeTable = seqStorePtr->mlCode; + BYTE* const ostart = (BYTE*)dst; + BYTE* const oend = ostart + dstCapacity; + BYTE* op = ostart; + size_t lastCountSize; + + entropyWorkspace = count + (MaxSeq + 1); + entropyWkspSize -= (MaxSeq + 1) * sizeof(*count); + + DEBUGLOG(4, "ZSTD_entropyCompressSeqStore_internal (nbSeq=%zu)", nbSeq); + ZSTD_STATIC_ASSERT(HUF_WORKSPACE_SIZE >= (1<= HUF_WORKSPACE_SIZE); + + /* Compress literals */ + { const BYTE* const literals = seqStorePtr->litStart; + size_t const numSequences = seqStorePtr->sequences - seqStorePtr->sequencesStart; + size_t const numLiterals = seqStorePtr->lit - seqStorePtr->litStart; + /* Base suspicion of uncompressibility on ratio of literals to sequences */ + unsigned const suspectUncompressible = (numSequences == 0) || (numLiterals / numSequences >= SUSPECT_UNCOMPRESSIBLE_LITERAL_RATIO); + size_t const litSize = (size_t)(seqStorePtr->lit - literals); + size_t const cSize = ZSTD_compressLiterals( + &prevEntropy->huf, &nextEntropy->huf, + cctxParams->cParams.strategy, + ZSTD_literalsCompressionIsDisabled(cctxParams), + op, dstCapacity, + literals, litSize, + entropyWorkspace, entropyWkspSize, + bmi2, suspectUncompressible); + FORWARD_IF_ERROR(cSize, "ZSTD_compressLiterals failed"); + assert(cSize <= dstCapacity); + op += cSize; + } + + /* Sequences Header */ + RETURN_ERROR_IF((oend-op) < 3 /*max nbSeq Size*/ + 1 /*seqHead*/, + dstSize_tooSmall, "Can't fit seq hdr in output buf!"); + if (nbSeq < 128) { + *op++ = (BYTE)nbSeq; + } else if (nbSeq < LONGNBSEQ) { + op[0] = (BYTE)((nbSeq>>8) + 0x80); + op[1] = (BYTE)nbSeq; + op+=2; + } else { + op[0]=0xFF; + MEM_writeLE16(op+1, (U16)(nbSeq - LONGNBSEQ)); + op+=3; + } + assert(op <= oend); + if (nbSeq==0) { + /* Copy the old tables over as if we repeated them */ + ZSTD_memcpy(&nextEntropy->fse, &prevEntropy->fse, sizeof(prevEntropy->fse)); + return (size_t)(op - ostart); + } + { + ZSTD_symbolEncodingTypeStats_t stats; + BYTE* seqHead = op++; + /* build stats for sequences */ + stats = ZSTD_buildSequencesStatistics(seqStorePtr, nbSeq, + &prevEntropy->fse, &nextEntropy->fse, + op, oend, + strategy, count, + entropyWorkspace, entropyWkspSize); + FORWARD_IF_ERROR(stats.size, "ZSTD_buildSequencesStatistics failed!"); + *seqHead = (BYTE)((stats.LLtype<<6) + (stats.Offtype<<4) + (stats.MLtype<<2)); + lastCountSize = stats.lastCountSize; + op += stats.size; + } + + { size_t const bitstreamSize = ZSTD_encodeSequences( + op, (size_t)(oend - op), + CTable_MatchLength, mlCodeTable, + CTable_OffsetBits, ofCodeTable, + CTable_LitLength, llCodeTable, + sequences, nbSeq, + longOffsets, bmi2); + FORWARD_IF_ERROR(bitstreamSize, "ZSTD_encodeSequences failed"); + op += bitstreamSize; + assert(op <= oend); + /* zstd versions <= 1.3.4 mistakenly report corruption when + * FSE_readNCount() receives a buffer < 4 bytes. + * Fixed by https://github.com/facebook/zstd/pull/1146. + * This can happen when the last set_compressed table present is 2 + * bytes and the bitstream is only one byte. + * In this exceedingly rare case, we will simply emit an uncompressed + * block, since it isn't worth optimizing. + */ + if (lastCountSize && (lastCountSize + bitstreamSize) < 4) { + /* lastCountSize >= 2 && bitstreamSize > 0 ==> lastCountSize == 3 */ + assert(lastCountSize + bitstreamSize == 3); + DEBUGLOG(5, "Avoiding bug in zstd decoder in versions <= 1.3.4 by " + "emitting an uncompressed block."); + return 0; + } + } + + DEBUGLOG(5, "compressed block size : %u", (unsigned)(op - ostart)); + return (size_t)(op - ostart); +} + +MEM_STATIC size_t +ZSTD_entropyCompressSeqStore(seqStore_t* seqStorePtr, + const ZSTD_entropyCTables_t* prevEntropy, + ZSTD_entropyCTables_t* nextEntropy, + const ZSTD_CCtx_params* cctxParams, + void* dst, size_t dstCapacity, + size_t srcSize, + void* entropyWorkspace, size_t entropyWkspSize, + int bmi2) +{ + size_t const cSize = ZSTD_entropyCompressSeqStore_internal( + seqStorePtr, prevEntropy, nextEntropy, cctxParams, + dst, dstCapacity, + entropyWorkspace, entropyWkspSize, bmi2); + if (cSize == 0) return 0; + /* When srcSize <= dstCapacity, there is enough space to write a raw uncompressed block. + * Since we ran out of space, block must be not compressible, so fall back to raw uncompressed block. + */ + if ((cSize == ERROR(dstSize_tooSmall)) & (srcSize <= dstCapacity)) + return 0; /* block not compressed */ + FORWARD_IF_ERROR(cSize, "ZSTD_entropyCompressSeqStore_internal failed"); + + /* Check compressibility */ + { size_t const maxCSize = srcSize - ZSTD_minGain(srcSize, cctxParams->cParams.strategy); + if (cSize >= maxCSize) return 0; /* block not compressed */ + } + DEBUGLOG(4, "ZSTD_entropyCompressSeqStore() cSize: %zu", cSize); + return cSize; +} + +/* ZSTD_selectBlockCompressor() : + * Not static, but internal use only (used by long distance matcher) + * assumption : strat is a valid strategy */ +ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, ZSTD_paramSwitch_e useRowMatchFinder, ZSTD_dictMode_e dictMode) +{ + static const ZSTD_blockCompressor blockCompressor[4][ZSTD_STRATEGY_MAX+1] = { + { ZSTD_compressBlock_fast /* default for 0 */, + ZSTD_compressBlock_fast, + ZSTD_compressBlock_doubleFast, + ZSTD_compressBlock_greedy, + ZSTD_compressBlock_lazy, + ZSTD_compressBlock_lazy2, + ZSTD_compressBlock_btlazy2, + ZSTD_compressBlock_btopt, + ZSTD_compressBlock_btultra, + ZSTD_compressBlock_btultra2 }, + { ZSTD_compressBlock_fast_extDict /* default for 0 */, + ZSTD_compressBlock_fast_extDict, + ZSTD_compressBlock_doubleFast_extDict, + ZSTD_compressBlock_greedy_extDict, + ZSTD_compressBlock_lazy_extDict, + ZSTD_compressBlock_lazy2_extDict, + ZSTD_compressBlock_btlazy2_extDict, + ZSTD_compressBlock_btopt_extDict, + ZSTD_compressBlock_btultra_extDict, + ZSTD_compressBlock_btultra_extDict }, + { ZSTD_compressBlock_fast_dictMatchState /* default for 0 */, + ZSTD_compressBlock_fast_dictMatchState, + ZSTD_compressBlock_doubleFast_dictMatchState, + ZSTD_compressBlock_greedy_dictMatchState, + ZSTD_compressBlock_lazy_dictMatchState, + ZSTD_compressBlock_lazy2_dictMatchState, + ZSTD_compressBlock_btlazy2_dictMatchState, + ZSTD_compressBlock_btopt_dictMatchState, + ZSTD_compressBlock_btultra_dictMatchState, + ZSTD_compressBlock_btultra_dictMatchState }, + { NULL /* default for 0 */, + NULL, + NULL, + ZSTD_compressBlock_greedy_dedicatedDictSearch, + ZSTD_compressBlock_lazy_dedicatedDictSearch, + ZSTD_compressBlock_lazy2_dedicatedDictSearch, + NULL, + NULL, + NULL, + NULL } + }; + ZSTD_blockCompressor selectedCompressor; + ZSTD_STATIC_ASSERT((unsigned)ZSTD_fast == 1); + + assert(ZSTD_cParam_withinBounds(ZSTD_c_strategy, strat)); + DEBUGLOG(4, "Selected block compressor: dictMode=%d strat=%d rowMatchfinder=%d", (int)dictMode, (int)strat, (int)useRowMatchFinder); + if (ZSTD_rowMatchFinderUsed(strat, useRowMatchFinder)) { + static const ZSTD_blockCompressor rowBasedBlockCompressors[4][3] = { + { ZSTD_compressBlock_greedy_row, + ZSTD_compressBlock_lazy_row, + ZSTD_compressBlock_lazy2_row }, + { ZSTD_compressBlock_greedy_extDict_row, + ZSTD_compressBlock_lazy_extDict_row, + ZSTD_compressBlock_lazy2_extDict_row }, + { ZSTD_compressBlock_greedy_dictMatchState_row, + ZSTD_compressBlock_lazy_dictMatchState_row, + ZSTD_compressBlock_lazy2_dictMatchState_row }, + { ZSTD_compressBlock_greedy_dedicatedDictSearch_row, + ZSTD_compressBlock_lazy_dedicatedDictSearch_row, + ZSTD_compressBlock_lazy2_dedicatedDictSearch_row } + }; + DEBUGLOG(4, "Selecting a row-based matchfinder"); + assert(useRowMatchFinder != ZSTD_ps_auto); + selectedCompressor = rowBasedBlockCompressors[(int)dictMode][(int)strat - (int)ZSTD_greedy]; + } else { + selectedCompressor = blockCompressor[(int)dictMode][(int)strat]; + } + assert(selectedCompressor != NULL); + return selectedCompressor; +} + +static void ZSTD_storeLastLiterals(seqStore_t* seqStorePtr, + const BYTE* anchor, size_t lastLLSize) +{ + ZSTD_memcpy(seqStorePtr->lit, anchor, lastLLSize); + seqStorePtr->lit += lastLLSize; +} + +void ZSTD_resetSeqStore(seqStore_t* ssPtr) +{ + ssPtr->lit = ssPtr->litStart; + ssPtr->sequences = ssPtr->sequencesStart; + ssPtr->longLengthType = ZSTD_llt_none; +} + +typedef enum { ZSTDbss_compress, ZSTDbss_noCompress } ZSTD_buildSeqStore_e; + +static size_t ZSTD_buildSeqStore(ZSTD_CCtx* zc, const void* src, size_t srcSize) +{ + ZSTD_matchState_t* const ms = &zc->blockState.matchState; + DEBUGLOG(5, "ZSTD_buildSeqStore (srcSize=%zu)", srcSize); + assert(srcSize <= ZSTD_BLOCKSIZE_MAX); + /* Assert that we have correctly flushed the ctx params into the ms's copy */ + ZSTD_assertEqualCParams(zc->appliedParams.cParams, ms->cParams); + if (srcSize < MIN_CBLOCK_SIZE+ZSTD_blockHeaderSize+1) { + if (zc->appliedParams.cParams.strategy >= ZSTD_btopt) { + ZSTD_ldm_skipRawSeqStoreBytes(&zc->externSeqStore, srcSize); + } else { + ZSTD_ldm_skipSequences(&zc->externSeqStore, srcSize, zc->appliedParams.cParams.minMatch); + } + return ZSTDbss_noCompress; /* don't even attempt compression below a certain srcSize */ + } + ZSTD_resetSeqStore(&(zc->seqStore)); + /* required for optimal parser to read stats from dictionary */ + ms->opt.symbolCosts = &zc->blockState.prevCBlock->entropy; + /* tell the optimal parser how we expect to compress literals */ + ms->opt.literalCompressionMode = zc->appliedParams.literalCompressionMode; + /* a gap between an attached dict and the current window is not safe, + * they must remain adjacent, + * and when that stops being the case, the dict must be unset */ + assert(ms->dictMatchState == NULL || ms->loadedDictEnd == ms->window.dictLimit); + + /* limited update after a very long match */ + { const BYTE* const base = ms->window.base; + const BYTE* const istart = (const BYTE*)src; + const U32 curr = (U32)(istart-base); + if (sizeof(ptrdiff_t)==8) assert(istart - base < (ptrdiff_t)(U32)(-1)); /* ensure no overflow */ + if (curr > ms->nextToUpdate + 384) + ms->nextToUpdate = curr - MIN(192, (U32)(curr - ms->nextToUpdate - 384)); + } + + /* select and store sequences */ + { ZSTD_dictMode_e const dictMode = ZSTD_matchState_dictMode(ms); + size_t lastLLSize; + { int i; + for (i = 0; i < ZSTD_REP_NUM; ++i) + zc->blockState.nextCBlock->rep[i] = zc->blockState.prevCBlock->rep[i]; + } + if (zc->externSeqStore.pos < zc->externSeqStore.size) { + assert(zc->appliedParams.ldmParams.enableLdm == ZSTD_ps_disable); + /* Updates ldmSeqStore.pos */ + lastLLSize = + ZSTD_ldm_blockCompress(&zc->externSeqStore, + ms, &zc->seqStore, + zc->blockState.nextCBlock->rep, + zc->appliedParams.useRowMatchFinder, + src, srcSize); + assert(zc->externSeqStore.pos <= zc->externSeqStore.size); + } else if (zc->appliedParams.ldmParams.enableLdm == ZSTD_ps_enable) { + rawSeqStore_t ldmSeqStore = kNullRawSeqStore; + + ldmSeqStore.seq = zc->ldmSequences; + ldmSeqStore.capacity = zc->maxNbLdmSequences; + /* Updates ldmSeqStore.size */ + FORWARD_IF_ERROR(ZSTD_ldm_generateSequences(&zc->ldmState, &ldmSeqStore, + &zc->appliedParams.ldmParams, + src, srcSize), ""); + /* Updates ldmSeqStore.pos */ + lastLLSize = + ZSTD_ldm_blockCompress(&ldmSeqStore, + ms, &zc->seqStore, + zc->blockState.nextCBlock->rep, + zc->appliedParams.useRowMatchFinder, + src, srcSize); + assert(ldmSeqStore.pos == ldmSeqStore.size); + } else { /* not long range mode */ + ZSTD_blockCompressor const blockCompressor = ZSTD_selectBlockCompressor(zc->appliedParams.cParams.strategy, + zc->appliedParams.useRowMatchFinder, + dictMode); + ms->ldmSeqStore = NULL; + lastLLSize = blockCompressor(ms, &zc->seqStore, zc->blockState.nextCBlock->rep, src, srcSize); + } + { const BYTE* const lastLiterals = (const BYTE*)src + srcSize - lastLLSize; + ZSTD_storeLastLiterals(&zc->seqStore, lastLiterals, lastLLSize); + } } + return ZSTDbss_compress; +} + +static void ZSTD_copyBlockSequences(ZSTD_CCtx* zc) +{ + const seqStore_t* seqStore = ZSTD_getSeqStore(zc); + const seqDef* seqStoreSeqs = seqStore->sequencesStart; + size_t seqStoreSeqSize = seqStore->sequences - seqStoreSeqs; + size_t seqStoreLiteralsSize = (size_t)(seqStore->lit - seqStore->litStart); + size_t literalsRead = 0; + size_t lastLLSize; + + ZSTD_Sequence* outSeqs = &zc->seqCollector.seqStart[zc->seqCollector.seqIndex]; + size_t i; + repcodes_t updatedRepcodes; + + assert(zc->seqCollector.seqIndex + 1 < zc->seqCollector.maxSequences); + /* Ensure we have enough space for last literals "sequence" */ + assert(zc->seqCollector.maxSequences >= seqStoreSeqSize + 1); + ZSTD_memcpy(updatedRepcodes.rep, zc->blockState.prevCBlock->rep, sizeof(repcodes_t)); + for (i = 0; i < seqStoreSeqSize; ++i) { + U32 rawOffset = seqStoreSeqs[i].offBase - ZSTD_REP_NUM; + outSeqs[i].litLength = seqStoreSeqs[i].litLength; + outSeqs[i].matchLength = seqStoreSeqs[i].mlBase + MINMATCH; + outSeqs[i].rep = 0; + + if (i == seqStore->longLengthPos) { + if (seqStore->longLengthType == ZSTD_llt_literalLength) { + outSeqs[i].litLength += 0x10000; + } else if (seqStore->longLengthType == ZSTD_llt_matchLength) { + outSeqs[i].matchLength += 0x10000; + } + } + + if (seqStoreSeqs[i].offBase <= ZSTD_REP_NUM) { + /* Derive the correct offset corresponding to a repcode */ + outSeqs[i].rep = seqStoreSeqs[i].offBase; + if (outSeqs[i].litLength != 0) { + rawOffset = updatedRepcodes.rep[outSeqs[i].rep - 1]; + } else { + if (outSeqs[i].rep == 3) { + rawOffset = updatedRepcodes.rep[0] - 1; + } else { + rawOffset = updatedRepcodes.rep[outSeqs[i].rep]; + } + } + } + outSeqs[i].offset = rawOffset; + /* seqStoreSeqs[i].offset == offCode+1, and ZSTD_updateRep() expects offCode + so we provide seqStoreSeqs[i].offset - 1 */ + ZSTD_updateRep(updatedRepcodes.rep, + seqStoreSeqs[i].offBase - 1, + seqStoreSeqs[i].litLength == 0); + literalsRead += outSeqs[i].litLength; + } + /* Insert last literals (if any exist) in the block as a sequence with ml == off == 0. + * If there are no last literals, then we'll emit (of: 0, ml: 0, ll: 0), which is a marker + * for the block boundary, according to the API. + */ + assert(seqStoreLiteralsSize >= literalsRead); + lastLLSize = seqStoreLiteralsSize - literalsRead; + outSeqs[i].litLength = (U32)lastLLSize; + outSeqs[i].matchLength = outSeqs[i].offset = outSeqs[i].rep = 0; + seqStoreSeqSize++; + zc->seqCollector.seqIndex += seqStoreSeqSize; +} + +size_t ZSTD_generateSequences(ZSTD_CCtx* zc, ZSTD_Sequence* outSeqs, + size_t outSeqsSize, const void* src, size_t srcSize) +{ + const size_t dstCapacity = ZSTD_compressBound(srcSize); + void* dst = ZSTD_customMalloc(dstCapacity, ZSTD_defaultCMem); + SeqCollector seqCollector; + + RETURN_ERROR_IF(dst == NULL, memory_allocation, "NULL pointer!"); + + seqCollector.collectSequences = 1; + seqCollector.seqStart = outSeqs; + seqCollector.seqIndex = 0; + seqCollector.maxSequences = outSeqsSize; + zc->seqCollector = seqCollector; + + ZSTD_compress2(zc, dst, dstCapacity, src, srcSize); + ZSTD_customFree(dst, ZSTD_defaultCMem); + return zc->seqCollector.seqIndex; +} + +size_t ZSTD_mergeBlockDelimiters(ZSTD_Sequence* sequences, size_t seqsSize) { + size_t in = 0; + size_t out = 0; + for (; in < seqsSize; ++in) { + if (sequences[in].offset == 0 && sequences[in].matchLength == 0) { + if (in != seqsSize - 1) { + sequences[in+1].litLength += sequences[in].litLength; + } + } else { + sequences[out] = sequences[in]; + ++out; + } + } + return out; +} + +/* Unrolled loop to read four size_ts of input at a time. Returns 1 if is RLE, 0 if not. */ +static int ZSTD_isRLE(const BYTE* src, size_t length) { + const BYTE* ip = src; + const BYTE value = ip[0]; + const size_t valueST = (size_t)((U64)value * 0x0101010101010101ULL); + const size_t unrollSize = sizeof(size_t) * 4; + const size_t unrollMask = unrollSize - 1; + const size_t prefixLength = length & unrollMask; + size_t i; + size_t u; + if (length == 1) return 1; + /* Check if prefix is RLE first before using unrolled loop */ + if (prefixLength && ZSTD_count(ip+1, ip, ip+prefixLength) != prefixLength-1) { + return 0; + } + for (i = prefixLength; i != length; i += unrollSize) { + for (u = 0; u < unrollSize; u += sizeof(size_t)) { + if (MEM_readST(ip + i + u) != valueST) { + return 0; + } + } + } + return 1; +} + +/* Returns true if the given block may be RLE. + * This is just a heuristic based on the compressibility. + * It may return both false positives and false negatives. + */ +static int ZSTD_maybeRLE(seqStore_t const* seqStore) +{ + size_t const nbSeqs = (size_t)(seqStore->sequences - seqStore->sequencesStart); + size_t const nbLits = (size_t)(seqStore->lit - seqStore->litStart); + + return nbSeqs < 4 && nbLits < 10; +} + +static void ZSTD_blockState_confirmRepcodesAndEntropyTables(ZSTD_blockState_t* const bs) +{ + ZSTD_compressedBlockState_t* const tmp = bs->prevCBlock; + bs->prevCBlock = bs->nextCBlock; + bs->nextCBlock = tmp; +} + +/* Writes the block header */ +static void writeBlockHeader(void* op, size_t cSize, size_t blockSize, U32 lastBlock) { + U32 const cBlockHeader = cSize == 1 ? + lastBlock + (((U32)bt_rle)<<1) + (U32)(blockSize << 3) : + lastBlock + (((U32)bt_compressed)<<1) + (U32)(cSize << 3); + MEM_writeLE24(op, cBlockHeader); + DEBUGLOG(3, "writeBlockHeader: cSize: %zu blockSize: %zu lastBlock: %u", cSize, blockSize, lastBlock); +} + +/** ZSTD_buildBlockEntropyStats_literals() : + * Builds entropy for the literals. + * Stores literals block type (raw, rle, compressed, repeat) and + * huffman description table to hufMetadata. + * Requires ENTROPY_WORKSPACE_SIZE workspace + * @return : size of huffman description table or error code */ +static size_t ZSTD_buildBlockEntropyStats_literals(void* const src, size_t srcSize, + const ZSTD_hufCTables_t* prevHuf, + ZSTD_hufCTables_t* nextHuf, + ZSTD_hufCTablesMetadata_t* hufMetadata, + const int literalsCompressionIsDisabled, + void* workspace, size_t wkspSize) +{ + BYTE* const wkspStart = (BYTE*)workspace; + BYTE* const wkspEnd = wkspStart + wkspSize; + BYTE* const countWkspStart = wkspStart; + unsigned* const countWksp = (unsigned*)workspace; + const size_t countWkspSize = (HUF_SYMBOLVALUE_MAX + 1) * sizeof(unsigned); + BYTE* const nodeWksp = countWkspStart + countWkspSize; + const size_t nodeWkspSize = wkspEnd-nodeWksp; + unsigned maxSymbolValue = HUF_SYMBOLVALUE_MAX; + unsigned huffLog = HUF_TABLELOG_DEFAULT; + HUF_repeat repeat = prevHuf->repeatMode; + DEBUGLOG(5, "ZSTD_buildBlockEntropyStats_literals (srcSize=%zu)", srcSize); + + /* Prepare nextEntropy assuming reusing the existing table */ + ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf)); + + if (literalsCompressionIsDisabled) { + DEBUGLOG(5, "set_basic - disabled"); + hufMetadata->hType = set_basic; + return 0; + } + + /* small ? don't even attempt compression (speed opt) */ +#ifndef COMPRESS_LITERALS_SIZE_MIN +#define COMPRESS_LITERALS_SIZE_MIN 63 +#endif + { size_t const minLitSize = (prevHuf->repeatMode == HUF_repeat_valid) ? 6 : COMPRESS_LITERALS_SIZE_MIN; + if (srcSize <= minLitSize) { + DEBUGLOG(5, "set_basic - too small"); + hufMetadata->hType = set_basic; + return 0; + } + } + + /* Scan input and build symbol stats */ + { size_t const largest = HIST_count_wksp (countWksp, &maxSymbolValue, (const BYTE*)src, srcSize, workspace, wkspSize); + FORWARD_IF_ERROR(largest, "HIST_count_wksp failed"); + if (largest == srcSize) { + DEBUGLOG(5, "set_rle"); + hufMetadata->hType = set_rle; + return 0; + } + if (largest <= (srcSize >> 7)+4) { + DEBUGLOG(5, "set_basic - no gain"); + hufMetadata->hType = set_basic; + return 0; + } + } + + /* Validate the previous Huffman table */ + if (repeat == HUF_repeat_check && !HUF_validateCTable((HUF_CElt const*)prevHuf->CTable, countWksp, maxSymbolValue)) { + repeat = HUF_repeat_none; + } + + /* Build Huffman Tree */ + ZSTD_memset(nextHuf->CTable, 0, sizeof(nextHuf->CTable)); + huffLog = HUF_optimalTableLog(huffLog, srcSize, maxSymbolValue); + { size_t const maxBits = HUF_buildCTable_wksp((HUF_CElt*)nextHuf->CTable, countWksp, + maxSymbolValue, huffLog, + nodeWksp, nodeWkspSize); + FORWARD_IF_ERROR(maxBits, "HUF_buildCTable_wksp"); + huffLog = (U32)maxBits; + { /* Build and write the CTable */ + size_t const newCSize = HUF_estimateCompressedSize( + (HUF_CElt*)nextHuf->CTable, countWksp, maxSymbolValue); + size_t const hSize = HUF_writeCTable_wksp( + hufMetadata->hufDesBuffer, sizeof(hufMetadata->hufDesBuffer), + (HUF_CElt*)nextHuf->CTable, maxSymbolValue, huffLog, + nodeWksp, nodeWkspSize); + /* Check against repeating the previous CTable */ + if (repeat != HUF_repeat_none) { + size_t const oldCSize = HUF_estimateCompressedSize( + (HUF_CElt const*)prevHuf->CTable, countWksp, maxSymbolValue); + if (oldCSize < srcSize && (oldCSize <= hSize + newCSize || hSize + 12 >= srcSize)) { + DEBUGLOG(5, "set_repeat - smaller"); + ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf)); + hufMetadata->hType = set_repeat; + return 0; + } + } + if (newCSize + hSize >= srcSize) { + DEBUGLOG(5, "set_basic - no gains"); + ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf)); + hufMetadata->hType = set_basic; + return 0; + } + DEBUGLOG(5, "set_compressed (hSize=%u)", (U32)hSize); + hufMetadata->hType = set_compressed; + nextHuf->repeatMode = HUF_repeat_check; + return hSize; + } + } +} + + +/* ZSTD_buildDummySequencesStatistics(): + * Returns a ZSTD_symbolEncodingTypeStats_t with all encoding types as set_basic, + * and updates nextEntropy to the appropriate repeatMode. + */ +static ZSTD_symbolEncodingTypeStats_t +ZSTD_buildDummySequencesStatistics(ZSTD_fseCTables_t* nextEntropy) { + ZSTD_symbolEncodingTypeStats_t stats = {set_basic, set_basic, set_basic, 0, 0}; + nextEntropy->litlength_repeatMode = FSE_repeat_none; + nextEntropy->offcode_repeatMode = FSE_repeat_none; + nextEntropy->matchlength_repeatMode = FSE_repeat_none; + return stats; +} + +/** ZSTD_buildBlockEntropyStats_sequences() : + * Builds entropy for the sequences. + * Stores symbol compression modes and fse table to fseMetadata. + * Requires ENTROPY_WORKSPACE_SIZE wksp. + * @return : size of fse tables or error code */ +static size_t ZSTD_buildBlockEntropyStats_sequences(seqStore_t* seqStorePtr, + const ZSTD_fseCTables_t* prevEntropy, + ZSTD_fseCTables_t* nextEntropy, + const ZSTD_CCtx_params* cctxParams, + ZSTD_fseCTablesMetadata_t* fseMetadata, + void* workspace, size_t wkspSize) +{ + ZSTD_strategy const strategy = cctxParams->cParams.strategy; + size_t const nbSeq = seqStorePtr->sequences - seqStorePtr->sequencesStart; + BYTE* const ostart = fseMetadata->fseTablesBuffer; + BYTE* const oend = ostart + sizeof(fseMetadata->fseTablesBuffer); + BYTE* op = ostart; + unsigned* countWorkspace = (unsigned*)workspace; + unsigned* entropyWorkspace = countWorkspace + (MaxSeq + 1); + size_t entropyWorkspaceSize = wkspSize - (MaxSeq + 1) * sizeof(*countWorkspace); + ZSTD_symbolEncodingTypeStats_t stats; + + DEBUGLOG(5, "ZSTD_buildBlockEntropyStats_sequences (nbSeq=%zu)", nbSeq); + stats = nbSeq != 0 ? ZSTD_buildSequencesStatistics(seqStorePtr, nbSeq, + prevEntropy, nextEntropy, op, oend, + strategy, countWorkspace, + entropyWorkspace, entropyWorkspaceSize) + : ZSTD_buildDummySequencesStatistics(nextEntropy); + FORWARD_IF_ERROR(stats.size, "ZSTD_buildSequencesStatistics failed!"); + fseMetadata->llType = (symbolEncodingType_e) stats.LLtype; + fseMetadata->ofType = (symbolEncodingType_e) stats.Offtype; + fseMetadata->mlType = (symbolEncodingType_e) stats.MLtype; + fseMetadata->lastCountSize = stats.lastCountSize; + return stats.size; +} + + +/** ZSTD_buildBlockEntropyStats() : + * Builds entropy for the block. + * Requires workspace size ENTROPY_WORKSPACE_SIZE + * + * @return : 0 on success or error code + */ +size_t ZSTD_buildBlockEntropyStats(seqStore_t* seqStorePtr, + const ZSTD_entropyCTables_t* prevEntropy, + ZSTD_entropyCTables_t* nextEntropy, + const ZSTD_CCtx_params* cctxParams, + ZSTD_entropyCTablesMetadata_t* entropyMetadata, + void* workspace, size_t wkspSize) +{ + size_t const litSize = seqStorePtr->lit - seqStorePtr->litStart; + entropyMetadata->hufMetadata.hufDesSize = + ZSTD_buildBlockEntropyStats_literals(seqStorePtr->litStart, litSize, + &prevEntropy->huf, &nextEntropy->huf, + &entropyMetadata->hufMetadata, + ZSTD_literalsCompressionIsDisabled(cctxParams), + workspace, wkspSize); + FORWARD_IF_ERROR(entropyMetadata->hufMetadata.hufDesSize, "ZSTD_buildBlockEntropyStats_literals failed"); + entropyMetadata->fseMetadata.fseTablesSize = + ZSTD_buildBlockEntropyStats_sequences(seqStorePtr, + &prevEntropy->fse, &nextEntropy->fse, + cctxParams, + &entropyMetadata->fseMetadata, + workspace, wkspSize); + FORWARD_IF_ERROR(entropyMetadata->fseMetadata.fseTablesSize, "ZSTD_buildBlockEntropyStats_sequences failed"); + return 0; +} + +/* Returns the size estimate for the literals section (header + content) of a block */ +static size_t ZSTD_estimateBlockSize_literal(const BYTE* literals, size_t litSize, + const ZSTD_hufCTables_t* huf, + const ZSTD_hufCTablesMetadata_t* hufMetadata, + void* workspace, size_t wkspSize, + int writeEntropy) +{ + unsigned* const countWksp = (unsigned*)workspace; + unsigned maxSymbolValue = HUF_SYMBOLVALUE_MAX; + size_t literalSectionHeaderSize = 3 + (litSize >= 1 KB) + (litSize >= 16 KB); + U32 singleStream = litSize < 256; + + if (hufMetadata->hType == set_basic) return litSize; + else if (hufMetadata->hType == set_rle) return 1; + else if (hufMetadata->hType == set_compressed || hufMetadata->hType == set_repeat) { + size_t const largest = HIST_count_wksp (countWksp, &maxSymbolValue, (const BYTE*)literals, litSize, workspace, wkspSize); + if (ZSTD_isError(largest)) return litSize; + { size_t cLitSizeEstimate = HUF_estimateCompressedSize((const HUF_CElt*)huf->CTable, countWksp, maxSymbolValue); + if (writeEntropy) cLitSizeEstimate += hufMetadata->hufDesSize; + if (!singleStream) cLitSizeEstimate += 6; /* multi-stream huffman uses 6-byte jump table */ + return cLitSizeEstimate + literalSectionHeaderSize; + } } + assert(0); /* impossible */ + return 0; +} + +/* Returns the size estimate for the FSE-compressed symbols (of, ml, ll) of a block */ +static size_t ZSTD_estimateBlockSize_symbolType(symbolEncodingType_e type, + const BYTE* codeTable, size_t nbSeq, unsigned maxCode, + const FSE_CTable* fseCTable, + const U8* additionalBits, + short const* defaultNorm, U32 defaultNormLog, U32 defaultMax, + void* workspace, size_t wkspSize) +{ + unsigned* const countWksp = (unsigned*)workspace; + const BYTE* ctp = codeTable; + const BYTE* const ctStart = ctp; + const BYTE* const ctEnd = ctStart + nbSeq; + size_t cSymbolTypeSizeEstimateInBits = 0; + unsigned max = maxCode; + + HIST_countFast_wksp(countWksp, &max, codeTable, nbSeq, workspace, wkspSize); /* can't fail */ + if (type == set_basic) { + /* We selected this encoding type, so it must be valid. */ + assert(max <= defaultMax); + (void)defaultMax; + cSymbolTypeSizeEstimateInBits = ZSTD_crossEntropyCost(defaultNorm, defaultNormLog, countWksp, max); + } else if (type == set_rle) { + cSymbolTypeSizeEstimateInBits = 0; + } else if (type == set_compressed || type == set_repeat) { + cSymbolTypeSizeEstimateInBits = ZSTD_fseBitCost(fseCTable, countWksp, max); + } + if (ZSTD_isError(cSymbolTypeSizeEstimateInBits)) { + return nbSeq * 10; + } + while (ctp < ctEnd) { + if (additionalBits) cSymbolTypeSizeEstimateInBits += additionalBits[*ctp]; + else cSymbolTypeSizeEstimateInBits += *ctp; /* for offset, offset code is also the number of additional bits */ + ctp++; + } + return cSymbolTypeSizeEstimateInBits >> 3; +} + +/* Returns the size estimate for the sequences section (header + content) of a block */ +static size_t ZSTD_estimateBlockSize_sequences(const BYTE* ofCodeTable, + const BYTE* llCodeTable, + const BYTE* mlCodeTable, + size_t nbSeq, + const ZSTD_fseCTables_t* fseTables, + const ZSTD_fseCTablesMetadata_t* fseMetadata, + void* workspace, size_t wkspSize, + int writeEntropy) +{ + size_t sequencesSectionHeaderSize = 1 /* seqHead */ + 1 /* min seqSize size */ + (nbSeq >= 128) + (nbSeq >= LONGNBSEQ); + size_t cSeqSizeEstimate = 0; + cSeqSizeEstimate += ZSTD_estimateBlockSize_symbolType(fseMetadata->ofType, ofCodeTable, nbSeq, MaxOff, + fseTables->offcodeCTable, NULL, + OF_defaultNorm, OF_defaultNormLog, DefaultMaxOff, + workspace, wkspSize); + cSeqSizeEstimate += ZSTD_estimateBlockSize_symbolType(fseMetadata->llType, llCodeTable, nbSeq, MaxLL, + fseTables->litlengthCTable, LL_bits, + LL_defaultNorm, LL_defaultNormLog, MaxLL, + workspace, wkspSize); + cSeqSizeEstimate += ZSTD_estimateBlockSize_symbolType(fseMetadata->mlType, mlCodeTable, nbSeq, MaxML, + fseTables->matchlengthCTable, ML_bits, + ML_defaultNorm, ML_defaultNormLog, MaxML, + workspace, wkspSize); + if (writeEntropy) cSeqSizeEstimate += fseMetadata->fseTablesSize; + return cSeqSizeEstimate + sequencesSectionHeaderSize; +} + +/* Returns the size estimate for a given stream of literals, of, ll, ml */ +static size_t ZSTD_estimateBlockSize(const BYTE* literals, size_t litSize, + const BYTE* ofCodeTable, + const BYTE* llCodeTable, + const BYTE* mlCodeTable, + size_t nbSeq, + const ZSTD_entropyCTables_t* entropy, + const ZSTD_entropyCTablesMetadata_t* entropyMetadata, + void* workspace, size_t wkspSize, + int writeLitEntropy, int writeSeqEntropy) { + size_t const literalsSize = ZSTD_estimateBlockSize_literal(literals, litSize, + &entropy->huf, &entropyMetadata->hufMetadata, + workspace, wkspSize, writeLitEntropy); + size_t const seqSize = ZSTD_estimateBlockSize_sequences(ofCodeTable, llCodeTable, mlCodeTable, + nbSeq, &entropy->fse, &entropyMetadata->fseMetadata, + workspace, wkspSize, writeSeqEntropy); + return seqSize + literalsSize + ZSTD_blockHeaderSize; +} + +/* Builds entropy statistics and uses them for blocksize estimation. + * + * Returns the estimated compressed size of the seqStore, or a zstd error. + */ +static size_t ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize(seqStore_t* seqStore, ZSTD_CCtx* zc) { + ZSTD_entropyCTablesMetadata_t* entropyMetadata = &zc->blockSplitCtx.entropyMetadata; + DEBUGLOG(6, "ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize()"); + FORWARD_IF_ERROR(ZSTD_buildBlockEntropyStats(seqStore, + &zc->blockState.prevCBlock->entropy, + &zc->blockState.nextCBlock->entropy, + &zc->appliedParams, + entropyMetadata, + zc->entropyWorkspace, ENTROPY_WORKSPACE_SIZE /* statically allocated in resetCCtx */), ""); + return ZSTD_estimateBlockSize(seqStore->litStart, (size_t)(seqStore->lit - seqStore->litStart), + seqStore->ofCode, seqStore->llCode, seqStore->mlCode, + (size_t)(seqStore->sequences - seqStore->sequencesStart), + &zc->blockState.nextCBlock->entropy, entropyMetadata, zc->entropyWorkspace, ENTROPY_WORKSPACE_SIZE, + (int)(entropyMetadata->hufMetadata.hType == set_compressed), 1); +} + +/* Returns literals bytes represented in a seqStore */ +static size_t ZSTD_countSeqStoreLiteralsBytes(const seqStore_t* const seqStore) { + size_t literalsBytes = 0; + size_t const nbSeqs = seqStore->sequences - seqStore->sequencesStart; + size_t i; + for (i = 0; i < nbSeqs; ++i) { + seqDef seq = seqStore->sequencesStart[i]; + literalsBytes += seq.litLength; + if (i == seqStore->longLengthPos && seqStore->longLengthType == ZSTD_llt_literalLength) { + literalsBytes += 0x10000; + } + } + return literalsBytes; +} + +/* Returns match bytes represented in a seqStore */ +static size_t ZSTD_countSeqStoreMatchBytes(const seqStore_t* const seqStore) { + size_t matchBytes = 0; + size_t const nbSeqs = seqStore->sequences - seqStore->sequencesStart; + size_t i; + for (i = 0; i < nbSeqs; ++i) { + seqDef seq = seqStore->sequencesStart[i]; + matchBytes += seq.mlBase + MINMATCH; + if (i == seqStore->longLengthPos && seqStore->longLengthType == ZSTD_llt_matchLength) { + matchBytes += 0x10000; + } + } + return matchBytes; +} + +/* Derives the seqStore that is a chunk of the originalSeqStore from [startIdx, endIdx). + * Stores the result in resultSeqStore. + */ +static void ZSTD_deriveSeqStoreChunk(seqStore_t* resultSeqStore, + const seqStore_t* originalSeqStore, + size_t startIdx, size_t endIdx) { + BYTE* const litEnd = originalSeqStore->lit; + size_t literalsBytes; + size_t literalsBytesPreceding = 0; + + *resultSeqStore = *originalSeqStore; + if (startIdx > 0) { + resultSeqStore->sequences = originalSeqStore->sequencesStart + startIdx; + literalsBytesPreceding = ZSTD_countSeqStoreLiteralsBytes(resultSeqStore); + } + + /* Move longLengthPos into the correct position if necessary */ + if (originalSeqStore->longLengthType != ZSTD_llt_none) { + if (originalSeqStore->longLengthPos < startIdx || originalSeqStore->longLengthPos > endIdx) { + resultSeqStore->longLengthType = ZSTD_llt_none; + } else { + resultSeqStore->longLengthPos -= (U32)startIdx; + } + } + resultSeqStore->sequencesStart = originalSeqStore->sequencesStart + startIdx; + resultSeqStore->sequences = originalSeqStore->sequencesStart + endIdx; + literalsBytes = ZSTD_countSeqStoreLiteralsBytes(resultSeqStore); + resultSeqStore->litStart += literalsBytesPreceding; + if (endIdx == (size_t)(originalSeqStore->sequences - originalSeqStore->sequencesStart)) { + /* This accounts for possible last literals if the derived chunk reaches the end of the block */ + resultSeqStore->lit = litEnd; + } else { + resultSeqStore->lit = resultSeqStore->litStart+literalsBytes; + } + resultSeqStore->llCode += startIdx; + resultSeqStore->mlCode += startIdx; + resultSeqStore->ofCode += startIdx; +} + +/** + * Returns the raw offset represented by the combination of offCode, ll0, and repcode history. + * offCode must represent a repcode in the numeric representation of ZSTD_storeSeq(). + */ +static U32 +ZSTD_resolveRepcodeToRawOffset(const U32 rep[ZSTD_REP_NUM], const U32 offCode, const U32 ll0) +{ + U32 const adjustedOffCode = STORED_REPCODE(offCode) - 1 + ll0; /* [ 0 - 3 ] */ + assert(STORED_IS_REPCODE(offCode)); + if (adjustedOffCode == ZSTD_REP_NUM) { + /* litlength == 0 and offCode == 2 implies selection of first repcode - 1 */ + assert(rep[0] > 0); + return rep[0] - 1; + } + return rep[adjustedOffCode]; +} + +/** + * ZSTD_seqStore_resolveOffCodes() reconciles any possible divergences in offset history that may arise + * due to emission of RLE/raw blocks that disturb the offset history, + * and replaces any repcodes within the seqStore that may be invalid. + * + * dRepcodes are updated as would be on the decompression side. + * cRepcodes are updated exactly in accordance with the seqStore. + * + * Note : this function assumes seq->offBase respects the following numbering scheme : + * 0 : invalid + * 1-3 : repcode 1-3 + * 4+ : real_offset+3 + */ +static void ZSTD_seqStore_resolveOffCodes(repcodes_t* const dRepcodes, repcodes_t* const cRepcodes, + seqStore_t* const seqStore, U32 const nbSeq) { + U32 idx = 0; + for (; idx < nbSeq; ++idx) { + seqDef* const seq = seqStore->sequencesStart + idx; + U32 const ll0 = (seq->litLength == 0); + U32 const offCode = OFFBASE_TO_STORED(seq->offBase); + assert(seq->offBase > 0); + if (STORED_IS_REPCODE(offCode)) { + U32 const dRawOffset = ZSTD_resolveRepcodeToRawOffset(dRepcodes->rep, offCode, ll0); + U32 const cRawOffset = ZSTD_resolveRepcodeToRawOffset(cRepcodes->rep, offCode, ll0); + /* Adjust simulated decompression repcode history if we come across a mismatch. Replace + * the repcode with the offset it actually references, determined by the compression + * repcode history. + */ + if (dRawOffset != cRawOffset) { + seq->offBase = cRawOffset + ZSTD_REP_NUM; + } + } + /* Compression repcode history is always updated with values directly from the unmodified seqStore. + * Decompression repcode history may use modified seq->offset value taken from compression repcode history. + */ + ZSTD_updateRep(dRepcodes->rep, OFFBASE_TO_STORED(seq->offBase), ll0); + ZSTD_updateRep(cRepcodes->rep, offCode, ll0); + } +} + +/* ZSTD_compressSeqStore_singleBlock(): + * Compresses a seqStore into a block with a block header, into the buffer dst. + * + * Returns the total size of that block (including header) or a ZSTD error code. + */ +static size_t +ZSTD_compressSeqStore_singleBlock(ZSTD_CCtx* zc, seqStore_t* const seqStore, + repcodes_t* const dRep, repcodes_t* const cRep, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + U32 lastBlock, U32 isPartition) +{ + const U32 rleMaxLength = 25; + BYTE* op = (BYTE*)dst; + const BYTE* ip = (const BYTE*)src; + size_t cSize; + size_t cSeqsSize; + + /* In case of an RLE or raw block, the simulated decompression repcode history must be reset */ + repcodes_t const dRepOriginal = *dRep; + DEBUGLOG(5, "ZSTD_compressSeqStore_singleBlock"); + if (isPartition) + ZSTD_seqStore_resolveOffCodes(dRep, cRep, seqStore, (U32)(seqStore->sequences - seqStore->sequencesStart)); + + RETURN_ERROR_IF(dstCapacity < ZSTD_blockHeaderSize, dstSize_tooSmall, "Block header doesn't fit"); + cSeqsSize = ZSTD_entropyCompressSeqStore(seqStore, + &zc->blockState.prevCBlock->entropy, &zc->blockState.nextCBlock->entropy, + &zc->appliedParams, + op + ZSTD_blockHeaderSize, dstCapacity - ZSTD_blockHeaderSize, + srcSize, + zc->entropyWorkspace, ENTROPY_WORKSPACE_SIZE /* statically allocated in resetCCtx */, + zc->bmi2); + FORWARD_IF_ERROR(cSeqsSize, "ZSTD_entropyCompressSeqStore failed!"); + + if (!zc->isFirstBlock && + cSeqsSize < rleMaxLength && + ZSTD_isRLE((BYTE const*)src, srcSize)) { + /* We don't want to emit our first block as a RLE even if it qualifies because + * doing so will cause the decoder (cli only) to throw a "should consume all input error." + * This is only an issue for zstd <= v1.4.3 + */ + cSeqsSize = 1; + } + + if (zc->seqCollector.collectSequences) { + ZSTD_copyBlockSequences(zc); + ZSTD_blockState_confirmRepcodesAndEntropyTables(&zc->blockState); + return 0; + } + + if (cSeqsSize == 0) { + cSize = ZSTD_noCompressBlock(op, dstCapacity, ip, srcSize, lastBlock); + FORWARD_IF_ERROR(cSize, "Nocompress block failed"); + DEBUGLOG(4, "Writing out nocompress block, size: %zu", cSize); + *dRep = dRepOriginal; /* reset simulated decompression repcode history */ + } else if (cSeqsSize == 1) { + cSize = ZSTD_rleCompressBlock(op, dstCapacity, *ip, srcSize, lastBlock); + FORWARD_IF_ERROR(cSize, "RLE compress block failed"); + DEBUGLOG(4, "Writing out RLE block, size: %zu", cSize); + *dRep = dRepOriginal; /* reset simulated decompression repcode history */ + } else { + ZSTD_blockState_confirmRepcodesAndEntropyTables(&zc->blockState); + writeBlockHeader(op, cSeqsSize, srcSize, lastBlock); + cSize = ZSTD_blockHeaderSize + cSeqsSize; + DEBUGLOG(4, "Writing out compressed block, size: %zu", cSize); + } + + if (zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode == FSE_repeat_valid) + zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode = FSE_repeat_check; + + return cSize; +} + +/* Struct to keep track of where we are in our recursive calls. */ +typedef struct { + U32* splitLocations; /* Array of split indices */ + size_t idx; /* The current index within splitLocations being worked on */ +} seqStoreSplits; + +#define MIN_SEQUENCES_BLOCK_SPLITTING 300 + +/* Helper function to perform the recursive search for block splits. + * Estimates the cost of seqStore prior to split, and estimates the cost of splitting the sequences in half. + * If advantageous to split, then we recurse down the two sub-blocks. If not, or if an error occurred in estimation, then + * we do not recurse. + * + * Note: The recursion depth is capped by a heuristic minimum number of sequences, defined by MIN_SEQUENCES_BLOCK_SPLITTING. + * In theory, this means the absolute largest recursion depth is 10 == log2(maxNbSeqInBlock/MIN_SEQUENCES_BLOCK_SPLITTING). + * In practice, recursion depth usually doesn't go beyond 4. + * + * Furthermore, the number of splits is capped by ZSTD_MAX_NB_BLOCK_SPLITS. At ZSTD_MAX_NB_BLOCK_SPLITS == 196 with the current existing blockSize + * maximum of 128 KB, this value is actually impossible to reach. + */ +static void +ZSTD_deriveBlockSplitsHelper(seqStoreSplits* splits, size_t startIdx, size_t endIdx, + ZSTD_CCtx* zc, const seqStore_t* origSeqStore) +{ + seqStore_t* fullSeqStoreChunk = &zc->blockSplitCtx.fullSeqStoreChunk; + seqStore_t* firstHalfSeqStore = &zc->blockSplitCtx.firstHalfSeqStore; + seqStore_t* secondHalfSeqStore = &zc->blockSplitCtx.secondHalfSeqStore; + size_t estimatedOriginalSize; + size_t estimatedFirstHalfSize; + size_t estimatedSecondHalfSize; + size_t midIdx = (startIdx + endIdx)/2; + + if (endIdx - startIdx < MIN_SEQUENCES_BLOCK_SPLITTING || splits->idx >= ZSTD_MAX_NB_BLOCK_SPLITS) { + DEBUGLOG(6, "ZSTD_deriveBlockSplitsHelper: Too few sequences"); + return; + } + DEBUGLOG(4, "ZSTD_deriveBlockSplitsHelper: startIdx=%zu endIdx=%zu", startIdx, endIdx); + ZSTD_deriveSeqStoreChunk(fullSeqStoreChunk, origSeqStore, startIdx, endIdx); + ZSTD_deriveSeqStoreChunk(firstHalfSeqStore, origSeqStore, startIdx, midIdx); + ZSTD_deriveSeqStoreChunk(secondHalfSeqStore, origSeqStore, midIdx, endIdx); + estimatedOriginalSize = ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize(fullSeqStoreChunk, zc); + estimatedFirstHalfSize = ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize(firstHalfSeqStore, zc); + estimatedSecondHalfSize = ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize(secondHalfSeqStore, zc); + DEBUGLOG(4, "Estimated original block size: %zu -- First half split: %zu -- Second half split: %zu", + estimatedOriginalSize, estimatedFirstHalfSize, estimatedSecondHalfSize); + if (ZSTD_isError(estimatedOriginalSize) || ZSTD_isError(estimatedFirstHalfSize) || ZSTD_isError(estimatedSecondHalfSize)) { + return; + } + if (estimatedFirstHalfSize + estimatedSecondHalfSize < estimatedOriginalSize) { + ZSTD_deriveBlockSplitsHelper(splits, startIdx, midIdx, zc, origSeqStore); + splits->splitLocations[splits->idx] = (U32)midIdx; + splits->idx++; + ZSTD_deriveBlockSplitsHelper(splits, midIdx, endIdx, zc, origSeqStore); + } +} + +/* Base recursive function. Populates a table with intra-block partition indices that can improve compression ratio. + * + * Returns the number of splits made (which equals the size of the partition table - 1). + */ +static size_t ZSTD_deriveBlockSplits(ZSTD_CCtx* zc, U32 partitions[], U32 nbSeq) { + seqStoreSplits splits = {partitions, 0}; + if (nbSeq <= 4) { + DEBUGLOG(4, "ZSTD_deriveBlockSplits: Too few sequences to split"); + /* Refuse to try and split anything with less than 4 sequences */ + return 0; + } + ZSTD_deriveBlockSplitsHelper(&splits, 0, nbSeq, zc, &zc->seqStore); + splits.splitLocations[splits.idx] = nbSeq; + DEBUGLOG(5, "ZSTD_deriveBlockSplits: final nb partitions: %zu", splits.idx+1); + return splits.idx; +} + +/* ZSTD_compressBlock_splitBlock(): + * Attempts to split a given block into multiple blocks to improve compression ratio. + * + * Returns combined size of all blocks (which includes headers), or a ZSTD error code. + */ +static size_t +ZSTD_compressBlock_splitBlock_internal(ZSTD_CCtx* zc, void* dst, size_t dstCapacity, + const void* src, size_t blockSize, U32 lastBlock, U32 nbSeq) +{ + size_t cSize = 0; + const BYTE* ip = (const BYTE*)src; + BYTE* op = (BYTE*)dst; + size_t i = 0; + size_t srcBytesTotal = 0; + U32* partitions = zc->blockSplitCtx.partitions; /* size == ZSTD_MAX_NB_BLOCK_SPLITS */ + seqStore_t* nextSeqStore = &zc->blockSplitCtx.nextSeqStore; + seqStore_t* currSeqStore = &zc->blockSplitCtx.currSeqStore; + size_t numSplits = ZSTD_deriveBlockSplits(zc, partitions, nbSeq); + + /* If a block is split and some partitions are emitted as RLE/uncompressed, then repcode history + * may become invalid. In order to reconcile potentially invalid repcodes, we keep track of two + * separate repcode histories that simulate repcode history on compression and decompression side, + * and use the histories to determine whether we must replace a particular repcode with its raw offset. + * + * 1) cRep gets updated for each partition, regardless of whether the block was emitted as uncompressed + * or RLE. This allows us to retrieve the offset value that an invalid repcode references within + * a nocompress/RLE block. + * 2) dRep gets updated only for compressed partitions, and when a repcode gets replaced, will use + * the replacement offset value rather than the original repcode to update the repcode history. + * dRep also will be the final repcode history sent to the next block. + * + * See ZSTD_seqStore_resolveOffCodes() for more details. + */ + repcodes_t dRep; + repcodes_t cRep; + ZSTD_memcpy(dRep.rep, zc->blockState.prevCBlock->rep, sizeof(repcodes_t)); + ZSTD_memcpy(cRep.rep, zc->blockState.prevCBlock->rep, sizeof(repcodes_t)); + ZSTD_memset(nextSeqStore, 0, sizeof(seqStore_t)); + + DEBUGLOG(4, "ZSTD_compressBlock_splitBlock_internal (dstCapacity=%u, dictLimit=%u, nextToUpdate=%u)", + (unsigned)dstCapacity, (unsigned)zc->blockState.matchState.window.dictLimit, + (unsigned)zc->blockState.matchState.nextToUpdate); + + if (numSplits == 0) { + size_t cSizeSingleBlock = ZSTD_compressSeqStore_singleBlock(zc, &zc->seqStore, + &dRep, &cRep, + op, dstCapacity, + ip, blockSize, + lastBlock, 0 /* isPartition */); + FORWARD_IF_ERROR(cSizeSingleBlock, "Compressing single block from splitBlock_internal() failed!"); + DEBUGLOG(5, "ZSTD_compressBlock_splitBlock_internal: No splits"); + assert(cSizeSingleBlock <= ZSTD_BLOCKSIZE_MAX + ZSTD_blockHeaderSize); + return cSizeSingleBlock; + } + + ZSTD_deriveSeqStoreChunk(currSeqStore, &zc->seqStore, 0, partitions[0]); + for (i = 0; i <= numSplits; ++i) { + size_t srcBytes; + size_t cSizeChunk; + U32 const lastPartition = (i == numSplits); + U32 lastBlockEntireSrc = 0; + + srcBytes = ZSTD_countSeqStoreLiteralsBytes(currSeqStore) + ZSTD_countSeqStoreMatchBytes(currSeqStore); + srcBytesTotal += srcBytes; + if (lastPartition) { + /* This is the final partition, need to account for possible last literals */ + srcBytes += blockSize - srcBytesTotal; + lastBlockEntireSrc = lastBlock; + } else { + ZSTD_deriveSeqStoreChunk(nextSeqStore, &zc->seqStore, partitions[i], partitions[i+1]); + } + + cSizeChunk = ZSTD_compressSeqStore_singleBlock(zc, currSeqStore, + &dRep, &cRep, + op, dstCapacity, + ip, srcBytes, + lastBlockEntireSrc, 1 /* isPartition */); + DEBUGLOG(5, "Estimated size: %zu actual size: %zu", ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize(currSeqStore, zc), cSizeChunk); + FORWARD_IF_ERROR(cSizeChunk, "Compressing chunk failed!"); + + ip += srcBytes; + op += cSizeChunk; + dstCapacity -= cSizeChunk; + cSize += cSizeChunk; + *currSeqStore = *nextSeqStore; + assert(cSizeChunk <= ZSTD_BLOCKSIZE_MAX + ZSTD_blockHeaderSize); + } + /* cRep and dRep may have diverged during the compression. If so, we use the dRep repcodes + * for the next block. + */ + ZSTD_memcpy(zc->blockState.prevCBlock->rep, dRep.rep, sizeof(repcodes_t)); + return cSize; +} + +static size_t +ZSTD_compressBlock_splitBlock(ZSTD_CCtx* zc, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, U32 lastBlock) +{ + const BYTE* ip = (const BYTE*)src; + BYTE* op = (BYTE*)dst; + U32 nbSeq; + size_t cSize; + DEBUGLOG(4, "ZSTD_compressBlock_splitBlock"); + assert(zc->appliedParams.useBlockSplitter == ZSTD_ps_enable); + + { const size_t bss = ZSTD_buildSeqStore(zc, src, srcSize); + FORWARD_IF_ERROR(bss, "ZSTD_buildSeqStore failed"); + if (bss == ZSTDbss_noCompress) { + if (zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode == FSE_repeat_valid) + zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode = FSE_repeat_check; + cSize = ZSTD_noCompressBlock(op, dstCapacity, ip, srcSize, lastBlock); + FORWARD_IF_ERROR(cSize, "ZSTD_noCompressBlock failed"); + DEBUGLOG(4, "ZSTD_compressBlock_splitBlock: Nocompress block"); + return cSize; + } + nbSeq = (U32)(zc->seqStore.sequences - zc->seqStore.sequencesStart); + } + + cSize = ZSTD_compressBlock_splitBlock_internal(zc, dst, dstCapacity, src, srcSize, lastBlock, nbSeq); + FORWARD_IF_ERROR(cSize, "Splitting blocks failed!"); + return cSize; +} + +static size_t +ZSTD_compressBlock_internal(ZSTD_CCtx* zc, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, U32 frame) +{ + /* This the upper bound for the length of an rle block. + * This isn't the actual upper bound. Finding the real threshold + * needs further investigation. + */ + const U32 rleMaxLength = 25; + size_t cSize; + const BYTE* ip = (const BYTE*)src; + BYTE* op = (BYTE*)dst; + DEBUGLOG(5, "ZSTD_compressBlock_internal (dstCapacity=%u, dictLimit=%u, nextToUpdate=%u)", + (unsigned)dstCapacity, (unsigned)zc->blockState.matchState.window.dictLimit, + (unsigned)zc->blockState.matchState.nextToUpdate); + + { const size_t bss = ZSTD_buildSeqStore(zc, src, srcSize); + FORWARD_IF_ERROR(bss, "ZSTD_buildSeqStore failed"); + if (bss == ZSTDbss_noCompress) { cSize = 0; goto out; } + } + + if (zc->seqCollector.collectSequences) { + ZSTD_copyBlockSequences(zc); + ZSTD_blockState_confirmRepcodesAndEntropyTables(&zc->blockState); + return 0; + } + + /* encode sequences and literals */ + cSize = ZSTD_entropyCompressSeqStore(&zc->seqStore, + &zc->blockState.prevCBlock->entropy, &zc->blockState.nextCBlock->entropy, + &zc->appliedParams, + dst, dstCapacity, + srcSize, + zc->entropyWorkspace, ENTROPY_WORKSPACE_SIZE /* statically allocated in resetCCtx */, + zc->bmi2); + + if (frame && + /* We don't want to emit our first block as a RLE even if it qualifies because + * doing so will cause the decoder (cli only) to throw a "should consume all input error." + * This is only an issue for zstd <= v1.4.3 + */ + !zc->isFirstBlock && + cSize < rleMaxLength && + ZSTD_isRLE(ip, srcSize)) + { + cSize = 1; + op[0] = ip[0]; + } + +out: + if (!ZSTD_isError(cSize) && cSize > 1) { + ZSTD_blockState_confirmRepcodesAndEntropyTables(&zc->blockState); + } + /* We check that dictionaries have offset codes available for the first + * block. After the first block, the offcode table might not have large + * enough codes to represent the offsets in the data. + */ + if (zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode == FSE_repeat_valid) + zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode = FSE_repeat_check; + + return cSize; +} + +static size_t ZSTD_compressBlock_targetCBlockSize_body(ZSTD_CCtx* zc, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const size_t bss, U32 lastBlock) +{ + DEBUGLOG(6, "Attempting ZSTD_compressSuperBlock()"); + if (bss == ZSTDbss_compress) { + if (/* We don't want to emit our first block as a RLE even if it qualifies because + * doing so will cause the decoder (cli only) to throw a "should consume all input error." + * This is only an issue for zstd <= v1.4.3 + */ + !zc->isFirstBlock && + ZSTD_maybeRLE(&zc->seqStore) && + ZSTD_isRLE((BYTE const*)src, srcSize)) + { + return ZSTD_rleCompressBlock(dst, dstCapacity, *(BYTE const*)src, srcSize, lastBlock); + } + /* Attempt superblock compression. + * + * Note that compressed size of ZSTD_compressSuperBlock() is not bound by the + * standard ZSTD_compressBound(). This is a problem, because even if we have + * space now, taking an extra byte now could cause us to run out of space later + * and violate ZSTD_compressBound(). + * + * Define blockBound(blockSize) = blockSize + ZSTD_blockHeaderSize. + * + * In order to respect ZSTD_compressBound() we must attempt to emit a raw + * uncompressed block in these cases: + * * cSize == 0: Return code for an uncompressed block. + * * cSize == dstSize_tooSmall: We may have expanded beyond blockBound(srcSize). + * ZSTD_noCompressBlock() will return dstSize_tooSmall if we are really out of + * output space. + * * cSize >= blockBound(srcSize): We have expanded the block too much so + * emit an uncompressed block. + */ + { + size_t const cSize = ZSTD_compressSuperBlock(zc, dst, dstCapacity, src, srcSize, lastBlock); + if (cSize != ERROR(dstSize_tooSmall)) { + size_t const maxCSize = srcSize - ZSTD_minGain(srcSize, zc->appliedParams.cParams.strategy); + FORWARD_IF_ERROR(cSize, "ZSTD_compressSuperBlock failed"); + if (cSize != 0 && cSize < maxCSize + ZSTD_blockHeaderSize) { + ZSTD_blockState_confirmRepcodesAndEntropyTables(&zc->blockState); + return cSize; + } + } + } + } + + DEBUGLOG(6, "Resorting to ZSTD_noCompressBlock()"); + /* Superblock compression failed, attempt to emit a single no compress block. + * The decoder will be able to stream this block since it is uncompressed. + */ + return ZSTD_noCompressBlock(dst, dstCapacity, src, srcSize, lastBlock); +} + +static size_t ZSTD_compressBlock_targetCBlockSize(ZSTD_CCtx* zc, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + U32 lastBlock) +{ + size_t cSize = 0; + const size_t bss = ZSTD_buildSeqStore(zc, src, srcSize); + DEBUGLOG(5, "ZSTD_compressBlock_targetCBlockSize (dstCapacity=%u, dictLimit=%u, nextToUpdate=%u, srcSize=%zu)", + (unsigned)dstCapacity, (unsigned)zc->blockState.matchState.window.dictLimit, (unsigned)zc->blockState.matchState.nextToUpdate, srcSize); + FORWARD_IF_ERROR(bss, "ZSTD_buildSeqStore failed"); + + cSize = ZSTD_compressBlock_targetCBlockSize_body(zc, dst, dstCapacity, src, srcSize, bss, lastBlock); + FORWARD_IF_ERROR(cSize, "ZSTD_compressBlock_targetCBlockSize_body failed"); + + if (zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode == FSE_repeat_valid) + zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode = FSE_repeat_check; + + return cSize; +} + +static void ZSTD_overflowCorrectIfNeeded(ZSTD_matchState_t* ms, + ZSTD_cwksp* ws, + ZSTD_CCtx_params const* params, + void const* ip, + void const* iend) +{ + U32 const cycleLog = ZSTD_cycleLog(params->cParams.chainLog, params->cParams.strategy); + U32 const maxDist = (U32)1 << params->cParams.windowLog; + if (ZSTD_window_needOverflowCorrection(ms->window, cycleLog, maxDist, ms->loadedDictEnd, ip, iend)) { + U32 const correction = ZSTD_window_correctOverflow(&ms->window, cycleLog, maxDist, ip); + ZSTD_STATIC_ASSERT(ZSTD_CHAINLOG_MAX <= 30); + ZSTD_STATIC_ASSERT(ZSTD_WINDOWLOG_MAX_32 <= 30); + ZSTD_STATIC_ASSERT(ZSTD_WINDOWLOG_MAX <= 31); + ZSTD_cwksp_mark_tables_dirty(ws); + ZSTD_reduceIndex(ms, params, correction); + ZSTD_cwksp_mark_tables_clean(ws); + if (ms->nextToUpdate < correction) ms->nextToUpdate = 0; + else ms->nextToUpdate -= correction; + /* invalidate dictionaries on overflow correction */ + ms->loadedDictEnd = 0; + ms->dictMatchState = NULL; + } +} + +/*! ZSTD_compress_frameChunk() : +* Compress a chunk of data into one or multiple blocks. +* All blocks will be terminated, all input will be consumed. +* Function will issue an error if there is not enough `dstCapacity` to hold the compressed content. +* Frame is supposed already started (header already produced) +* @return : compressed size, or an error code +*/ +static size_t ZSTD_compress_frameChunk(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + U32 lastFrameChunk) +{ + size_t blockSize = cctx->blockSize; + size_t remaining = srcSize; + const BYTE* ip = (const BYTE*)src; + BYTE* const ostart = (BYTE*)dst; + BYTE* op = ostart; + U32 const maxDist = (U32)1 << cctx->appliedParams.cParams.windowLog; + + assert(cctx->appliedParams.cParams.windowLog <= ZSTD_WINDOWLOG_MAX); + + DEBUGLOG(4, "ZSTD_compress_frameChunk (blockSize=%u)", (unsigned)blockSize); + if (cctx->appliedParams.fParams.checksumFlag && srcSize) + XXH64_update(&cctx->xxhState, src, srcSize); + + while (remaining) { + ZSTD_matchState_t* const ms = &cctx->blockState.matchState; + U32 const lastBlock = lastFrameChunk & (blockSize >= remaining); + + RETURN_ERROR_IF(dstCapacity < ZSTD_blockHeaderSize + MIN_CBLOCK_SIZE, + dstSize_tooSmall, + "not enough space to store compressed block"); + if (remaining < blockSize) blockSize = remaining; + + ZSTD_overflowCorrectIfNeeded( + ms, &cctx->workspace, &cctx->appliedParams, ip, ip + blockSize); + ZSTD_checkDictValidity(&ms->window, ip + blockSize, maxDist, &ms->loadedDictEnd, &ms->dictMatchState); + ZSTD_window_enforceMaxDist(&ms->window, ip, maxDist, &ms->loadedDictEnd, &ms->dictMatchState); + + /* Ensure hash/chain table insertion resumes no sooner than lowlimit */ + if (ms->nextToUpdate < ms->window.lowLimit) ms->nextToUpdate = ms->window.lowLimit; + + { size_t cSize; + if (ZSTD_useTargetCBlockSize(&cctx->appliedParams)) { + cSize = ZSTD_compressBlock_targetCBlockSize(cctx, op, dstCapacity, ip, blockSize, lastBlock); + FORWARD_IF_ERROR(cSize, "ZSTD_compressBlock_targetCBlockSize failed"); + assert(cSize > 0); + assert(cSize <= blockSize + ZSTD_blockHeaderSize); + } else if (ZSTD_blockSplitterEnabled(&cctx->appliedParams)) { + cSize = ZSTD_compressBlock_splitBlock(cctx, op, dstCapacity, ip, blockSize, lastBlock); + FORWARD_IF_ERROR(cSize, "ZSTD_compressBlock_splitBlock failed"); + assert(cSize > 0 || cctx->seqCollector.collectSequences == 1); + } else { + cSize = ZSTD_compressBlock_internal(cctx, + op+ZSTD_blockHeaderSize, dstCapacity-ZSTD_blockHeaderSize, + ip, blockSize, 1 /* frame */); + FORWARD_IF_ERROR(cSize, "ZSTD_compressBlock_internal failed"); + + if (cSize == 0) { /* block is not compressible */ + cSize = ZSTD_noCompressBlock(op, dstCapacity, ip, blockSize, lastBlock); + FORWARD_IF_ERROR(cSize, "ZSTD_noCompressBlock failed"); + } else { + U32 const cBlockHeader = cSize == 1 ? + lastBlock + (((U32)bt_rle)<<1) + (U32)(blockSize << 3) : + lastBlock + (((U32)bt_compressed)<<1) + (U32)(cSize << 3); + MEM_writeLE24(op, cBlockHeader); + cSize += ZSTD_blockHeaderSize; + } + } + + + ip += blockSize; + assert(remaining >= blockSize); + remaining -= blockSize; + op += cSize; + assert(dstCapacity >= cSize); + dstCapacity -= cSize; + cctx->isFirstBlock = 0; + DEBUGLOG(5, "ZSTD_compress_frameChunk: adding a block of size %u", + (unsigned)cSize); + } } + + if (lastFrameChunk && (op>ostart)) cctx->stage = ZSTDcs_ending; + return (size_t)(op-ostart); +} + + +static size_t ZSTD_writeFrameHeader(void* dst, size_t dstCapacity, + const ZSTD_CCtx_params* params, U64 pledgedSrcSize, U32 dictID) +{ BYTE* const op = (BYTE*)dst; + U32 const dictIDSizeCodeLength = (dictID>0) + (dictID>=256) + (dictID>=65536); /* 0-3 */ + U32 const dictIDSizeCode = params->fParams.noDictIDFlag ? 0 : dictIDSizeCodeLength; /* 0-3 */ + U32 const checksumFlag = params->fParams.checksumFlag>0; + U32 const windowSize = (U32)1 << params->cParams.windowLog; + U32 const singleSegment = params->fParams.contentSizeFlag && (windowSize >= pledgedSrcSize); + BYTE const windowLogByte = (BYTE)((params->cParams.windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN) << 3); + U32 const fcsCode = params->fParams.contentSizeFlag ? + (pledgedSrcSize>=256) + (pledgedSrcSize>=65536+256) + (pledgedSrcSize>=0xFFFFFFFFU) : 0; /* 0-3 */ + BYTE const frameHeaderDescriptionByte = (BYTE)(dictIDSizeCode + (checksumFlag<<2) + (singleSegment<<5) + (fcsCode<<6) ); + size_t pos=0; + + assert(!(params->fParams.contentSizeFlag && pledgedSrcSize == ZSTD_CONTENTSIZE_UNKNOWN)); + RETURN_ERROR_IF(dstCapacity < ZSTD_FRAMEHEADERSIZE_MAX, dstSize_tooSmall, + "dst buf is too small to fit worst-case frame header size."); + DEBUGLOG(4, "ZSTD_writeFrameHeader : dictIDFlag : %u ; dictID : %u ; dictIDSizeCode : %u", + !params->fParams.noDictIDFlag, (unsigned)dictID, (unsigned)dictIDSizeCode); + if (params->format == ZSTD_f_zstd1) { + MEM_writeLE32(dst, ZSTD_MAGICNUMBER); + pos = 4; + } + op[pos++] = frameHeaderDescriptionByte; + if (!singleSegment) op[pos++] = windowLogByte; + switch(dictIDSizeCode) + { + default: + assert(0); /* impossible */ + ZSTD_FALLTHROUGH; + case 0 : break; + case 1 : op[pos] = (BYTE)(dictID); pos++; break; + case 2 : MEM_writeLE16(op+pos, (U16)dictID); pos+=2; break; + case 3 : MEM_writeLE32(op+pos, dictID); pos+=4; break; + } + switch(fcsCode) + { + default: + assert(0); /* impossible */ + ZSTD_FALLTHROUGH; + case 0 : if (singleSegment) op[pos++] = (BYTE)(pledgedSrcSize); break; + case 1 : MEM_writeLE16(op+pos, (U16)(pledgedSrcSize-256)); pos+=2; break; + case 2 : MEM_writeLE32(op+pos, (U32)(pledgedSrcSize)); pos+=4; break; + case 3 : MEM_writeLE64(op+pos, (U64)(pledgedSrcSize)); pos+=8; break; + } + return pos; +} + +/* ZSTD_writeSkippableFrame_advanced() : + * Writes out a skippable frame with the specified magic number variant (16 are supported), + * from ZSTD_MAGIC_SKIPPABLE_START to ZSTD_MAGIC_SKIPPABLE_START+15, and the desired source data. + * + * Returns the total number of bytes written, or a ZSTD error code. + */ +size_t ZSTD_writeSkippableFrame(void* dst, size_t dstCapacity, + const void* src, size_t srcSize, unsigned magicVariant) { + BYTE* op = (BYTE*)dst; + RETURN_ERROR_IF(dstCapacity < srcSize + ZSTD_SKIPPABLEHEADERSIZE /* Skippable frame overhead */, + dstSize_tooSmall, "Not enough room for skippable frame"); + RETURN_ERROR_IF(srcSize > (unsigned)0xFFFFFFFF, srcSize_wrong, "Src size too large for skippable frame"); + RETURN_ERROR_IF(magicVariant > 15, parameter_outOfBound, "Skippable frame magic number variant not supported"); + + MEM_writeLE32(op, (U32)(ZSTD_MAGIC_SKIPPABLE_START + magicVariant)); + MEM_writeLE32(op+4, (U32)srcSize); + ZSTD_memcpy(op+8, src, srcSize); + return srcSize + ZSTD_SKIPPABLEHEADERSIZE; +} + +/* ZSTD_writeLastEmptyBlock() : + * output an empty Block with end-of-frame mark to complete a frame + * @return : size of data written into `dst` (== ZSTD_blockHeaderSize (defined in zstd_internal.h)) + * or an error code if `dstCapacity` is too small (stage != ZSTDcs_init, stage_wrong, + "wrong cctx stage"); + RETURN_ERROR_IF(cctx->appliedParams.ldmParams.enableLdm == ZSTD_ps_enable, + parameter_unsupported, + "incompatible with ldm"); + cctx->externSeqStore.seq = seq; + cctx->externSeqStore.size = nbSeq; + cctx->externSeqStore.capacity = nbSeq; + cctx->externSeqStore.pos = 0; + cctx->externSeqStore.posInSequence = 0; + return 0; +} + + +static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + U32 frame, U32 lastFrameChunk) +{ + ZSTD_matchState_t* const ms = &cctx->blockState.matchState; + size_t fhSize = 0; + + DEBUGLOG(5, "ZSTD_compressContinue_internal, stage: %u, srcSize: %u", + cctx->stage, (unsigned)srcSize); + RETURN_ERROR_IF(cctx->stage==ZSTDcs_created, stage_wrong, + "missing init (ZSTD_compressBegin)"); + + if (frame && (cctx->stage==ZSTDcs_init)) { + fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, &cctx->appliedParams, + cctx->pledgedSrcSizePlusOne-1, cctx->dictID); + FORWARD_IF_ERROR(fhSize, "ZSTD_writeFrameHeader failed"); + assert(fhSize <= dstCapacity); + dstCapacity -= fhSize; + dst = (char*)dst + fhSize; + cctx->stage = ZSTDcs_ongoing; + } + + if (!srcSize) return fhSize; /* do not generate an empty block if no input */ + + if (!ZSTD_window_update(&ms->window, src, srcSize, ms->forceNonContiguous)) { + ms->forceNonContiguous = 0; + ms->nextToUpdate = ms->window.dictLimit; + } + if (cctx->appliedParams.ldmParams.enableLdm == ZSTD_ps_enable) { + ZSTD_window_update(&cctx->ldmState.window, src, srcSize, /* forceNonContiguous */ 0); + } + + if (!frame) { + /* overflow check and correction for block mode */ + ZSTD_overflowCorrectIfNeeded( + ms, &cctx->workspace, &cctx->appliedParams, + src, (BYTE const*)src + srcSize); + } + + DEBUGLOG(5, "ZSTD_compressContinue_internal (blockSize=%u)", (unsigned)cctx->blockSize); + { size_t const cSize = frame ? + ZSTD_compress_frameChunk (cctx, dst, dstCapacity, src, srcSize, lastFrameChunk) : + ZSTD_compressBlock_internal (cctx, dst, dstCapacity, src, srcSize, 0 /* frame */); + FORWARD_IF_ERROR(cSize, "%s", frame ? "ZSTD_compress_frameChunk failed" : "ZSTD_compressBlock_internal failed"); + cctx->consumedSrcSize += srcSize; + cctx->producedCSize += (cSize + fhSize); + assert(!(cctx->appliedParams.fParams.contentSizeFlag && cctx->pledgedSrcSizePlusOne == 0)); + if (cctx->pledgedSrcSizePlusOne != 0) { /* control src size */ + ZSTD_STATIC_ASSERT(ZSTD_CONTENTSIZE_UNKNOWN == (unsigned long long)-1); + RETURN_ERROR_IF( + cctx->consumedSrcSize+1 > cctx->pledgedSrcSizePlusOne, + srcSize_wrong, + "error : pledgedSrcSize = %u, while realSrcSize >= %u", + (unsigned)cctx->pledgedSrcSizePlusOne-1, + (unsigned)cctx->consumedSrcSize); + } + return cSize + fhSize; + } +} + +size_t ZSTD_compressContinue (ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize) +{ + DEBUGLOG(5, "ZSTD_compressContinue (srcSize=%u)", (unsigned)srcSize); + return ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize, 1 /* frame mode */, 0 /* last chunk */); +} + + +size_t ZSTD_getBlockSize(const ZSTD_CCtx* cctx) +{ + ZSTD_compressionParameters const cParams = cctx->appliedParams.cParams; + assert(!ZSTD_checkCParams(cParams)); + return MIN (ZSTD_BLOCKSIZE_MAX, (U32)1 << cParams.windowLog); +} + +size_t ZSTD_compressBlock(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ + DEBUGLOG(5, "ZSTD_compressBlock: srcSize = %u", (unsigned)srcSize); + { size_t const blockSizeMax = ZSTD_getBlockSize(cctx); + RETURN_ERROR_IF(srcSize > blockSizeMax, srcSize_wrong, "input is larger than a block"); } + + return ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize, 0 /* frame mode */, 0 /* last chunk */); +} + +/*! ZSTD_loadDictionaryContent() : + * @return : 0, or an error code + */ +static size_t ZSTD_loadDictionaryContent(ZSTD_matchState_t* ms, + ldmState_t* ls, + ZSTD_cwksp* ws, + ZSTD_CCtx_params const* params, + const void* src, size_t srcSize, + ZSTD_dictTableLoadMethod_e dtlm) +{ + const BYTE* ip = (const BYTE*) src; + const BYTE* const iend = ip + srcSize; + int const loadLdmDict = params->ldmParams.enableLdm == ZSTD_ps_enable && ls != NULL; + + /* Assert that we the ms params match the params we're being given */ + ZSTD_assertEqualCParams(params->cParams, ms->cParams); + + if (srcSize > ZSTD_CHUNKSIZE_MAX) { + /* Allow the dictionary to set indices up to exactly ZSTD_CURRENT_MAX. + * Dictionaries right at the edge will immediately trigger overflow + * correction, but I don't want to insert extra constraints here. + */ + U32 const maxDictSize = ZSTD_CURRENT_MAX - 1; + /* We must have cleared our windows when our source is this large. */ + assert(ZSTD_window_isEmpty(ms->window)); + if (loadLdmDict) + assert(ZSTD_window_isEmpty(ls->window)); + /* If the dictionary is too large, only load the suffix of the dictionary. */ + if (srcSize > maxDictSize) { + ip = iend - maxDictSize; + src = ip; + srcSize = maxDictSize; + } + } + + DEBUGLOG(4, "ZSTD_loadDictionaryContent(): useRowMatchFinder=%d", (int)params->useRowMatchFinder); + ZSTD_window_update(&ms->window, src, srcSize, /* forceNonContiguous */ 0); + ms->loadedDictEnd = params->forceWindow ? 0 : (U32)(iend - ms->window.base); + ms->forceNonContiguous = params->deterministicRefPrefix; + + if (loadLdmDict) { + ZSTD_window_update(&ls->window, src, srcSize, /* forceNonContiguous */ 0); + ls->loadedDictEnd = params->forceWindow ? 0 : (U32)(iend - ls->window.base); + } + + if (srcSize <= HASH_READ_SIZE) return 0; + + ZSTD_overflowCorrectIfNeeded(ms, ws, params, ip, iend); + + if (loadLdmDict) + ZSTD_ldm_fillHashTable(ls, ip, iend, ¶ms->ldmParams); + + switch(params->cParams.strategy) + { + case ZSTD_fast: + ZSTD_fillHashTable(ms, iend, dtlm); + break; + case ZSTD_dfast: + ZSTD_fillDoubleHashTable(ms, iend, dtlm); + break; + + case ZSTD_greedy: + case ZSTD_lazy: + case ZSTD_lazy2: + assert(srcSize >= HASH_READ_SIZE); + if (ms->dedicatedDictSearch) { + assert(ms->chainTable != NULL); + ZSTD_dedicatedDictSearch_lazy_loadDictionary(ms, iend-HASH_READ_SIZE); + } else { + assert(params->useRowMatchFinder != ZSTD_ps_auto); + if (params->useRowMatchFinder == ZSTD_ps_enable) { + size_t const tagTableSize = ((size_t)1 << params->cParams.hashLog) * sizeof(U16); + ZSTD_memset(ms->tagTable, 0, tagTableSize); + ZSTD_row_update(ms, iend-HASH_READ_SIZE); + DEBUGLOG(4, "Using row-based hash table for lazy dict"); + } else { + ZSTD_insertAndFindFirstIndex(ms, iend-HASH_READ_SIZE); + DEBUGLOG(4, "Using chain-based hash table for lazy dict"); + } + } + break; + + case ZSTD_btlazy2: /* we want the dictionary table fully sorted */ + case ZSTD_btopt: + case ZSTD_btultra: + case ZSTD_btultra2: + assert(srcSize >= HASH_READ_SIZE); + ZSTD_updateTree(ms, iend-HASH_READ_SIZE, iend); + break; + + default: + assert(0); /* not possible : not a valid strategy id */ + } + + ms->nextToUpdate = (U32)(iend - ms->window.base); + return 0; +} + + +/* Dictionaries that assign zero probability to symbols that show up causes problems + * when FSE encoding. Mark dictionaries with zero probability symbols as FSE_repeat_check + * and only dictionaries with 100% valid symbols can be assumed valid. + */ +static FSE_repeat ZSTD_dictNCountRepeat(short* normalizedCounter, unsigned dictMaxSymbolValue, unsigned maxSymbolValue) +{ + U32 s; + if (dictMaxSymbolValue < maxSymbolValue) { + return FSE_repeat_check; + } + for (s = 0; s <= maxSymbolValue; ++s) { + if (normalizedCounter[s] == 0) { + return FSE_repeat_check; + } + } + return FSE_repeat_valid; +} + +size_t ZSTD_loadCEntropy(ZSTD_compressedBlockState_t* bs, void* workspace, + const void* const dict, size_t dictSize) +{ + short offcodeNCount[MaxOff+1]; + unsigned offcodeMaxValue = MaxOff; + const BYTE* dictPtr = (const BYTE*)dict; /* skip magic num and dict ID */ + const BYTE* const dictEnd = dictPtr + dictSize; + dictPtr += 8; + bs->entropy.huf.repeatMode = HUF_repeat_check; + + { unsigned maxSymbolValue = 255; + unsigned hasZeroWeights = 1; + size_t const hufHeaderSize = HUF_readCTable((HUF_CElt*)bs->entropy.huf.CTable, &maxSymbolValue, dictPtr, + dictEnd-dictPtr, &hasZeroWeights); + + /* We only set the loaded table as valid if it contains all non-zero + * weights. Otherwise, we set it to check */ + if (!hasZeroWeights) + bs->entropy.huf.repeatMode = HUF_repeat_valid; + + RETURN_ERROR_IF(HUF_isError(hufHeaderSize), dictionary_corrupted, ""); + RETURN_ERROR_IF(maxSymbolValue < 255, dictionary_corrupted, ""); + dictPtr += hufHeaderSize; + } + + { unsigned offcodeLog; + size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, dictEnd-dictPtr); + RETURN_ERROR_IF(FSE_isError(offcodeHeaderSize), dictionary_corrupted, ""); + RETURN_ERROR_IF(offcodeLog > OffFSELog, dictionary_corrupted, ""); + /* fill all offset symbols to avoid garbage at end of table */ + RETURN_ERROR_IF(FSE_isError(FSE_buildCTable_wksp( + bs->entropy.fse.offcodeCTable, + offcodeNCount, MaxOff, offcodeLog, + workspace, HUF_WORKSPACE_SIZE)), + dictionary_corrupted, ""); + /* Defer checking offcodeMaxValue because we need to know the size of the dictionary content */ + dictPtr += offcodeHeaderSize; + } + + { short matchlengthNCount[MaxML+1]; + unsigned matchlengthMaxValue = MaxML, matchlengthLog; + size_t const matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dictPtr, dictEnd-dictPtr); + RETURN_ERROR_IF(FSE_isError(matchlengthHeaderSize), dictionary_corrupted, ""); + RETURN_ERROR_IF(matchlengthLog > MLFSELog, dictionary_corrupted, ""); + RETURN_ERROR_IF(FSE_isError(FSE_buildCTable_wksp( + bs->entropy.fse.matchlengthCTable, + matchlengthNCount, matchlengthMaxValue, matchlengthLog, + workspace, HUF_WORKSPACE_SIZE)), + dictionary_corrupted, ""); + bs->entropy.fse.matchlength_repeatMode = ZSTD_dictNCountRepeat(matchlengthNCount, matchlengthMaxValue, MaxML); + dictPtr += matchlengthHeaderSize; + } + + { short litlengthNCount[MaxLL+1]; + unsigned litlengthMaxValue = MaxLL, litlengthLog; + size_t const litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dictPtr, dictEnd-dictPtr); + RETURN_ERROR_IF(FSE_isError(litlengthHeaderSize), dictionary_corrupted, ""); + RETURN_ERROR_IF(litlengthLog > LLFSELog, dictionary_corrupted, ""); + RETURN_ERROR_IF(FSE_isError(FSE_buildCTable_wksp( + bs->entropy.fse.litlengthCTable, + litlengthNCount, litlengthMaxValue, litlengthLog, + workspace, HUF_WORKSPACE_SIZE)), + dictionary_corrupted, ""); + bs->entropy.fse.litlength_repeatMode = ZSTD_dictNCountRepeat(litlengthNCount, litlengthMaxValue, MaxLL); + dictPtr += litlengthHeaderSize; + } + + RETURN_ERROR_IF(dictPtr+12 > dictEnd, dictionary_corrupted, ""); + bs->rep[0] = MEM_readLE32(dictPtr+0); + bs->rep[1] = MEM_readLE32(dictPtr+4); + bs->rep[2] = MEM_readLE32(dictPtr+8); + dictPtr += 12; + + { size_t const dictContentSize = (size_t)(dictEnd - dictPtr); + U32 offcodeMax = MaxOff; + if (dictContentSize <= ((U32)-1) - 128 KB) { + U32 const maxOffset = (U32)dictContentSize + 128 KB; /* The maximum offset that must be supported */ + offcodeMax = ZSTD_highbit32(maxOffset); /* Calculate minimum offset code required to represent maxOffset */ + } + /* All offset values <= dictContentSize + 128 KB must be representable for a valid table */ + bs->entropy.fse.offcode_repeatMode = ZSTD_dictNCountRepeat(offcodeNCount, offcodeMaxValue, MIN(offcodeMax, MaxOff)); + + /* All repCodes must be <= dictContentSize and != 0 */ + { U32 u; + for (u=0; u<3; u++) { + RETURN_ERROR_IF(bs->rep[u] == 0, dictionary_corrupted, ""); + RETURN_ERROR_IF(bs->rep[u] > dictContentSize, dictionary_corrupted, ""); + } } } + + return dictPtr - (const BYTE*)dict; +} + +/* Dictionary format : + * See : + * https://github.com/facebook/zstd/blob/release/doc/zstd_compression_format.md#dictionary-format + */ +/*! ZSTD_loadZstdDictionary() : + * @return : dictID, or an error code + * assumptions : magic number supposed already checked + * dictSize supposed >= 8 + */ +static size_t ZSTD_loadZstdDictionary(ZSTD_compressedBlockState_t* bs, + ZSTD_matchState_t* ms, + ZSTD_cwksp* ws, + ZSTD_CCtx_params const* params, + const void* dict, size_t dictSize, + ZSTD_dictTableLoadMethod_e dtlm, + void* workspace) +{ + const BYTE* dictPtr = (const BYTE*)dict; + const BYTE* const dictEnd = dictPtr + dictSize; + size_t dictID; + size_t eSize; + ZSTD_STATIC_ASSERT(HUF_WORKSPACE_SIZE >= (1<= 8); + assert(MEM_readLE32(dictPtr) == ZSTD_MAGIC_DICTIONARY); + + dictID = params->fParams.noDictIDFlag ? 0 : MEM_readLE32(dictPtr + 4 /* skip magic number */ ); + eSize = ZSTD_loadCEntropy(bs, workspace, dict, dictSize); + FORWARD_IF_ERROR(eSize, "ZSTD_loadCEntropy failed"); + dictPtr += eSize; + + { + size_t const dictContentSize = (size_t)(dictEnd - dictPtr); + FORWARD_IF_ERROR(ZSTD_loadDictionaryContent( + ms, NULL, ws, params, dictPtr, dictContentSize, dtlm), ""); + } + return dictID; +} + +/** ZSTD_compress_insertDictionary() : +* @return : dictID, or an error code */ +static size_t +ZSTD_compress_insertDictionary(ZSTD_compressedBlockState_t* bs, + ZSTD_matchState_t* ms, + ldmState_t* ls, + ZSTD_cwksp* ws, + const ZSTD_CCtx_params* params, + const void* dict, size_t dictSize, + ZSTD_dictContentType_e dictContentType, + ZSTD_dictTableLoadMethod_e dtlm, + void* workspace) +{ + DEBUGLOG(4, "ZSTD_compress_insertDictionary (dictSize=%u)", (U32)dictSize); + if ((dict==NULL) || (dictSize<8)) { + RETURN_ERROR_IF(dictContentType == ZSTD_dct_fullDict, dictionary_wrong, ""); + return 0; + } + + ZSTD_reset_compressedBlockState(bs); + + /* dict restricted modes */ + if (dictContentType == ZSTD_dct_rawContent) + return ZSTD_loadDictionaryContent(ms, ls, ws, params, dict, dictSize, dtlm); + + if (MEM_readLE32(dict) != ZSTD_MAGIC_DICTIONARY) { + if (dictContentType == ZSTD_dct_auto) { + DEBUGLOG(4, "raw content dictionary detected"); + return ZSTD_loadDictionaryContent( + ms, ls, ws, params, dict, dictSize, dtlm); + } + RETURN_ERROR_IF(dictContentType == ZSTD_dct_fullDict, dictionary_wrong, ""); + assert(0); /* impossible */ + } + + /* dict as full zstd dictionary */ + return ZSTD_loadZstdDictionary( + bs, ms, ws, params, dict, dictSize, dtlm, workspace); +} + +#define ZSTD_USE_CDICT_PARAMS_SRCSIZE_CUTOFF (128 KB) +#define ZSTD_USE_CDICT_PARAMS_DICTSIZE_MULTIPLIER (6ULL) + +/*! ZSTD_compressBegin_internal() : + * @return : 0, or an error code */ +static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx, + const void* dict, size_t dictSize, + ZSTD_dictContentType_e dictContentType, + ZSTD_dictTableLoadMethod_e dtlm, + const ZSTD_CDict* cdict, + const ZSTD_CCtx_params* params, U64 pledgedSrcSize, + ZSTD_buffered_policy_e zbuff) +{ + size_t const dictContentSize = cdict ? cdict->dictContentSize : dictSize; +#if ZSTD_TRACE + cctx->traceCtx = (ZSTD_trace_compress_begin != NULL) ? ZSTD_trace_compress_begin(cctx) : 0; +#endif + DEBUGLOG(4, "ZSTD_compressBegin_internal: wlog=%u", params->cParams.windowLog); + /* params are supposed to be fully validated at this point */ + assert(!ZSTD_isError(ZSTD_checkCParams(params->cParams))); + assert(!((dict) && (cdict))); /* either dict or cdict, not both */ + if ( (cdict) + && (cdict->dictContentSize > 0) + && ( pledgedSrcSize < ZSTD_USE_CDICT_PARAMS_SRCSIZE_CUTOFF + || pledgedSrcSize < cdict->dictContentSize * ZSTD_USE_CDICT_PARAMS_DICTSIZE_MULTIPLIER + || pledgedSrcSize == ZSTD_CONTENTSIZE_UNKNOWN + || cdict->compressionLevel == 0) + && (params->attachDictPref != ZSTD_dictForceLoad) ) { + return ZSTD_resetCCtx_usingCDict(cctx, cdict, params, pledgedSrcSize, zbuff); + } + + FORWARD_IF_ERROR( ZSTD_resetCCtx_internal(cctx, params, pledgedSrcSize, + dictContentSize, + ZSTDcrp_makeClean, zbuff) , ""); + { size_t const dictID = cdict ? + ZSTD_compress_insertDictionary( + cctx->blockState.prevCBlock, &cctx->blockState.matchState, + &cctx->ldmState, &cctx->workspace, &cctx->appliedParams, cdict->dictContent, + cdict->dictContentSize, cdict->dictContentType, dtlm, + cctx->entropyWorkspace) + : ZSTD_compress_insertDictionary( + cctx->blockState.prevCBlock, &cctx->blockState.matchState, + &cctx->ldmState, &cctx->workspace, &cctx->appliedParams, dict, dictSize, + dictContentType, dtlm, cctx->entropyWorkspace); + FORWARD_IF_ERROR(dictID, "ZSTD_compress_insertDictionary failed"); + assert(dictID <= UINT_MAX); + cctx->dictID = (U32)dictID; + cctx->dictContentSize = dictContentSize; + } + return 0; +} + +size_t ZSTD_compressBegin_advanced_internal(ZSTD_CCtx* cctx, + const void* dict, size_t dictSize, + ZSTD_dictContentType_e dictContentType, + ZSTD_dictTableLoadMethod_e dtlm, + const ZSTD_CDict* cdict, + const ZSTD_CCtx_params* params, + unsigned long long pledgedSrcSize) +{ + DEBUGLOG(4, "ZSTD_compressBegin_advanced_internal: wlog=%u", params->cParams.windowLog); + /* compression parameters verification and optimization */ + FORWARD_IF_ERROR( ZSTD_checkCParams(params->cParams) , ""); + return ZSTD_compressBegin_internal(cctx, + dict, dictSize, dictContentType, dtlm, + cdict, + params, pledgedSrcSize, + ZSTDb_not_buffered); +} + +/*! ZSTD_compressBegin_advanced() : +* @return : 0, or an error code */ +size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, + const void* dict, size_t dictSize, + ZSTD_parameters params, unsigned long long pledgedSrcSize) +{ + ZSTD_CCtx_params cctxParams; + ZSTD_CCtxParams_init_internal(&cctxParams, ¶ms, ZSTD_NO_CLEVEL); + return ZSTD_compressBegin_advanced_internal(cctx, + dict, dictSize, ZSTD_dct_auto, ZSTD_dtlm_fast, + NULL /*cdict*/, + &cctxParams, pledgedSrcSize); +} + +size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel) +{ + ZSTD_CCtx_params cctxParams; + { + ZSTD_parameters const params = ZSTD_getParams_internal(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, dictSize, ZSTD_cpm_noAttachDict); + ZSTD_CCtxParams_init_internal(&cctxParams, ¶ms, (compressionLevel == 0) ? ZSTD_CLEVEL_DEFAULT : compressionLevel); + } + DEBUGLOG(4, "ZSTD_compressBegin_usingDict (dictSize=%u)", (unsigned)dictSize); + return ZSTD_compressBegin_internal(cctx, dict, dictSize, ZSTD_dct_auto, ZSTD_dtlm_fast, NULL, + &cctxParams, ZSTD_CONTENTSIZE_UNKNOWN, ZSTDb_not_buffered); +} + +size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel) +{ + return ZSTD_compressBegin_usingDict(cctx, NULL, 0, compressionLevel); +} + + +/*! ZSTD_writeEpilogue() : +* Ends a frame. +* @return : nb of bytes written into dst (or an error code) */ +static size_t ZSTD_writeEpilogue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity) +{ + BYTE* const ostart = (BYTE*)dst; + BYTE* op = ostart; + size_t fhSize = 0; + + DEBUGLOG(4, "ZSTD_writeEpilogue"); + RETURN_ERROR_IF(cctx->stage == ZSTDcs_created, stage_wrong, "init missing"); + + /* special case : empty frame */ + if (cctx->stage == ZSTDcs_init) { + fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, &cctx->appliedParams, 0, 0); + FORWARD_IF_ERROR(fhSize, "ZSTD_writeFrameHeader failed"); + dstCapacity -= fhSize; + op += fhSize; + cctx->stage = ZSTDcs_ongoing; + } + + if (cctx->stage != ZSTDcs_ending) { + /* write one last empty block, make it the "last" block */ + U32 const cBlockHeader24 = 1 /* last block */ + (((U32)bt_raw)<<1) + 0; + RETURN_ERROR_IF(dstCapacity<4, dstSize_tooSmall, "no room for epilogue"); + MEM_writeLE32(op, cBlockHeader24); + op += ZSTD_blockHeaderSize; + dstCapacity -= ZSTD_blockHeaderSize; + } + + if (cctx->appliedParams.fParams.checksumFlag) { + U32 const checksum = (U32) XXH64_digest(&cctx->xxhState); + RETURN_ERROR_IF(dstCapacity<4, dstSize_tooSmall, "no room for checksum"); + DEBUGLOG(4, "ZSTD_writeEpilogue: write checksum : %08X", (unsigned)checksum); + MEM_writeLE32(op, checksum); + op += 4; + } + + cctx->stage = ZSTDcs_created; /* return to "created but no init" status */ + return op-ostart; +} + +void ZSTD_CCtx_trace(ZSTD_CCtx* cctx, size_t extraCSize) +{ +#if ZSTD_TRACE + if (cctx->traceCtx && ZSTD_trace_compress_end != NULL) { + int const streaming = cctx->inBuffSize > 0 || cctx->outBuffSize > 0 || cctx->appliedParams.nbWorkers > 0; + ZSTD_Trace trace; + ZSTD_memset(&trace, 0, sizeof(trace)); + trace.version = ZSTD_VERSION_NUMBER; + trace.streaming = streaming; + trace.dictionaryID = cctx->dictID; + trace.dictionarySize = cctx->dictContentSize; + trace.uncompressedSize = cctx->consumedSrcSize; + trace.compressedSize = cctx->producedCSize + extraCSize; + trace.params = &cctx->appliedParams; + trace.cctx = cctx; + ZSTD_trace_compress_end(cctx->traceCtx, &trace); + } + cctx->traceCtx = 0; +#else + (void)cctx; + (void)extraCSize; +#endif +} + +size_t ZSTD_compressEnd (ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize) +{ + size_t endResult; + size_t const cSize = ZSTD_compressContinue_internal(cctx, + dst, dstCapacity, src, srcSize, + 1 /* frame mode */, 1 /* last chunk */); + FORWARD_IF_ERROR(cSize, "ZSTD_compressContinue_internal failed"); + endResult = ZSTD_writeEpilogue(cctx, (char*)dst + cSize, dstCapacity-cSize); + FORWARD_IF_ERROR(endResult, "ZSTD_writeEpilogue failed"); + assert(!(cctx->appliedParams.fParams.contentSizeFlag && cctx->pledgedSrcSizePlusOne == 0)); + if (cctx->pledgedSrcSizePlusOne != 0) { /* control src size */ + ZSTD_STATIC_ASSERT(ZSTD_CONTENTSIZE_UNKNOWN == (unsigned long long)-1); + DEBUGLOG(4, "end of frame : controlling src size"); + RETURN_ERROR_IF( + cctx->pledgedSrcSizePlusOne != cctx->consumedSrcSize+1, + srcSize_wrong, + "error : pledgedSrcSize = %u, while realSrcSize = %u", + (unsigned)cctx->pledgedSrcSizePlusOne-1, + (unsigned)cctx->consumedSrcSize); + } + ZSTD_CCtx_trace(cctx, endResult); + return cSize + endResult; +} + +size_t ZSTD_compress_advanced (ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const void* dict,size_t dictSize, + ZSTD_parameters params) +{ + DEBUGLOG(4, "ZSTD_compress_advanced"); + FORWARD_IF_ERROR(ZSTD_checkCParams(params.cParams), ""); + ZSTD_CCtxParams_init_internal(&cctx->simpleApiParams, ¶ms, ZSTD_NO_CLEVEL); + return ZSTD_compress_advanced_internal(cctx, + dst, dstCapacity, + src, srcSize, + dict, dictSize, + &cctx->simpleApiParams); +} + +/* Internal */ +size_t ZSTD_compress_advanced_internal( + ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const void* dict,size_t dictSize, + const ZSTD_CCtx_params* params) +{ + DEBUGLOG(4, "ZSTD_compress_advanced_internal (srcSize:%u)", (unsigned)srcSize); + FORWARD_IF_ERROR( ZSTD_compressBegin_internal(cctx, + dict, dictSize, ZSTD_dct_auto, ZSTD_dtlm_fast, NULL, + params, srcSize, ZSTDb_not_buffered) , ""); + return ZSTD_compressEnd(cctx, dst, dstCapacity, src, srcSize); +} + +size_t ZSTD_compress_usingDict(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const void* dict, size_t dictSize, + int compressionLevel) +{ + { + ZSTD_parameters const params = ZSTD_getParams_internal(compressionLevel, srcSize, dict ? dictSize : 0, ZSTD_cpm_noAttachDict); + assert(params.fParams.contentSizeFlag == 1); + ZSTD_CCtxParams_init_internal(&cctx->simpleApiParams, ¶ms, (compressionLevel == 0) ? ZSTD_CLEVEL_DEFAULT: compressionLevel); + } + DEBUGLOG(4, "ZSTD_compress_usingDict (srcSize=%u)", (unsigned)srcSize); + return ZSTD_compress_advanced_internal(cctx, dst, dstCapacity, src, srcSize, dict, dictSize, &cctx->simpleApiParams); +} + +size_t ZSTD_compressCCtx(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + int compressionLevel) +{ + DEBUGLOG(4, "ZSTD_compressCCtx (srcSize=%u)", (unsigned)srcSize); + assert(cctx != NULL); + return ZSTD_compress_usingDict(cctx, dst, dstCapacity, src, srcSize, NULL, 0, compressionLevel); +} + +size_t ZSTD_compress(void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + int compressionLevel) +{ + size_t result; +#if ZSTD_COMPRESS_HEAPMODE + ZSTD_CCtx* cctx = ZSTD_createCCtx(); + RETURN_ERROR_IF(!cctx, memory_allocation, "ZSTD_createCCtx failed"); + result = ZSTD_compressCCtx(cctx, dst, dstCapacity, src, srcSize, compressionLevel); + ZSTD_freeCCtx(cctx); +#else + ZSTD_CCtx ctxBody; + ZSTD_initCCtx(&ctxBody, ZSTD_defaultCMem); + result = ZSTD_compressCCtx(&ctxBody, dst, dstCapacity, src, srcSize, compressionLevel); + ZSTD_freeCCtxContent(&ctxBody); /* can't free ctxBody itself, as it's on stack; free only heap content */ +#endif + return result; +} + + +/* ===== Dictionary API ===== */ + +/*! ZSTD_estimateCDictSize_advanced() : + * Estimate amount of memory that will be needed to create a dictionary with following arguments */ +size_t ZSTD_estimateCDictSize_advanced( + size_t dictSize, ZSTD_compressionParameters cParams, + ZSTD_dictLoadMethod_e dictLoadMethod) +{ + DEBUGLOG(5, "sizeof(ZSTD_CDict) : %u", (unsigned)sizeof(ZSTD_CDict)); + return ZSTD_cwksp_alloc_size(sizeof(ZSTD_CDict)) + + ZSTD_cwksp_alloc_size(HUF_WORKSPACE_SIZE) + /* enableDedicatedDictSearch == 1 ensures that CDict estimation will not be too small + * in case we are using DDS with row-hash. */ + + ZSTD_sizeof_matchState(&cParams, ZSTD_resolveRowMatchFinderMode(ZSTD_ps_auto, &cParams), + /* enableDedicatedDictSearch */ 1, /* forCCtx */ 0) + + (dictLoadMethod == ZSTD_dlm_byRef ? 0 + : ZSTD_cwksp_alloc_size(ZSTD_cwksp_align(dictSize, sizeof(void *)))); +} + +size_t ZSTD_estimateCDictSize(size_t dictSize, int compressionLevel) +{ + ZSTD_compressionParameters const cParams = ZSTD_getCParams_internal(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, dictSize, ZSTD_cpm_createCDict); + return ZSTD_estimateCDictSize_advanced(dictSize, cParams, ZSTD_dlm_byCopy); +} + +size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict) +{ + if (cdict==NULL) return 0; /* support sizeof on NULL */ + DEBUGLOG(5, "sizeof(*cdict) : %u", (unsigned)sizeof(*cdict)); + /* cdict may be in the workspace */ + return (cdict->workspace.workspace == cdict ? 0 : sizeof(*cdict)) + + ZSTD_cwksp_sizeof(&cdict->workspace); +} + +static size_t ZSTD_initCDict_internal( + ZSTD_CDict* cdict, + const void* dictBuffer, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType, + ZSTD_CCtx_params params) +{ + DEBUGLOG(3, "ZSTD_initCDict_internal (dictContentType:%u)", (unsigned)dictContentType); + assert(!ZSTD_checkCParams(params.cParams)); + cdict->matchState.cParams = params.cParams; + cdict->matchState.dedicatedDictSearch = params.enableDedicatedDictSearch; + if ((dictLoadMethod == ZSTD_dlm_byRef) || (!dictBuffer) || (!dictSize)) { + cdict->dictContent = dictBuffer; + } else { + void *internalBuffer = ZSTD_cwksp_reserve_object(&cdict->workspace, ZSTD_cwksp_align(dictSize, sizeof(void*))); + RETURN_ERROR_IF(!internalBuffer, memory_allocation, "NULL pointer!"); + cdict->dictContent = internalBuffer; + ZSTD_memcpy(internalBuffer, dictBuffer, dictSize); + } + cdict->dictContentSize = dictSize; + cdict->dictContentType = dictContentType; + + cdict->entropyWorkspace = (U32*)ZSTD_cwksp_reserve_object(&cdict->workspace, HUF_WORKSPACE_SIZE); + + + /* Reset the state to no dictionary */ + ZSTD_reset_compressedBlockState(&cdict->cBlockState); + FORWARD_IF_ERROR(ZSTD_reset_matchState( + &cdict->matchState, + &cdict->workspace, + ¶ms.cParams, + params.useRowMatchFinder, + ZSTDcrp_makeClean, + ZSTDirp_reset, + ZSTD_resetTarget_CDict), ""); + /* (Maybe) load the dictionary + * Skips loading the dictionary if it is < 8 bytes. + */ + { params.compressionLevel = ZSTD_CLEVEL_DEFAULT; + params.fParams.contentSizeFlag = 1; + { size_t const dictID = ZSTD_compress_insertDictionary( + &cdict->cBlockState, &cdict->matchState, NULL, &cdict->workspace, + ¶ms, cdict->dictContent, cdict->dictContentSize, + dictContentType, ZSTD_dtlm_full, cdict->entropyWorkspace); + FORWARD_IF_ERROR(dictID, "ZSTD_compress_insertDictionary failed"); + assert(dictID <= (size_t)(U32)-1); + cdict->dictID = (U32)dictID; + } + } + + return 0; +} + +static ZSTD_CDict* ZSTD_createCDict_advanced_internal(size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_compressionParameters cParams, + ZSTD_paramSwitch_e useRowMatchFinder, + U32 enableDedicatedDictSearch, + ZSTD_customMem customMem) +{ + if ((!customMem.customAlloc) ^ (!customMem.customFree)) return NULL; + + { size_t const workspaceSize = + ZSTD_cwksp_alloc_size(sizeof(ZSTD_CDict)) + + ZSTD_cwksp_alloc_size(HUF_WORKSPACE_SIZE) + + ZSTD_sizeof_matchState(&cParams, useRowMatchFinder, enableDedicatedDictSearch, /* forCCtx */ 0) + + (dictLoadMethod == ZSTD_dlm_byRef ? 0 + : ZSTD_cwksp_alloc_size(ZSTD_cwksp_align(dictSize, sizeof(void*)))); + void* const workspace = ZSTD_customMalloc(workspaceSize, customMem); + ZSTD_cwksp ws; + ZSTD_CDict* cdict; + + if (!workspace) { + ZSTD_customFree(workspace, customMem); + return NULL; + } + + ZSTD_cwksp_init(&ws, workspace, workspaceSize, ZSTD_cwksp_dynamic_alloc); + + cdict = (ZSTD_CDict*)ZSTD_cwksp_reserve_object(&ws, sizeof(ZSTD_CDict)); + assert(cdict != NULL); + ZSTD_cwksp_move(&cdict->workspace, &ws); + cdict->customMem = customMem; + cdict->compressionLevel = ZSTD_NO_CLEVEL; /* signals advanced API usage */ + cdict->useRowMatchFinder = useRowMatchFinder; + return cdict; + } +} + +ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType, + ZSTD_compressionParameters cParams, + ZSTD_customMem customMem) +{ + ZSTD_CCtx_params cctxParams; + ZSTD_memset(&cctxParams, 0, sizeof(cctxParams)); + ZSTD_CCtxParams_init(&cctxParams, 0); + cctxParams.cParams = cParams; + cctxParams.customMem = customMem; + return ZSTD_createCDict_advanced2( + dictBuffer, dictSize, + dictLoadMethod, dictContentType, + &cctxParams, customMem); +} + +ZSTD_CDict* ZSTD_createCDict_advanced2( + const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType, + const ZSTD_CCtx_params* originalCctxParams, + ZSTD_customMem customMem) +{ + ZSTD_CCtx_params cctxParams = *originalCctxParams; + ZSTD_compressionParameters cParams; + ZSTD_CDict* cdict; + + DEBUGLOG(3, "ZSTD_createCDict_advanced2, mode %u", (unsigned)dictContentType); + if (!customMem.customAlloc ^ !customMem.customFree) return NULL; + + if (cctxParams.enableDedicatedDictSearch) { + cParams = ZSTD_dedicatedDictSearch_getCParams( + cctxParams.compressionLevel, dictSize); + ZSTD_overrideCParams(&cParams, &cctxParams.cParams); + } else { + cParams = ZSTD_getCParamsFromCCtxParams( + &cctxParams, ZSTD_CONTENTSIZE_UNKNOWN, dictSize, ZSTD_cpm_createCDict); + } + + if (!ZSTD_dedicatedDictSearch_isSupported(&cParams)) { + /* Fall back to non-DDSS params */ + cctxParams.enableDedicatedDictSearch = 0; + cParams = ZSTD_getCParamsFromCCtxParams( + &cctxParams, ZSTD_CONTENTSIZE_UNKNOWN, dictSize, ZSTD_cpm_createCDict); + } + + DEBUGLOG(3, "ZSTD_createCDict_advanced2: DDS: %u", cctxParams.enableDedicatedDictSearch); + cctxParams.cParams = cParams; + cctxParams.useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(cctxParams.useRowMatchFinder, &cParams); + + cdict = ZSTD_createCDict_advanced_internal(dictSize, + dictLoadMethod, cctxParams.cParams, + cctxParams.useRowMatchFinder, cctxParams.enableDedicatedDictSearch, + customMem); + + if (ZSTD_isError( ZSTD_initCDict_internal(cdict, + dict, dictSize, + dictLoadMethod, dictContentType, + cctxParams) )) { + ZSTD_freeCDict(cdict); + return NULL; + } + + return cdict; +} + +ZSTD_CDict* ZSTD_createCDict(const void* dict, size_t dictSize, int compressionLevel) +{ + ZSTD_compressionParameters cParams = ZSTD_getCParams_internal(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, dictSize, ZSTD_cpm_createCDict); + ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dict, dictSize, + ZSTD_dlm_byCopy, ZSTD_dct_auto, + cParams, ZSTD_defaultCMem); + if (cdict) + cdict->compressionLevel = (compressionLevel == 0) ? ZSTD_CLEVEL_DEFAULT : compressionLevel; + return cdict; +} + +ZSTD_CDict* ZSTD_createCDict_byReference(const void* dict, size_t dictSize, int compressionLevel) +{ + ZSTD_compressionParameters cParams = ZSTD_getCParams_internal(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, dictSize, ZSTD_cpm_createCDict); + ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dict, dictSize, + ZSTD_dlm_byRef, ZSTD_dct_auto, + cParams, ZSTD_defaultCMem); + if (cdict) + cdict->compressionLevel = (compressionLevel == 0) ? ZSTD_CLEVEL_DEFAULT : compressionLevel; + return cdict; +} + +size_t ZSTD_freeCDict(ZSTD_CDict* cdict) +{ + if (cdict==NULL) return 0; /* support free on NULL */ + { ZSTD_customMem const cMem = cdict->customMem; + int cdictInWorkspace = ZSTD_cwksp_owns_buffer(&cdict->workspace, cdict); + ZSTD_cwksp_free(&cdict->workspace, cMem); + if (!cdictInWorkspace) { + ZSTD_customFree(cdict, cMem); + } + return 0; + } +} + +/*! ZSTD_initStaticCDict_advanced() : + * Generate a digested dictionary in provided memory area. + * workspace: The memory area to emplace the dictionary into. + * Provided pointer must 8-bytes aligned. + * It must outlive dictionary usage. + * workspaceSize: Use ZSTD_estimateCDictSize() + * to determine how large workspace must be. + * cParams : use ZSTD_getCParams() to transform a compression level + * into its relevants cParams. + * @return : pointer to ZSTD_CDict*, or NULL if error (size too small) + * Note : there is no corresponding "free" function. + * Since workspace was allocated externally, it must be freed externally. + */ +const ZSTD_CDict* ZSTD_initStaticCDict( + void* workspace, size_t workspaceSize, + const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType, + ZSTD_compressionParameters cParams) +{ + ZSTD_paramSwitch_e const useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(ZSTD_ps_auto, &cParams); + /* enableDedicatedDictSearch == 1 ensures matchstate is not too small in case this CDict will be used for DDS + row hash */ + size_t const matchStateSize = ZSTD_sizeof_matchState(&cParams, useRowMatchFinder, /* enableDedicatedDictSearch */ 1, /* forCCtx */ 0); + size_t const neededSize = ZSTD_cwksp_alloc_size(sizeof(ZSTD_CDict)) + + (dictLoadMethod == ZSTD_dlm_byRef ? 0 + : ZSTD_cwksp_alloc_size(ZSTD_cwksp_align(dictSize, sizeof(void*)))) + + ZSTD_cwksp_alloc_size(HUF_WORKSPACE_SIZE) + + matchStateSize; + ZSTD_CDict* cdict; + ZSTD_CCtx_params params; + + if ((size_t)workspace & 7) return NULL; /* 8-aligned */ + + { + ZSTD_cwksp ws; + ZSTD_cwksp_init(&ws, workspace, workspaceSize, ZSTD_cwksp_static_alloc); + cdict = (ZSTD_CDict*)ZSTD_cwksp_reserve_object(&ws, sizeof(ZSTD_CDict)); + if (cdict == NULL) return NULL; + ZSTD_cwksp_move(&cdict->workspace, &ws); + } + + DEBUGLOG(4, "(workspaceSize < neededSize) : (%u < %u) => %u", + (unsigned)workspaceSize, (unsigned)neededSize, (unsigned)(workspaceSize < neededSize)); + if (workspaceSize < neededSize) return NULL; + + ZSTD_CCtxParams_init(¶ms, 0); + params.cParams = cParams; + params.useRowMatchFinder = useRowMatchFinder; + cdict->useRowMatchFinder = useRowMatchFinder; + + if (ZSTD_isError( ZSTD_initCDict_internal(cdict, + dict, dictSize, + dictLoadMethod, dictContentType, + params) )) + return NULL; + + return cdict; +} + +ZSTD_compressionParameters ZSTD_getCParamsFromCDict(const ZSTD_CDict* cdict) +{ + assert(cdict != NULL); + return cdict->matchState.cParams; +} + +/*! ZSTD_getDictID_fromCDict() : + * Provides the dictID of the dictionary loaded into `cdict`. + * If @return == 0, the dictionary is not conformant to Zstandard specification, or empty. + * Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */ +unsigned ZSTD_getDictID_fromCDict(const ZSTD_CDict* cdict) +{ + if (cdict==NULL) return 0; + return cdict->dictID; +} + +/* ZSTD_compressBegin_usingCDict_internal() : + * Implementation of various ZSTD_compressBegin_usingCDict* functions. + */ +static size_t ZSTD_compressBegin_usingCDict_internal( + ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, + ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize) +{ + ZSTD_CCtx_params cctxParams; + DEBUGLOG(4, "ZSTD_compressBegin_usingCDict_internal"); + RETURN_ERROR_IF(cdict==NULL, dictionary_wrong, "NULL pointer!"); + /* Initialize the cctxParams from the cdict */ + { + ZSTD_parameters params; + params.fParams = fParams; + params.cParams = ( pledgedSrcSize < ZSTD_USE_CDICT_PARAMS_SRCSIZE_CUTOFF + || pledgedSrcSize < cdict->dictContentSize * ZSTD_USE_CDICT_PARAMS_DICTSIZE_MULTIPLIER + || pledgedSrcSize == ZSTD_CONTENTSIZE_UNKNOWN + || cdict->compressionLevel == 0 ) ? + ZSTD_getCParamsFromCDict(cdict) + : ZSTD_getCParams(cdict->compressionLevel, + pledgedSrcSize, + cdict->dictContentSize); + ZSTD_CCtxParams_init_internal(&cctxParams, ¶ms, cdict->compressionLevel); + } + /* Increase window log to fit the entire dictionary and source if the + * source size is known. Limit the increase to 19, which is the + * window log for compression level 1 with the largest source size. + */ + if (pledgedSrcSize != ZSTD_CONTENTSIZE_UNKNOWN) { + U32 const limitedSrcSize = (U32)MIN(pledgedSrcSize, 1U << 19); + U32 const limitedSrcLog = limitedSrcSize > 1 ? ZSTD_highbit32(limitedSrcSize - 1) + 1 : 1; + cctxParams.cParams.windowLog = MAX(cctxParams.cParams.windowLog, limitedSrcLog); + } + return ZSTD_compressBegin_internal(cctx, + NULL, 0, ZSTD_dct_auto, ZSTD_dtlm_fast, + cdict, + &cctxParams, pledgedSrcSize, + ZSTDb_not_buffered); +} + + +/* ZSTD_compressBegin_usingCDict_advanced() : + * This function is DEPRECATED. + * cdict must be != NULL */ +size_t ZSTD_compressBegin_usingCDict_advanced( + ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, + ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize) +{ + return ZSTD_compressBegin_usingCDict_internal(cctx, cdict, fParams, pledgedSrcSize); +} + +/* ZSTD_compressBegin_usingCDict() : + * cdict must be != NULL */ +size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict) +{ + ZSTD_frameParameters const fParams = { 0 /*content*/, 0 /*checksum*/, 0 /*noDictID*/ }; + return ZSTD_compressBegin_usingCDict_internal(cctx, cdict, fParams, ZSTD_CONTENTSIZE_UNKNOWN); +} + +/*! ZSTD_compress_usingCDict_internal(): + * Implementation of various ZSTD_compress_usingCDict* functions. + */ +static size_t ZSTD_compress_usingCDict_internal(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const ZSTD_CDict* cdict, ZSTD_frameParameters fParams) +{ + FORWARD_IF_ERROR(ZSTD_compressBegin_usingCDict_internal(cctx, cdict, fParams, srcSize), ""); /* will check if cdict != NULL */ + return ZSTD_compressEnd(cctx, dst, dstCapacity, src, srcSize); +} + +/*! ZSTD_compress_usingCDict_advanced(): + * This function is DEPRECATED. + */ +size_t ZSTD_compress_usingCDict_advanced(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const ZSTD_CDict* cdict, ZSTD_frameParameters fParams) +{ + return ZSTD_compress_usingCDict_internal(cctx, dst, dstCapacity, src, srcSize, cdict, fParams); +} + +/*! ZSTD_compress_usingCDict() : + * Compression using a digested Dictionary. + * Faster startup than ZSTD_compress_usingDict(), recommended when same dictionary is used multiple times. + * Note that compression parameters are decided at CDict creation time + * while frame parameters are hardcoded */ +size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const ZSTD_CDict* cdict) +{ + ZSTD_frameParameters const fParams = { 1 /*content*/, 0 /*checksum*/, 0 /*noDictID*/ }; + return ZSTD_compress_usingCDict_internal(cctx, dst, dstCapacity, src, srcSize, cdict, fParams); +} + + + +/* ****************************************************************** +* Streaming +********************************************************************/ + +ZSTD_CStream* ZSTD_createCStream(void) +{ + DEBUGLOG(3, "ZSTD_createCStream"); + return ZSTD_createCStream_advanced(ZSTD_defaultCMem); +} + +ZSTD_CStream* ZSTD_initStaticCStream(void *workspace, size_t workspaceSize) +{ + return ZSTD_initStaticCCtx(workspace, workspaceSize); +} + +ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem) +{ /* CStream and CCtx are now same object */ + return ZSTD_createCCtx_advanced(customMem); +} + +size_t ZSTD_freeCStream(ZSTD_CStream* zcs) +{ + return ZSTD_freeCCtx(zcs); /* same object */ +} + + + +/*====== Initialization ======*/ + +size_t ZSTD_CStreamInSize(void) { return ZSTD_BLOCKSIZE_MAX; } + +size_t ZSTD_CStreamOutSize(void) +{ + return ZSTD_compressBound(ZSTD_BLOCKSIZE_MAX) + ZSTD_blockHeaderSize + 4 /* 32-bits hash */ ; +} + +static ZSTD_cParamMode_e ZSTD_getCParamMode(ZSTD_CDict const* cdict, ZSTD_CCtx_params const* params, U64 pledgedSrcSize) +{ + if (cdict != NULL && ZSTD_shouldAttachDict(cdict, params, pledgedSrcSize)) + return ZSTD_cpm_attachDict; + else + return ZSTD_cpm_noAttachDict; +} + +/* ZSTD_resetCStream(): + * pledgedSrcSize == 0 means "unknown" */ +size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pss) +{ + /* temporary : 0 interpreted as "unknown" during transition period. + * Users willing to specify "unknown" **must** use ZSTD_CONTENTSIZE_UNKNOWN. + * 0 will be interpreted as "empty" in the future. + */ + U64 const pledgedSrcSize = (pss==0) ? ZSTD_CONTENTSIZE_UNKNOWN : pss; + DEBUGLOG(4, "ZSTD_resetCStream: pledgedSrcSize = %u", (unsigned)pledgedSrcSize); + FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) , ""); + FORWARD_IF_ERROR( ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize) , ""); + return 0; +} + +/*! ZSTD_initCStream_internal() : + * Note : for lib/compress only. Used by zstdmt_compress.c. + * Assumption 1 : params are valid + * Assumption 2 : either dict, or cdict, is defined, not both */ +size_t ZSTD_initCStream_internal(ZSTD_CStream* zcs, + const void* dict, size_t dictSize, const ZSTD_CDict* cdict, + const ZSTD_CCtx_params* params, + unsigned long long pledgedSrcSize) +{ + DEBUGLOG(4, "ZSTD_initCStream_internal"); + FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) , ""); + FORWARD_IF_ERROR( ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize) , ""); + assert(!ZSTD_isError(ZSTD_checkCParams(params->cParams))); + zcs->requestedParams = *params; + assert(!((dict) && (cdict))); /* either dict or cdict, not both */ + if (dict) { + FORWARD_IF_ERROR( ZSTD_CCtx_loadDictionary(zcs, dict, dictSize) , ""); + } else { + /* Dictionary is cleared if !cdict */ + FORWARD_IF_ERROR( ZSTD_CCtx_refCDict(zcs, cdict) , ""); + } + return 0; +} + +/* ZSTD_initCStream_usingCDict_advanced() : + * same as ZSTD_initCStream_usingCDict(), with control over frame parameters */ +size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, + const ZSTD_CDict* cdict, + ZSTD_frameParameters fParams, + unsigned long long pledgedSrcSize) +{ + DEBUGLOG(4, "ZSTD_initCStream_usingCDict_advanced"); + FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) , ""); + FORWARD_IF_ERROR( ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize) , ""); + zcs->requestedParams.fParams = fParams; + FORWARD_IF_ERROR( ZSTD_CCtx_refCDict(zcs, cdict) , ""); + return 0; +} + +/* note : cdict must outlive compression session */ +size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict) +{ + DEBUGLOG(4, "ZSTD_initCStream_usingCDict"); + FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) , ""); + FORWARD_IF_ERROR( ZSTD_CCtx_refCDict(zcs, cdict) , ""); + return 0; +} + + +/* ZSTD_initCStream_advanced() : + * pledgedSrcSize must be exact. + * if srcSize is not known at init time, use value ZSTD_CONTENTSIZE_UNKNOWN. + * dict is loaded with default parameters ZSTD_dct_auto and ZSTD_dlm_byCopy. */ +size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, + const void* dict, size_t dictSize, + ZSTD_parameters params, unsigned long long pss) +{ + /* for compatibility with older programs relying on this behavior. + * Users should now specify ZSTD_CONTENTSIZE_UNKNOWN. + * This line will be removed in the future. + */ + U64 const pledgedSrcSize = (pss==0 && params.fParams.contentSizeFlag==0) ? ZSTD_CONTENTSIZE_UNKNOWN : pss; + DEBUGLOG(4, "ZSTD_initCStream_advanced"); + FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) , ""); + FORWARD_IF_ERROR( ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize) , ""); + FORWARD_IF_ERROR( ZSTD_checkCParams(params.cParams) , ""); + ZSTD_CCtxParams_setZstdParams(&zcs->requestedParams, ¶ms); + FORWARD_IF_ERROR( ZSTD_CCtx_loadDictionary(zcs, dict, dictSize) , ""); + return 0; +} + +size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel) +{ + DEBUGLOG(4, "ZSTD_initCStream_usingDict"); + FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) , ""); + FORWARD_IF_ERROR( ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel) , ""); + FORWARD_IF_ERROR( ZSTD_CCtx_loadDictionary(zcs, dict, dictSize) , ""); + return 0; +} + +size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pss) +{ + /* temporary : 0 interpreted as "unknown" during transition period. + * Users willing to specify "unknown" **must** use ZSTD_CONTENTSIZE_UNKNOWN. + * 0 will be interpreted as "empty" in the future. + */ + U64 const pledgedSrcSize = (pss==0) ? ZSTD_CONTENTSIZE_UNKNOWN : pss; + DEBUGLOG(4, "ZSTD_initCStream_srcSize"); + FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) , ""); + FORWARD_IF_ERROR( ZSTD_CCtx_refCDict(zcs, NULL) , ""); + FORWARD_IF_ERROR( ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel) , ""); + FORWARD_IF_ERROR( ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize) , ""); + return 0; +} + +size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel) +{ + DEBUGLOG(4, "ZSTD_initCStream"); + FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) , ""); + FORWARD_IF_ERROR( ZSTD_CCtx_refCDict(zcs, NULL) , ""); + FORWARD_IF_ERROR( ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel) , ""); + return 0; +} + +/*====== Compression ======*/ + +static size_t ZSTD_nextInputSizeHint(const ZSTD_CCtx* cctx) +{ + size_t hintInSize = cctx->inBuffTarget - cctx->inBuffPos; + if (hintInSize==0) hintInSize = cctx->blockSize; + return hintInSize; +} + +/** ZSTD_compressStream_generic(): + * internal function for all *compressStream*() variants + * non-static, because can be called from zstdmt_compress.c + * @return : hint size for next input */ +static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs, + ZSTD_outBuffer* output, + ZSTD_inBuffer* input, + ZSTD_EndDirective const flushMode) +{ + const char* const istart = (const char*)input->src; + const char* const iend = input->size != 0 ? istart + input->size : istart; + const char* ip = input->pos != 0 ? istart + input->pos : istart; + char* const ostart = (char*)output->dst; + char* const oend = output->size != 0 ? ostart + output->size : ostart; + char* op = output->pos != 0 ? ostart + output->pos : ostart; + U32 someMoreWork = 1; + + /* check expectations */ + DEBUGLOG(5, "ZSTD_compressStream_generic, flush=%u", (unsigned)flushMode); + if (zcs->appliedParams.inBufferMode == ZSTD_bm_buffered) { + assert(zcs->inBuff != NULL); + assert(zcs->inBuffSize > 0); + } + if (zcs->appliedParams.outBufferMode == ZSTD_bm_buffered) { + assert(zcs->outBuff != NULL); + assert(zcs->outBuffSize > 0); + } + assert(output->pos <= output->size); + assert(input->pos <= input->size); + assert((U32)flushMode <= (U32)ZSTD_e_end); + + while (someMoreWork) { + switch(zcs->streamStage) + { + case zcss_init: + RETURN_ERROR(init_missing, "call ZSTD_initCStream() first!"); + + case zcss_load: + if ( (flushMode == ZSTD_e_end) + && ( (size_t)(oend-op) >= ZSTD_compressBound(iend-ip) /* Enough output space */ + || zcs->appliedParams.outBufferMode == ZSTD_bm_stable) /* OR we are allowed to return dstSizeTooSmall */ + && (zcs->inBuffPos == 0) ) { + /* shortcut to compression pass directly into output buffer */ + size_t const cSize = ZSTD_compressEnd(zcs, + op, oend-op, ip, iend-ip); + DEBUGLOG(4, "ZSTD_compressEnd : cSize=%u", (unsigned)cSize); + FORWARD_IF_ERROR(cSize, "ZSTD_compressEnd failed"); + ip = iend; + op += cSize; + zcs->frameEnded = 1; + ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); + someMoreWork = 0; break; + } + /* complete loading into inBuffer in buffered mode */ + if (zcs->appliedParams.inBufferMode == ZSTD_bm_buffered) { + size_t const toLoad = zcs->inBuffTarget - zcs->inBuffPos; + size_t const loaded = ZSTD_limitCopy( + zcs->inBuff + zcs->inBuffPos, toLoad, + ip, iend-ip); + zcs->inBuffPos += loaded; + if (loaded != 0) + ip += loaded; + if ( (flushMode == ZSTD_e_continue) + && (zcs->inBuffPos < zcs->inBuffTarget) ) { + /* not enough input to fill full block : stop here */ + someMoreWork = 0; break; + } + if ( (flushMode == ZSTD_e_flush) + && (zcs->inBuffPos == zcs->inToCompress) ) { + /* empty */ + someMoreWork = 0; break; + } + } + /* compress current block (note : this stage cannot be stopped in the middle) */ + DEBUGLOG(5, "stream compression stage (flushMode==%u)", flushMode); + { int const inputBuffered = (zcs->appliedParams.inBufferMode == ZSTD_bm_buffered); + void* cDst; + size_t cSize; + size_t oSize = oend-op; + size_t const iSize = inputBuffered + ? zcs->inBuffPos - zcs->inToCompress + : MIN((size_t)(iend - ip), zcs->blockSize); + if (oSize >= ZSTD_compressBound(iSize) || zcs->appliedParams.outBufferMode == ZSTD_bm_stable) + cDst = op; /* compress into output buffer, to skip flush stage */ + else + cDst = zcs->outBuff, oSize = zcs->outBuffSize; + if (inputBuffered) { + unsigned const lastBlock = (flushMode == ZSTD_e_end) && (ip==iend); + cSize = lastBlock ? + ZSTD_compressEnd(zcs, cDst, oSize, + zcs->inBuff + zcs->inToCompress, iSize) : + ZSTD_compressContinue(zcs, cDst, oSize, + zcs->inBuff + zcs->inToCompress, iSize); + FORWARD_IF_ERROR(cSize, "%s", lastBlock ? "ZSTD_compressEnd failed" : "ZSTD_compressContinue failed"); + zcs->frameEnded = lastBlock; + /* prepare next block */ + zcs->inBuffTarget = zcs->inBuffPos + zcs->blockSize; + if (zcs->inBuffTarget > zcs->inBuffSize) + zcs->inBuffPos = 0, zcs->inBuffTarget = zcs->blockSize; + DEBUGLOG(5, "inBuffTarget:%u / inBuffSize:%u", + (unsigned)zcs->inBuffTarget, (unsigned)zcs->inBuffSize); + if (!lastBlock) + assert(zcs->inBuffTarget <= zcs->inBuffSize); + zcs->inToCompress = zcs->inBuffPos; + } else { + unsigned const lastBlock = (ip + iSize == iend); + assert(flushMode == ZSTD_e_end /* Already validated */); + cSize = lastBlock ? + ZSTD_compressEnd(zcs, cDst, oSize, ip, iSize) : + ZSTD_compressContinue(zcs, cDst, oSize, ip, iSize); + /* Consume the input prior to error checking to mirror buffered mode. */ + if (iSize > 0) + ip += iSize; + FORWARD_IF_ERROR(cSize, "%s", lastBlock ? "ZSTD_compressEnd failed" : "ZSTD_compressContinue failed"); + zcs->frameEnded = lastBlock; + if (lastBlock) + assert(ip == iend); + } + if (cDst == op) { /* no need to flush */ + op += cSize; + if (zcs->frameEnded) { + DEBUGLOG(5, "Frame completed directly in outBuffer"); + someMoreWork = 0; + ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); + } + break; + } + zcs->outBuffContentSize = cSize; + zcs->outBuffFlushedSize = 0; + zcs->streamStage = zcss_flush; /* pass-through to flush stage */ + } + ZSTD_FALLTHROUGH; + case zcss_flush: + DEBUGLOG(5, "flush stage"); + assert(zcs->appliedParams.outBufferMode == ZSTD_bm_buffered); + { size_t const toFlush = zcs->outBuffContentSize - zcs->outBuffFlushedSize; + size_t const flushed = ZSTD_limitCopy(op, (size_t)(oend-op), + zcs->outBuff + zcs->outBuffFlushedSize, toFlush); + DEBUGLOG(5, "toFlush: %u into %u ==> flushed: %u", + (unsigned)toFlush, (unsigned)(oend-op), (unsigned)flushed); + if (flushed) + op += flushed; + zcs->outBuffFlushedSize += flushed; + if (toFlush!=flushed) { + /* flush not fully completed, presumably because dst is too small */ + assert(op==oend); + someMoreWork = 0; + break; + } + zcs->outBuffContentSize = zcs->outBuffFlushedSize = 0; + if (zcs->frameEnded) { + DEBUGLOG(5, "Frame completed on flush"); + someMoreWork = 0; + ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); + break; + } + zcs->streamStage = zcss_load; + break; + } + + default: /* impossible */ + assert(0); + } + } + + input->pos = ip - istart; + output->pos = op - ostart; + if (zcs->frameEnded) return 0; + return ZSTD_nextInputSizeHint(zcs); +} + +static size_t ZSTD_nextInputSizeHint_MTorST(const ZSTD_CCtx* cctx) +{ +#ifdef ZSTD_MULTITHREAD + if (cctx->appliedParams.nbWorkers >= 1) { + assert(cctx->mtctx != NULL); + return ZSTDMT_nextInputSizeHint(cctx->mtctx); + } +#endif + return ZSTD_nextInputSizeHint(cctx); + +} + +size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input) +{ + FORWARD_IF_ERROR( ZSTD_compressStream2(zcs, output, input, ZSTD_e_continue) , ""); + return ZSTD_nextInputSizeHint_MTorST(zcs); +} + +/* After a compression call set the expected input/output buffer. + * This is validated at the start of the next compression call. + */ +static void ZSTD_setBufferExpectations(ZSTD_CCtx* cctx, ZSTD_outBuffer const* output, ZSTD_inBuffer const* input) +{ + if (cctx->appliedParams.inBufferMode == ZSTD_bm_stable) { + cctx->expectedInBuffer = *input; + } + if (cctx->appliedParams.outBufferMode == ZSTD_bm_stable) { + cctx->expectedOutBufferSize = output->size - output->pos; + } +} + +/* Validate that the input/output buffers match the expectations set by + * ZSTD_setBufferExpectations. + */ +static size_t ZSTD_checkBufferStability(ZSTD_CCtx const* cctx, + ZSTD_outBuffer const* output, + ZSTD_inBuffer const* input, + ZSTD_EndDirective endOp) +{ + if (cctx->appliedParams.inBufferMode == ZSTD_bm_stable) { + ZSTD_inBuffer const expect = cctx->expectedInBuffer; + if (expect.src != input->src || expect.pos != input->pos || expect.size != input->size) + RETURN_ERROR(srcBuffer_wrong, "ZSTD_c_stableInBuffer enabled but input differs!"); + if (endOp != ZSTD_e_end) + RETURN_ERROR(srcBuffer_wrong, "ZSTD_c_stableInBuffer can only be used with ZSTD_e_end!"); + } + if (cctx->appliedParams.outBufferMode == ZSTD_bm_stable) { + size_t const outBufferSize = output->size - output->pos; + if (cctx->expectedOutBufferSize != outBufferSize) + RETURN_ERROR(dstBuffer_wrong, "ZSTD_c_stableOutBuffer enabled but output size differs!"); + } + return 0; +} + +static size_t ZSTD_CCtx_init_compressStream2(ZSTD_CCtx* cctx, + ZSTD_EndDirective endOp, + size_t inSize) { + ZSTD_CCtx_params params = cctx->requestedParams; + ZSTD_prefixDict const prefixDict = cctx->prefixDict; + FORWARD_IF_ERROR( ZSTD_initLocalDict(cctx) , ""); /* Init the local dict if present. */ + ZSTD_memset(&cctx->prefixDict, 0, sizeof(cctx->prefixDict)); /* single usage */ + assert(prefixDict.dict==NULL || cctx->cdict==NULL); /* only one can be set */ + if (cctx->cdict && !cctx->localDict.cdict) { + /* Let the cdict's compression level take priority over the requested params. + * But do not take the cdict's compression level if the "cdict" is actually a localDict + * generated from ZSTD_initLocalDict(). + */ + params.compressionLevel = cctx->cdict->compressionLevel; + } + DEBUGLOG(4, "ZSTD_compressStream2 : transparent init stage"); + if (endOp == ZSTD_e_end) cctx->pledgedSrcSizePlusOne = inSize + 1; /* auto-fix pledgedSrcSize */ + { + size_t const dictSize = prefixDict.dict + ? prefixDict.dictSize + : (cctx->cdict ? cctx->cdict->dictContentSize : 0); + ZSTD_cParamMode_e const mode = ZSTD_getCParamMode(cctx->cdict, ¶ms, cctx->pledgedSrcSizePlusOne - 1); + params.cParams = ZSTD_getCParamsFromCCtxParams( + ¶ms, cctx->pledgedSrcSizePlusOne-1, + dictSize, mode); + } + + params.useBlockSplitter = ZSTD_resolveBlockSplitterMode(params.useBlockSplitter, ¶ms.cParams); + params.ldmParams.enableLdm = ZSTD_resolveEnableLdm(params.ldmParams.enableLdm, ¶ms.cParams); + params.useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(params.useRowMatchFinder, ¶ms.cParams); + +#ifdef ZSTD_MULTITHREAD + if ((cctx->pledgedSrcSizePlusOne-1) <= ZSTDMT_JOBSIZE_MIN) { + params.nbWorkers = 0; /* do not invoke multi-threading when src size is too small */ + } + if (params.nbWorkers > 0) { +#if ZSTD_TRACE + cctx->traceCtx = (ZSTD_trace_compress_begin != NULL) ? ZSTD_trace_compress_begin(cctx) : 0; +#endif + /* mt context creation */ + if (cctx->mtctx == NULL) { + DEBUGLOG(4, "ZSTD_compressStream2: creating new mtctx for nbWorkers=%u", + params.nbWorkers); + cctx->mtctx = ZSTDMT_createCCtx_advanced((U32)params.nbWorkers, cctx->customMem, cctx->pool); + RETURN_ERROR_IF(cctx->mtctx == NULL, memory_allocation, "NULL pointer!"); + } + /* mt compression */ + DEBUGLOG(4, "call ZSTDMT_initCStream_internal as nbWorkers=%u", params.nbWorkers); + FORWARD_IF_ERROR( ZSTDMT_initCStream_internal( + cctx->mtctx, + prefixDict.dict, prefixDict.dictSize, prefixDict.dictContentType, + cctx->cdict, params, cctx->pledgedSrcSizePlusOne-1) , ""); + cctx->dictID = cctx->cdict ? cctx->cdict->dictID : 0; + cctx->dictContentSize = cctx->cdict ? cctx->cdict->dictContentSize : prefixDict.dictSize; + cctx->consumedSrcSize = 0; + cctx->producedCSize = 0; + cctx->streamStage = zcss_load; + cctx->appliedParams = params; + } else +#endif + { U64 const pledgedSrcSize = cctx->pledgedSrcSizePlusOne - 1; + assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams))); + FORWARD_IF_ERROR( ZSTD_compressBegin_internal(cctx, + prefixDict.dict, prefixDict.dictSize, prefixDict.dictContentType, ZSTD_dtlm_fast, + cctx->cdict, + ¶ms, pledgedSrcSize, + ZSTDb_buffered) , ""); + assert(cctx->appliedParams.nbWorkers == 0); + cctx->inToCompress = 0; + cctx->inBuffPos = 0; + if (cctx->appliedParams.inBufferMode == ZSTD_bm_buffered) { + /* for small input: avoid automatic flush on reaching end of block, since + * it would require to add a 3-bytes null block to end frame + */ + cctx->inBuffTarget = cctx->blockSize + (cctx->blockSize == pledgedSrcSize); + } else { + cctx->inBuffTarget = 0; + } + cctx->outBuffContentSize = cctx->outBuffFlushedSize = 0; + cctx->streamStage = zcss_load; + cctx->frameEnded = 0; + } + return 0; +} + +size_t ZSTD_compressStream2( ZSTD_CCtx* cctx, + ZSTD_outBuffer* output, + ZSTD_inBuffer* input, + ZSTD_EndDirective endOp) +{ + DEBUGLOG(5, "ZSTD_compressStream2, endOp=%u ", (unsigned)endOp); + /* check conditions */ + RETURN_ERROR_IF(output->pos > output->size, dstSize_tooSmall, "invalid output buffer"); + RETURN_ERROR_IF(input->pos > input->size, srcSize_wrong, "invalid input buffer"); + RETURN_ERROR_IF((U32)endOp > (U32)ZSTD_e_end, parameter_outOfBound, "invalid endDirective"); + assert(cctx != NULL); + + /* transparent initialization stage */ + if (cctx->streamStage == zcss_init) { + FORWARD_IF_ERROR(ZSTD_CCtx_init_compressStream2(cctx, endOp, input->size), "CompressStream2 initialization failed"); + ZSTD_setBufferExpectations(cctx, output, input); /* Set initial buffer expectations now that we've initialized */ + } + /* end of transparent initialization stage */ + + FORWARD_IF_ERROR(ZSTD_checkBufferStability(cctx, output, input, endOp), "invalid buffers"); + /* compression stage */ +#ifdef ZSTD_MULTITHREAD + if (cctx->appliedParams.nbWorkers > 0) { + size_t flushMin; + if (cctx->cParamsChanged) { + ZSTDMT_updateCParams_whileCompressing(cctx->mtctx, &cctx->requestedParams); + cctx->cParamsChanged = 0; + } + for (;;) { + size_t const ipos = input->pos; + size_t const opos = output->pos; + flushMin = ZSTDMT_compressStream_generic(cctx->mtctx, output, input, endOp); + cctx->consumedSrcSize += (U64)(input->pos - ipos); + cctx->producedCSize += (U64)(output->pos - opos); + if ( ZSTD_isError(flushMin) + || (endOp == ZSTD_e_end && flushMin == 0) ) { /* compression completed */ + if (flushMin == 0) + ZSTD_CCtx_trace(cctx, 0); + ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only); + } + FORWARD_IF_ERROR(flushMin, "ZSTDMT_compressStream_generic failed"); + + if (endOp == ZSTD_e_continue) { + /* We only require some progress with ZSTD_e_continue, not maximal progress. + * We're done if we've consumed or produced any bytes, or either buffer is + * full. + */ + if (input->pos != ipos || output->pos != opos || input->pos == input->size || output->pos == output->size) + break; + } else { + assert(endOp == ZSTD_e_flush || endOp == ZSTD_e_end); + /* We require maximal progress. We're done when the flush is complete or the + * output buffer is full. + */ + if (flushMin == 0 || output->pos == output->size) + break; + } + } + DEBUGLOG(5, "completed ZSTD_compressStream2 delegating to ZSTDMT_compressStream_generic"); + /* Either we don't require maximum forward progress, we've finished the + * flush, or we are out of output space. + */ + assert(endOp == ZSTD_e_continue || flushMin == 0 || output->pos == output->size); + ZSTD_setBufferExpectations(cctx, output, input); + return flushMin; + } +#endif + FORWARD_IF_ERROR( ZSTD_compressStream_generic(cctx, output, input, endOp) , ""); + DEBUGLOG(5, "completed ZSTD_compressStream2"); + ZSTD_setBufferExpectations(cctx, output, input); + return cctx->outBuffContentSize - cctx->outBuffFlushedSize; /* remaining to flush */ +} + +size_t ZSTD_compressStream2_simpleArgs ( + ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, size_t* dstPos, + const void* src, size_t srcSize, size_t* srcPos, + ZSTD_EndDirective endOp) +{ + ZSTD_outBuffer output = { dst, dstCapacity, *dstPos }; + ZSTD_inBuffer input = { src, srcSize, *srcPos }; + /* ZSTD_compressStream2() will check validity of dstPos and srcPos */ + size_t const cErr = ZSTD_compressStream2(cctx, &output, &input, endOp); + *dstPos = output.pos; + *srcPos = input.pos; + return cErr; +} + +size_t ZSTD_compress2(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize) +{ + ZSTD_bufferMode_e const originalInBufferMode = cctx->requestedParams.inBufferMode; + ZSTD_bufferMode_e const originalOutBufferMode = cctx->requestedParams.outBufferMode; + DEBUGLOG(4, "ZSTD_compress2 (srcSize=%u)", (unsigned)srcSize); + ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only); + /* Enable stable input/output buffers. */ + cctx->requestedParams.inBufferMode = ZSTD_bm_stable; + cctx->requestedParams.outBufferMode = ZSTD_bm_stable; + { size_t oPos = 0; + size_t iPos = 0; + size_t const result = ZSTD_compressStream2_simpleArgs(cctx, + dst, dstCapacity, &oPos, + src, srcSize, &iPos, + ZSTD_e_end); + /* Reset to the original values. */ + cctx->requestedParams.inBufferMode = originalInBufferMode; + cctx->requestedParams.outBufferMode = originalOutBufferMode; + FORWARD_IF_ERROR(result, "ZSTD_compressStream2_simpleArgs failed"); + if (result != 0) { /* compression not completed, due to lack of output space */ + assert(oPos == dstCapacity); + RETURN_ERROR(dstSize_tooSmall, ""); + } + assert(iPos == srcSize); /* all input is expected consumed */ + return oPos; + } +} + +typedef struct { + U32 idx; /* Index in array of ZSTD_Sequence */ + U32 posInSequence; /* Position within sequence at idx */ + size_t posInSrc; /* Number of bytes given by sequences provided so far */ +} ZSTD_sequencePosition; + +/* ZSTD_validateSequence() : + * @offCode : is presumed to follow format required by ZSTD_storeSeq() + * @returns a ZSTD error code if sequence is not valid + */ +static size_t +ZSTD_validateSequence(U32 offCode, U32 matchLength, + size_t posInSrc, U32 windowLog, size_t dictSize) +{ + U32 const windowSize = 1 << windowLog; + /* posInSrc represents the amount of data the the decoder would decode up to this point. + * As long as the amount of data decoded is less than or equal to window size, offsets may be + * larger than the total length of output decoded in order to reference the dict, even larger than + * window size. After output surpasses windowSize, we're limited to windowSize offsets again. + */ + size_t const offsetBound = posInSrc > windowSize ? (size_t)windowSize : posInSrc + (size_t)dictSize; + RETURN_ERROR_IF(offCode > STORE_OFFSET(offsetBound), corruption_detected, "Offset too large!"); + RETURN_ERROR_IF(matchLength < MINMATCH, corruption_detected, "Matchlength too small"); + return 0; +} + +/* Returns an offset code, given a sequence's raw offset, the ongoing repcode array, and whether litLength == 0 */ +static U32 ZSTD_finalizeOffCode(U32 rawOffset, const U32 rep[ZSTD_REP_NUM], U32 ll0) +{ + U32 offCode = STORE_OFFSET(rawOffset); + + if (!ll0 && rawOffset == rep[0]) { + offCode = STORE_REPCODE_1; + } else if (rawOffset == rep[1]) { + offCode = STORE_REPCODE(2 - ll0); + } else if (rawOffset == rep[2]) { + offCode = STORE_REPCODE(3 - ll0); + } else if (ll0 && rawOffset == rep[0] - 1) { + offCode = STORE_REPCODE_3; + } + return offCode; +} + +/* Returns 0 on success, and a ZSTD_error otherwise. This function scans through an array of + * ZSTD_Sequence, storing the sequences it finds, until it reaches a block delimiter. + */ +static size_t +ZSTD_copySequencesToSeqStoreExplicitBlockDelim(ZSTD_CCtx* cctx, + ZSTD_sequencePosition* seqPos, + const ZSTD_Sequence* const inSeqs, size_t inSeqsSize, + const void* src, size_t blockSize) +{ + U32 idx = seqPos->idx; + BYTE const* ip = (BYTE const*)(src); + const BYTE* const iend = ip + blockSize; + repcodes_t updatedRepcodes; + U32 dictSize; + + if (cctx->cdict) { + dictSize = (U32)cctx->cdict->dictContentSize; + } else if (cctx->prefixDict.dict) { + dictSize = (U32)cctx->prefixDict.dictSize; + } else { + dictSize = 0; + } + ZSTD_memcpy(updatedRepcodes.rep, cctx->blockState.prevCBlock->rep, sizeof(repcodes_t)); + for (; (inSeqs[idx].matchLength != 0 || inSeqs[idx].offset != 0) && idx < inSeqsSize; ++idx) { + U32 const litLength = inSeqs[idx].litLength; + U32 const ll0 = (litLength == 0); + U32 const matchLength = inSeqs[idx].matchLength; + U32 const offCode = ZSTD_finalizeOffCode(inSeqs[idx].offset, updatedRepcodes.rep, ll0); + ZSTD_updateRep(updatedRepcodes.rep, offCode, ll0); + + DEBUGLOG(6, "Storing sequence: (of: %u, ml: %u, ll: %u)", offCode, matchLength, litLength); + if (cctx->appliedParams.validateSequences) { + seqPos->posInSrc += litLength + matchLength; + FORWARD_IF_ERROR(ZSTD_validateSequence(offCode, matchLength, seqPos->posInSrc, + cctx->appliedParams.cParams.windowLog, dictSize), + "Sequence validation failed"); + } + RETURN_ERROR_IF(idx - seqPos->idx > cctx->seqStore.maxNbSeq, memory_allocation, + "Not enough memory allocated. Try adjusting ZSTD_c_minMatch."); + ZSTD_storeSeq(&cctx->seqStore, litLength, ip, iend, offCode, matchLength); + ip += matchLength + litLength; + } + ZSTD_memcpy(cctx->blockState.nextCBlock->rep, updatedRepcodes.rep, sizeof(repcodes_t)); + + if (inSeqs[idx].litLength) { + DEBUGLOG(6, "Storing last literals of size: %u", inSeqs[idx].litLength); + ZSTD_storeLastLiterals(&cctx->seqStore, ip, inSeqs[idx].litLength); + ip += inSeqs[idx].litLength; + seqPos->posInSrc += inSeqs[idx].litLength; + } + RETURN_ERROR_IF(ip != iend, corruption_detected, "Blocksize doesn't agree with block delimiter!"); + seqPos->idx = idx+1; + return 0; +} + +/* Returns the number of bytes to move the current read position back by. Only non-zero + * if we ended up splitting a sequence. Otherwise, it may return a ZSTD error if something + * went wrong. + * + * This function will attempt to scan through blockSize bytes represented by the sequences + * in inSeqs, storing any (partial) sequences. + * + * Occasionally, we may want to change the actual number of bytes we consumed from inSeqs to + * avoid splitting a match, or to avoid splitting a match such that it would produce a match + * smaller than MINMATCH. In this case, we return the number of bytes that we didn't read from this block. + */ +static size_t +ZSTD_copySequencesToSeqStoreNoBlockDelim(ZSTD_CCtx* cctx, ZSTD_sequencePosition* seqPos, + const ZSTD_Sequence* const inSeqs, size_t inSeqsSize, + const void* src, size_t blockSize) +{ + U32 idx = seqPos->idx; + U32 startPosInSequence = seqPos->posInSequence; + U32 endPosInSequence = seqPos->posInSequence + (U32)blockSize; + size_t dictSize; + BYTE const* ip = (BYTE const*)(src); + BYTE const* iend = ip + blockSize; /* May be adjusted if we decide to process fewer than blockSize bytes */ + repcodes_t updatedRepcodes; + U32 bytesAdjustment = 0; + U32 finalMatchSplit = 0; + + if (cctx->cdict) { + dictSize = cctx->cdict->dictContentSize; + } else if (cctx->prefixDict.dict) { + dictSize = cctx->prefixDict.dictSize; + } else { + dictSize = 0; + } + DEBUGLOG(5, "ZSTD_copySequencesToSeqStore: idx: %u PIS: %u blockSize: %zu", idx, startPosInSequence, blockSize); + DEBUGLOG(5, "Start seq: idx: %u (of: %u ml: %u ll: %u)", idx, inSeqs[idx].offset, inSeqs[idx].matchLength, inSeqs[idx].litLength); + ZSTD_memcpy(updatedRepcodes.rep, cctx->blockState.prevCBlock->rep, sizeof(repcodes_t)); + while (endPosInSequence && idx < inSeqsSize && !finalMatchSplit) { + const ZSTD_Sequence currSeq = inSeqs[idx]; + U32 litLength = currSeq.litLength; + U32 matchLength = currSeq.matchLength; + U32 const rawOffset = currSeq.offset; + U32 offCode; + + /* Modify the sequence depending on where endPosInSequence lies */ + if (endPosInSequence >= currSeq.litLength + currSeq.matchLength) { + if (startPosInSequence >= litLength) { + startPosInSequence -= litLength; + litLength = 0; + matchLength -= startPosInSequence; + } else { + litLength -= startPosInSequence; + } + /* Move to the next sequence */ + endPosInSequence -= currSeq.litLength + currSeq.matchLength; + startPosInSequence = 0; + idx++; + } else { + /* This is the final (partial) sequence we're adding from inSeqs, and endPosInSequence + does not reach the end of the match. So, we have to split the sequence */ + DEBUGLOG(6, "Require a split: diff: %u, idx: %u PIS: %u", + currSeq.litLength + currSeq.matchLength - endPosInSequence, idx, endPosInSequence); + if (endPosInSequence > litLength) { + U32 firstHalfMatchLength; + litLength = startPosInSequence >= litLength ? 0 : litLength - startPosInSequence; + firstHalfMatchLength = endPosInSequence - startPosInSequence - litLength; + if (matchLength > blockSize && firstHalfMatchLength >= cctx->appliedParams.cParams.minMatch) { + /* Only ever split the match if it is larger than the block size */ + U32 secondHalfMatchLength = currSeq.matchLength + currSeq.litLength - endPosInSequence; + if (secondHalfMatchLength < cctx->appliedParams.cParams.minMatch) { + /* Move the endPosInSequence backward so that it creates match of minMatch length */ + endPosInSequence -= cctx->appliedParams.cParams.minMatch - secondHalfMatchLength; + bytesAdjustment = cctx->appliedParams.cParams.minMatch - secondHalfMatchLength; + firstHalfMatchLength -= bytesAdjustment; + } + matchLength = firstHalfMatchLength; + /* Flag that we split the last match - after storing the sequence, exit the loop, + but keep the value of endPosInSequence */ + finalMatchSplit = 1; + } else { + /* Move the position in sequence backwards so that we don't split match, and break to store + * the last literals. We use the original currSeq.litLength as a marker for where endPosInSequence + * should go. We prefer to do this whenever it is not necessary to split the match, or if doing so + * would cause the first half of the match to be too small + */ + bytesAdjustment = endPosInSequence - currSeq.litLength; + endPosInSequence = currSeq.litLength; + break; + } + } else { + /* This sequence ends inside the literals, break to store the last literals */ + break; + } + } + /* Check if this offset can be represented with a repcode */ + { U32 const ll0 = (litLength == 0); + offCode = ZSTD_finalizeOffCode(rawOffset, updatedRepcodes.rep, ll0); + ZSTD_updateRep(updatedRepcodes.rep, offCode, ll0); + } + + if (cctx->appliedParams.validateSequences) { + seqPos->posInSrc += litLength + matchLength; + FORWARD_IF_ERROR(ZSTD_validateSequence(offCode, matchLength, seqPos->posInSrc, + cctx->appliedParams.cParams.windowLog, dictSize), + "Sequence validation failed"); + } + DEBUGLOG(6, "Storing sequence: (of: %u, ml: %u, ll: %u)", offCode, matchLength, litLength); + RETURN_ERROR_IF(idx - seqPos->idx > cctx->seqStore.maxNbSeq, memory_allocation, + "Not enough memory allocated. Try adjusting ZSTD_c_minMatch."); + ZSTD_storeSeq(&cctx->seqStore, litLength, ip, iend, offCode, matchLength); + ip += matchLength + litLength; + } + DEBUGLOG(5, "Ending seq: idx: %u (of: %u ml: %u ll: %u)", idx, inSeqs[idx].offset, inSeqs[idx].matchLength, inSeqs[idx].litLength); + assert(idx == inSeqsSize || endPosInSequence <= inSeqs[idx].litLength + inSeqs[idx].matchLength); + seqPos->idx = idx; + seqPos->posInSequence = endPosInSequence; + ZSTD_memcpy(cctx->blockState.nextCBlock->rep, updatedRepcodes.rep, sizeof(repcodes_t)); + + iend -= bytesAdjustment; + if (ip != iend) { + /* Store any last literals */ + U32 lastLLSize = (U32)(iend - ip); + assert(ip <= iend); + DEBUGLOG(6, "Storing last literals of size: %u", lastLLSize); + ZSTD_storeLastLiterals(&cctx->seqStore, ip, lastLLSize); + seqPos->posInSrc += lastLLSize; + } + + return bytesAdjustment; +} + +typedef size_t (*ZSTD_sequenceCopier) (ZSTD_CCtx* cctx, ZSTD_sequencePosition* seqPos, + const ZSTD_Sequence* const inSeqs, size_t inSeqsSize, + const void* src, size_t blockSize); +static ZSTD_sequenceCopier ZSTD_selectSequenceCopier(ZSTD_sequenceFormat_e mode) +{ + ZSTD_sequenceCopier sequenceCopier = NULL; + assert(ZSTD_cParam_withinBounds(ZSTD_c_blockDelimiters, mode)); + if (mode == ZSTD_sf_explicitBlockDelimiters) { + return ZSTD_copySequencesToSeqStoreExplicitBlockDelim; + } else if (mode == ZSTD_sf_noBlockDelimiters) { + return ZSTD_copySequencesToSeqStoreNoBlockDelim; + } + assert(sequenceCopier != NULL); + return sequenceCopier; +} + +/* Compress, block-by-block, all of the sequences given. + * + * Returns the cumulative size of all compressed blocks (including their headers), + * otherwise a ZSTD error. + */ +static size_t +ZSTD_compressSequences_internal(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const ZSTD_Sequence* inSeqs, size_t inSeqsSize, + const void* src, size_t srcSize) +{ + size_t cSize = 0; + U32 lastBlock; + size_t blockSize; + size_t compressedSeqsSize; + size_t remaining = srcSize; + ZSTD_sequencePosition seqPos = {0, 0, 0}; + + BYTE const* ip = (BYTE const*)src; + BYTE* op = (BYTE*)dst; + ZSTD_sequenceCopier const sequenceCopier = ZSTD_selectSequenceCopier(cctx->appliedParams.blockDelimiters); + + DEBUGLOG(4, "ZSTD_compressSequences_internal srcSize: %zu, inSeqsSize: %zu", srcSize, inSeqsSize); + /* Special case: empty frame */ + if (remaining == 0) { + U32 const cBlockHeader24 = 1 /* last block */ + (((U32)bt_raw)<<1); + RETURN_ERROR_IF(dstCapacity<4, dstSize_tooSmall, "No room for empty frame block header"); + MEM_writeLE32(op, cBlockHeader24); + op += ZSTD_blockHeaderSize; + dstCapacity -= ZSTD_blockHeaderSize; + cSize += ZSTD_blockHeaderSize; + } + + while (remaining) { + size_t cBlockSize; + size_t additionalByteAdjustment; + lastBlock = remaining <= cctx->blockSize; + blockSize = lastBlock ? (U32)remaining : (U32)cctx->blockSize; + ZSTD_resetSeqStore(&cctx->seqStore); + DEBUGLOG(4, "Working on new block. Blocksize: %zu", blockSize); + + additionalByteAdjustment = sequenceCopier(cctx, &seqPos, inSeqs, inSeqsSize, ip, blockSize); + FORWARD_IF_ERROR(additionalByteAdjustment, "Bad sequence copy"); + blockSize -= additionalByteAdjustment; + + /* If blocks are too small, emit as a nocompress block */ + if (blockSize < MIN_CBLOCK_SIZE+ZSTD_blockHeaderSize+1) { + cBlockSize = ZSTD_noCompressBlock(op, dstCapacity, ip, blockSize, lastBlock); + FORWARD_IF_ERROR(cBlockSize, "Nocompress block failed"); + DEBUGLOG(4, "Block too small, writing out nocompress block: cSize: %zu", cBlockSize); + cSize += cBlockSize; + ip += blockSize; + op += cBlockSize; + remaining -= blockSize; + dstCapacity -= cBlockSize; + continue; + } + + compressedSeqsSize = ZSTD_entropyCompressSeqStore(&cctx->seqStore, + &cctx->blockState.prevCBlock->entropy, &cctx->blockState.nextCBlock->entropy, + &cctx->appliedParams, + op + ZSTD_blockHeaderSize /* Leave space for block header */, dstCapacity - ZSTD_blockHeaderSize, + blockSize, + cctx->entropyWorkspace, ENTROPY_WORKSPACE_SIZE /* statically allocated in resetCCtx */, + cctx->bmi2); + FORWARD_IF_ERROR(compressedSeqsSize, "Compressing sequences of block failed"); + DEBUGLOG(4, "Compressed sequences size: %zu", compressedSeqsSize); + + if (!cctx->isFirstBlock && + ZSTD_maybeRLE(&cctx->seqStore) && + ZSTD_isRLE((BYTE const*)src, srcSize)) { + /* We don't want to emit our first block as a RLE even if it qualifies because + * doing so will cause the decoder (cli only) to throw a "should consume all input error." + * This is only an issue for zstd <= v1.4.3 + */ + compressedSeqsSize = 1; + } + + if (compressedSeqsSize == 0) { + /* ZSTD_noCompressBlock writes the block header as well */ + cBlockSize = ZSTD_noCompressBlock(op, dstCapacity, ip, blockSize, lastBlock); + FORWARD_IF_ERROR(cBlockSize, "Nocompress block failed"); + DEBUGLOG(4, "Writing out nocompress block, size: %zu", cBlockSize); + } else if (compressedSeqsSize == 1) { + cBlockSize = ZSTD_rleCompressBlock(op, dstCapacity, *ip, blockSize, lastBlock); + FORWARD_IF_ERROR(cBlockSize, "RLE compress block failed"); + DEBUGLOG(4, "Writing out RLE block, size: %zu", cBlockSize); + } else { + U32 cBlockHeader; + /* Error checking and repcodes update */ + ZSTD_blockState_confirmRepcodesAndEntropyTables(&cctx->blockState); + if (cctx->blockState.prevCBlock->entropy.fse.offcode_repeatMode == FSE_repeat_valid) + cctx->blockState.prevCBlock->entropy.fse.offcode_repeatMode = FSE_repeat_check; + + /* Write block header into beginning of block*/ + cBlockHeader = lastBlock + (((U32)bt_compressed)<<1) + (U32)(compressedSeqsSize << 3); + MEM_writeLE24(op, cBlockHeader); + cBlockSize = ZSTD_blockHeaderSize + compressedSeqsSize; + DEBUGLOG(4, "Writing out compressed block, size: %zu", cBlockSize); + } + + cSize += cBlockSize; + DEBUGLOG(4, "cSize running total: %zu", cSize); + + if (lastBlock) { + break; + } else { + ip += blockSize; + op += cBlockSize; + remaining -= blockSize; + dstCapacity -= cBlockSize; + cctx->isFirstBlock = 0; + } + } + + return cSize; +} + +size_t ZSTD_compressSequences(ZSTD_CCtx* const cctx, void* dst, size_t dstCapacity, + const ZSTD_Sequence* inSeqs, size_t inSeqsSize, + const void* src, size_t srcSize) +{ + BYTE* op = (BYTE*)dst; + size_t cSize = 0; + size_t compressedBlocksSize = 0; + size_t frameHeaderSize = 0; + + /* Transparent initialization stage, same as compressStream2() */ + DEBUGLOG(3, "ZSTD_compressSequences()"); + assert(cctx != NULL); + FORWARD_IF_ERROR(ZSTD_CCtx_init_compressStream2(cctx, ZSTD_e_end, srcSize), "CCtx initialization failed"); + /* Begin writing output, starting with frame header */ + frameHeaderSize = ZSTD_writeFrameHeader(op, dstCapacity, &cctx->appliedParams, srcSize, cctx->dictID); + op += frameHeaderSize; + dstCapacity -= frameHeaderSize; + cSize += frameHeaderSize; + if (cctx->appliedParams.fParams.checksumFlag && srcSize) { + XXH64_update(&cctx->xxhState, src, srcSize); + } + /* cSize includes block header size and compressed sequences size */ + compressedBlocksSize = ZSTD_compressSequences_internal(cctx, + op, dstCapacity, + inSeqs, inSeqsSize, + src, srcSize); + FORWARD_IF_ERROR(compressedBlocksSize, "Compressing blocks failed!"); + cSize += compressedBlocksSize; + dstCapacity -= compressedBlocksSize; + + if (cctx->appliedParams.fParams.checksumFlag) { + U32 const checksum = (U32) XXH64_digest(&cctx->xxhState); + RETURN_ERROR_IF(dstCapacity<4, dstSize_tooSmall, "no room for checksum"); + DEBUGLOG(4, "Write checksum : %08X", (unsigned)checksum); + MEM_writeLE32((char*)dst + cSize, checksum); + cSize += 4; + } + + DEBUGLOG(3, "Final compressed size: %zu", cSize); + return cSize; +} + +/*====== Finalize ======*/ + +/*! ZSTD_flushStream() : + * @return : amount of data remaining to flush */ +size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output) +{ + ZSTD_inBuffer input = { NULL, 0, 0 }; + return ZSTD_compressStream2(zcs, output, &input, ZSTD_e_flush); +} + + +size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output) +{ + ZSTD_inBuffer input = { NULL, 0, 0 }; + size_t const remainingToFlush = ZSTD_compressStream2(zcs, output, &input, ZSTD_e_end); + FORWARD_IF_ERROR( remainingToFlush , "ZSTD_compressStream2 failed"); + if (zcs->appliedParams.nbWorkers > 0) return remainingToFlush; /* minimal estimation */ + /* single thread mode : attempt to calculate remaining to flush more precisely */ + { size_t const lastBlockSize = zcs->frameEnded ? 0 : ZSTD_BLOCKHEADERSIZE; + size_t const checksumSize = (size_t)(zcs->frameEnded ? 0 : zcs->appliedParams.fParams.checksumFlag * 4); + size_t const toFlush = remainingToFlush + lastBlockSize + checksumSize; + DEBUGLOG(4, "ZSTD_endStream : remaining to flush : %u", (unsigned)toFlush); + return toFlush; + } +} + + +/*-===== Pre-defined compression levels =====-*/ +#include "clevels.h" + +int ZSTD_maxCLevel(void) { return ZSTD_MAX_CLEVEL; } +int ZSTD_minCLevel(void) { return (int)-ZSTD_TARGETLENGTH_MAX; } +int ZSTD_defaultCLevel(void) { return ZSTD_CLEVEL_DEFAULT; } + +static ZSTD_compressionParameters ZSTD_dedicatedDictSearch_getCParams(int const compressionLevel, size_t const dictSize) +{ + ZSTD_compressionParameters cParams = ZSTD_getCParams_internal(compressionLevel, 0, dictSize, ZSTD_cpm_createCDict); + switch (cParams.strategy) { + case ZSTD_fast: + case ZSTD_dfast: + break; + case ZSTD_greedy: + case ZSTD_lazy: + case ZSTD_lazy2: + cParams.hashLog += ZSTD_LAZY_DDSS_BUCKET_LOG; + break; + case ZSTD_btlazy2: + case ZSTD_btopt: + case ZSTD_btultra: + case ZSTD_btultra2: + break; + } + return cParams; +} + +static int ZSTD_dedicatedDictSearch_isSupported( + ZSTD_compressionParameters const* cParams) +{ + return (cParams->strategy >= ZSTD_greedy) + && (cParams->strategy <= ZSTD_lazy2) + && (cParams->hashLog > cParams->chainLog) + && (cParams->chainLog <= 24); +} + +/** + * Reverses the adjustment applied to cparams when enabling dedicated dict + * search. This is used to recover the params set to be used in the working + * context. (Otherwise, those tables would also grow.) + */ +static void ZSTD_dedicatedDictSearch_revertCParams( + ZSTD_compressionParameters* cParams) { + switch (cParams->strategy) { + case ZSTD_fast: + case ZSTD_dfast: + break; + case ZSTD_greedy: + case ZSTD_lazy: + case ZSTD_lazy2: + cParams->hashLog -= ZSTD_LAZY_DDSS_BUCKET_LOG; + if (cParams->hashLog < ZSTD_HASHLOG_MIN) { + cParams->hashLog = ZSTD_HASHLOG_MIN; + } + break; + case ZSTD_btlazy2: + case ZSTD_btopt: + case ZSTD_btultra: + case ZSTD_btultra2: + break; + } +} + +static U64 ZSTD_getCParamRowSize(U64 srcSizeHint, size_t dictSize, ZSTD_cParamMode_e mode) +{ + switch (mode) { + case ZSTD_cpm_unknown: + case ZSTD_cpm_noAttachDict: + case ZSTD_cpm_createCDict: + break; + case ZSTD_cpm_attachDict: + dictSize = 0; + break; + default: + assert(0); + break; + } + { int const unknown = srcSizeHint == ZSTD_CONTENTSIZE_UNKNOWN; + size_t const addedSize = unknown && dictSize > 0 ? 500 : 0; + return unknown && dictSize == 0 ? ZSTD_CONTENTSIZE_UNKNOWN : srcSizeHint+dictSize+addedSize; + } +} + +/*! ZSTD_getCParams_internal() : + * @return ZSTD_compressionParameters structure for a selected compression level, srcSize and dictSize. + * Note: srcSizeHint 0 means 0, use ZSTD_CONTENTSIZE_UNKNOWN for unknown. + * Use dictSize == 0 for unknown or unused. + * Note: `mode` controls how we treat the `dictSize`. See docs for `ZSTD_cParamMode_e`. */ +static ZSTD_compressionParameters ZSTD_getCParams_internal(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize, ZSTD_cParamMode_e mode) +{ + U64 const rSize = ZSTD_getCParamRowSize(srcSizeHint, dictSize, mode); + U32 const tableID = (rSize <= 256 KB) + (rSize <= 128 KB) + (rSize <= 16 KB); + int row; + DEBUGLOG(5, "ZSTD_getCParams_internal (cLevel=%i)", compressionLevel); + + /* row */ + if (compressionLevel == 0) row = ZSTD_CLEVEL_DEFAULT; /* 0 == default */ + else if (compressionLevel < 0) row = 0; /* entry 0 is baseline for fast mode */ + else if (compressionLevel > ZSTD_MAX_CLEVEL) row = ZSTD_MAX_CLEVEL; + else row = compressionLevel; + + { ZSTD_compressionParameters cp = ZSTD_defaultCParameters[tableID][row]; + DEBUGLOG(5, "ZSTD_getCParams_internal selected tableID: %u row: %u strat: %u", tableID, row, (U32)cp.strategy); + /* acceleration factor */ + if (compressionLevel < 0) { + int const clampedCompressionLevel = MAX(ZSTD_minCLevel(), compressionLevel); + cp.targetLength = (unsigned)(-clampedCompressionLevel); + } + /* refine parameters based on srcSize & dictSize */ + return ZSTD_adjustCParams_internal(cp, srcSizeHint, dictSize, mode); + } +} + +/*! ZSTD_getCParams() : + * @return ZSTD_compressionParameters structure for a selected compression level, srcSize and dictSize. + * Size values are optional, provide 0 if not known or unused */ +ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize) +{ + if (srcSizeHint == 0) srcSizeHint = ZSTD_CONTENTSIZE_UNKNOWN; + return ZSTD_getCParams_internal(compressionLevel, srcSizeHint, dictSize, ZSTD_cpm_unknown); +} + +/*! ZSTD_getParams() : + * same idea as ZSTD_getCParams() + * @return a `ZSTD_parameters` structure (instead of `ZSTD_compressionParameters`). + * Fields of `ZSTD_frameParameters` are set to default values */ +static ZSTD_parameters ZSTD_getParams_internal(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize, ZSTD_cParamMode_e mode) { + ZSTD_parameters params; + ZSTD_compressionParameters const cParams = ZSTD_getCParams_internal(compressionLevel, srcSizeHint, dictSize, mode); + DEBUGLOG(5, "ZSTD_getParams (cLevel=%i)", compressionLevel); + ZSTD_memset(¶ms, 0, sizeof(params)); + params.cParams = cParams; + params.fParams.contentSizeFlag = 1; + return params; +} + +/*! ZSTD_getParams() : + * same idea as ZSTD_getCParams() + * @return a `ZSTD_parameters` structure (instead of `ZSTD_compressionParameters`). + * Fields of `ZSTD_frameParameters` are set to default values */ +ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize) { + if (srcSizeHint == 0) srcSizeHint = ZSTD_CONTENTSIZE_UNKNOWN; + return ZSTD_getParams_internal(compressionLevel, srcSizeHint, dictSize, ZSTD_cpm_unknown); +} diff --git a/src/blosc2/internal-complibs/zstd-1.5.2/compress/zstd_compress_internal.h b/src/blosc2/internal-complibs/zstd-1.5.2/compress/zstd_compress_internal.h new file mode 100644 index 0000000..c406e79 --- /dev/null +++ b/src/blosc2/internal-complibs/zstd-1.5.2/compress/zstd_compress_internal.h @@ -0,0 +1,1458 @@ +/* + * Copyright (c) Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/* This header contains definitions + * that shall **only** be used by modules within lib/compress. + */ + +#ifndef ZSTD_COMPRESS_H +#define ZSTD_COMPRESS_H + +/*-************************************* +* Dependencies +***************************************/ +#include "../common/zstd_internal.h" +#include "zstd_cwksp.h" +#ifdef ZSTD_MULTITHREAD +# include "zstdmt_compress.h" +#endif + +#if defined (__cplusplus) +extern "C" { +#endif + +/*-************************************* +* Constants +***************************************/ +#define kSearchStrength 8 +#define HASH_READ_SIZE 8 +#define ZSTD_DUBT_UNSORTED_MARK 1 /* For btlazy2 strategy, index ZSTD_DUBT_UNSORTED_MARK==1 means "unsorted". + It could be confused for a real successor at index "1", if sorted as larger than its predecessor. + It's not a big deal though : candidate will just be sorted again. + Additionally, candidate position 1 will be lost. + But candidate 1 cannot hide a large tree of candidates, so it's a minimal loss. + The benefit is that ZSTD_DUBT_UNSORTED_MARK cannot be mishandled after table re-use with a different strategy. + This constant is required by ZSTD_compressBlock_btlazy2() and ZSTD_reduceTable_internal() */ + + +/*-************************************* +* Context memory management +***************************************/ +typedef enum { ZSTDcs_created=0, ZSTDcs_init, ZSTDcs_ongoing, ZSTDcs_ending } ZSTD_compressionStage_e; +typedef enum { zcss_init=0, zcss_load, zcss_flush } ZSTD_cStreamStage; + +typedef struct ZSTD_prefixDict_s { + const void* dict; + size_t dictSize; + ZSTD_dictContentType_e dictContentType; +} ZSTD_prefixDict; + +typedef struct { + void* dictBuffer; + void const* dict; + size_t dictSize; + ZSTD_dictContentType_e dictContentType; + ZSTD_CDict* cdict; +} ZSTD_localDict; + +typedef struct { + HUF_CElt CTable[HUF_CTABLE_SIZE_ST(255)]; + HUF_repeat repeatMode; +} ZSTD_hufCTables_t; + +typedef struct { + FSE_CTable offcodeCTable[FSE_CTABLE_SIZE_U32(OffFSELog, MaxOff)]; + FSE_CTable matchlengthCTable[FSE_CTABLE_SIZE_U32(MLFSELog, MaxML)]; + FSE_CTable litlengthCTable[FSE_CTABLE_SIZE_U32(LLFSELog, MaxLL)]; + FSE_repeat offcode_repeatMode; + FSE_repeat matchlength_repeatMode; + FSE_repeat litlength_repeatMode; +} ZSTD_fseCTables_t; + +typedef struct { + ZSTD_hufCTables_t huf; + ZSTD_fseCTables_t fse; +} ZSTD_entropyCTables_t; + +/*********************************************** +* Entropy buffer statistics structs and funcs * +***********************************************/ +/** ZSTD_hufCTablesMetadata_t : + * Stores Literals Block Type for a super-block in hType, and + * huffman tree description in hufDesBuffer. + * hufDesSize refers to the size of huffman tree description in bytes. + * This metadata is populated in ZSTD_buildBlockEntropyStats_literals() */ +typedef struct { + symbolEncodingType_e hType; + BYTE hufDesBuffer[ZSTD_MAX_HUF_HEADER_SIZE]; + size_t hufDesSize; +} ZSTD_hufCTablesMetadata_t; + +/** ZSTD_fseCTablesMetadata_t : + * Stores symbol compression modes for a super-block in {ll, ol, ml}Type, and + * fse tables in fseTablesBuffer. + * fseTablesSize refers to the size of fse tables in bytes. + * This metadata is populated in ZSTD_buildBlockEntropyStats_sequences() */ +typedef struct { + symbolEncodingType_e llType; + symbolEncodingType_e ofType; + symbolEncodingType_e mlType; + BYTE fseTablesBuffer[ZSTD_MAX_FSE_HEADERS_SIZE]; + size_t fseTablesSize; + size_t lastCountSize; /* This is to account for bug in 1.3.4. More detail in ZSTD_entropyCompressSeqStore_internal() */ +} ZSTD_fseCTablesMetadata_t; + +typedef struct { + ZSTD_hufCTablesMetadata_t hufMetadata; + ZSTD_fseCTablesMetadata_t fseMetadata; +} ZSTD_entropyCTablesMetadata_t; + +/** ZSTD_buildBlockEntropyStats() : + * Builds entropy for the block. + * @return : 0 on success or error code */ +size_t ZSTD_buildBlockEntropyStats(seqStore_t* seqStorePtr, + const ZSTD_entropyCTables_t* prevEntropy, + ZSTD_entropyCTables_t* nextEntropy, + const ZSTD_CCtx_params* cctxParams, + ZSTD_entropyCTablesMetadata_t* entropyMetadata, + void* workspace, size_t wkspSize); + +/********************************* +* Compression internals structs * +*********************************/ + +typedef struct { + U32 off; /* Offset sumtype code for the match, using ZSTD_storeSeq() format */ + U32 len; /* Raw length of match */ +} ZSTD_match_t; + +typedef struct { + U32 offset; /* Offset of sequence */ + U32 litLength; /* Length of literals prior to match */ + U32 matchLength; /* Raw length of match */ +} rawSeq; + +typedef struct { + rawSeq* seq; /* The start of the sequences */ + size_t pos; /* The index in seq where reading stopped. pos <= size. */ + size_t posInSequence; /* The position within the sequence at seq[pos] where reading + stopped. posInSequence <= seq[pos].litLength + seq[pos].matchLength */ + size_t size; /* The number of sequences. <= capacity. */ + size_t capacity; /* The capacity starting from `seq` pointer */ +} rawSeqStore_t; + +UNUSED_ATTR static const rawSeqStore_t kNullRawSeqStore = {NULL, 0, 0, 0, 0}; + +typedef struct { + int price; + U32 off; + U32 mlen; + U32 litlen; + U32 rep[ZSTD_REP_NUM]; +} ZSTD_optimal_t; + +typedef enum { zop_dynamic=0, zop_predef } ZSTD_OptPrice_e; + +typedef struct { + /* All tables are allocated inside cctx->workspace by ZSTD_resetCCtx_internal() */ + unsigned* litFreq; /* table of literals statistics, of size 256 */ + unsigned* litLengthFreq; /* table of litLength statistics, of size (MaxLL+1) */ + unsigned* matchLengthFreq; /* table of matchLength statistics, of size (MaxML+1) */ + unsigned* offCodeFreq; /* table of offCode statistics, of size (MaxOff+1) */ + ZSTD_match_t* matchTable; /* list of found matches, of size ZSTD_OPT_NUM+1 */ + ZSTD_optimal_t* priceTable; /* All positions tracked by optimal parser, of size ZSTD_OPT_NUM+1 */ + + U32 litSum; /* nb of literals */ + U32 litLengthSum; /* nb of litLength codes */ + U32 matchLengthSum; /* nb of matchLength codes */ + U32 offCodeSum; /* nb of offset codes */ + U32 litSumBasePrice; /* to compare to log2(litfreq) */ + U32 litLengthSumBasePrice; /* to compare to log2(llfreq) */ + U32 matchLengthSumBasePrice;/* to compare to log2(mlfreq) */ + U32 offCodeSumBasePrice; /* to compare to log2(offreq) */ + ZSTD_OptPrice_e priceType; /* prices can be determined dynamically, or follow a pre-defined cost structure */ + const ZSTD_entropyCTables_t* symbolCosts; /* pre-calculated dictionary statistics */ + ZSTD_paramSwitch_e literalCompressionMode; +} optState_t; + +typedef struct { + ZSTD_entropyCTables_t entropy; + U32 rep[ZSTD_REP_NUM]; +} ZSTD_compressedBlockState_t; + +typedef struct { + BYTE const* nextSrc; /* next block here to continue on current prefix */ + BYTE const* base; /* All regular indexes relative to this position */ + BYTE const* dictBase; /* extDict indexes relative to this position */ + U32 dictLimit; /* below that point, need extDict */ + U32 lowLimit; /* below that point, no more valid data */ + U32 nbOverflowCorrections; /* Number of times overflow correction has run since + * ZSTD_window_init(). Useful for debugging coredumps + * and for ZSTD_WINDOW_OVERFLOW_CORRECT_FREQUENTLY. + */ +} ZSTD_window_t; + +#define ZSTD_WINDOW_START_INDEX 2 + +typedef struct ZSTD_matchState_t ZSTD_matchState_t; + +#define ZSTD_ROW_HASH_CACHE_SIZE 8 /* Size of prefetching hash cache for row-based matchfinder */ + +struct ZSTD_matchState_t { + ZSTD_window_t window; /* State for window round buffer management */ + U32 loadedDictEnd; /* index of end of dictionary, within context's referential. + * When loadedDictEnd != 0, a dictionary is in use, and still valid. + * This relies on a mechanism to set loadedDictEnd=0 when dictionary is no longer within distance. + * Such mechanism is provided within ZSTD_window_enforceMaxDist() and ZSTD_checkDictValidity(). + * When dict referential is copied into active context (i.e. not attached), + * loadedDictEnd == dictSize, since referential starts from zero. + */ + U32 nextToUpdate; /* index from which to continue table update */ + U32 hashLog3; /* dispatch table for matches of len==3 : larger == faster, more memory */ + + U32 rowHashLog; /* For row-based matchfinder: Hashlog based on nb of rows in the hashTable.*/ + U16* tagTable; /* For row-based matchFinder: A row-based table containing the hashes and head index. */ + U32 hashCache[ZSTD_ROW_HASH_CACHE_SIZE]; /* For row-based matchFinder: a cache of hashes to improve speed */ + + U32* hashTable; + U32* hashTable3; + U32* chainTable; + + U32 forceNonContiguous; /* Non-zero if we should force non-contiguous load for the next window update. */ + + int dedicatedDictSearch; /* Indicates whether this matchState is using the + * dedicated dictionary search structure. + */ + optState_t opt; /* optimal parser state */ + const ZSTD_matchState_t* dictMatchState; + ZSTD_compressionParameters cParams; + const rawSeqStore_t* ldmSeqStore; +}; + +typedef struct { + ZSTD_compressedBlockState_t* prevCBlock; + ZSTD_compressedBlockState_t* nextCBlock; + ZSTD_matchState_t matchState; +} ZSTD_blockState_t; + +typedef struct { + U32 offset; + U32 checksum; +} ldmEntry_t; + +typedef struct { + BYTE const* split; + U32 hash; + U32 checksum; + ldmEntry_t* bucket; +} ldmMatchCandidate_t; + +#define LDM_BATCH_SIZE 64 + +typedef struct { + ZSTD_window_t window; /* State for the window round buffer management */ + ldmEntry_t* hashTable; + U32 loadedDictEnd; + BYTE* bucketOffsets; /* Next position in bucket to insert entry */ + size_t splitIndices[LDM_BATCH_SIZE]; + ldmMatchCandidate_t matchCandidates[LDM_BATCH_SIZE]; +} ldmState_t; + +typedef struct { + ZSTD_paramSwitch_e enableLdm; /* ZSTD_ps_enable to enable LDM. ZSTD_ps_auto by default */ + U32 hashLog; /* Log size of hashTable */ + U32 bucketSizeLog; /* Log bucket size for collision resolution, at most 8 */ + U32 minMatchLength; /* Minimum match length */ + U32 hashRateLog; /* Log number of entries to skip */ + U32 windowLog; /* Window log for the LDM */ +} ldmParams_t; + +typedef struct { + int collectSequences; + ZSTD_Sequence* seqStart; + size_t seqIndex; + size_t maxSequences; +} SeqCollector; + +struct ZSTD_CCtx_params_s { + ZSTD_format_e format; + ZSTD_compressionParameters cParams; + ZSTD_frameParameters fParams; + + int compressionLevel; + int forceWindow; /* force back-references to respect limit of + * 1< 63) ? ZSTD_highbit32(litLength) + LL_deltaCode : LL_Code[litLength]; +} + +/* ZSTD_MLcode() : + * note : mlBase = matchLength - MINMATCH; + * because it's the format it's stored in seqStore->sequences */ +MEM_STATIC U32 ZSTD_MLcode(U32 mlBase) +{ + static const BYTE ML_Code[128] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, + 32, 32, 33, 33, 34, 34, 35, 35, 36, 36, 36, 36, 37, 37, 37, 37, + 38, 38, 38, 38, 38, 38, 38, 38, 39, 39, 39, 39, 39, 39, 39, 39, + 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, + 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, + 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, + 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42 }; + static const U32 ML_deltaCode = 36; + return (mlBase > 127) ? ZSTD_highbit32(mlBase) + ML_deltaCode : ML_Code[mlBase]; +} + +/* ZSTD_cParam_withinBounds: + * @return 1 if value is within cParam bounds, + * 0 otherwise */ +MEM_STATIC int ZSTD_cParam_withinBounds(ZSTD_cParameter cParam, int value) +{ + ZSTD_bounds const bounds = ZSTD_cParam_getBounds(cParam); + if (ZSTD_isError(bounds.error)) return 0; + if (value < bounds.lowerBound) return 0; + if (value > bounds.upperBound) return 0; + return 1; +} + +/* ZSTD_noCompressBlock() : + * Writes uncompressed block to dst buffer from given src. + * Returns the size of the block */ +MEM_STATIC size_t ZSTD_noCompressBlock (void* dst, size_t dstCapacity, const void* src, size_t srcSize, U32 lastBlock) +{ + U32 const cBlockHeader24 = lastBlock + (((U32)bt_raw)<<1) + (U32)(srcSize << 3); + RETURN_ERROR_IF(srcSize + ZSTD_blockHeaderSize > dstCapacity, + dstSize_tooSmall, "dst buf too small for uncompressed block"); + MEM_writeLE24(dst, cBlockHeader24); + ZSTD_memcpy((BYTE*)dst + ZSTD_blockHeaderSize, src, srcSize); + return ZSTD_blockHeaderSize + srcSize; +} + +MEM_STATIC size_t ZSTD_rleCompressBlock (void* dst, size_t dstCapacity, BYTE src, size_t srcSize, U32 lastBlock) +{ + BYTE* const op = (BYTE*)dst; + U32 const cBlockHeader = lastBlock + (((U32)bt_rle)<<1) + (U32)(srcSize << 3); + RETURN_ERROR_IF(dstCapacity < 4, dstSize_tooSmall, ""); + MEM_writeLE24(op, cBlockHeader); + op[3] = src; + return 4; +} + + +/* ZSTD_minGain() : + * minimum compression required + * to generate a compress block or a compressed literals section. + * note : use same formula for both situations */ +MEM_STATIC size_t ZSTD_minGain(size_t srcSize, ZSTD_strategy strat) +{ + U32 const minlog = (strat>=ZSTD_btultra) ? (U32)(strat) - 1 : 6; + ZSTD_STATIC_ASSERT(ZSTD_btultra == 8); + assert(ZSTD_cParam_withinBounds(ZSTD_c_strategy, strat)); + return (srcSize >> minlog) + 2; +} + +MEM_STATIC int ZSTD_literalsCompressionIsDisabled(const ZSTD_CCtx_params* cctxParams) +{ + switch (cctxParams->literalCompressionMode) { + case ZSTD_ps_enable: + return 0; + case ZSTD_ps_disable: + return 1; + default: + assert(0 /* impossible: pre-validated */); + ZSTD_FALLTHROUGH; + case ZSTD_ps_auto: + return (cctxParams->cParams.strategy == ZSTD_fast) && (cctxParams->cParams.targetLength > 0); + } +} + +/*! ZSTD_safecopyLiterals() : + * memcpy() function that won't read beyond more than WILDCOPY_OVERLENGTH bytes past ilimit_w. + * Only called when the sequence ends past ilimit_w, so it only needs to be optimized for single + * large copies. + */ +static void +ZSTD_safecopyLiterals(BYTE* op, BYTE const* ip, BYTE const* const iend, BYTE const* ilimit_w) +{ + assert(iend > ilimit_w); + if (ip <= ilimit_w) { + ZSTD_wildcopy(op, ip, ilimit_w - ip, ZSTD_no_overlap); + op += ilimit_w - ip; + ip = ilimit_w; + } + while (ip < iend) *op++ = *ip++; +} + +#define ZSTD_REP_MOVE (ZSTD_REP_NUM-1) +#define STORE_REPCODE_1 STORE_REPCODE(1) +#define STORE_REPCODE_2 STORE_REPCODE(2) +#define STORE_REPCODE_3 STORE_REPCODE(3) +#define STORE_REPCODE(r) (assert((r)>=1), assert((r)<=3), (r)-1) +#define STORE_OFFSET(o) (assert((o)>0), o + ZSTD_REP_MOVE) +#define STORED_IS_OFFSET(o) ((o) > ZSTD_REP_MOVE) +#define STORED_IS_REPCODE(o) ((o) <= ZSTD_REP_MOVE) +#define STORED_OFFSET(o) (assert(STORED_IS_OFFSET(o)), (o)-ZSTD_REP_MOVE) +#define STORED_REPCODE(o) (assert(STORED_IS_REPCODE(o)), (o)+1) /* returns ID 1,2,3 */ +#define STORED_TO_OFFBASE(o) ((o)+1) +#define OFFBASE_TO_STORED(o) ((o)-1) + +/*! ZSTD_storeSeq() : + * Store a sequence (litlen, litPtr, offCode and matchLength) into seqStore_t. + * @offBase_minus1 : Users should use employ macros STORE_REPCODE_X and STORE_OFFSET(). + * @matchLength : must be >= MINMATCH + * Allowed to overread literals up to litLimit. +*/ +HINT_INLINE UNUSED_ATTR void +ZSTD_storeSeq(seqStore_t* seqStorePtr, + size_t litLength, const BYTE* literals, const BYTE* litLimit, + U32 offBase_minus1, + size_t matchLength) +{ + BYTE const* const litLimit_w = litLimit - WILDCOPY_OVERLENGTH; + BYTE const* const litEnd = literals + litLength; +#if defined(DEBUGLEVEL) && (DEBUGLEVEL >= 6) + static const BYTE* g_start = NULL; + if (g_start==NULL) g_start = (const BYTE*)literals; /* note : index only works for compression within a single segment */ + { U32 const pos = (U32)((const BYTE*)literals - g_start); + DEBUGLOG(6, "Cpos%7u :%3u literals, match%4u bytes at offCode%7u", + pos, (U32)litLength, (U32)matchLength, (U32)offBase_minus1); + } +#endif + assert((size_t)(seqStorePtr->sequences - seqStorePtr->sequencesStart) < seqStorePtr->maxNbSeq); + /* copy Literals */ + assert(seqStorePtr->maxNbLit <= 128 KB); + assert(seqStorePtr->lit + litLength <= seqStorePtr->litStart + seqStorePtr->maxNbLit); + assert(literals + litLength <= litLimit); + if (litEnd <= litLimit_w) { + /* Common case we can use wildcopy. + * First copy 16 bytes, because literals are likely short. + */ + assert(WILDCOPY_OVERLENGTH >= 16); + ZSTD_copy16(seqStorePtr->lit, literals); + if (litLength > 16) { + ZSTD_wildcopy(seqStorePtr->lit+16, literals+16, (ptrdiff_t)litLength-16, ZSTD_no_overlap); + } + } else { + ZSTD_safecopyLiterals(seqStorePtr->lit, literals, litEnd, litLimit_w); + } + seqStorePtr->lit += litLength; + + /* literal Length */ + if (litLength>0xFFFF) { + assert(seqStorePtr->longLengthType == ZSTD_llt_none); /* there can only be a single long length */ + seqStorePtr->longLengthType = ZSTD_llt_literalLength; + seqStorePtr->longLengthPos = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart); + } + seqStorePtr->sequences[0].litLength = (U16)litLength; + + /* match offset */ + seqStorePtr->sequences[0].offBase = STORED_TO_OFFBASE(offBase_minus1); + + /* match Length */ + assert(matchLength >= MINMATCH); + { size_t const mlBase = matchLength - MINMATCH; + if (mlBase>0xFFFF) { + assert(seqStorePtr->longLengthType == ZSTD_llt_none); /* there can only be a single long length */ + seqStorePtr->longLengthType = ZSTD_llt_matchLength; + seqStorePtr->longLengthPos = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart); + } + seqStorePtr->sequences[0].mlBase = (U16)mlBase; + } + + seqStorePtr->sequences++; +} + +/* ZSTD_updateRep() : + * updates in-place @rep (array of repeat offsets) + * @offBase_minus1 : sum-type, with same numeric representation as ZSTD_storeSeq() + */ +MEM_STATIC void +ZSTD_updateRep(U32 rep[ZSTD_REP_NUM], U32 const offBase_minus1, U32 const ll0) +{ + if (STORED_IS_OFFSET(offBase_minus1)) { /* full offset */ + rep[2] = rep[1]; + rep[1] = rep[0]; + rep[0] = STORED_OFFSET(offBase_minus1); + } else { /* repcode */ + U32 const repCode = STORED_REPCODE(offBase_minus1) - 1 + ll0; + if (repCode > 0) { /* note : if repCode==0, no change */ + U32 const currentOffset = (repCode==ZSTD_REP_NUM) ? (rep[0] - 1) : rep[repCode]; + rep[2] = (repCode >= 2) ? rep[1] : rep[2]; + rep[1] = rep[0]; + rep[0] = currentOffset; + } else { /* repCode == 0 */ + /* nothing to do */ + } + } +} + +typedef struct repcodes_s { + U32 rep[3]; +} repcodes_t; + +MEM_STATIC repcodes_t +ZSTD_newRep(U32 const rep[ZSTD_REP_NUM], U32 const offBase_minus1, U32 const ll0) +{ + repcodes_t newReps; + ZSTD_memcpy(&newReps, rep, sizeof(newReps)); + ZSTD_updateRep(newReps.rep, offBase_minus1, ll0); + return newReps; +} + + +/*-************************************* +* Match length counter +***************************************/ +static unsigned ZSTD_NbCommonBytes (size_t val) +{ + if (MEM_isLittleEndian()) { + if (MEM_64bits()) { +# if defined(_MSC_VER) && defined(_WIN64) +# if STATIC_BMI2 + return _tzcnt_u64(val) >> 3; +# else + if (val != 0) { + unsigned long r; + _BitScanForward64(&r, (U64)val); + return (unsigned)(r >> 3); + } else { + /* Should not reach this code path */ + __assume(0); + } +# endif +# elif defined(__GNUC__) && (__GNUC__ >= 4) + return (__builtin_ctzll((U64)val) >> 3); +# else + static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, + 0, 3, 1, 3, 1, 4, 2, 7, + 0, 2, 3, 6, 1, 5, 3, 5, + 1, 3, 4, 4, 2, 5, 6, 7, + 7, 0, 1, 2, 3, 3, 4, 6, + 2, 6, 5, 5, 3, 4, 5, 6, + 7, 1, 2, 4, 6, 4, 4, 5, + 7, 2, 6, 5, 7, 6, 7, 7 }; + return DeBruijnBytePos[((U64)((val & -(long long)val) * 0x0218A392CDABBD3FULL)) >> 58]; +# endif + } else { /* 32 bits */ +# if defined(_MSC_VER) + if (val != 0) { + unsigned long r; + _BitScanForward(&r, (U32)val); + return (unsigned)(r >> 3); + } else { + /* Should not reach this code path */ + __assume(0); + } +# elif defined(__GNUC__) && (__GNUC__ >= 3) + return (__builtin_ctz((U32)val) >> 3); +# else + static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, + 3, 2, 2, 1, 3, 2, 0, 1, + 3, 3, 1, 2, 2, 2, 2, 0, + 3, 1, 2, 0, 1, 0, 1, 1 }; + return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27]; +# endif + } + } else { /* Big Endian CPU */ + if (MEM_64bits()) { +# if defined(_MSC_VER) && defined(_WIN64) +# if STATIC_BMI2 + return _lzcnt_u64(val) >> 3; +# else + if (val != 0) { + unsigned long r; + _BitScanReverse64(&r, (U64)val); + return (unsigned)(r >> 3); + } else { + /* Should not reach this code path */ + __assume(0); + } +# endif +# elif defined(__GNUC__) && (__GNUC__ >= 4) + return (__builtin_clzll(val) >> 3); +# else + unsigned r; + const unsigned n32 = sizeof(size_t)*4; /* calculate this way due to compiler complaining in 32-bits mode */ + if (!(val>>n32)) { r=4; } else { r=0; val>>=n32; } + if (!(val>>16)) { r+=2; val>>=8; } else { val>>=24; } + r += (!val); + return r; +# endif + } else { /* 32 bits */ +# if defined(_MSC_VER) + if (val != 0) { + unsigned long r; + _BitScanReverse(&r, (unsigned long)val); + return (unsigned)(r >> 3); + } else { + /* Should not reach this code path */ + __assume(0); + } +# elif defined(__GNUC__) && (__GNUC__ >= 3) + return (__builtin_clz((U32)val) >> 3); +# else + unsigned r; + if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; } + r += (!val); + return r; +# endif + } } +} + + +MEM_STATIC size_t ZSTD_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* const pInLimit) +{ + const BYTE* const pStart = pIn; + const BYTE* const pInLoopLimit = pInLimit - (sizeof(size_t)-1); + + if (pIn < pInLoopLimit) { + { size_t const diff = MEM_readST(pMatch) ^ MEM_readST(pIn); + if (diff) return ZSTD_NbCommonBytes(diff); } + pIn+=sizeof(size_t); pMatch+=sizeof(size_t); + while (pIn < pInLoopLimit) { + size_t const diff = MEM_readST(pMatch) ^ MEM_readST(pIn); + if (!diff) { pIn+=sizeof(size_t); pMatch+=sizeof(size_t); continue; } + pIn += ZSTD_NbCommonBytes(diff); + return (size_t)(pIn - pStart); + } } + if (MEM_64bits() && (pIn<(pInLimit-3)) && (MEM_read32(pMatch) == MEM_read32(pIn))) { pIn+=4; pMatch+=4; } + if ((pIn<(pInLimit-1)) && (MEM_read16(pMatch) == MEM_read16(pIn))) { pIn+=2; pMatch+=2; } + if ((pIn> (32-h) ; } +MEM_STATIC size_t ZSTD_hash3Ptr(const void* ptr, U32 h) { return ZSTD_hash3(MEM_readLE32(ptr), h); } /* only in zstd_opt.h */ + +static const U32 prime4bytes = 2654435761U; +static U32 ZSTD_hash4(U32 u, U32 h) { return (u * prime4bytes) >> (32-h) ; } +static size_t ZSTD_hash4Ptr(const void* ptr, U32 h) { return ZSTD_hash4(MEM_read32(ptr), h); } + +static const U64 prime5bytes = 889523592379ULL; +static size_t ZSTD_hash5(U64 u, U32 h) { return (size_t)(((u << (64-40)) * prime5bytes) >> (64-h)) ; } +static size_t ZSTD_hash5Ptr(const void* p, U32 h) { return ZSTD_hash5(MEM_readLE64(p), h); } + +static const U64 prime6bytes = 227718039650203ULL; +static size_t ZSTD_hash6(U64 u, U32 h) { return (size_t)(((u << (64-48)) * prime6bytes) >> (64-h)) ; } +static size_t ZSTD_hash6Ptr(const void* p, U32 h) { return ZSTD_hash6(MEM_readLE64(p), h); } + +static const U64 prime7bytes = 58295818150454627ULL; +static size_t ZSTD_hash7(U64 u, U32 h) { return (size_t)(((u << (64-56)) * prime7bytes) >> (64-h)) ; } +static size_t ZSTD_hash7Ptr(const void* p, U32 h) { return ZSTD_hash7(MEM_readLE64(p), h); } + +static const U64 prime8bytes = 0xCF1BBCDCB7A56463ULL; +static size_t ZSTD_hash8(U64 u, U32 h) { return (size_t)(((u) * prime8bytes) >> (64-h)) ; } +static size_t ZSTD_hash8Ptr(const void* p, U32 h) { return ZSTD_hash8(MEM_readLE64(p), h); } + +MEM_STATIC FORCE_INLINE_ATTR +size_t ZSTD_hashPtr(const void* p, U32 hBits, U32 mls) +{ + switch(mls) + { + default: + case 4: return ZSTD_hash4Ptr(p, hBits); + case 5: return ZSTD_hash5Ptr(p, hBits); + case 6: return ZSTD_hash6Ptr(p, hBits); + case 7: return ZSTD_hash7Ptr(p, hBits); + case 8: return ZSTD_hash8Ptr(p, hBits); + } +} + +/** ZSTD_ipow() : + * Return base^exponent. + */ +static U64 ZSTD_ipow(U64 base, U64 exponent) +{ + U64 power = 1; + while (exponent) { + if (exponent & 1) power *= base; + exponent >>= 1; + base *= base; + } + return power; +} + +#define ZSTD_ROLL_HASH_CHAR_OFFSET 10 + +/** ZSTD_rollingHash_append() : + * Add the buffer to the hash value. + */ +static U64 ZSTD_rollingHash_append(U64 hash, void const* buf, size_t size) +{ + BYTE const* istart = (BYTE const*)buf; + size_t pos; + for (pos = 0; pos < size; ++pos) { + hash *= prime8bytes; + hash += istart[pos] + ZSTD_ROLL_HASH_CHAR_OFFSET; + } + return hash; +} + +/** ZSTD_rollingHash_compute() : + * Compute the rolling hash value of the buffer. + */ +MEM_STATIC U64 ZSTD_rollingHash_compute(void const* buf, size_t size) +{ + return ZSTD_rollingHash_append(0, buf, size); +} + +/** ZSTD_rollingHash_primePower() : + * Compute the primePower to be passed to ZSTD_rollingHash_rotate() for a hash + * over a window of length bytes. + */ +MEM_STATIC U64 ZSTD_rollingHash_primePower(U32 length) +{ + return ZSTD_ipow(prime8bytes, length - 1); +} + +/** ZSTD_rollingHash_rotate() : + * Rotate the rolling hash by one byte. + */ +MEM_STATIC U64 ZSTD_rollingHash_rotate(U64 hash, BYTE toRemove, BYTE toAdd, U64 primePower) +{ + hash -= (toRemove + ZSTD_ROLL_HASH_CHAR_OFFSET) * primePower; + hash *= prime8bytes; + hash += toAdd + ZSTD_ROLL_HASH_CHAR_OFFSET; + return hash; +} + +/*-************************************* +* Round buffer management +***************************************/ +#if (ZSTD_WINDOWLOG_MAX_64 > 31) +# error "ZSTD_WINDOWLOG_MAX is too large : would overflow ZSTD_CURRENT_MAX" +#endif +/* Max current allowed */ +#define ZSTD_CURRENT_MAX ((3U << 29) + (1U << ZSTD_WINDOWLOG_MAX)) +/* Maximum chunk size before overflow correction needs to be called again */ +#define ZSTD_CHUNKSIZE_MAX \ + ( ((U32)-1) /* Maximum ending current index */ \ + - ZSTD_CURRENT_MAX) /* Maximum beginning lowLimit */ + +/** + * ZSTD_window_clear(): + * Clears the window containing the history by simply setting it to empty. + */ +MEM_STATIC void ZSTD_window_clear(ZSTD_window_t* window) +{ + size_t const endT = (size_t)(window->nextSrc - window->base); + U32 const end = (U32)endT; + + window->lowLimit = end; + window->dictLimit = end; +} + +MEM_STATIC U32 ZSTD_window_isEmpty(ZSTD_window_t const window) +{ + return window.dictLimit == ZSTD_WINDOW_START_INDEX && + window.lowLimit == ZSTD_WINDOW_START_INDEX && + (window.nextSrc - window.base) == ZSTD_WINDOW_START_INDEX; +} + +/** + * ZSTD_window_hasExtDict(): + * Returns non-zero if the window has a non-empty extDict. + */ +MEM_STATIC U32 ZSTD_window_hasExtDict(ZSTD_window_t const window) +{ + return window.lowLimit < window.dictLimit; +} + +/** + * ZSTD_matchState_dictMode(): + * Inspects the provided matchState and figures out what dictMode should be + * passed to the compressor. + */ +MEM_STATIC ZSTD_dictMode_e ZSTD_matchState_dictMode(const ZSTD_matchState_t *ms) +{ + return ZSTD_window_hasExtDict(ms->window) ? + ZSTD_extDict : + ms->dictMatchState != NULL ? + (ms->dictMatchState->dedicatedDictSearch ? ZSTD_dedicatedDictSearch : ZSTD_dictMatchState) : + ZSTD_noDict; +} + +/* Defining this macro to non-zero tells zstd to run the overflow correction + * code much more frequently. This is very inefficient, and should only be + * used for tests and fuzzers. + */ +#ifndef ZSTD_WINDOW_OVERFLOW_CORRECT_FREQUENTLY +# ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION +# define ZSTD_WINDOW_OVERFLOW_CORRECT_FREQUENTLY 1 +# else +# define ZSTD_WINDOW_OVERFLOW_CORRECT_FREQUENTLY 0 +# endif +#endif + +/** + * ZSTD_window_canOverflowCorrect(): + * Returns non-zero if the indices are large enough for overflow correction + * to work correctly without impacting compression ratio. + */ +MEM_STATIC U32 ZSTD_window_canOverflowCorrect(ZSTD_window_t const window, + U32 cycleLog, + U32 maxDist, + U32 loadedDictEnd, + void const* src) +{ + U32 const cycleSize = 1u << cycleLog; + U32 const curr = (U32)((BYTE const*)src - window.base); + U32 const minIndexToOverflowCorrect = cycleSize + + MAX(maxDist, cycleSize) + + ZSTD_WINDOW_START_INDEX; + + /* Adjust the min index to backoff the overflow correction frequency, + * so we don't waste too much CPU in overflow correction. If this + * computation overflows we don't really care, we just need to make + * sure it is at least minIndexToOverflowCorrect. + */ + U32 const adjustment = window.nbOverflowCorrections + 1; + U32 const adjustedIndex = MAX(minIndexToOverflowCorrect * adjustment, + minIndexToOverflowCorrect); + U32 const indexLargeEnough = curr > adjustedIndex; + + /* Only overflow correct early if the dictionary is invalidated already, + * so we don't hurt compression ratio. + */ + U32 const dictionaryInvalidated = curr > maxDist + loadedDictEnd; + + return indexLargeEnough && dictionaryInvalidated; +} + +/** + * ZSTD_window_needOverflowCorrection(): + * Returns non-zero if the indices are getting too large and need overflow + * protection. + */ +MEM_STATIC U32 ZSTD_window_needOverflowCorrection(ZSTD_window_t const window, + U32 cycleLog, + U32 maxDist, + U32 loadedDictEnd, + void const* src, + void const* srcEnd) +{ + U32 const curr = (U32)((BYTE const*)srcEnd - window.base); + if (ZSTD_WINDOW_OVERFLOW_CORRECT_FREQUENTLY) { + if (ZSTD_window_canOverflowCorrect(window, cycleLog, maxDist, loadedDictEnd, src)) { + return 1; + } + } + return curr > ZSTD_CURRENT_MAX; +} + +/** + * ZSTD_window_correctOverflow(): + * Reduces the indices to protect from index overflow. + * Returns the correction made to the indices, which must be applied to every + * stored index. + * + * The least significant cycleLog bits of the indices must remain the same, + * which may be 0. Every index up to maxDist in the past must be valid. + */ +MEM_STATIC U32 ZSTD_window_correctOverflow(ZSTD_window_t* window, U32 cycleLog, + U32 maxDist, void const* src) +{ + /* preemptive overflow correction: + * 1. correction is large enough: + * lowLimit > (3<<29) ==> current > 3<<29 + 1< (3<<29 + 1< (3<<29) - (1< (3<<29) - (1<<30) (NOTE: chainLog <= 30) + * > 1<<29 + * + * 2. (ip+ZSTD_CHUNKSIZE_MAX - cctx->base) doesn't overflow: + * After correction, current is less than (1<base < 1<<32. + * 3. (cctx->lowLimit + 1< 3<<29 + 1<base); + U32 const currentCycle = curr & cycleMask; + /* Ensure newCurrent - maxDist >= ZSTD_WINDOW_START_INDEX. */ + U32 const currentCycleCorrection = currentCycle < ZSTD_WINDOW_START_INDEX + ? MAX(cycleSize, ZSTD_WINDOW_START_INDEX) + : 0; + U32 const newCurrent = currentCycle + + currentCycleCorrection + + MAX(maxDist, cycleSize); + U32 const correction = curr - newCurrent; + /* maxDist must be a power of two so that: + * (newCurrent & cycleMask) == (curr & cycleMask) + * This is required to not corrupt the chains / binary tree. + */ + assert((maxDist & (maxDist - 1)) == 0); + assert((curr & cycleMask) == (newCurrent & cycleMask)); + assert(curr > newCurrent); + if (!ZSTD_WINDOW_OVERFLOW_CORRECT_FREQUENTLY) { + /* Loose bound, should be around 1<<29 (see above) */ + assert(correction > 1<<28); + } + + window->base += correction; + window->dictBase += correction; + if (window->lowLimit < correction + ZSTD_WINDOW_START_INDEX) { + window->lowLimit = ZSTD_WINDOW_START_INDEX; + } else { + window->lowLimit -= correction; + } + if (window->dictLimit < correction + ZSTD_WINDOW_START_INDEX) { + window->dictLimit = ZSTD_WINDOW_START_INDEX; + } else { + window->dictLimit -= correction; + } + + /* Ensure we can still reference the full window. */ + assert(newCurrent >= maxDist); + assert(newCurrent - maxDist >= ZSTD_WINDOW_START_INDEX); + /* Ensure that lowLimit and dictLimit didn't underflow. */ + assert(window->lowLimit <= newCurrent); + assert(window->dictLimit <= newCurrent); + + ++window->nbOverflowCorrections; + + DEBUGLOG(4, "Correction of 0x%x bytes to lowLimit=0x%x", correction, + window->lowLimit); + return correction; +} + +/** + * ZSTD_window_enforceMaxDist(): + * Updates lowLimit so that: + * (srcEnd - base) - lowLimit == maxDist + loadedDictEnd + * + * It ensures index is valid as long as index >= lowLimit. + * This must be called before a block compression call. + * + * loadedDictEnd is only defined if a dictionary is in use for current compression. + * As the name implies, loadedDictEnd represents the index at end of dictionary. + * The value lies within context's referential, it can be directly compared to blockEndIdx. + * + * If loadedDictEndPtr is NULL, no dictionary is in use, and we use loadedDictEnd == 0. + * If loadedDictEndPtr is not NULL, we set it to zero after updating lowLimit. + * This is because dictionaries are allowed to be referenced fully + * as long as the last byte of the dictionary is in the window. + * Once input has progressed beyond window size, dictionary cannot be referenced anymore. + * + * In normal dict mode, the dictionary lies between lowLimit and dictLimit. + * In dictMatchState mode, lowLimit and dictLimit are the same, + * and the dictionary is below them. + * forceWindow and dictMatchState are therefore incompatible. + */ +MEM_STATIC void +ZSTD_window_enforceMaxDist(ZSTD_window_t* window, + const void* blockEnd, + U32 maxDist, + U32* loadedDictEndPtr, + const ZSTD_matchState_t** dictMatchStatePtr) +{ + U32 const blockEndIdx = (U32)((BYTE const*)blockEnd - window->base); + U32 const loadedDictEnd = (loadedDictEndPtr != NULL) ? *loadedDictEndPtr : 0; + DEBUGLOG(5, "ZSTD_window_enforceMaxDist: blockEndIdx=%u, maxDist=%u, loadedDictEnd=%u", + (unsigned)blockEndIdx, (unsigned)maxDist, (unsigned)loadedDictEnd); + + /* - When there is no dictionary : loadedDictEnd == 0. + In which case, the test (blockEndIdx > maxDist) is merely to avoid + overflowing next operation `newLowLimit = blockEndIdx - maxDist`. + - When there is a standard dictionary : + Index referential is copied from the dictionary, + which means it starts from 0. + In which case, loadedDictEnd == dictSize, + and it makes sense to compare `blockEndIdx > maxDist + dictSize` + since `blockEndIdx` also starts from zero. + - When there is an attached dictionary : + loadedDictEnd is expressed within the referential of the context, + so it can be directly compared against blockEndIdx. + */ + if (blockEndIdx > maxDist + loadedDictEnd) { + U32 const newLowLimit = blockEndIdx - maxDist; + if (window->lowLimit < newLowLimit) window->lowLimit = newLowLimit; + if (window->dictLimit < window->lowLimit) { + DEBUGLOG(5, "Update dictLimit to match lowLimit, from %u to %u", + (unsigned)window->dictLimit, (unsigned)window->lowLimit); + window->dictLimit = window->lowLimit; + } + /* On reaching window size, dictionaries are invalidated */ + if (loadedDictEndPtr) *loadedDictEndPtr = 0; + if (dictMatchStatePtr) *dictMatchStatePtr = NULL; + } +} + +/* Similar to ZSTD_window_enforceMaxDist(), + * but only invalidates dictionary + * when input progresses beyond window size. + * assumption : loadedDictEndPtr and dictMatchStatePtr are valid (non NULL) + * loadedDictEnd uses same referential as window->base + * maxDist is the window size */ +MEM_STATIC void +ZSTD_checkDictValidity(const ZSTD_window_t* window, + const void* blockEnd, + U32 maxDist, + U32* loadedDictEndPtr, + const ZSTD_matchState_t** dictMatchStatePtr) +{ + assert(loadedDictEndPtr != NULL); + assert(dictMatchStatePtr != NULL); + { U32 const blockEndIdx = (U32)((BYTE const*)blockEnd - window->base); + U32 const loadedDictEnd = *loadedDictEndPtr; + DEBUGLOG(5, "ZSTD_checkDictValidity: blockEndIdx=%u, maxDist=%u, loadedDictEnd=%u", + (unsigned)blockEndIdx, (unsigned)maxDist, (unsigned)loadedDictEnd); + assert(blockEndIdx >= loadedDictEnd); + + if (blockEndIdx > loadedDictEnd + maxDist) { + /* On reaching window size, dictionaries are invalidated. + * For simplification, if window size is reached anywhere within next block, + * the dictionary is invalidated for the full block. + */ + DEBUGLOG(6, "invalidating dictionary for current block (distance > windowSize)"); + *loadedDictEndPtr = 0; + *dictMatchStatePtr = NULL; + } else { + if (*loadedDictEndPtr != 0) { + DEBUGLOG(6, "dictionary considered valid for current block"); + } } } +} + +MEM_STATIC void ZSTD_window_init(ZSTD_window_t* window) { + ZSTD_memset(window, 0, sizeof(*window)); + window->base = (BYTE const*)" "; + window->dictBase = (BYTE const*)" "; + ZSTD_STATIC_ASSERT(ZSTD_DUBT_UNSORTED_MARK < ZSTD_WINDOW_START_INDEX); /* Start above ZSTD_DUBT_UNSORTED_MARK */ + window->dictLimit = ZSTD_WINDOW_START_INDEX; /* start from >0, so that 1st position is valid */ + window->lowLimit = ZSTD_WINDOW_START_INDEX; /* it ensures first and later CCtx usages compress the same */ + window->nextSrc = window->base + ZSTD_WINDOW_START_INDEX; /* see issue #1241 */ + window->nbOverflowCorrections = 0; +} + +/** + * ZSTD_window_update(): + * Updates the window by appending [src, src + srcSize) to the window. + * If it is not contiguous, the current prefix becomes the extDict, and we + * forget about the extDict. Handles overlap of the prefix and extDict. + * Returns non-zero if the segment is contiguous. + */ +MEM_STATIC U32 ZSTD_window_update(ZSTD_window_t* window, + void const* src, size_t srcSize, + int forceNonContiguous) +{ + BYTE const* const ip = (BYTE const*)src; + U32 contiguous = 1; + DEBUGLOG(5, "ZSTD_window_update"); + if (srcSize == 0) + return contiguous; + assert(window->base != NULL); + assert(window->dictBase != NULL); + /* Check if blocks follow each other */ + if (src != window->nextSrc || forceNonContiguous) { + /* not contiguous */ + size_t const distanceFromBase = (size_t)(window->nextSrc - window->base); + DEBUGLOG(5, "Non contiguous blocks, new segment starts at %u", window->dictLimit); + window->lowLimit = window->dictLimit; + assert(distanceFromBase == (size_t)(U32)distanceFromBase); /* should never overflow */ + window->dictLimit = (U32)distanceFromBase; + window->dictBase = window->base; + window->base = ip - distanceFromBase; + /* ms->nextToUpdate = window->dictLimit; */ + if (window->dictLimit - window->lowLimit < HASH_READ_SIZE) window->lowLimit = window->dictLimit; /* too small extDict */ + contiguous = 0; + } + window->nextSrc = ip + srcSize; + /* if input and dictionary overlap : reduce dictionary (area presumed modified by input) */ + if ( (ip+srcSize > window->dictBase + window->lowLimit) + & (ip < window->dictBase + window->dictLimit)) { + ptrdiff_t const highInputIdx = (ip + srcSize) - window->dictBase; + U32 const lowLimitMax = (highInputIdx > (ptrdiff_t)window->dictLimit) ? window->dictLimit : (U32)highInputIdx; + window->lowLimit = lowLimitMax; + DEBUGLOG(5, "Overlapping extDict and input : new lowLimit = %u", window->lowLimit); + } + return contiguous; +} + +/** + * Returns the lowest allowed match index. It may either be in the ext-dict or the prefix. + */ +MEM_STATIC U32 ZSTD_getLowestMatchIndex(const ZSTD_matchState_t* ms, U32 curr, unsigned windowLog) +{ + U32 const maxDistance = 1U << windowLog; + U32 const lowestValid = ms->window.lowLimit; + U32 const withinWindow = (curr - lowestValid > maxDistance) ? curr - maxDistance : lowestValid; + U32 const isDictionary = (ms->loadedDictEnd != 0); + /* When using a dictionary the entire dictionary is valid if a single byte of the dictionary + * is within the window. We invalidate the dictionary (and set loadedDictEnd to 0) when it isn't + * valid for the entire block. So this check is sufficient to find the lowest valid match index. + */ + U32 const matchLowest = isDictionary ? lowestValid : withinWindow; + return matchLowest; +} + +/** + * Returns the lowest allowed match index in the prefix. + */ +MEM_STATIC U32 ZSTD_getLowestPrefixIndex(const ZSTD_matchState_t* ms, U32 curr, unsigned windowLog) +{ + U32 const maxDistance = 1U << windowLog; + U32 const lowestValid = ms->window.dictLimit; + U32 const withinWindow = (curr - lowestValid > maxDistance) ? curr - maxDistance : lowestValid; + U32 const isDictionary = (ms->loadedDictEnd != 0); + /* When computing the lowest prefix index we need to take the dictionary into account to handle + * the edge case where the dictionary and the source are contiguous in memory. + */ + U32 const matchLowest = isDictionary ? lowestValid : withinWindow; + return matchLowest; +} + + + +/* debug functions */ +#if (DEBUGLEVEL>=2) + +MEM_STATIC double ZSTD_fWeight(U32 rawStat) +{ + U32 const fp_accuracy = 8; + U32 const fp_multiplier = (1 << fp_accuracy); + U32 const newStat = rawStat + 1; + U32 const hb = ZSTD_highbit32(newStat); + U32 const BWeight = hb * fp_multiplier; + U32 const FWeight = (newStat << fp_accuracy) >> hb; + U32 const weight = BWeight + FWeight; + assert(hb + fp_accuracy < 31); + return (double)weight / fp_multiplier; +} + +/* display a table content, + * listing each element, its frequency, and its predicted bit cost */ +MEM_STATIC void ZSTD_debugTable(const U32* table, U32 max) +{ + unsigned u, sum; + for (u=0, sum=0; u<=max; u++) sum += table[u]; + DEBUGLOG(2, "total nb elts: %u", sum); + for (u=0; u<=max; u++) { + DEBUGLOG(2, "%2u: %5u (%.2f)", + u, table[u], ZSTD_fWeight(sum) - ZSTD_fWeight(table[u]) ); + } +} + +#endif + + +#if defined (__cplusplus) +} +#endif + +/* =============================================================== + * Shared internal declarations + * These prototypes may be called from sources not in lib/compress + * =============================================================== */ + +/* ZSTD_loadCEntropy() : + * dict : must point at beginning of a valid zstd dictionary. + * return : size of dictionary header (size of magic number + dict ID + entropy tables) + * assumptions : magic number supposed already checked + * and dictSize >= 8 */ +size_t ZSTD_loadCEntropy(ZSTD_compressedBlockState_t* bs, void* workspace, + const void* const dict, size_t dictSize); + +void ZSTD_reset_compressedBlockState(ZSTD_compressedBlockState_t* bs); + +/* ============================================================== + * Private declarations + * These prototypes shall only be called from within lib/compress + * ============================================================== */ + +/* ZSTD_getCParamsFromCCtxParams() : + * cParams are built depending on compressionLevel, src size hints, + * LDM and manually set compression parameters. + * Note: srcSizeHint == 0 means 0! + */ +ZSTD_compressionParameters ZSTD_getCParamsFromCCtxParams( + const ZSTD_CCtx_params* CCtxParams, U64 srcSizeHint, size_t dictSize, ZSTD_cParamMode_e mode); + +/*! ZSTD_initCStream_internal() : + * Private use only. Init streaming operation. + * expects params to be valid. + * must receive dict, or cdict, or none, but not both. + * @return : 0, or an error code */ +size_t ZSTD_initCStream_internal(ZSTD_CStream* zcs, + const void* dict, size_t dictSize, + const ZSTD_CDict* cdict, + const ZSTD_CCtx_params* params, unsigned long long pledgedSrcSize); + +void ZSTD_resetSeqStore(seqStore_t* ssPtr); + +/*! ZSTD_getCParamsFromCDict() : + * as the name implies */ +ZSTD_compressionParameters ZSTD_getCParamsFromCDict(const ZSTD_CDict* cdict); + +/* ZSTD_compressBegin_advanced_internal() : + * Private use only. To be called from zstdmt_compress.c. */ +size_t ZSTD_compressBegin_advanced_internal(ZSTD_CCtx* cctx, + const void* dict, size_t dictSize, + ZSTD_dictContentType_e dictContentType, + ZSTD_dictTableLoadMethod_e dtlm, + const ZSTD_CDict* cdict, + const ZSTD_CCtx_params* params, + unsigned long long pledgedSrcSize); + +/* ZSTD_compress_advanced_internal() : + * Private use only. To be called from zstdmt_compress.c. */ +size_t ZSTD_compress_advanced_internal(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const void* dict,size_t dictSize, + const ZSTD_CCtx_params* params); + + +/* ZSTD_writeLastEmptyBlock() : + * output an empty Block with end-of-frame mark to complete a frame + * @return : size of data written into `dst` (== ZSTD_blockHeaderSize (defined in zstd_internal.h)) + * or an error code if `dstCapacity` is too small ( 1 */ +U32 ZSTD_cycleLog(U32 hashLog, ZSTD_strategy strat); + +/** ZSTD_CCtx_trace() : + * Trace the end of a compression call. + */ +void ZSTD_CCtx_trace(ZSTD_CCtx* cctx, size_t extraCSize); + +#endif /* ZSTD_COMPRESS_H */ diff --git a/src/blosc2/internal-complibs/zstd-1.5.2/compress/zstd_compress_literals.c b/src/blosc2/internal-complibs/zstd-1.5.2/compress/zstd_compress_literals.c new file mode 100644 index 0000000..52b0a80 --- /dev/null +++ b/src/blosc2/internal-complibs/zstd-1.5.2/compress/zstd_compress_literals.c @@ -0,0 +1,159 @@ +/* + * Copyright (c) Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + + /*-************************************* + * Dependencies + ***************************************/ +#include "zstd_compress_literals.h" + +size_t ZSTD_noCompressLiterals (void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ + BYTE* const ostart = (BYTE*)dst; + U32 const flSize = 1 + (srcSize>31) + (srcSize>4095); + + RETURN_ERROR_IF(srcSize + flSize > dstCapacity, dstSize_tooSmall, ""); + + switch(flSize) + { + case 1: /* 2 - 1 - 5 */ + ostart[0] = (BYTE)((U32)set_basic + (srcSize<<3)); + break; + case 2: /* 2 - 2 - 12 */ + MEM_writeLE16(ostart, (U16)((U32)set_basic + (1<<2) + (srcSize<<4))); + break; + case 3: /* 2 - 2 - 20 */ + MEM_writeLE32(ostart, (U32)((U32)set_basic + (3<<2) + (srcSize<<4))); + break; + default: /* not necessary : flSize is {1,2,3} */ + assert(0); + } + + ZSTD_memcpy(ostart + flSize, src, srcSize); + DEBUGLOG(5, "Raw literals: %u -> %u", (U32)srcSize, (U32)(srcSize + flSize)); + return srcSize + flSize; +} + +size_t ZSTD_compressRleLiteralsBlock (void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ + BYTE* const ostart = (BYTE*)dst; + U32 const flSize = 1 + (srcSize>31) + (srcSize>4095); + + (void)dstCapacity; /* dstCapacity already guaranteed to be >=4, hence large enough */ + + switch(flSize) + { + case 1: /* 2 - 1 - 5 */ + ostart[0] = (BYTE)((U32)set_rle + (srcSize<<3)); + break; + case 2: /* 2 - 2 - 12 */ + MEM_writeLE16(ostart, (U16)((U32)set_rle + (1<<2) + (srcSize<<4))); + break; + case 3: /* 2 - 2 - 20 */ + MEM_writeLE32(ostart, (U32)((U32)set_rle + (3<<2) + (srcSize<<4))); + break; + default: /* not necessary : flSize is {1,2,3} */ + assert(0); + } + + ostart[flSize] = *(const BYTE*)src; + DEBUGLOG(5, "RLE literals: %u -> %u", (U32)srcSize, (U32)flSize + 1); + return flSize+1; +} + +size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf, + ZSTD_hufCTables_t* nextHuf, + ZSTD_strategy strategy, int disableLiteralCompression, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + void* entropyWorkspace, size_t entropyWorkspaceSize, + const int bmi2, + unsigned suspectUncompressible) +{ + size_t const minGain = ZSTD_minGain(srcSize, strategy); + size_t const lhSize = 3 + (srcSize >= 1 KB) + (srcSize >= 16 KB); + BYTE* const ostart = (BYTE*)dst; + U32 singleStream = srcSize < 256; + symbolEncodingType_e hType = set_compressed; + size_t cLitSize; + + DEBUGLOG(5,"ZSTD_compressLiterals (disableLiteralCompression=%i srcSize=%u)", + disableLiteralCompression, (U32)srcSize); + + /* Prepare nextEntropy assuming reusing the existing table */ + ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf)); + + if (disableLiteralCompression) + return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize); + + /* small ? don't even attempt compression (speed opt) */ +# define COMPRESS_LITERALS_SIZE_MIN 63 + { size_t const minLitSize = (prevHuf->repeatMode == HUF_repeat_valid) ? 6 : COMPRESS_LITERALS_SIZE_MIN; + if (srcSize <= minLitSize) return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize); + } + + RETURN_ERROR_IF(dstCapacity < lhSize+1, dstSize_tooSmall, "not enough space for compression"); + { HUF_repeat repeat = prevHuf->repeatMode; + int const preferRepeat = strategy < ZSTD_lazy ? srcSize <= 1024 : 0; + if (repeat == HUF_repeat_valid && lhSize == 3) singleStream = 1; + cLitSize = singleStream ? + HUF_compress1X_repeat( + ostart+lhSize, dstCapacity-lhSize, src, srcSize, + HUF_SYMBOLVALUE_MAX, HUF_TABLELOG_DEFAULT, entropyWorkspace, entropyWorkspaceSize, + (HUF_CElt*)nextHuf->CTable, &repeat, preferRepeat, bmi2, suspectUncompressible) : + HUF_compress4X_repeat( + ostart+lhSize, dstCapacity-lhSize, src, srcSize, + HUF_SYMBOLVALUE_MAX, HUF_TABLELOG_DEFAULT, entropyWorkspace, entropyWorkspaceSize, + (HUF_CElt*)nextHuf->CTable, &repeat, preferRepeat, bmi2, suspectUncompressible); + if (repeat != HUF_repeat_none) { + /* reused the existing table */ + DEBUGLOG(5, "Reusing previous huffman table"); + hType = set_repeat; + } + } + + if ((cLitSize==0) || (cLitSize >= srcSize - minGain) || ERR_isError(cLitSize)) { + ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf)); + return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize); + } + if (cLitSize==1) { + ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf)); + return ZSTD_compressRleLiteralsBlock(dst, dstCapacity, src, srcSize); + } + + if (hType == set_compressed) { + /* using a newly constructed table */ + nextHuf->repeatMode = HUF_repeat_check; + } + + /* Build header */ + switch(lhSize) + { + case 3: /* 2 - 2 - 10 - 10 */ + { U32 const lhc = hType + ((!singleStream) << 2) + ((U32)srcSize<<4) + ((U32)cLitSize<<14); + MEM_writeLE24(ostart, lhc); + break; + } + case 4: /* 2 - 2 - 14 - 14 */ + { U32 const lhc = hType + (2 << 2) + ((U32)srcSize<<4) + ((U32)cLitSize<<18); + MEM_writeLE32(ostart, lhc); + break; + } + case 5: /* 2 - 2 - 18 - 18 */ + { U32 const lhc = hType + (3 << 2) + ((U32)srcSize<<4) + ((U32)cLitSize<<22); + MEM_writeLE32(ostart, lhc); + ostart[4] = (BYTE)(cLitSize >> 10); + break; + } + default: /* not possible : lhSize is {3,4,5} */ + assert(0); + } + DEBUGLOG(5, "Compressed literals: %u -> %u", (U32)srcSize, (U32)(lhSize+cLitSize)); + return lhSize+cLitSize; +} diff --git a/src/blosc2/internal-complibs/zstd-1.5.2/compress/zstd_compress_literals.h b/src/blosc2/internal-complibs/zstd-1.5.2/compress/zstd_compress_literals.h new file mode 100644 index 0000000..9775fb9 --- /dev/null +++ b/src/blosc2/internal-complibs/zstd-1.5.2/compress/zstd_compress_literals.h @@ -0,0 +1,31 @@ +/* + * Copyright (c) Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_COMPRESS_LITERALS_H +#define ZSTD_COMPRESS_LITERALS_H + +#include "zstd_compress_internal.h" /* ZSTD_hufCTables_t, ZSTD_minGain() */ + + +size_t ZSTD_noCompressLiterals (void* dst, size_t dstCapacity, const void* src, size_t srcSize); + +size_t ZSTD_compressRleLiteralsBlock (void* dst, size_t dstCapacity, const void* src, size_t srcSize); + +/* If suspectUncompressible then some sampling checks will be run to potentially skip huffman coding */ +size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf, + ZSTD_hufCTables_t* nextHuf, + ZSTD_strategy strategy, int disableLiteralCompression, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + void* entropyWorkspace, size_t entropyWorkspaceSize, + const int bmi2, + unsigned suspectUncompressible); + +#endif /* ZSTD_COMPRESS_LITERALS_H */ diff --git a/src/blosc2/internal-complibs/zstd-1.5.2/compress/zstd_compress_sequences.c b/src/blosc2/internal-complibs/zstd-1.5.2/compress/zstd_compress_sequences.c new file mode 100644 index 0000000..f1e40af --- /dev/null +++ b/src/blosc2/internal-complibs/zstd-1.5.2/compress/zstd_compress_sequences.c @@ -0,0 +1,442 @@ +/* + * Copyright (c) Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + + /*-************************************* + * Dependencies + ***************************************/ +#include "zstd_compress_sequences.h" + +/** + * -log2(x / 256) lookup table for x in [0, 256). + * If x == 0: Return 0 + * Else: Return floor(-log2(x / 256) * 256) + */ +static unsigned const kInverseProbabilityLog256[256] = { + 0, 2048, 1792, 1642, 1536, 1453, 1386, 1329, 1280, 1236, 1197, 1162, + 1130, 1100, 1073, 1047, 1024, 1001, 980, 960, 941, 923, 906, 889, + 874, 859, 844, 830, 817, 804, 791, 779, 768, 756, 745, 734, + 724, 714, 704, 694, 685, 676, 667, 658, 650, 642, 633, 626, + 618, 610, 603, 595, 588, 581, 574, 567, 561, 554, 548, 542, + 535, 529, 523, 517, 512, 506, 500, 495, 489, 484, 478, 473, + 468, 463, 458, 453, 448, 443, 438, 434, 429, 424, 420, 415, + 411, 407, 402, 398, 394, 390, 386, 382, 377, 373, 370, 366, + 362, 358, 354, 350, 347, 343, 339, 336, 332, 329, 325, 322, + 318, 315, 311, 308, 305, 302, 298, 295, 292, 289, 286, 282, + 279, 276, 273, 270, 267, 264, 261, 258, 256, 253, 250, 247, + 244, 241, 239, 236, 233, 230, 228, 225, 222, 220, 217, 215, + 212, 209, 207, 204, 202, 199, 197, 194, 192, 190, 187, 185, + 182, 180, 178, 175, 173, 171, 168, 166, 164, 162, 159, 157, + 155, 153, 151, 149, 146, 144, 142, 140, 138, 136, 134, 132, + 130, 128, 126, 123, 121, 119, 117, 115, 114, 112, 110, 108, + 106, 104, 102, 100, 98, 96, 94, 93, 91, 89, 87, 85, + 83, 82, 80, 78, 76, 74, 73, 71, 69, 67, 66, 64, + 62, 61, 59, 57, 55, 54, 52, 50, 49, 47, 46, 44, + 42, 41, 39, 37, 36, 34, 33, 31, 30, 28, 26, 25, + 23, 22, 20, 19, 17, 16, 14, 13, 11, 10, 8, 7, + 5, 4, 2, 1, +}; + +static unsigned ZSTD_getFSEMaxSymbolValue(FSE_CTable const* ctable) { + void const* ptr = ctable; + U16 const* u16ptr = (U16 const*)ptr; + U32 const maxSymbolValue = MEM_read16(u16ptr + 1); + return maxSymbolValue; +} + +/** + * Returns true if we should use ncount=-1 else we should + * use ncount=1 for low probability symbols instead. + */ +static unsigned ZSTD_useLowProbCount(size_t const nbSeq) +{ + /* Heuristic: This should cover most blocks <= 16K and + * start to fade out after 16K to about 32K depending on + * comprssibility. + */ + return nbSeq >= 2048; +} + +/** + * Returns the cost in bytes of encoding the normalized count header. + * Returns an error if any of the helper functions return an error. + */ +static size_t ZSTD_NCountCost(unsigned const* count, unsigned const max, + size_t const nbSeq, unsigned const FSELog) +{ + BYTE wksp[FSE_NCOUNTBOUND]; + S16 norm[MaxSeq + 1]; + const U32 tableLog = FSE_optimalTableLog(FSELog, nbSeq, max); + FORWARD_IF_ERROR(FSE_normalizeCount(norm, tableLog, count, nbSeq, max, ZSTD_useLowProbCount(nbSeq)), ""); + return FSE_writeNCount(wksp, sizeof(wksp), norm, max, tableLog); +} + +/** + * Returns the cost in bits of encoding the distribution described by count + * using the entropy bound. + */ +static size_t ZSTD_entropyCost(unsigned const* count, unsigned const max, size_t const total) +{ + unsigned cost = 0; + unsigned s; + + assert(total > 0); + for (s = 0; s <= max; ++s) { + unsigned norm = (unsigned)((256 * count[s]) / total); + if (count[s] != 0 && norm == 0) + norm = 1; + assert(count[s] < total); + cost += count[s] * kInverseProbabilityLog256[norm]; + } + return cost >> 8; +} + +/** + * Returns the cost in bits of encoding the distribution in count using ctable. + * Returns an error if ctable cannot represent all the symbols in count. + */ +size_t ZSTD_fseBitCost( + FSE_CTable const* ctable, + unsigned const* count, + unsigned const max) +{ + unsigned const kAccuracyLog = 8; + size_t cost = 0; + unsigned s; + FSE_CState_t cstate; + FSE_initCState(&cstate, ctable); + if (ZSTD_getFSEMaxSymbolValue(ctable) < max) { + DEBUGLOG(5, "Repeat FSE_CTable has maxSymbolValue %u < %u", + ZSTD_getFSEMaxSymbolValue(ctable), max); + return ERROR(GENERIC); + } + for (s = 0; s <= max; ++s) { + unsigned const tableLog = cstate.stateLog; + unsigned const badCost = (tableLog + 1) << kAccuracyLog; + unsigned const bitCost = FSE_bitCost(cstate.symbolTT, tableLog, s, kAccuracyLog); + if (count[s] == 0) + continue; + if (bitCost >= badCost) { + DEBUGLOG(5, "Repeat FSE_CTable has Prob[%u] == 0", s); + return ERROR(GENERIC); + } + cost += (size_t)count[s] * bitCost; + } + return cost >> kAccuracyLog; +} + +/** + * Returns the cost in bits of encoding the distribution in count using the + * table described by norm. The max symbol support by norm is assumed >= max. + * norm must be valid for every symbol with non-zero probability in count. + */ +size_t ZSTD_crossEntropyCost(short const* norm, unsigned accuracyLog, + unsigned const* count, unsigned const max) +{ + unsigned const shift = 8 - accuracyLog; + size_t cost = 0; + unsigned s; + assert(accuracyLog <= 8); + for (s = 0; s <= max; ++s) { + unsigned const normAcc = (norm[s] != -1) ? (unsigned)norm[s] : 1; + unsigned const norm256 = normAcc << shift; + assert(norm256 > 0); + assert(norm256 < 256); + cost += count[s] * kInverseProbabilityLog256[norm256]; + } + return cost >> 8; +} + +symbolEncodingType_e +ZSTD_selectEncodingType( + FSE_repeat* repeatMode, unsigned const* count, unsigned const max, + size_t const mostFrequent, size_t nbSeq, unsigned const FSELog, + FSE_CTable const* prevCTable, + short const* defaultNorm, U32 defaultNormLog, + ZSTD_defaultPolicy_e const isDefaultAllowed, + ZSTD_strategy const strategy) +{ + ZSTD_STATIC_ASSERT(ZSTD_defaultDisallowed == 0 && ZSTD_defaultAllowed != 0); + if (mostFrequent == nbSeq) { + *repeatMode = FSE_repeat_none; + if (isDefaultAllowed && nbSeq <= 2) { + /* Prefer set_basic over set_rle when there are 2 or less symbols, + * since RLE uses 1 byte, but set_basic uses 5-6 bits per symbol. + * If basic encoding isn't possible, always choose RLE. + */ + DEBUGLOG(5, "Selected set_basic"); + return set_basic; + } + DEBUGLOG(5, "Selected set_rle"); + return set_rle; + } + if (strategy < ZSTD_lazy) { + if (isDefaultAllowed) { + size_t const staticFse_nbSeq_max = 1000; + size_t const mult = 10 - strategy; + size_t const baseLog = 3; + size_t const dynamicFse_nbSeq_min = (((size_t)1 << defaultNormLog) * mult) >> baseLog; /* 28-36 for offset, 56-72 for lengths */ + assert(defaultNormLog >= 5 && defaultNormLog <= 6); /* xx_DEFAULTNORMLOG */ + assert(mult <= 9 && mult >= 7); + if ( (*repeatMode == FSE_repeat_valid) + && (nbSeq < staticFse_nbSeq_max) ) { + DEBUGLOG(5, "Selected set_repeat"); + return set_repeat; + } + if ( (nbSeq < dynamicFse_nbSeq_min) + || (mostFrequent < (nbSeq >> (defaultNormLog-1))) ) { + DEBUGLOG(5, "Selected set_basic"); + /* The format allows default tables to be repeated, but it isn't useful. + * When using simple heuristics to select encoding type, we don't want + * to confuse these tables with dictionaries. When running more careful + * analysis, we don't need to waste time checking both repeating tables + * and default tables. + */ + *repeatMode = FSE_repeat_none; + return set_basic; + } + } + } else { + size_t const basicCost = isDefaultAllowed ? ZSTD_crossEntropyCost(defaultNorm, defaultNormLog, count, max) : ERROR(GENERIC); + size_t const repeatCost = *repeatMode != FSE_repeat_none ? ZSTD_fseBitCost(prevCTable, count, max) : ERROR(GENERIC); + size_t const NCountCost = ZSTD_NCountCost(count, max, nbSeq, FSELog); + size_t const compressedCost = (NCountCost << 3) + ZSTD_entropyCost(count, max, nbSeq); + + if (isDefaultAllowed) { + assert(!ZSTD_isError(basicCost)); + assert(!(*repeatMode == FSE_repeat_valid && ZSTD_isError(repeatCost))); + } + assert(!ZSTD_isError(NCountCost)); + assert(compressedCost < ERROR(maxCode)); + DEBUGLOG(5, "Estimated bit costs: basic=%u\trepeat=%u\tcompressed=%u", + (unsigned)basicCost, (unsigned)repeatCost, (unsigned)compressedCost); + if (basicCost <= repeatCost && basicCost <= compressedCost) { + DEBUGLOG(5, "Selected set_basic"); + assert(isDefaultAllowed); + *repeatMode = FSE_repeat_none; + return set_basic; + } + if (repeatCost <= compressedCost) { + DEBUGLOG(5, "Selected set_repeat"); + assert(!ZSTD_isError(repeatCost)); + return set_repeat; + } + assert(compressedCost < basicCost && compressedCost < repeatCost); + } + DEBUGLOG(5, "Selected set_compressed"); + *repeatMode = FSE_repeat_check; + return set_compressed; +} + +typedef struct { + S16 norm[MaxSeq + 1]; + U32 wksp[FSE_BUILD_CTABLE_WORKSPACE_SIZE_U32(MaxSeq, MaxFSELog)]; +} ZSTD_BuildCTableWksp; + +size_t +ZSTD_buildCTable(void* dst, size_t dstCapacity, + FSE_CTable* nextCTable, U32 FSELog, symbolEncodingType_e type, + unsigned* count, U32 max, + const BYTE* codeTable, size_t nbSeq, + const S16* defaultNorm, U32 defaultNormLog, U32 defaultMax, + const FSE_CTable* prevCTable, size_t prevCTableSize, + void* entropyWorkspace, size_t entropyWorkspaceSize) +{ + BYTE* op = (BYTE*)dst; + const BYTE* const oend = op + dstCapacity; + DEBUGLOG(6, "ZSTD_buildCTable (dstCapacity=%u)", (unsigned)dstCapacity); + + switch (type) { + case set_rle: + FORWARD_IF_ERROR(FSE_buildCTable_rle(nextCTable, (BYTE)max), ""); + RETURN_ERROR_IF(dstCapacity==0, dstSize_tooSmall, "not enough space"); + *op = codeTable[0]; + return 1; + case set_repeat: + ZSTD_memcpy(nextCTable, prevCTable, prevCTableSize); + return 0; + case set_basic: + FORWARD_IF_ERROR(FSE_buildCTable_wksp(nextCTable, defaultNorm, defaultMax, defaultNormLog, entropyWorkspace, entropyWorkspaceSize), ""); /* note : could be pre-calculated */ + return 0; + case set_compressed: { + ZSTD_BuildCTableWksp* wksp = (ZSTD_BuildCTableWksp*)entropyWorkspace; + size_t nbSeq_1 = nbSeq; + const U32 tableLog = FSE_optimalTableLog(FSELog, nbSeq, max); + if (count[codeTable[nbSeq-1]] > 1) { + count[codeTable[nbSeq-1]]--; + nbSeq_1--; + } + assert(nbSeq_1 > 1); + assert(entropyWorkspaceSize >= sizeof(ZSTD_BuildCTableWksp)); + (void)entropyWorkspaceSize; + FORWARD_IF_ERROR(FSE_normalizeCount(wksp->norm, tableLog, count, nbSeq_1, max, ZSTD_useLowProbCount(nbSeq_1)), "FSE_normalizeCount failed"); + assert(oend >= op); + { size_t const NCountSize = FSE_writeNCount(op, (size_t)(oend - op), wksp->norm, max, tableLog); /* overflow protected */ + FORWARD_IF_ERROR(NCountSize, "FSE_writeNCount failed"); + FORWARD_IF_ERROR(FSE_buildCTable_wksp(nextCTable, wksp->norm, max, tableLog, wksp->wksp, sizeof(wksp->wksp)), "FSE_buildCTable_wksp failed"); + return NCountSize; + } + } + default: assert(0); RETURN_ERROR(GENERIC, "impossible to reach"); + } +} + +FORCE_INLINE_TEMPLATE size_t +ZSTD_encodeSequences_body( + void* dst, size_t dstCapacity, + FSE_CTable const* CTable_MatchLength, BYTE const* mlCodeTable, + FSE_CTable const* CTable_OffsetBits, BYTE const* ofCodeTable, + FSE_CTable const* CTable_LitLength, BYTE const* llCodeTable, + seqDef const* sequences, size_t nbSeq, int longOffsets) +{ + BIT_CStream_t blockStream; + FSE_CState_t stateMatchLength; + FSE_CState_t stateOffsetBits; + FSE_CState_t stateLitLength; + + RETURN_ERROR_IF( + ERR_isError(BIT_initCStream(&blockStream, dst, dstCapacity)), + dstSize_tooSmall, "not enough space remaining"); + DEBUGLOG(6, "available space for bitstream : %i (dstCapacity=%u)", + (int)(blockStream.endPtr - blockStream.startPtr), + (unsigned)dstCapacity); + + /* first symbols */ + FSE_initCState2(&stateMatchLength, CTable_MatchLength, mlCodeTable[nbSeq-1]); + FSE_initCState2(&stateOffsetBits, CTable_OffsetBits, ofCodeTable[nbSeq-1]); + FSE_initCState2(&stateLitLength, CTable_LitLength, llCodeTable[nbSeq-1]); + BIT_addBits(&blockStream, sequences[nbSeq-1].litLength, LL_bits[llCodeTable[nbSeq-1]]); + if (MEM_32bits()) BIT_flushBits(&blockStream); + BIT_addBits(&blockStream, sequences[nbSeq-1].mlBase, ML_bits[mlCodeTable[nbSeq-1]]); + if (MEM_32bits()) BIT_flushBits(&blockStream); + if (longOffsets) { + U32 const ofBits = ofCodeTable[nbSeq-1]; + unsigned const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN-1); + if (extraBits) { + BIT_addBits(&blockStream, sequences[nbSeq-1].offBase, extraBits); + BIT_flushBits(&blockStream); + } + BIT_addBits(&blockStream, sequences[nbSeq-1].offBase >> extraBits, + ofBits - extraBits); + } else { + BIT_addBits(&blockStream, sequences[nbSeq-1].offBase, ofCodeTable[nbSeq-1]); + } + BIT_flushBits(&blockStream); + + { size_t n; + for (n=nbSeq-2 ; n= 64-7-(LLFSELog+MLFSELog+OffFSELog))) + BIT_flushBits(&blockStream); /* (7)*/ + BIT_addBits(&blockStream, sequences[n].litLength, llBits); + if (MEM_32bits() && ((llBits+mlBits)>24)) BIT_flushBits(&blockStream); + BIT_addBits(&blockStream, sequences[n].mlBase, mlBits); + if (MEM_32bits() || (ofBits+mlBits+llBits > 56)) BIT_flushBits(&blockStream); + if (longOffsets) { + unsigned const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN-1); + if (extraBits) { + BIT_addBits(&blockStream, sequences[n].offBase, extraBits); + BIT_flushBits(&blockStream); /* (7)*/ + } + BIT_addBits(&blockStream, sequences[n].offBase >> extraBits, + ofBits - extraBits); /* 31 */ + } else { + BIT_addBits(&blockStream, sequences[n].offBase, ofBits); /* 31 */ + } + BIT_flushBits(&blockStream); /* (7)*/ + DEBUGLOG(7, "remaining space : %i", (int)(blockStream.endPtr - blockStream.ptr)); + } } + + DEBUGLOG(6, "ZSTD_encodeSequences: flushing ML state with %u bits", stateMatchLength.stateLog); + FSE_flushCState(&blockStream, &stateMatchLength); + DEBUGLOG(6, "ZSTD_encodeSequences: flushing Off state with %u bits", stateOffsetBits.stateLog); + FSE_flushCState(&blockStream, &stateOffsetBits); + DEBUGLOG(6, "ZSTD_encodeSequences: flushing LL state with %u bits", stateLitLength.stateLog); + FSE_flushCState(&blockStream, &stateLitLength); + + { size_t const streamSize = BIT_closeCStream(&blockStream); + RETURN_ERROR_IF(streamSize==0, dstSize_tooSmall, "not enough space"); + return streamSize; + } +} + +static size_t +ZSTD_encodeSequences_default( + void* dst, size_t dstCapacity, + FSE_CTable const* CTable_MatchLength, BYTE const* mlCodeTable, + FSE_CTable const* CTable_OffsetBits, BYTE const* ofCodeTable, + FSE_CTable const* CTable_LitLength, BYTE const* llCodeTable, + seqDef const* sequences, size_t nbSeq, int longOffsets) +{ + return ZSTD_encodeSequences_body(dst, dstCapacity, + CTable_MatchLength, mlCodeTable, + CTable_OffsetBits, ofCodeTable, + CTable_LitLength, llCodeTable, + sequences, nbSeq, longOffsets); +} + + +#if DYNAMIC_BMI2 + +static BMI2_TARGET_ATTRIBUTE size_t +ZSTD_encodeSequences_bmi2( + void* dst, size_t dstCapacity, + FSE_CTable const* CTable_MatchLength, BYTE const* mlCodeTable, + FSE_CTable const* CTable_OffsetBits, BYTE const* ofCodeTable, + FSE_CTable const* CTable_LitLength, BYTE const* llCodeTable, + seqDef const* sequences, size_t nbSeq, int longOffsets) +{ + return ZSTD_encodeSequences_body(dst, dstCapacity, + CTable_MatchLength, mlCodeTable, + CTable_OffsetBits, ofCodeTable, + CTable_LitLength, llCodeTable, + sequences, nbSeq, longOffsets); +} + +#endif + +size_t ZSTD_encodeSequences( + void* dst, size_t dstCapacity, + FSE_CTable const* CTable_MatchLength, BYTE const* mlCodeTable, + FSE_CTable const* CTable_OffsetBits, BYTE const* ofCodeTable, + FSE_CTable const* CTable_LitLength, BYTE const* llCodeTable, + seqDef const* sequences, size_t nbSeq, int longOffsets, int bmi2) +{ + DEBUGLOG(5, "ZSTD_encodeSequences: dstCapacity = %u", (unsigned)dstCapacity); +#if DYNAMIC_BMI2 + if (bmi2) { + return ZSTD_encodeSequences_bmi2(dst, dstCapacity, + CTable_MatchLength, mlCodeTable, + CTable_OffsetBits, ofCodeTable, + CTable_LitLength, llCodeTable, + sequences, nbSeq, longOffsets); + } +#endif + (void)bmi2; + return ZSTD_encodeSequences_default(dst, dstCapacity, + CTable_MatchLength, mlCodeTable, + CTable_OffsetBits, ofCodeTable, + CTable_LitLength, llCodeTable, + sequences, nbSeq, longOffsets); +} diff --git a/src/blosc2/internal-complibs/zstd-1.5.2/compress/zstd_compress_sequences.h b/src/blosc2/internal-complibs/zstd-1.5.2/compress/zstd_compress_sequences.h new file mode 100644 index 0000000..7991364 --- /dev/null +++ b/src/blosc2/internal-complibs/zstd-1.5.2/compress/zstd_compress_sequences.h @@ -0,0 +1,54 @@ +/* + * Copyright (c) Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_COMPRESS_SEQUENCES_H +#define ZSTD_COMPRESS_SEQUENCES_H + +#include "../common/fse.h" /* FSE_repeat, FSE_CTable */ +#include "../common/zstd_internal.h" /* symbolEncodingType_e, ZSTD_strategy */ + +typedef enum { + ZSTD_defaultDisallowed = 0, + ZSTD_defaultAllowed = 1 +} ZSTD_defaultPolicy_e; + +symbolEncodingType_e +ZSTD_selectEncodingType( + FSE_repeat* repeatMode, unsigned const* count, unsigned const max, + size_t const mostFrequent, size_t nbSeq, unsigned const FSELog, + FSE_CTable const* prevCTable, + short const* defaultNorm, U32 defaultNormLog, + ZSTD_defaultPolicy_e const isDefaultAllowed, + ZSTD_strategy const strategy); + +size_t +ZSTD_buildCTable(void* dst, size_t dstCapacity, + FSE_CTable* nextCTable, U32 FSELog, symbolEncodingType_e type, + unsigned* count, U32 max, + const BYTE* codeTable, size_t nbSeq, + const S16* defaultNorm, U32 defaultNormLog, U32 defaultMax, + const FSE_CTable* prevCTable, size_t prevCTableSize, + void* entropyWorkspace, size_t entropyWorkspaceSize); + +size_t ZSTD_encodeSequences( + void* dst, size_t dstCapacity, + FSE_CTable const* CTable_MatchLength, BYTE const* mlCodeTable, + FSE_CTable const* CTable_OffsetBits, BYTE const* ofCodeTable, + FSE_CTable const* CTable_LitLength, BYTE const* llCodeTable, + seqDef const* sequences, size_t nbSeq, int longOffsets, int bmi2); + +size_t ZSTD_fseBitCost( + FSE_CTable const* ctable, + unsigned const* count, + unsigned const max); + +size_t ZSTD_crossEntropyCost(short const* norm, unsigned accuracyLog, + unsigned const* count, unsigned const max); +#endif /* ZSTD_COMPRESS_SEQUENCES_H */ diff --git a/src/blosc2/internal-complibs/zstd-1.5.2/compress/zstd_compress_superblock.c b/src/blosc2/internal-complibs/zstd-1.5.2/compress/zstd_compress_superblock.c new file mode 100644 index 0000000..10e3378 --- /dev/null +++ b/src/blosc2/internal-complibs/zstd-1.5.2/compress/zstd_compress_superblock.c @@ -0,0 +1,573 @@ +/* + * Copyright (c) Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + + /*-************************************* + * Dependencies + ***************************************/ +#include "zstd_compress_superblock.h" + +#include "../common/zstd_internal.h" /* ZSTD_getSequenceLength */ +#include "hist.h" /* HIST_countFast_wksp */ +#include "zstd_compress_internal.h" /* ZSTD_[huf|fse|entropy]CTablesMetadata_t */ +#include "zstd_compress_sequences.h" +#include "zstd_compress_literals.h" + +/** ZSTD_compressSubBlock_literal() : + * Compresses literals section for a sub-block. + * When we have to write the Huffman table we will sometimes choose a header + * size larger than necessary. This is because we have to pick the header size + * before we know the table size + compressed size, so we have a bound on the + * table size. If we guessed incorrectly, we fall back to uncompressed literals. + * + * We write the header when writeEntropy=1 and set entropyWritten=1 when we succeeded + * in writing the header, otherwise it is set to 0. + * + * hufMetadata->hType has literals block type info. + * If it is set_basic, all sub-blocks literals section will be Raw_Literals_Block. + * If it is set_rle, all sub-blocks literals section will be RLE_Literals_Block. + * If it is set_compressed, first sub-block's literals section will be Compressed_Literals_Block + * If it is set_compressed, first sub-block's literals section will be Treeless_Literals_Block + * and the following sub-blocks' literals sections will be Treeless_Literals_Block. + * @return : compressed size of literals section of a sub-block + * Or 0 if it unable to compress. + * Or error code */ +static size_t ZSTD_compressSubBlock_literal(const HUF_CElt* hufTable, + const ZSTD_hufCTablesMetadata_t* hufMetadata, + const BYTE* literals, size_t litSize, + void* dst, size_t dstSize, + const int bmi2, int writeEntropy, int* entropyWritten) +{ + size_t const header = writeEntropy ? 200 : 0; + size_t const lhSize = 3 + (litSize >= (1 KB - header)) + (litSize >= (16 KB - header)); + BYTE* const ostart = (BYTE*)dst; + BYTE* const oend = ostart + dstSize; + BYTE* op = ostart + lhSize; + U32 const singleStream = lhSize == 3; + symbolEncodingType_e hType = writeEntropy ? hufMetadata->hType : set_repeat; + size_t cLitSize = 0; + + (void)bmi2; /* TODO bmi2... */ + + DEBUGLOG(5, "ZSTD_compressSubBlock_literal (litSize=%zu, lhSize=%zu, writeEntropy=%d)", litSize, lhSize, writeEntropy); + + *entropyWritten = 0; + if (litSize == 0 || hufMetadata->hType == set_basic) { + DEBUGLOG(5, "ZSTD_compressSubBlock_literal using raw literal"); + return ZSTD_noCompressLiterals(dst, dstSize, literals, litSize); + } else if (hufMetadata->hType == set_rle) { + DEBUGLOG(5, "ZSTD_compressSubBlock_literal using rle literal"); + return ZSTD_compressRleLiteralsBlock(dst, dstSize, literals, litSize); + } + + assert(litSize > 0); + assert(hufMetadata->hType == set_compressed || hufMetadata->hType == set_repeat); + + if (writeEntropy && hufMetadata->hType == set_compressed) { + ZSTD_memcpy(op, hufMetadata->hufDesBuffer, hufMetadata->hufDesSize); + op += hufMetadata->hufDesSize; + cLitSize += hufMetadata->hufDesSize; + DEBUGLOG(5, "ZSTD_compressSubBlock_literal (hSize=%zu)", hufMetadata->hufDesSize); + } + + /* TODO bmi2 */ + { const size_t cSize = singleStream ? HUF_compress1X_usingCTable(op, oend-op, literals, litSize, hufTable) + : HUF_compress4X_usingCTable(op, oend-op, literals, litSize, hufTable); + op += cSize; + cLitSize += cSize; + if (cSize == 0 || ERR_isError(cSize)) { + DEBUGLOG(5, "Failed to write entropy tables %s", ZSTD_getErrorName(cSize)); + return 0; + } + /* If we expand and we aren't writing a header then emit uncompressed */ + if (!writeEntropy && cLitSize >= litSize) { + DEBUGLOG(5, "ZSTD_compressSubBlock_literal using raw literal because uncompressible"); + return ZSTD_noCompressLiterals(dst, dstSize, literals, litSize); + } + /* If we are writing headers then allow expansion that doesn't change our header size. */ + if (lhSize < (size_t)(3 + (cLitSize >= 1 KB) + (cLitSize >= 16 KB))) { + assert(cLitSize > litSize); + DEBUGLOG(5, "Literals expanded beyond allowed header size"); + return ZSTD_noCompressLiterals(dst, dstSize, literals, litSize); + } + DEBUGLOG(5, "ZSTD_compressSubBlock_literal (cSize=%zu)", cSize); + } + + /* Build header */ + switch(lhSize) + { + case 3: /* 2 - 2 - 10 - 10 */ + { U32 const lhc = hType + ((!singleStream) << 2) + ((U32)litSize<<4) + ((U32)cLitSize<<14); + MEM_writeLE24(ostart, lhc); + break; + } + case 4: /* 2 - 2 - 14 - 14 */ + { U32 const lhc = hType + (2 << 2) + ((U32)litSize<<4) + ((U32)cLitSize<<18); + MEM_writeLE32(ostart, lhc); + break; + } + case 5: /* 2 - 2 - 18 - 18 */ + { U32 const lhc = hType + (3 << 2) + ((U32)litSize<<4) + ((U32)cLitSize<<22); + MEM_writeLE32(ostart, lhc); + ostart[4] = (BYTE)(cLitSize >> 10); + break; + } + default: /* not possible : lhSize is {3,4,5} */ + assert(0); + } + *entropyWritten = 1; + DEBUGLOG(5, "Compressed literals: %u -> %u", (U32)litSize, (U32)(op-ostart)); + return op-ostart; +} + +static size_t ZSTD_seqDecompressedSize(seqStore_t const* seqStore, const seqDef* sequences, size_t nbSeq, size_t litSize, int lastSequence) { + const seqDef* const sstart = sequences; + const seqDef* const send = sequences + nbSeq; + const seqDef* sp = sstart; + size_t matchLengthSum = 0; + size_t litLengthSum = 0; + (void)(litLengthSum); /* suppress unused variable warning on some environments */ + while (send-sp > 0) { + ZSTD_sequenceLength const seqLen = ZSTD_getSequenceLength(seqStore, sp); + litLengthSum += seqLen.litLength; + matchLengthSum += seqLen.matchLength; + sp++; + } + assert(litLengthSum <= litSize); + if (!lastSequence) { + assert(litLengthSum == litSize); + } + return matchLengthSum + litSize; +} + +/** ZSTD_compressSubBlock_sequences() : + * Compresses sequences section for a sub-block. + * fseMetadata->llType, fseMetadata->ofType, and fseMetadata->mlType have + * symbol compression modes for the super-block. + * The first successfully compressed block will have these in its header. + * We set entropyWritten=1 when we succeed in compressing the sequences. + * The following sub-blocks will always have repeat mode. + * @return : compressed size of sequences section of a sub-block + * Or 0 if it is unable to compress + * Or error code. */ +static size_t ZSTD_compressSubBlock_sequences(const ZSTD_fseCTables_t* fseTables, + const ZSTD_fseCTablesMetadata_t* fseMetadata, + const seqDef* sequences, size_t nbSeq, + const BYTE* llCode, const BYTE* mlCode, const BYTE* ofCode, + const ZSTD_CCtx_params* cctxParams, + void* dst, size_t dstCapacity, + const int bmi2, int writeEntropy, int* entropyWritten) +{ + const int longOffsets = cctxParams->cParams.windowLog > STREAM_ACCUMULATOR_MIN; + BYTE* const ostart = (BYTE*)dst; + BYTE* const oend = ostart + dstCapacity; + BYTE* op = ostart; + BYTE* seqHead; + + DEBUGLOG(5, "ZSTD_compressSubBlock_sequences (nbSeq=%zu, writeEntropy=%d, longOffsets=%d)", nbSeq, writeEntropy, longOffsets); + + *entropyWritten = 0; + /* Sequences Header */ + RETURN_ERROR_IF((oend-op) < 3 /*max nbSeq Size*/ + 1 /*seqHead*/, + dstSize_tooSmall, ""); + if (nbSeq < 0x7F) + *op++ = (BYTE)nbSeq; + else if (nbSeq < LONGNBSEQ) + op[0] = (BYTE)((nbSeq>>8) + 0x80), op[1] = (BYTE)nbSeq, op+=2; + else + op[0]=0xFF, MEM_writeLE16(op+1, (U16)(nbSeq - LONGNBSEQ)), op+=3; + if (nbSeq==0) { + return op - ostart; + } + + /* seqHead : flags for FSE encoding type */ + seqHead = op++; + + DEBUGLOG(5, "ZSTD_compressSubBlock_sequences (seqHeadSize=%u)", (unsigned)(op-ostart)); + + if (writeEntropy) { + const U32 LLtype = fseMetadata->llType; + const U32 Offtype = fseMetadata->ofType; + const U32 MLtype = fseMetadata->mlType; + DEBUGLOG(5, "ZSTD_compressSubBlock_sequences (fseTablesSize=%zu)", fseMetadata->fseTablesSize); + *seqHead = (BYTE)((LLtype<<6) + (Offtype<<4) + (MLtype<<2)); + ZSTD_memcpy(op, fseMetadata->fseTablesBuffer, fseMetadata->fseTablesSize); + op += fseMetadata->fseTablesSize; + } else { + const U32 repeat = set_repeat; + *seqHead = (BYTE)((repeat<<6) + (repeat<<4) + (repeat<<2)); + } + + { size_t const bitstreamSize = ZSTD_encodeSequences( + op, oend - op, + fseTables->matchlengthCTable, mlCode, + fseTables->offcodeCTable, ofCode, + fseTables->litlengthCTable, llCode, + sequences, nbSeq, + longOffsets, bmi2); + FORWARD_IF_ERROR(bitstreamSize, "ZSTD_encodeSequences failed"); + op += bitstreamSize; + /* zstd versions <= 1.3.4 mistakenly report corruption when + * FSE_readNCount() receives a buffer < 4 bytes. + * Fixed by https://github.com/facebook/zstd/pull/1146. + * This can happen when the last set_compressed table present is 2 + * bytes and the bitstream is only one byte. + * In this exceedingly rare case, we will simply emit an uncompressed + * block, since it isn't worth optimizing. + */ +#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION + if (writeEntropy && fseMetadata->lastCountSize && fseMetadata->lastCountSize + bitstreamSize < 4) { + /* NCountSize >= 2 && bitstreamSize > 0 ==> lastCountSize == 3 */ + assert(fseMetadata->lastCountSize + bitstreamSize == 3); + DEBUGLOG(5, "Avoiding bug in zstd decoder in versions <= 1.3.4 by " + "emitting an uncompressed block."); + return 0; + } +#endif + DEBUGLOG(5, "ZSTD_compressSubBlock_sequences (bitstreamSize=%zu)", bitstreamSize); + } + + /* zstd versions <= 1.4.0 mistakenly report error when + * sequences section body size is less than 3 bytes. + * Fixed by https://github.com/facebook/zstd/pull/1664. + * This can happen when the previous sequences section block is compressed + * with rle mode and the current block's sequences section is compressed + * with repeat mode where sequences section body size can be 1 byte. + */ +#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION + if (op-seqHead < 4) { + DEBUGLOG(5, "Avoiding bug in zstd decoder in versions <= 1.4.0 by emitting " + "an uncompressed block when sequences are < 4 bytes"); + return 0; + } +#endif + + *entropyWritten = 1; + return op - ostart; +} + +/** ZSTD_compressSubBlock() : + * Compresses a single sub-block. + * @return : compressed size of the sub-block + * Or 0 if it failed to compress. */ +static size_t ZSTD_compressSubBlock(const ZSTD_entropyCTables_t* entropy, + const ZSTD_entropyCTablesMetadata_t* entropyMetadata, + const seqDef* sequences, size_t nbSeq, + const BYTE* literals, size_t litSize, + const BYTE* llCode, const BYTE* mlCode, const BYTE* ofCode, + const ZSTD_CCtx_params* cctxParams, + void* dst, size_t dstCapacity, + const int bmi2, + int writeLitEntropy, int writeSeqEntropy, + int* litEntropyWritten, int* seqEntropyWritten, + U32 lastBlock) +{ + BYTE* const ostart = (BYTE*)dst; + BYTE* const oend = ostart + dstCapacity; + BYTE* op = ostart + ZSTD_blockHeaderSize; + DEBUGLOG(5, "ZSTD_compressSubBlock (litSize=%zu, nbSeq=%zu, writeLitEntropy=%d, writeSeqEntropy=%d, lastBlock=%d)", + litSize, nbSeq, writeLitEntropy, writeSeqEntropy, lastBlock); + { size_t cLitSize = ZSTD_compressSubBlock_literal((const HUF_CElt*)entropy->huf.CTable, + &entropyMetadata->hufMetadata, literals, litSize, + op, oend-op, bmi2, writeLitEntropy, litEntropyWritten); + FORWARD_IF_ERROR(cLitSize, "ZSTD_compressSubBlock_literal failed"); + if (cLitSize == 0) return 0; + op += cLitSize; + } + { size_t cSeqSize = ZSTD_compressSubBlock_sequences(&entropy->fse, + &entropyMetadata->fseMetadata, + sequences, nbSeq, + llCode, mlCode, ofCode, + cctxParams, + op, oend-op, + bmi2, writeSeqEntropy, seqEntropyWritten); + FORWARD_IF_ERROR(cSeqSize, "ZSTD_compressSubBlock_sequences failed"); + if (cSeqSize == 0) return 0; + op += cSeqSize; + } + /* Write block header */ + { size_t cSize = (op-ostart)-ZSTD_blockHeaderSize; + U32 const cBlockHeader24 = lastBlock + (((U32)bt_compressed)<<1) + (U32)(cSize << 3); + MEM_writeLE24(ostart, cBlockHeader24); + } + return op-ostart; +} + +static size_t ZSTD_estimateSubBlockSize_literal(const BYTE* literals, size_t litSize, + const ZSTD_hufCTables_t* huf, + const ZSTD_hufCTablesMetadata_t* hufMetadata, + void* workspace, size_t wkspSize, + int writeEntropy) +{ + unsigned* const countWksp = (unsigned*)workspace; + unsigned maxSymbolValue = 255; + size_t literalSectionHeaderSize = 3; /* Use hard coded size of 3 bytes */ + + if (hufMetadata->hType == set_basic) return litSize; + else if (hufMetadata->hType == set_rle) return 1; + else if (hufMetadata->hType == set_compressed || hufMetadata->hType == set_repeat) { + size_t const largest = HIST_count_wksp (countWksp, &maxSymbolValue, (const BYTE*)literals, litSize, workspace, wkspSize); + if (ZSTD_isError(largest)) return litSize; + { size_t cLitSizeEstimate = HUF_estimateCompressedSize((const HUF_CElt*)huf->CTable, countWksp, maxSymbolValue); + if (writeEntropy) cLitSizeEstimate += hufMetadata->hufDesSize; + return cLitSizeEstimate + literalSectionHeaderSize; + } } + assert(0); /* impossible */ + return 0; +} + +static size_t ZSTD_estimateSubBlockSize_symbolType(symbolEncodingType_e type, + const BYTE* codeTable, unsigned maxCode, + size_t nbSeq, const FSE_CTable* fseCTable, + const U8* additionalBits, + short const* defaultNorm, U32 defaultNormLog, U32 defaultMax, + void* workspace, size_t wkspSize) +{ + unsigned* const countWksp = (unsigned*)workspace; + const BYTE* ctp = codeTable; + const BYTE* const ctStart = ctp; + const BYTE* const ctEnd = ctStart + nbSeq; + size_t cSymbolTypeSizeEstimateInBits = 0; + unsigned max = maxCode; + + HIST_countFast_wksp(countWksp, &max, codeTable, nbSeq, workspace, wkspSize); /* can't fail */ + if (type == set_basic) { + /* We selected this encoding type, so it must be valid. */ + assert(max <= defaultMax); + cSymbolTypeSizeEstimateInBits = max <= defaultMax + ? ZSTD_crossEntropyCost(defaultNorm, defaultNormLog, countWksp, max) + : ERROR(GENERIC); + } else if (type == set_rle) { + cSymbolTypeSizeEstimateInBits = 0; + } else if (type == set_compressed || type == set_repeat) { + cSymbolTypeSizeEstimateInBits = ZSTD_fseBitCost(fseCTable, countWksp, max); + } + if (ZSTD_isError(cSymbolTypeSizeEstimateInBits)) return nbSeq * 10; + while (ctp < ctEnd) { + if (additionalBits) cSymbolTypeSizeEstimateInBits += additionalBits[*ctp]; + else cSymbolTypeSizeEstimateInBits += *ctp; /* for offset, offset code is also the number of additional bits */ + ctp++; + } + return cSymbolTypeSizeEstimateInBits / 8; +} + +static size_t ZSTD_estimateSubBlockSize_sequences(const BYTE* ofCodeTable, + const BYTE* llCodeTable, + const BYTE* mlCodeTable, + size_t nbSeq, + const ZSTD_fseCTables_t* fseTables, + const ZSTD_fseCTablesMetadata_t* fseMetadata, + void* workspace, size_t wkspSize, + int writeEntropy) +{ + size_t const sequencesSectionHeaderSize = 3; /* Use hard coded size of 3 bytes */ + size_t cSeqSizeEstimate = 0; + if (nbSeq == 0) return sequencesSectionHeaderSize; + cSeqSizeEstimate += ZSTD_estimateSubBlockSize_symbolType(fseMetadata->ofType, ofCodeTable, MaxOff, + nbSeq, fseTables->offcodeCTable, NULL, + OF_defaultNorm, OF_defaultNormLog, DefaultMaxOff, + workspace, wkspSize); + cSeqSizeEstimate += ZSTD_estimateSubBlockSize_symbolType(fseMetadata->llType, llCodeTable, MaxLL, + nbSeq, fseTables->litlengthCTable, LL_bits, + LL_defaultNorm, LL_defaultNormLog, MaxLL, + workspace, wkspSize); + cSeqSizeEstimate += ZSTD_estimateSubBlockSize_symbolType(fseMetadata->mlType, mlCodeTable, MaxML, + nbSeq, fseTables->matchlengthCTable, ML_bits, + ML_defaultNorm, ML_defaultNormLog, MaxML, + workspace, wkspSize); + if (writeEntropy) cSeqSizeEstimate += fseMetadata->fseTablesSize; + return cSeqSizeEstimate + sequencesSectionHeaderSize; +} + +static size_t ZSTD_estimateSubBlockSize(const BYTE* literals, size_t litSize, + const BYTE* ofCodeTable, + const BYTE* llCodeTable, + const BYTE* mlCodeTable, + size_t nbSeq, + const ZSTD_entropyCTables_t* entropy, + const ZSTD_entropyCTablesMetadata_t* entropyMetadata, + void* workspace, size_t wkspSize, + int writeLitEntropy, int writeSeqEntropy) { + size_t cSizeEstimate = 0; + cSizeEstimate += ZSTD_estimateSubBlockSize_literal(literals, litSize, + &entropy->huf, &entropyMetadata->hufMetadata, + workspace, wkspSize, writeLitEntropy); + cSizeEstimate += ZSTD_estimateSubBlockSize_sequences(ofCodeTable, llCodeTable, mlCodeTable, + nbSeq, &entropy->fse, &entropyMetadata->fseMetadata, + workspace, wkspSize, writeSeqEntropy); + return cSizeEstimate + ZSTD_blockHeaderSize; +} + +static int ZSTD_needSequenceEntropyTables(ZSTD_fseCTablesMetadata_t const* fseMetadata) +{ + if (fseMetadata->llType == set_compressed || fseMetadata->llType == set_rle) + return 1; + if (fseMetadata->mlType == set_compressed || fseMetadata->mlType == set_rle) + return 1; + if (fseMetadata->ofType == set_compressed || fseMetadata->ofType == set_rle) + return 1; + return 0; +} + +/** ZSTD_compressSubBlock_multi() : + * Breaks super-block into multiple sub-blocks and compresses them. + * Entropy will be written to the first block. + * The following blocks will use repeat mode to compress. + * All sub-blocks are compressed blocks (no raw or rle blocks). + * @return : compressed size of the super block (which is multiple ZSTD blocks) + * Or 0 if it failed to compress. */ +static size_t ZSTD_compressSubBlock_multi(const seqStore_t* seqStorePtr, + const ZSTD_compressedBlockState_t* prevCBlock, + ZSTD_compressedBlockState_t* nextCBlock, + const ZSTD_entropyCTablesMetadata_t* entropyMetadata, + const ZSTD_CCtx_params* cctxParams, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const int bmi2, U32 lastBlock, + void* workspace, size_t wkspSize) +{ + const seqDef* const sstart = seqStorePtr->sequencesStart; + const seqDef* const send = seqStorePtr->sequences; + const seqDef* sp = sstart; + const BYTE* const lstart = seqStorePtr->litStart; + const BYTE* const lend = seqStorePtr->lit; + const BYTE* lp = lstart; + BYTE const* ip = (BYTE const*)src; + BYTE const* const iend = ip + srcSize; + BYTE* const ostart = (BYTE*)dst; + BYTE* const oend = ostart + dstCapacity; + BYTE* op = ostart; + const BYTE* llCodePtr = seqStorePtr->llCode; + const BYTE* mlCodePtr = seqStorePtr->mlCode; + const BYTE* ofCodePtr = seqStorePtr->ofCode; + size_t targetCBlockSize = cctxParams->targetCBlockSize; + size_t litSize, seqCount; + int writeLitEntropy = entropyMetadata->hufMetadata.hType == set_compressed; + int writeSeqEntropy = 1; + int lastSequence = 0; + + DEBUGLOG(5, "ZSTD_compressSubBlock_multi (litSize=%u, nbSeq=%u)", + (unsigned)(lend-lp), (unsigned)(send-sstart)); + + litSize = 0; + seqCount = 0; + do { + size_t cBlockSizeEstimate = 0; + if (sstart == send) { + lastSequence = 1; + } else { + const seqDef* const sequence = sp + seqCount; + lastSequence = sequence == send - 1; + litSize += ZSTD_getSequenceLength(seqStorePtr, sequence).litLength; + seqCount++; + } + if (lastSequence) { + assert(lp <= lend); + assert(litSize <= (size_t)(lend - lp)); + litSize = (size_t)(lend - lp); + } + /* I think there is an optimization opportunity here. + * Calling ZSTD_estimateSubBlockSize for every sequence can be wasteful + * since it recalculates estimate from scratch. + * For example, it would recount literal distribution and symbol codes every time. + */ + cBlockSizeEstimate = ZSTD_estimateSubBlockSize(lp, litSize, ofCodePtr, llCodePtr, mlCodePtr, seqCount, + &nextCBlock->entropy, entropyMetadata, + workspace, wkspSize, writeLitEntropy, writeSeqEntropy); + if (cBlockSizeEstimate > targetCBlockSize || lastSequence) { + int litEntropyWritten = 0; + int seqEntropyWritten = 0; + const size_t decompressedSize = ZSTD_seqDecompressedSize(seqStorePtr, sp, seqCount, litSize, lastSequence); + const size_t cSize = ZSTD_compressSubBlock(&nextCBlock->entropy, entropyMetadata, + sp, seqCount, + lp, litSize, + llCodePtr, mlCodePtr, ofCodePtr, + cctxParams, + op, oend-op, + bmi2, writeLitEntropy, writeSeqEntropy, + &litEntropyWritten, &seqEntropyWritten, + lastBlock && lastSequence); + FORWARD_IF_ERROR(cSize, "ZSTD_compressSubBlock failed"); + if (cSize > 0 && cSize < decompressedSize) { + DEBUGLOG(5, "Committed the sub-block"); + assert(ip + decompressedSize <= iend); + ip += decompressedSize; + sp += seqCount; + lp += litSize; + op += cSize; + llCodePtr += seqCount; + mlCodePtr += seqCount; + ofCodePtr += seqCount; + litSize = 0; + seqCount = 0; + /* Entropy only needs to be written once */ + if (litEntropyWritten) { + writeLitEntropy = 0; + } + if (seqEntropyWritten) { + writeSeqEntropy = 0; + } + } + } + } while (!lastSequence); + if (writeLitEntropy) { + DEBUGLOG(5, "ZSTD_compressSubBlock_multi has literal entropy tables unwritten"); + ZSTD_memcpy(&nextCBlock->entropy.huf, &prevCBlock->entropy.huf, sizeof(prevCBlock->entropy.huf)); + } + if (writeSeqEntropy && ZSTD_needSequenceEntropyTables(&entropyMetadata->fseMetadata)) { + /* If we haven't written our entropy tables, then we've violated our contract and + * must emit an uncompressed block. + */ + DEBUGLOG(5, "ZSTD_compressSubBlock_multi has sequence entropy tables unwritten"); + return 0; + } + if (ip < iend) { + size_t const cSize = ZSTD_noCompressBlock(op, oend - op, ip, iend - ip, lastBlock); + DEBUGLOG(5, "ZSTD_compressSubBlock_multi last sub-block uncompressed, %zu bytes", (size_t)(iend - ip)); + FORWARD_IF_ERROR(cSize, "ZSTD_noCompressBlock failed"); + assert(cSize != 0); + op += cSize; + /* We have to regenerate the repcodes because we've skipped some sequences */ + if (sp < send) { + seqDef const* seq; + repcodes_t rep; + ZSTD_memcpy(&rep, prevCBlock->rep, sizeof(rep)); + for (seq = sstart; seq < sp; ++seq) { + ZSTD_updateRep(rep.rep, seq->offBase - 1, ZSTD_getSequenceLength(seqStorePtr, seq).litLength == 0); + } + ZSTD_memcpy(nextCBlock->rep, &rep, sizeof(rep)); + } + } + DEBUGLOG(5, "ZSTD_compressSubBlock_multi compressed"); + return op-ostart; +} + +size_t ZSTD_compressSuperBlock(ZSTD_CCtx* zc, + void* dst, size_t dstCapacity, + void const* src, size_t srcSize, + unsigned lastBlock) { + ZSTD_entropyCTablesMetadata_t entropyMetadata; + + FORWARD_IF_ERROR(ZSTD_buildBlockEntropyStats(&zc->seqStore, + &zc->blockState.prevCBlock->entropy, + &zc->blockState.nextCBlock->entropy, + &zc->appliedParams, + &entropyMetadata, + zc->entropyWorkspace, ENTROPY_WORKSPACE_SIZE /* statically allocated in resetCCtx */), ""); + + return ZSTD_compressSubBlock_multi(&zc->seqStore, + zc->blockState.prevCBlock, + zc->blockState.nextCBlock, + &entropyMetadata, + &zc->appliedParams, + dst, dstCapacity, + src, srcSize, + zc->bmi2, lastBlock, + zc->entropyWorkspace, ENTROPY_WORKSPACE_SIZE /* statically allocated in resetCCtx */); +} diff --git a/src/blosc2/internal-complibs/zstd-1.5.2/compress/zstd_compress_superblock.h b/src/blosc2/internal-complibs/zstd-1.5.2/compress/zstd_compress_superblock.h new file mode 100644 index 0000000..176f9b1 --- /dev/null +++ b/src/blosc2/internal-complibs/zstd-1.5.2/compress/zstd_compress_superblock.h @@ -0,0 +1,32 @@ +/* + * Copyright (c) Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_COMPRESS_ADVANCED_H +#define ZSTD_COMPRESS_ADVANCED_H + +/*-************************************* +* Dependencies +***************************************/ + +#include "../zstd.h" /* ZSTD_CCtx */ + +/*-************************************* +* Target Compressed Block Size +***************************************/ + +/* ZSTD_compressSuperBlock() : + * Used to compress a super block when targetCBlockSize is being used. + * The given block will be compressed into multiple sub blocks that are around targetCBlockSize. */ +size_t ZSTD_compressSuperBlock(ZSTD_CCtx* zc, + void* dst, size_t dstCapacity, + void const* src, size_t srcSize, + unsigned lastBlock); + +#endif /* ZSTD_COMPRESS_ADVANCED_H */ diff --git a/src/blosc2/internal-complibs/zstd-1.5.2/compress/zstd_cwksp.h b/src/blosc2/internal-complibs/zstd-1.5.2/compress/zstd_cwksp.h new file mode 100644 index 0000000..dc3f40c --- /dev/null +++ b/src/blosc2/internal-complibs/zstd-1.5.2/compress/zstd_cwksp.h @@ -0,0 +1,676 @@ +/* + * Copyright (c) Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_CWKSP_H +#define ZSTD_CWKSP_H + +/*-************************************* +* Dependencies +***************************************/ +#include "../common/zstd_internal.h" + +#if defined (__cplusplus) +extern "C" { +#endif + +/*-************************************* +* Constants +***************************************/ + +/* Since the workspace is effectively its own little malloc implementation / + * arena, when we run under ASAN, we should similarly insert redzones between + * each internal element of the workspace, so ASAN will catch overruns that + * reach outside an object but that stay inside the workspace. + * + * This defines the size of that redzone. + */ +#ifndef ZSTD_CWKSP_ASAN_REDZONE_SIZE +#define ZSTD_CWKSP_ASAN_REDZONE_SIZE 128 +#endif + + +/* Set our tables and aligneds to align by 64 bytes */ +#define ZSTD_CWKSP_ALIGNMENT_BYTES 64 + +/*-************************************* +* Structures +***************************************/ +typedef enum { + ZSTD_cwksp_alloc_objects, + ZSTD_cwksp_alloc_buffers, + ZSTD_cwksp_alloc_aligned +} ZSTD_cwksp_alloc_phase_e; + +/** + * Used to describe whether the workspace is statically allocated (and will not + * necessarily ever be freed), or if it's dynamically allocated and we can + * expect a well-formed caller to free this. + */ +typedef enum { + ZSTD_cwksp_dynamic_alloc, + ZSTD_cwksp_static_alloc +} ZSTD_cwksp_static_alloc_e; + +/** + * Zstd fits all its internal datastructures into a single continuous buffer, + * so that it only needs to perform a single OS allocation (or so that a buffer + * can be provided to it and it can perform no allocations at all). This buffer + * is called the workspace. + * + * Several optimizations complicate that process of allocating memory ranges + * from this workspace for each internal datastructure: + * + * - These different internal datastructures have different setup requirements: + * + * - The static objects need to be cleared once and can then be trivially + * reused for each compression. + * + * - Various buffers don't need to be initialized at all--they are always + * written into before they're read. + * + * - The matchstate tables have a unique requirement that they don't need + * their memory to be totally cleared, but they do need the memory to have + * some bound, i.e., a guarantee that all values in the memory they've been + * allocated is less than some maximum value (which is the starting value + * for the indices that they will then use for compression). When this + * guarantee is provided to them, they can use the memory without any setup + * work. When it can't, they have to clear the area. + * + * - These buffers also have different alignment requirements. + * + * - We would like to reuse the objects in the workspace for multiple + * compressions without having to perform any expensive reallocation or + * reinitialization work. + * + * - We would like to be able to efficiently reuse the workspace across + * multiple compressions **even when the compression parameters change** and + * we need to resize some of the objects (where possible). + * + * To attempt to manage this buffer, given these constraints, the ZSTD_cwksp + * abstraction was created. It works as follows: + * + * Workspace Layout: + * + * [ ... workspace ... ] + * [objects][tables ... ->] free space [<- ... aligned][<- ... buffers] + * + * The various objects that live in the workspace are divided into the + * following categories, and are allocated separately: + * + * - Static objects: this is optionally the enclosing ZSTD_CCtx or ZSTD_CDict, + * so that literally everything fits in a single buffer. Note: if present, + * this must be the first object in the workspace, since ZSTD_customFree{CCtx, + * CDict}() rely on a pointer comparison to see whether one or two frees are + * required. + * + * - Fixed size objects: these are fixed-size, fixed-count objects that are + * nonetheless "dynamically" allocated in the workspace so that we can + * control how they're initialized separately from the broader ZSTD_CCtx. + * Examples: + * - Entropy Workspace + * - 2 x ZSTD_compressedBlockState_t + * - CDict dictionary contents + * + * - Tables: these are any of several different datastructures (hash tables, + * chain tables, binary trees) that all respect a common format: they are + * uint32_t arrays, all of whose values are between 0 and (nextSrc - base). + * Their sizes depend on the cparams. These tables are 64-byte aligned. + * + * - Aligned: these buffers are used for various purposes that require 4 byte + * alignment, but don't require any initialization before they're used. These + * buffers are each aligned to 64 bytes. + * + * - Buffers: these buffers are used for various purposes that don't require + * any alignment or initialization before they're used. This means they can + * be moved around at no cost for a new compression. + * + * Allocating Memory: + * + * The various types of objects must be allocated in order, so they can be + * correctly packed into the workspace buffer. That order is: + * + * 1. Objects + * 2. Buffers + * 3. Aligned/Tables + * + * Attempts to reserve objects of different types out of order will fail. + */ +typedef struct { + void* workspace; + void* workspaceEnd; + + void* objectEnd; + void* tableEnd; + void* tableValidEnd; + void* allocStart; + + BYTE allocFailed; + int workspaceOversizedDuration; + ZSTD_cwksp_alloc_phase_e phase; + ZSTD_cwksp_static_alloc_e isStatic; +} ZSTD_cwksp; + +/*-************************************* +* Functions +***************************************/ + +MEM_STATIC size_t ZSTD_cwksp_available_space(ZSTD_cwksp* ws); + +MEM_STATIC void ZSTD_cwksp_assert_internal_consistency(ZSTD_cwksp* ws) { + (void)ws; + assert(ws->workspace <= ws->objectEnd); + assert(ws->objectEnd <= ws->tableEnd); + assert(ws->objectEnd <= ws->tableValidEnd); + assert(ws->tableEnd <= ws->allocStart); + assert(ws->tableValidEnd <= ws->allocStart); + assert(ws->allocStart <= ws->workspaceEnd); +} + +/** + * Align must be a power of 2. + */ +MEM_STATIC size_t ZSTD_cwksp_align(size_t size, size_t const align) { + size_t const mask = align - 1; + assert((align & mask) == 0); + return (size + mask) & ~mask; +} + +/** + * Use this to determine how much space in the workspace we will consume to + * allocate this object. (Normally it should be exactly the size of the object, + * but under special conditions, like ASAN, where we pad each object, it might + * be larger.) + * + * Since tables aren't currently redzoned, you don't need to call through this + * to figure out how much space you need for the matchState tables. Everything + * else is though. + * + * Do not use for sizing aligned buffers. Instead, use ZSTD_cwksp_aligned_alloc_size(). + */ +MEM_STATIC size_t ZSTD_cwksp_alloc_size(size_t size) { + if (size == 0) + return 0; +#if ZSTD_ADDRESS_SANITIZER && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE) + return size + 2 * ZSTD_CWKSP_ASAN_REDZONE_SIZE; +#else + return size; +#endif +} + +/** + * Returns an adjusted alloc size that is the nearest larger multiple of 64 bytes. + * Used to determine the number of bytes required for a given "aligned". + */ +MEM_STATIC size_t ZSTD_cwksp_aligned_alloc_size(size_t size) { + return ZSTD_cwksp_alloc_size(ZSTD_cwksp_align(size, ZSTD_CWKSP_ALIGNMENT_BYTES)); +} + +/** + * Returns the amount of additional space the cwksp must allocate + * for internal purposes (currently only alignment). + */ +MEM_STATIC size_t ZSTD_cwksp_slack_space_required(void) { + /* For alignment, the wksp will always allocate an additional n_1=[1, 64] bytes + * to align the beginning of tables section, as well as another n_2=[0, 63] bytes + * to align the beginning of the aligned section. + * + * n_1 + n_2 == 64 bytes if the cwksp is freshly allocated, due to tables and + * aligneds being sized in multiples of 64 bytes. + */ + size_t const slackSpace = ZSTD_CWKSP_ALIGNMENT_BYTES; + return slackSpace; +} + + +/** + * Return the number of additional bytes required to align a pointer to the given number of bytes. + * alignBytes must be a power of two. + */ +MEM_STATIC size_t ZSTD_cwksp_bytes_to_align_ptr(void* ptr, const size_t alignBytes) { + size_t const alignBytesMask = alignBytes - 1; + size_t const bytes = (alignBytes - ((size_t)ptr & (alignBytesMask))) & alignBytesMask; + assert((alignBytes & alignBytesMask) == 0); + assert(bytes != ZSTD_CWKSP_ALIGNMENT_BYTES); + return bytes; +} + +/** + * Internal function. Do not use directly. + * Reserves the given number of bytes within the aligned/buffer segment of the wksp, + * which counts from the end of the wksp (as opposed to the object/table segment). + * + * Returns a pointer to the beginning of that space. + */ +MEM_STATIC void* +ZSTD_cwksp_reserve_internal_buffer_space(ZSTD_cwksp* ws, size_t const bytes) +{ + void* const alloc = (BYTE*)ws->allocStart - bytes; + void* const bottom = ws->tableEnd; + DEBUGLOG(5, "cwksp: reserving %p %zd bytes, %zd bytes remaining", + alloc, bytes, ZSTD_cwksp_available_space(ws) - bytes); + ZSTD_cwksp_assert_internal_consistency(ws); + assert(alloc >= bottom); + if (alloc < bottom) { + DEBUGLOG(4, "cwksp: alloc failed!"); + ws->allocFailed = 1; + return NULL; + } + /* the area is reserved from the end of wksp. + * If it overlaps with tableValidEnd, it voids guarantees on values' range */ + if (alloc < ws->tableValidEnd) { + ws->tableValidEnd = alloc; + } + ws->allocStart = alloc; + return alloc; +} + +/** + * Moves the cwksp to the next phase, and does any necessary allocations. + * cwksp initialization must necessarily go through each phase in order. + * Returns a 0 on success, or zstd error + */ +MEM_STATIC size_t +ZSTD_cwksp_internal_advance_phase(ZSTD_cwksp* ws, ZSTD_cwksp_alloc_phase_e phase) +{ + assert(phase >= ws->phase); + if (phase > ws->phase) { + /* Going from allocating objects to allocating buffers */ + if (ws->phase < ZSTD_cwksp_alloc_buffers && + phase >= ZSTD_cwksp_alloc_buffers) { + ws->tableValidEnd = ws->objectEnd; + } + + /* Going from allocating buffers to allocating aligneds/tables */ + if (ws->phase < ZSTD_cwksp_alloc_aligned && + phase >= ZSTD_cwksp_alloc_aligned) { + { /* Align the start of the "aligned" to 64 bytes. Use [1, 64] bytes. */ + size_t const bytesToAlign = + ZSTD_CWKSP_ALIGNMENT_BYTES - ZSTD_cwksp_bytes_to_align_ptr(ws->allocStart, ZSTD_CWKSP_ALIGNMENT_BYTES); + DEBUGLOG(5, "reserving aligned alignment addtl space: %zu", bytesToAlign); + ZSTD_STATIC_ASSERT((ZSTD_CWKSP_ALIGNMENT_BYTES & (ZSTD_CWKSP_ALIGNMENT_BYTES - 1)) == 0); /* power of 2 */ + RETURN_ERROR_IF(!ZSTD_cwksp_reserve_internal_buffer_space(ws, bytesToAlign), + memory_allocation, "aligned phase - alignment initial allocation failed!"); + } + { /* Align the start of the tables to 64 bytes. Use [0, 63] bytes */ + void* const alloc = ws->objectEnd; + size_t const bytesToAlign = ZSTD_cwksp_bytes_to_align_ptr(alloc, ZSTD_CWKSP_ALIGNMENT_BYTES); + void* const objectEnd = (BYTE*)alloc + bytesToAlign; + DEBUGLOG(5, "reserving table alignment addtl space: %zu", bytesToAlign); + RETURN_ERROR_IF(objectEnd > ws->workspaceEnd, memory_allocation, + "table phase - alignment initial allocation failed!"); + ws->objectEnd = objectEnd; + ws->tableEnd = objectEnd; /* table area starts being empty */ + if (ws->tableValidEnd < ws->tableEnd) { + ws->tableValidEnd = ws->tableEnd; + } } } + ws->phase = phase; + ZSTD_cwksp_assert_internal_consistency(ws); + } + return 0; +} + +/** + * Returns whether this object/buffer/etc was allocated in this workspace. + */ +MEM_STATIC int ZSTD_cwksp_owns_buffer(const ZSTD_cwksp* ws, const void* ptr) +{ + return (ptr != NULL) && (ws->workspace <= ptr) && (ptr <= ws->workspaceEnd); +} + +/** + * Internal function. Do not use directly. + */ +MEM_STATIC void* +ZSTD_cwksp_reserve_internal(ZSTD_cwksp* ws, size_t bytes, ZSTD_cwksp_alloc_phase_e phase) +{ + void* alloc; + if (ZSTD_isError(ZSTD_cwksp_internal_advance_phase(ws, phase)) || bytes == 0) { + return NULL; + } + +#if ZSTD_ADDRESS_SANITIZER && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE) + /* over-reserve space */ + bytes += 2 * ZSTD_CWKSP_ASAN_REDZONE_SIZE; +#endif + + alloc = ZSTD_cwksp_reserve_internal_buffer_space(ws, bytes); + +#if ZSTD_ADDRESS_SANITIZER && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE) + /* Move alloc so there's ZSTD_CWKSP_ASAN_REDZONE_SIZE unused space on + * either size. */ + if (alloc) { + alloc = (BYTE *)alloc + ZSTD_CWKSP_ASAN_REDZONE_SIZE; + if (ws->isStatic == ZSTD_cwksp_dynamic_alloc) { + __asan_unpoison_memory_region(alloc, bytes); + } + } +#endif + + return alloc; +} + +/** + * Reserves and returns unaligned memory. + */ +MEM_STATIC BYTE* ZSTD_cwksp_reserve_buffer(ZSTD_cwksp* ws, size_t bytes) +{ + return (BYTE*)ZSTD_cwksp_reserve_internal(ws, bytes, ZSTD_cwksp_alloc_buffers); +} + +/** + * Reserves and returns memory sized on and aligned on ZSTD_CWKSP_ALIGNMENT_BYTES (64 bytes). + */ +MEM_STATIC void* ZSTD_cwksp_reserve_aligned(ZSTD_cwksp* ws, size_t bytes) +{ + void* ptr = ZSTD_cwksp_reserve_internal(ws, ZSTD_cwksp_align(bytes, ZSTD_CWKSP_ALIGNMENT_BYTES), + ZSTD_cwksp_alloc_aligned); + assert(((size_t)ptr & (ZSTD_CWKSP_ALIGNMENT_BYTES-1))== 0); + return ptr; +} + +/** + * Aligned on 64 bytes. These buffers have the special property that + * their values remain constrained, allowing us to re-use them without + * memset()-ing them. + */ +MEM_STATIC void* ZSTD_cwksp_reserve_table(ZSTD_cwksp* ws, size_t bytes) +{ + const ZSTD_cwksp_alloc_phase_e phase = ZSTD_cwksp_alloc_aligned; + void* alloc; + void* end; + void* top; + + if (ZSTD_isError(ZSTD_cwksp_internal_advance_phase(ws, phase))) { + return NULL; + } + alloc = ws->tableEnd; + end = (BYTE *)alloc + bytes; + top = ws->allocStart; + + DEBUGLOG(5, "cwksp: reserving %p table %zd bytes, %zd bytes remaining", + alloc, bytes, ZSTD_cwksp_available_space(ws) - bytes); + assert((bytes & (sizeof(U32)-1)) == 0); + ZSTD_cwksp_assert_internal_consistency(ws); + assert(end <= top); + if (end > top) { + DEBUGLOG(4, "cwksp: table alloc failed!"); + ws->allocFailed = 1; + return NULL; + } + ws->tableEnd = end; + +#if ZSTD_ADDRESS_SANITIZER && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE) + if (ws->isStatic == ZSTD_cwksp_dynamic_alloc) { + __asan_unpoison_memory_region(alloc, bytes); + } +#endif + + assert((bytes & (ZSTD_CWKSP_ALIGNMENT_BYTES-1)) == 0); + assert(((size_t)alloc & (ZSTD_CWKSP_ALIGNMENT_BYTES-1))== 0); + return alloc; +} + +/** + * Aligned on sizeof(void*). + * Note : should happen only once, at workspace first initialization + */ +MEM_STATIC void* ZSTD_cwksp_reserve_object(ZSTD_cwksp* ws, size_t bytes) +{ + size_t const roundedBytes = ZSTD_cwksp_align(bytes, sizeof(void*)); + void* alloc = ws->objectEnd; + void* end = (BYTE*)alloc + roundedBytes; + +#if ZSTD_ADDRESS_SANITIZER && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE) + /* over-reserve space */ + end = (BYTE *)end + 2 * ZSTD_CWKSP_ASAN_REDZONE_SIZE; +#endif + + DEBUGLOG(4, + "cwksp: reserving %p object %zd bytes (rounded to %zd), %zd bytes remaining", + alloc, bytes, roundedBytes, ZSTD_cwksp_available_space(ws) - roundedBytes); + assert((size_t)alloc % ZSTD_ALIGNOF(void*) == 0); + assert(bytes % ZSTD_ALIGNOF(void*) == 0); + ZSTD_cwksp_assert_internal_consistency(ws); + /* we must be in the first phase, no advance is possible */ + if (ws->phase != ZSTD_cwksp_alloc_objects || end > ws->workspaceEnd) { + DEBUGLOG(3, "cwksp: object alloc failed!"); + ws->allocFailed = 1; + return NULL; + } + ws->objectEnd = end; + ws->tableEnd = end; + ws->tableValidEnd = end; + +#if ZSTD_ADDRESS_SANITIZER && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE) + /* Move alloc so there's ZSTD_CWKSP_ASAN_REDZONE_SIZE unused space on + * either size. */ + alloc = (BYTE*)alloc + ZSTD_CWKSP_ASAN_REDZONE_SIZE; + if (ws->isStatic == ZSTD_cwksp_dynamic_alloc) { + __asan_unpoison_memory_region(alloc, bytes); + } +#endif + + return alloc; +} + +MEM_STATIC void ZSTD_cwksp_mark_tables_dirty(ZSTD_cwksp* ws) +{ + DEBUGLOG(4, "cwksp: ZSTD_cwksp_mark_tables_dirty"); + +#if ZSTD_MEMORY_SANITIZER && !defined (ZSTD_MSAN_DONT_POISON_WORKSPACE) + /* To validate that the table re-use logic is sound, and that we don't + * access table space that we haven't cleaned, we re-"poison" the table + * space every time we mark it dirty. */ + { + size_t size = (BYTE*)ws->tableValidEnd - (BYTE*)ws->objectEnd; + assert(__msan_test_shadow(ws->objectEnd, size) == -1); + __msan_poison(ws->objectEnd, size); + } +#endif + + assert(ws->tableValidEnd >= ws->objectEnd); + assert(ws->tableValidEnd <= ws->allocStart); + ws->tableValidEnd = ws->objectEnd; + ZSTD_cwksp_assert_internal_consistency(ws); +} + +MEM_STATIC void ZSTD_cwksp_mark_tables_clean(ZSTD_cwksp* ws) { + DEBUGLOG(4, "cwksp: ZSTD_cwksp_mark_tables_clean"); + assert(ws->tableValidEnd >= ws->objectEnd); + assert(ws->tableValidEnd <= ws->allocStart); + if (ws->tableValidEnd < ws->tableEnd) { + ws->tableValidEnd = ws->tableEnd; + } + ZSTD_cwksp_assert_internal_consistency(ws); +} + +/** + * Zero the part of the allocated tables not already marked clean. + */ +MEM_STATIC void ZSTD_cwksp_clean_tables(ZSTD_cwksp* ws) { + DEBUGLOG(4, "cwksp: ZSTD_cwksp_clean_tables"); + assert(ws->tableValidEnd >= ws->objectEnd); + assert(ws->tableValidEnd <= ws->allocStart); + if (ws->tableValidEnd < ws->tableEnd) { + ZSTD_memset(ws->tableValidEnd, 0, (BYTE*)ws->tableEnd - (BYTE*)ws->tableValidEnd); + } + ZSTD_cwksp_mark_tables_clean(ws); +} + +/** + * Invalidates table allocations. + * All other allocations remain valid. + */ +MEM_STATIC void ZSTD_cwksp_clear_tables(ZSTD_cwksp* ws) { + DEBUGLOG(4, "cwksp: clearing tables!"); + +#if ZSTD_ADDRESS_SANITIZER && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE) + /* We don't do this when the workspace is statically allocated, because + * when that is the case, we have no capability to hook into the end of the + * workspace's lifecycle to unpoison the memory. + */ + if (ws->isStatic == ZSTD_cwksp_dynamic_alloc) { + size_t size = (BYTE*)ws->tableValidEnd - (BYTE*)ws->objectEnd; + __asan_poison_memory_region(ws->objectEnd, size); + } +#endif + + ws->tableEnd = ws->objectEnd; + ZSTD_cwksp_assert_internal_consistency(ws); +} + +/** + * Invalidates all buffer, aligned, and table allocations. + * Object allocations remain valid. + */ +MEM_STATIC void ZSTD_cwksp_clear(ZSTD_cwksp* ws) { + DEBUGLOG(4, "cwksp: clearing!"); + +#if ZSTD_MEMORY_SANITIZER && !defined (ZSTD_MSAN_DONT_POISON_WORKSPACE) + /* To validate that the context re-use logic is sound, and that we don't + * access stuff that this compression hasn't initialized, we re-"poison" + * the workspace (or at least the non-static, non-table parts of it) + * every time we start a new compression. */ + { + size_t size = (BYTE*)ws->workspaceEnd - (BYTE*)ws->tableValidEnd; + __msan_poison(ws->tableValidEnd, size); + } +#endif + +#if ZSTD_ADDRESS_SANITIZER && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE) + /* We don't do this when the workspace is statically allocated, because + * when that is the case, we have no capability to hook into the end of the + * workspace's lifecycle to unpoison the memory. + */ + if (ws->isStatic == ZSTD_cwksp_dynamic_alloc) { + size_t size = (BYTE*)ws->workspaceEnd - (BYTE*)ws->objectEnd; + __asan_poison_memory_region(ws->objectEnd, size); + } +#endif + + ws->tableEnd = ws->objectEnd; + ws->allocStart = ws->workspaceEnd; + ws->allocFailed = 0; + if (ws->phase > ZSTD_cwksp_alloc_buffers) { + ws->phase = ZSTD_cwksp_alloc_buffers; + } + ZSTD_cwksp_assert_internal_consistency(ws); +} + +/** + * The provided workspace takes ownership of the buffer [start, start+size). + * Any existing values in the workspace are ignored (the previously managed + * buffer, if present, must be separately freed). + */ +MEM_STATIC void ZSTD_cwksp_init(ZSTD_cwksp* ws, void* start, size_t size, ZSTD_cwksp_static_alloc_e isStatic) { + DEBUGLOG(4, "cwksp: init'ing workspace with %zd bytes", size); + assert(((size_t)start & (sizeof(void*)-1)) == 0); /* ensure correct alignment */ + ws->workspace = start; + ws->workspaceEnd = (BYTE*)start + size; + ws->objectEnd = ws->workspace; + ws->tableValidEnd = ws->objectEnd; + ws->phase = ZSTD_cwksp_alloc_objects; + ws->isStatic = isStatic; + ZSTD_cwksp_clear(ws); + ws->workspaceOversizedDuration = 0; + ZSTD_cwksp_assert_internal_consistency(ws); +} + +MEM_STATIC size_t ZSTD_cwksp_create(ZSTD_cwksp* ws, size_t size, ZSTD_customMem customMem) { + void* workspace = ZSTD_customMalloc(size, customMem); + DEBUGLOG(4, "cwksp: creating new workspace with %zd bytes", size); + RETURN_ERROR_IF(workspace == NULL, memory_allocation, "NULL pointer!"); + ZSTD_cwksp_init(ws, workspace, size, ZSTD_cwksp_dynamic_alloc); + return 0; +} + +MEM_STATIC void ZSTD_cwksp_free(ZSTD_cwksp* ws, ZSTD_customMem customMem) { + void *ptr = ws->workspace; + DEBUGLOG(4, "cwksp: freeing workspace"); + ZSTD_memset(ws, 0, sizeof(ZSTD_cwksp)); + ZSTD_customFree(ptr, customMem); +} + +/** + * Moves the management of a workspace from one cwksp to another. The src cwksp + * is left in an invalid state (src must be re-init()'ed before it's used again). + */ +MEM_STATIC void ZSTD_cwksp_move(ZSTD_cwksp* dst, ZSTD_cwksp* src) { + *dst = *src; + ZSTD_memset(src, 0, sizeof(ZSTD_cwksp)); +} + +MEM_STATIC size_t ZSTD_cwksp_sizeof(const ZSTD_cwksp* ws) { + return (size_t)((BYTE*)ws->workspaceEnd - (BYTE*)ws->workspace); +} + +MEM_STATIC size_t ZSTD_cwksp_used(const ZSTD_cwksp* ws) { + return (size_t)((BYTE*)ws->tableEnd - (BYTE*)ws->workspace) + + (size_t)((BYTE*)ws->workspaceEnd - (BYTE*)ws->allocStart); +} + +MEM_STATIC int ZSTD_cwksp_reserve_failed(const ZSTD_cwksp* ws) { + return ws->allocFailed; +} + +/*-************************************* +* Functions Checking Free Space +***************************************/ + +/* ZSTD_alignmentSpaceWithinBounds() : + * Returns if the estimated space needed for a wksp is within an acceptable limit of the + * actual amount of space used. + */ +MEM_STATIC int ZSTD_cwksp_estimated_space_within_bounds(const ZSTD_cwksp* const ws, + size_t const estimatedSpace, int resizedWorkspace) { + if (resizedWorkspace) { + /* Resized/newly allocated wksp should have exact bounds */ + return ZSTD_cwksp_used(ws) == estimatedSpace; + } else { + /* Due to alignment, when reusing a workspace, we can actually consume 63 fewer or more bytes + * than estimatedSpace. See the comments in zstd_cwksp.h for details. + */ + return (ZSTD_cwksp_used(ws) >= estimatedSpace - 63) && (ZSTD_cwksp_used(ws) <= estimatedSpace + 63); + } +} + + +MEM_STATIC size_t ZSTD_cwksp_available_space(ZSTD_cwksp* ws) { + return (size_t)((BYTE*)ws->allocStart - (BYTE*)ws->tableEnd); +} + +MEM_STATIC int ZSTD_cwksp_check_available(ZSTD_cwksp* ws, size_t additionalNeededSpace) { + return ZSTD_cwksp_available_space(ws) >= additionalNeededSpace; +} + +MEM_STATIC int ZSTD_cwksp_check_too_large(ZSTD_cwksp* ws, size_t additionalNeededSpace) { + return ZSTD_cwksp_check_available( + ws, additionalNeededSpace * ZSTD_WORKSPACETOOLARGE_FACTOR); +} + +MEM_STATIC int ZSTD_cwksp_check_wasteful(ZSTD_cwksp* ws, size_t additionalNeededSpace) { + return ZSTD_cwksp_check_too_large(ws, additionalNeededSpace) + && ws->workspaceOversizedDuration > ZSTD_WORKSPACETOOLARGE_MAXDURATION; +} + +MEM_STATIC void ZSTD_cwksp_bump_oversized_duration( + ZSTD_cwksp* ws, size_t additionalNeededSpace) { + if (ZSTD_cwksp_check_too_large(ws, additionalNeededSpace)) { + ws->workspaceOversizedDuration++; + } else { + ws->workspaceOversizedDuration = 0; + } +} + +#if defined (__cplusplus) +} +#endif + +#endif /* ZSTD_CWKSP_H */ diff --git a/src/blosc2/internal-complibs/zstd-1.5.2/compress/zstd_double_fast.c b/src/blosc2/internal-complibs/zstd-1.5.2/compress/zstd_double_fast.c new file mode 100644 index 0000000..76933de --- /dev/null +++ b/src/blosc2/internal-complibs/zstd-1.5.2/compress/zstd_double_fast.c @@ -0,0 +1,696 @@ +/* + * Copyright (c) Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#include "zstd_compress_internal.h" +#include "zstd_double_fast.h" + + +void ZSTD_fillDoubleHashTable(ZSTD_matchState_t* ms, + void const* end, ZSTD_dictTableLoadMethod_e dtlm) +{ + const ZSTD_compressionParameters* const cParams = &ms->cParams; + U32* const hashLarge = ms->hashTable; + U32 const hBitsL = cParams->hashLog; + U32 const mls = cParams->minMatch; + U32* const hashSmall = ms->chainTable; + U32 const hBitsS = cParams->chainLog; + const BYTE* const base = ms->window.base; + const BYTE* ip = base + ms->nextToUpdate; + const BYTE* const iend = ((const BYTE*)end) - HASH_READ_SIZE; + const U32 fastHashFillStep = 3; + + /* Always insert every fastHashFillStep position into the hash tables. + * Insert the other positions into the large hash table if their entry + * is empty. + */ + for (; ip + fastHashFillStep - 1 <= iend; ip += fastHashFillStep) { + U32 const curr = (U32)(ip - base); + U32 i; + for (i = 0; i < fastHashFillStep; ++i) { + size_t const smHash = ZSTD_hashPtr(ip + i, hBitsS, mls); + size_t const lgHash = ZSTD_hashPtr(ip + i, hBitsL, 8); + if (i == 0) + hashSmall[smHash] = curr + i; + if (i == 0 || hashLarge[lgHash] == 0) + hashLarge[lgHash] = curr + i; + /* Only load extra positions for ZSTD_dtlm_full */ + if (dtlm == ZSTD_dtlm_fast) + break; + } } +} + + +FORCE_INLINE_TEMPLATE +size_t ZSTD_compressBlock_doubleFast_noDict_generic( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize, U32 const mls /* template */) +{ + ZSTD_compressionParameters const* cParams = &ms->cParams; + U32* const hashLong = ms->hashTable; + const U32 hBitsL = cParams->hashLog; + U32* const hashSmall = ms->chainTable; + const U32 hBitsS = cParams->chainLog; + const BYTE* const base = ms->window.base; + const BYTE* const istart = (const BYTE*)src; + const BYTE* anchor = istart; + const U32 endIndex = (U32)((size_t)(istart - base) + srcSize); + /* presumes that, if there is a dictionary, it must be using Attach mode */ + const U32 prefixLowestIndex = ZSTD_getLowestPrefixIndex(ms, endIndex, cParams->windowLog); + const BYTE* const prefixLowest = base + prefixLowestIndex; + const BYTE* const iend = istart + srcSize; + const BYTE* const ilimit = iend - HASH_READ_SIZE; + U32 offset_1=rep[0], offset_2=rep[1]; + U32 offsetSaved = 0; + + size_t mLength; + U32 offset; + U32 curr; + + /* how many positions to search before increasing step size */ + const size_t kStepIncr = 1 << kSearchStrength; + /* the position at which to increment the step size if no match is found */ + const BYTE* nextStep; + size_t step; /* the current step size */ + + size_t hl0; /* the long hash at ip */ + size_t hl1; /* the long hash at ip1 */ + + U32 idxl0; /* the long match index for ip */ + U32 idxl1; /* the long match index for ip1 */ + + const BYTE* matchl0; /* the long match for ip */ + const BYTE* matchs0; /* the short match for ip */ + const BYTE* matchl1; /* the long match for ip1 */ + + const BYTE* ip = istart; /* the current position */ + const BYTE* ip1; /* the next position */ + + DEBUGLOG(5, "ZSTD_compressBlock_doubleFast_noDict_generic"); + + /* init */ + ip += ((ip - prefixLowest) == 0); + { + U32 const current = (U32)(ip - base); + U32 const windowLow = ZSTD_getLowestPrefixIndex(ms, current, cParams->windowLog); + U32 const maxRep = current - windowLow; + if (offset_2 > maxRep) offsetSaved = offset_2, offset_2 = 0; + if (offset_1 > maxRep) offsetSaved = offset_1, offset_1 = 0; + } + + /* Outer Loop: one iteration per match found and stored */ + while (1) { + step = 1; + nextStep = ip + kStepIncr; + ip1 = ip + step; + + if (ip1 > ilimit) { + goto _cleanup; + } + + hl0 = ZSTD_hashPtr(ip, hBitsL, 8); + idxl0 = hashLong[hl0]; + matchl0 = base + idxl0; + + /* Inner Loop: one iteration per search / position */ + do { + const size_t hs0 = ZSTD_hashPtr(ip, hBitsS, mls); + const U32 idxs0 = hashSmall[hs0]; + curr = (U32)(ip-base); + matchs0 = base + idxs0; + + hashLong[hl0] = hashSmall[hs0] = curr; /* update hash tables */ + + /* check noDict repcode */ + if ((offset_1 > 0) & (MEM_read32(ip+1-offset_1) == MEM_read32(ip+1))) { + mLength = ZSTD_count(ip+1+4, ip+1+4-offset_1, iend) + 4; + ip++; + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, STORE_REPCODE_1, mLength); + goto _match_stored; + } + + hl1 = ZSTD_hashPtr(ip1, hBitsL, 8); + + if (idxl0 > prefixLowestIndex) { + /* check prefix long match */ + if (MEM_read64(matchl0) == MEM_read64(ip)) { + mLength = ZSTD_count(ip+8, matchl0+8, iend) + 8; + offset = (U32)(ip-matchl0); + while (((ip>anchor) & (matchl0>prefixLowest)) && (ip[-1] == matchl0[-1])) { ip--; matchl0--; mLength++; } /* catch up */ + goto _match_found; + } + } + + idxl1 = hashLong[hl1]; + matchl1 = base + idxl1; + + if (idxs0 > prefixLowestIndex) { + /* check prefix short match */ + if (MEM_read32(matchs0) == MEM_read32(ip)) { + goto _search_next_long; + } + } + + if (ip1 >= nextStep) { + PREFETCH_L1(ip1 + 64); + PREFETCH_L1(ip1 + 128); + step++; + nextStep += kStepIncr; + } + ip = ip1; + ip1 += step; + + hl0 = hl1; + idxl0 = idxl1; + matchl0 = matchl1; + #if defined(__aarch64__) + PREFETCH_L1(ip+256); + #endif + } while (ip1 <= ilimit); + +_cleanup: + /* save reps for next block */ + rep[0] = offset_1 ? offset_1 : offsetSaved; + rep[1] = offset_2 ? offset_2 : offsetSaved; + + /* Return the last literals size */ + return (size_t)(iend - anchor); + +_search_next_long: + + /* check prefix long +1 match */ + if (idxl1 > prefixLowestIndex) { + if (MEM_read64(matchl1) == MEM_read64(ip1)) { + ip = ip1; + mLength = ZSTD_count(ip+8, matchl1+8, iend) + 8; + offset = (U32)(ip-matchl1); + while (((ip>anchor) & (matchl1>prefixLowest)) && (ip[-1] == matchl1[-1])) { ip--; matchl1--; mLength++; } /* catch up */ + goto _match_found; + } + } + + /* if no long +1 match, explore the short match we found */ + mLength = ZSTD_count(ip+4, matchs0+4, iend) + 4; + offset = (U32)(ip - matchs0); + while (((ip>anchor) & (matchs0>prefixLowest)) && (ip[-1] == matchs0[-1])) { ip--; matchs0--; mLength++; } /* catch up */ + + /* fall-through */ + +_match_found: /* requires ip, offset, mLength */ + offset_2 = offset_1; + offset_1 = offset; + + if (step < 4) { + /* It is unsafe to write this value back to the hashtable when ip1 is + * greater than or equal to the new ip we will have after we're done + * processing this match. Rather than perform that test directly + * (ip1 >= ip + mLength), which costs speed in practice, we do a simpler + * more predictable test. The minmatch even if we take a short match is + * 4 bytes, so as long as step, the distance between ip and ip1 + * (initially) is less than 4, we know ip1 < new ip. */ + hashLong[hl1] = (U32)(ip1 - base); + } + + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, STORE_OFFSET(offset), mLength); + +_match_stored: + /* match found */ + ip += mLength; + anchor = ip; + + if (ip <= ilimit) { + /* Complementary insertion */ + /* done after iLimit test, as candidates could be > iend-8 */ + { U32 const indexToInsert = curr+2; + hashLong[ZSTD_hashPtr(base+indexToInsert, hBitsL, 8)] = indexToInsert; + hashLong[ZSTD_hashPtr(ip-2, hBitsL, 8)] = (U32)(ip-2-base); + hashSmall[ZSTD_hashPtr(base+indexToInsert, hBitsS, mls)] = indexToInsert; + hashSmall[ZSTD_hashPtr(ip-1, hBitsS, mls)] = (U32)(ip-1-base); + } + + /* check immediate repcode */ + while ( (ip <= ilimit) + && ( (offset_2>0) + & (MEM_read32(ip) == MEM_read32(ip - offset_2)) )) { + /* store sequence */ + size_t const rLength = ZSTD_count(ip+4, ip+4-offset_2, iend) + 4; + U32 const tmpOff = offset_2; offset_2 = offset_1; offset_1 = tmpOff; /* swap offset_2 <=> offset_1 */ + hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = (U32)(ip-base); + hashLong[ZSTD_hashPtr(ip, hBitsL, 8)] = (U32)(ip-base); + ZSTD_storeSeq(seqStore, 0, anchor, iend, STORE_REPCODE_1, rLength); + ip += rLength; + anchor = ip; + continue; /* faster when present ... (?) */ + } + } + } +} + + +FORCE_INLINE_TEMPLATE +size_t ZSTD_compressBlock_doubleFast_dictMatchState_generic( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize, + U32 const mls /* template */) +{ + ZSTD_compressionParameters const* cParams = &ms->cParams; + U32* const hashLong = ms->hashTable; + const U32 hBitsL = cParams->hashLog; + U32* const hashSmall = ms->chainTable; + const U32 hBitsS = cParams->chainLog; + const BYTE* const base = ms->window.base; + const BYTE* const istart = (const BYTE*)src; + const BYTE* ip = istart; + const BYTE* anchor = istart; + const U32 endIndex = (U32)((size_t)(istart - base) + srcSize); + /* presumes that, if there is a dictionary, it must be using Attach mode */ + const U32 prefixLowestIndex = ZSTD_getLowestPrefixIndex(ms, endIndex, cParams->windowLog); + const BYTE* const prefixLowest = base + prefixLowestIndex; + const BYTE* const iend = istart + srcSize; + const BYTE* const ilimit = iend - HASH_READ_SIZE; + U32 offset_1=rep[0], offset_2=rep[1]; + U32 offsetSaved = 0; + + const ZSTD_matchState_t* const dms = ms->dictMatchState; + const ZSTD_compressionParameters* const dictCParams = &dms->cParams; + const U32* const dictHashLong = dms->hashTable; + const U32* const dictHashSmall = dms->chainTable; + const U32 dictStartIndex = dms->window.dictLimit; + const BYTE* const dictBase = dms->window.base; + const BYTE* const dictStart = dictBase + dictStartIndex; + const BYTE* const dictEnd = dms->window.nextSrc; + const U32 dictIndexDelta = prefixLowestIndex - (U32)(dictEnd - dictBase); + const U32 dictHBitsL = dictCParams->hashLog; + const U32 dictHBitsS = dictCParams->chainLog; + const U32 dictAndPrefixLength = (U32)((ip - prefixLowest) + (dictEnd - dictStart)); + + DEBUGLOG(5, "ZSTD_compressBlock_doubleFast_dictMatchState_generic"); + + /* if a dictionary is attached, it must be within window range */ + assert(ms->window.dictLimit + (1U << cParams->windowLog) >= endIndex); + + /* init */ + ip += (dictAndPrefixLength == 0); + + /* dictMatchState repCode checks don't currently handle repCode == 0 + * disabling. */ + assert(offset_1 <= dictAndPrefixLength); + assert(offset_2 <= dictAndPrefixLength); + + /* Main Search Loop */ + while (ip < ilimit) { /* < instead of <=, because repcode check at (ip+1) */ + size_t mLength; + U32 offset; + size_t const h2 = ZSTD_hashPtr(ip, hBitsL, 8); + size_t const h = ZSTD_hashPtr(ip, hBitsS, mls); + size_t const dictHL = ZSTD_hashPtr(ip, dictHBitsL, 8); + size_t const dictHS = ZSTD_hashPtr(ip, dictHBitsS, mls); + U32 const curr = (U32)(ip-base); + U32 const matchIndexL = hashLong[h2]; + U32 matchIndexS = hashSmall[h]; + const BYTE* matchLong = base + matchIndexL; + const BYTE* match = base + matchIndexS; + const U32 repIndex = curr + 1 - offset_1; + const BYTE* repMatch = (repIndex < prefixLowestIndex) ? + dictBase + (repIndex - dictIndexDelta) : + base + repIndex; + hashLong[h2] = hashSmall[h] = curr; /* update hash tables */ + + /* check repcode */ + if (((U32)((prefixLowestIndex-1) - repIndex) >= 3 /* intentional underflow */) + && (MEM_read32(repMatch) == MEM_read32(ip+1)) ) { + const BYTE* repMatchEnd = repIndex < prefixLowestIndex ? dictEnd : iend; + mLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, prefixLowest) + 4; + ip++; + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, STORE_REPCODE_1, mLength); + goto _match_stored; + } + + if (matchIndexL > prefixLowestIndex) { + /* check prefix long match */ + if (MEM_read64(matchLong) == MEM_read64(ip)) { + mLength = ZSTD_count(ip+8, matchLong+8, iend) + 8; + offset = (U32)(ip-matchLong); + while (((ip>anchor) & (matchLong>prefixLowest)) && (ip[-1] == matchLong[-1])) { ip--; matchLong--; mLength++; } /* catch up */ + goto _match_found; + } + } else { + /* check dictMatchState long match */ + U32 const dictMatchIndexL = dictHashLong[dictHL]; + const BYTE* dictMatchL = dictBase + dictMatchIndexL; + assert(dictMatchL < dictEnd); + + if (dictMatchL > dictStart && MEM_read64(dictMatchL) == MEM_read64(ip)) { + mLength = ZSTD_count_2segments(ip+8, dictMatchL+8, iend, dictEnd, prefixLowest) + 8; + offset = (U32)(curr - dictMatchIndexL - dictIndexDelta); + while (((ip>anchor) & (dictMatchL>dictStart)) && (ip[-1] == dictMatchL[-1])) { ip--; dictMatchL--; mLength++; } /* catch up */ + goto _match_found; + } } + + if (matchIndexS > prefixLowestIndex) { + /* check prefix short match */ + if (MEM_read32(match) == MEM_read32(ip)) { + goto _search_next_long; + } + } else { + /* check dictMatchState short match */ + U32 const dictMatchIndexS = dictHashSmall[dictHS]; + match = dictBase + dictMatchIndexS; + matchIndexS = dictMatchIndexS + dictIndexDelta; + + if (match > dictStart && MEM_read32(match) == MEM_read32(ip)) { + goto _search_next_long; + } } + + ip += ((ip-anchor) >> kSearchStrength) + 1; +#if defined(__aarch64__) + PREFETCH_L1(ip+256); +#endif + continue; + +_search_next_long: + + { size_t const hl3 = ZSTD_hashPtr(ip+1, hBitsL, 8); + size_t const dictHLNext = ZSTD_hashPtr(ip+1, dictHBitsL, 8); + U32 const matchIndexL3 = hashLong[hl3]; + const BYTE* matchL3 = base + matchIndexL3; + hashLong[hl3] = curr + 1; + + /* check prefix long +1 match */ + if (matchIndexL3 > prefixLowestIndex) { + if (MEM_read64(matchL3) == MEM_read64(ip+1)) { + mLength = ZSTD_count(ip+9, matchL3+8, iend) + 8; + ip++; + offset = (U32)(ip-matchL3); + while (((ip>anchor) & (matchL3>prefixLowest)) && (ip[-1] == matchL3[-1])) { ip--; matchL3--; mLength++; } /* catch up */ + goto _match_found; + } + } else { + /* check dict long +1 match */ + U32 const dictMatchIndexL3 = dictHashLong[dictHLNext]; + const BYTE* dictMatchL3 = dictBase + dictMatchIndexL3; + assert(dictMatchL3 < dictEnd); + if (dictMatchL3 > dictStart && MEM_read64(dictMatchL3) == MEM_read64(ip+1)) { + mLength = ZSTD_count_2segments(ip+1+8, dictMatchL3+8, iend, dictEnd, prefixLowest) + 8; + ip++; + offset = (U32)(curr + 1 - dictMatchIndexL3 - dictIndexDelta); + while (((ip>anchor) & (dictMatchL3>dictStart)) && (ip[-1] == dictMatchL3[-1])) { ip--; dictMatchL3--; mLength++; } /* catch up */ + goto _match_found; + } } } + + /* if no long +1 match, explore the short match we found */ + if (matchIndexS < prefixLowestIndex) { + mLength = ZSTD_count_2segments(ip+4, match+4, iend, dictEnd, prefixLowest) + 4; + offset = (U32)(curr - matchIndexS); + while (((ip>anchor) & (match>dictStart)) && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */ + } else { + mLength = ZSTD_count(ip+4, match+4, iend) + 4; + offset = (U32)(ip - match); + while (((ip>anchor) & (match>prefixLowest)) && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */ + } + +_match_found: + offset_2 = offset_1; + offset_1 = offset; + + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, STORE_OFFSET(offset), mLength); + +_match_stored: + /* match found */ + ip += mLength; + anchor = ip; + + if (ip <= ilimit) { + /* Complementary insertion */ + /* done after iLimit test, as candidates could be > iend-8 */ + { U32 const indexToInsert = curr+2; + hashLong[ZSTD_hashPtr(base+indexToInsert, hBitsL, 8)] = indexToInsert; + hashLong[ZSTD_hashPtr(ip-2, hBitsL, 8)] = (U32)(ip-2-base); + hashSmall[ZSTD_hashPtr(base+indexToInsert, hBitsS, mls)] = indexToInsert; + hashSmall[ZSTD_hashPtr(ip-1, hBitsS, mls)] = (U32)(ip-1-base); + } + + /* check immediate repcode */ + while (ip <= ilimit) { + U32 const current2 = (U32)(ip-base); + U32 const repIndex2 = current2 - offset_2; + const BYTE* repMatch2 = repIndex2 < prefixLowestIndex ? + dictBase + repIndex2 - dictIndexDelta : + base + repIndex2; + if ( ((U32)((prefixLowestIndex-1) - (U32)repIndex2) >= 3 /* intentional overflow */) + && (MEM_read32(repMatch2) == MEM_read32(ip)) ) { + const BYTE* const repEnd2 = repIndex2 < prefixLowestIndex ? dictEnd : iend; + size_t const repLength2 = ZSTD_count_2segments(ip+4, repMatch2+4, iend, repEnd2, prefixLowest) + 4; + U32 tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */ + ZSTD_storeSeq(seqStore, 0, anchor, iend, STORE_REPCODE_1, repLength2); + hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = current2; + hashLong[ZSTD_hashPtr(ip, hBitsL, 8)] = current2; + ip += repLength2; + anchor = ip; + continue; + } + break; + } + } + } /* while (ip < ilimit) */ + + /* save reps for next block */ + rep[0] = offset_1 ? offset_1 : offsetSaved; + rep[1] = offset_2 ? offset_2 : offsetSaved; + + /* Return the last literals size */ + return (size_t)(iend - anchor); +} + +#define ZSTD_GEN_DFAST_FN(dictMode, mls) \ + static size_t ZSTD_compressBlock_doubleFast_##dictMode##_##mls( \ + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], \ + void const* src, size_t srcSize) \ + { \ + return ZSTD_compressBlock_doubleFast_##dictMode##_generic(ms, seqStore, rep, src, srcSize, mls); \ + } + +ZSTD_GEN_DFAST_FN(noDict, 4) +ZSTD_GEN_DFAST_FN(noDict, 5) +ZSTD_GEN_DFAST_FN(noDict, 6) +ZSTD_GEN_DFAST_FN(noDict, 7) + +ZSTD_GEN_DFAST_FN(dictMatchState, 4) +ZSTD_GEN_DFAST_FN(dictMatchState, 5) +ZSTD_GEN_DFAST_FN(dictMatchState, 6) +ZSTD_GEN_DFAST_FN(dictMatchState, 7) + + +size_t ZSTD_compressBlock_doubleFast( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + const U32 mls = ms->cParams.minMatch; + switch(mls) + { + default: /* includes case 3 */ + case 4 : + return ZSTD_compressBlock_doubleFast_noDict_4(ms, seqStore, rep, src, srcSize); + case 5 : + return ZSTD_compressBlock_doubleFast_noDict_5(ms, seqStore, rep, src, srcSize); + case 6 : + return ZSTD_compressBlock_doubleFast_noDict_6(ms, seqStore, rep, src, srcSize); + case 7 : + return ZSTD_compressBlock_doubleFast_noDict_7(ms, seqStore, rep, src, srcSize); + } +} + + +size_t ZSTD_compressBlock_doubleFast_dictMatchState( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + const U32 mls = ms->cParams.minMatch; + switch(mls) + { + default: /* includes case 3 */ + case 4 : + return ZSTD_compressBlock_doubleFast_dictMatchState_4(ms, seqStore, rep, src, srcSize); + case 5 : + return ZSTD_compressBlock_doubleFast_dictMatchState_5(ms, seqStore, rep, src, srcSize); + case 6 : + return ZSTD_compressBlock_doubleFast_dictMatchState_6(ms, seqStore, rep, src, srcSize); + case 7 : + return ZSTD_compressBlock_doubleFast_dictMatchState_7(ms, seqStore, rep, src, srcSize); + } +} + + +static size_t ZSTD_compressBlock_doubleFast_extDict_generic( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize, + U32 const mls /* template */) +{ + ZSTD_compressionParameters const* cParams = &ms->cParams; + U32* const hashLong = ms->hashTable; + U32 const hBitsL = cParams->hashLog; + U32* const hashSmall = ms->chainTable; + U32 const hBitsS = cParams->chainLog; + const BYTE* const istart = (const BYTE*)src; + const BYTE* ip = istart; + const BYTE* anchor = istart; + const BYTE* const iend = istart + srcSize; + const BYTE* const ilimit = iend - 8; + const BYTE* const base = ms->window.base; + const U32 endIndex = (U32)((size_t)(istart - base) + srcSize); + const U32 lowLimit = ZSTD_getLowestMatchIndex(ms, endIndex, cParams->windowLog); + const U32 dictStartIndex = lowLimit; + const U32 dictLimit = ms->window.dictLimit; + const U32 prefixStartIndex = (dictLimit > lowLimit) ? dictLimit : lowLimit; + const BYTE* const prefixStart = base + prefixStartIndex; + const BYTE* const dictBase = ms->window.dictBase; + const BYTE* const dictStart = dictBase + dictStartIndex; + const BYTE* const dictEnd = dictBase + prefixStartIndex; + U32 offset_1=rep[0], offset_2=rep[1]; + + DEBUGLOG(5, "ZSTD_compressBlock_doubleFast_extDict_generic (srcSize=%zu)", srcSize); + + /* if extDict is invalidated due to maxDistance, switch to "regular" variant */ + if (prefixStartIndex == dictStartIndex) + return ZSTD_compressBlock_doubleFast(ms, seqStore, rep, src, srcSize); + + /* Search Loop */ + while (ip < ilimit) { /* < instead of <=, because (ip+1) */ + const size_t hSmall = ZSTD_hashPtr(ip, hBitsS, mls); + const U32 matchIndex = hashSmall[hSmall]; + const BYTE* const matchBase = matchIndex < prefixStartIndex ? dictBase : base; + const BYTE* match = matchBase + matchIndex; + + const size_t hLong = ZSTD_hashPtr(ip, hBitsL, 8); + const U32 matchLongIndex = hashLong[hLong]; + const BYTE* const matchLongBase = matchLongIndex < prefixStartIndex ? dictBase : base; + const BYTE* matchLong = matchLongBase + matchLongIndex; + + const U32 curr = (U32)(ip-base); + const U32 repIndex = curr + 1 - offset_1; /* offset_1 expected <= curr +1 */ + const BYTE* const repBase = repIndex < prefixStartIndex ? dictBase : base; + const BYTE* const repMatch = repBase + repIndex; + size_t mLength; + hashSmall[hSmall] = hashLong[hLong] = curr; /* update hash table */ + + if ((((U32)((prefixStartIndex-1) - repIndex) >= 3) /* intentional underflow : ensure repIndex doesn't overlap dict + prefix */ + & (offset_1 <= curr+1 - dictStartIndex)) /* note: we are searching at curr+1 */ + && (MEM_read32(repMatch) == MEM_read32(ip+1)) ) { + const BYTE* repMatchEnd = repIndex < prefixStartIndex ? dictEnd : iend; + mLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, prefixStart) + 4; + ip++; + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, STORE_REPCODE_1, mLength); + } else { + if ((matchLongIndex > dictStartIndex) && (MEM_read64(matchLong) == MEM_read64(ip))) { + const BYTE* const matchEnd = matchLongIndex < prefixStartIndex ? dictEnd : iend; + const BYTE* const lowMatchPtr = matchLongIndex < prefixStartIndex ? dictStart : prefixStart; + U32 offset; + mLength = ZSTD_count_2segments(ip+8, matchLong+8, iend, matchEnd, prefixStart) + 8; + offset = curr - matchLongIndex; + while (((ip>anchor) & (matchLong>lowMatchPtr)) && (ip[-1] == matchLong[-1])) { ip--; matchLong--; mLength++; } /* catch up */ + offset_2 = offset_1; + offset_1 = offset; + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, STORE_OFFSET(offset), mLength); + + } else if ((matchIndex > dictStartIndex) && (MEM_read32(match) == MEM_read32(ip))) { + size_t const h3 = ZSTD_hashPtr(ip+1, hBitsL, 8); + U32 const matchIndex3 = hashLong[h3]; + const BYTE* const match3Base = matchIndex3 < prefixStartIndex ? dictBase : base; + const BYTE* match3 = match3Base + matchIndex3; + U32 offset; + hashLong[h3] = curr + 1; + if ( (matchIndex3 > dictStartIndex) && (MEM_read64(match3) == MEM_read64(ip+1)) ) { + const BYTE* const matchEnd = matchIndex3 < prefixStartIndex ? dictEnd : iend; + const BYTE* const lowMatchPtr = matchIndex3 < prefixStartIndex ? dictStart : prefixStart; + mLength = ZSTD_count_2segments(ip+9, match3+8, iend, matchEnd, prefixStart) + 8; + ip++; + offset = curr+1 - matchIndex3; + while (((ip>anchor) & (match3>lowMatchPtr)) && (ip[-1] == match3[-1])) { ip--; match3--; mLength++; } /* catch up */ + } else { + const BYTE* const matchEnd = matchIndex < prefixStartIndex ? dictEnd : iend; + const BYTE* const lowMatchPtr = matchIndex < prefixStartIndex ? dictStart : prefixStart; + mLength = ZSTD_count_2segments(ip+4, match+4, iend, matchEnd, prefixStart) + 4; + offset = curr - matchIndex; + while (((ip>anchor) & (match>lowMatchPtr)) && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */ + } + offset_2 = offset_1; + offset_1 = offset; + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, STORE_OFFSET(offset), mLength); + + } else { + ip += ((ip-anchor) >> kSearchStrength) + 1; + continue; + } } + + /* move to next sequence start */ + ip += mLength; + anchor = ip; + + if (ip <= ilimit) { + /* Complementary insertion */ + /* done after iLimit test, as candidates could be > iend-8 */ + { U32 const indexToInsert = curr+2; + hashLong[ZSTD_hashPtr(base+indexToInsert, hBitsL, 8)] = indexToInsert; + hashLong[ZSTD_hashPtr(ip-2, hBitsL, 8)] = (U32)(ip-2-base); + hashSmall[ZSTD_hashPtr(base+indexToInsert, hBitsS, mls)] = indexToInsert; + hashSmall[ZSTD_hashPtr(ip-1, hBitsS, mls)] = (U32)(ip-1-base); + } + + /* check immediate repcode */ + while (ip <= ilimit) { + U32 const current2 = (U32)(ip-base); + U32 const repIndex2 = current2 - offset_2; + const BYTE* repMatch2 = repIndex2 < prefixStartIndex ? dictBase + repIndex2 : base + repIndex2; + if ( (((U32)((prefixStartIndex-1) - repIndex2) >= 3) /* intentional overflow : ensure repIndex2 doesn't overlap dict + prefix */ + & (offset_2 <= current2 - dictStartIndex)) + && (MEM_read32(repMatch2) == MEM_read32(ip)) ) { + const BYTE* const repEnd2 = repIndex2 < prefixStartIndex ? dictEnd : iend; + size_t const repLength2 = ZSTD_count_2segments(ip+4, repMatch2+4, iend, repEnd2, prefixStart) + 4; + U32 const tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */ + ZSTD_storeSeq(seqStore, 0, anchor, iend, STORE_REPCODE_1, repLength2); + hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = current2; + hashLong[ZSTD_hashPtr(ip, hBitsL, 8)] = current2; + ip += repLength2; + anchor = ip; + continue; + } + break; + } } } + + /* save reps for next block */ + rep[0] = offset_1; + rep[1] = offset_2; + + /* Return the last literals size */ + return (size_t)(iend - anchor); +} + +ZSTD_GEN_DFAST_FN(extDict, 4) +ZSTD_GEN_DFAST_FN(extDict, 5) +ZSTD_GEN_DFAST_FN(extDict, 6) +ZSTD_GEN_DFAST_FN(extDict, 7) + +size_t ZSTD_compressBlock_doubleFast_extDict( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + U32 const mls = ms->cParams.minMatch; + switch(mls) + { + default: /* includes case 3 */ + case 4 : + return ZSTD_compressBlock_doubleFast_extDict_4(ms, seqStore, rep, src, srcSize); + case 5 : + return ZSTD_compressBlock_doubleFast_extDict_5(ms, seqStore, rep, src, srcSize); + case 6 : + return ZSTD_compressBlock_doubleFast_extDict_6(ms, seqStore, rep, src, srcSize); + case 7 : + return ZSTD_compressBlock_doubleFast_extDict_7(ms, seqStore, rep, src, srcSize); + } +} diff --git a/src/blosc2/internal-complibs/zstd-1.5.2/compress/zstd_double_fast.h b/src/blosc2/internal-complibs/zstd-1.5.2/compress/zstd_double_fast.h new file mode 100644 index 0000000..e16b7b0 --- /dev/null +++ b/src/blosc2/internal-complibs/zstd-1.5.2/compress/zstd_double_fast.h @@ -0,0 +1,38 @@ +/* + * Copyright (c) Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_DOUBLE_FAST_H +#define ZSTD_DOUBLE_FAST_H + +#if defined (__cplusplus) +extern "C" { +#endif + +#include "../common/mem.h" /* U32 */ +#include "zstd_compress_internal.h" /* ZSTD_CCtx, size_t */ + +void ZSTD_fillDoubleHashTable(ZSTD_matchState_t* ms, + void const* end, ZSTD_dictTableLoadMethod_e dtlm); +size_t ZSTD_compressBlock_doubleFast( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_doubleFast_dictMatchState( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_doubleFast_extDict( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); + + +#if defined (__cplusplus) +} +#endif + +#endif /* ZSTD_DOUBLE_FAST_H */ diff --git a/src/blosc2/internal-complibs/zstd-1.5.2/compress/zstd_fast.c b/src/blosc2/internal-complibs/zstd-1.5.2/compress/zstd_fast.c new file mode 100644 index 0000000..802fc31 --- /dev/null +++ b/src/blosc2/internal-complibs/zstd-1.5.2/compress/zstd_fast.c @@ -0,0 +1,675 @@ +/* + * Copyright (c) Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#include "zstd_compress_internal.h" /* ZSTD_hashPtr, ZSTD_count, ZSTD_storeSeq */ +#include "zstd_fast.h" + + +void ZSTD_fillHashTable(ZSTD_matchState_t* ms, + const void* const end, + ZSTD_dictTableLoadMethod_e dtlm) +{ + const ZSTD_compressionParameters* const cParams = &ms->cParams; + U32* const hashTable = ms->hashTable; + U32 const hBits = cParams->hashLog; + U32 const mls = cParams->minMatch; + const BYTE* const base = ms->window.base; + const BYTE* ip = base + ms->nextToUpdate; + const BYTE* const iend = ((const BYTE*)end) - HASH_READ_SIZE; + const U32 fastHashFillStep = 3; + + /* Always insert every fastHashFillStep position into the hash table. + * Insert the other positions if their hash entry is empty. + */ + for ( ; ip + fastHashFillStep < iend + 2; ip += fastHashFillStep) { + U32 const curr = (U32)(ip - base); + size_t const hash0 = ZSTD_hashPtr(ip, hBits, mls); + hashTable[hash0] = curr; + if (dtlm == ZSTD_dtlm_fast) continue; + /* Only load extra positions for ZSTD_dtlm_full */ + { U32 p; + for (p = 1; p < fastHashFillStep; ++p) { + size_t const hash = ZSTD_hashPtr(ip + p, hBits, mls); + if (hashTable[hash] == 0) { /* not yet filled */ + hashTable[hash] = curr + p; + } } } } +} + + +/** + * If you squint hard enough (and ignore repcodes), the search operation at any + * given position is broken into 4 stages: + * + * 1. Hash (map position to hash value via input read) + * 2. Lookup (map hash val to index via hashtable read) + * 3. Load (map index to value at that position via input read) + * 4. Compare + * + * Each of these steps involves a memory read at an address which is computed + * from the previous step. This means these steps must be sequenced and their + * latencies are cumulative. + * + * Rather than do 1->2->3->4 sequentially for a single position before moving + * onto the next, this implementation interleaves these operations across the + * next few positions: + * + * R = Repcode Read & Compare + * H = Hash + * T = Table Lookup + * M = Match Read & Compare + * + * Pos | Time --> + * ----+------------------- + * N | ... M + * N+1 | ... TM + * N+2 | R H T M + * N+3 | H TM + * N+4 | R H T M + * N+5 | H ... + * N+6 | R ... + * + * This is very much analogous to the pipelining of execution in a CPU. And just + * like a CPU, we have to dump the pipeline when we find a match (i.e., take a + * branch). + * + * When this happens, we throw away our current state, and do the following prep + * to re-enter the loop: + * + * Pos | Time --> + * ----+------------------- + * N | H T + * N+1 | H + * + * This is also the work we do at the beginning to enter the loop initially. + */ +FORCE_INLINE_TEMPLATE size_t +ZSTD_compressBlock_fast_noDict_generic( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize, + U32 const mls, U32 const hasStep) +{ + const ZSTD_compressionParameters* const cParams = &ms->cParams; + U32* const hashTable = ms->hashTable; + U32 const hlog = cParams->hashLog; + /* support stepSize of 0 */ + size_t const stepSize = hasStep ? (cParams->targetLength + !(cParams->targetLength) + 1) : 2; + const BYTE* const base = ms->window.base; + const BYTE* const istart = (const BYTE*)src; + const U32 endIndex = (U32)((size_t)(istart - base) + srcSize); + const U32 prefixStartIndex = ZSTD_getLowestPrefixIndex(ms, endIndex, cParams->windowLog); + const BYTE* const prefixStart = base + prefixStartIndex; + const BYTE* const iend = istart + srcSize; + const BYTE* const ilimit = iend - HASH_READ_SIZE; + + const BYTE* anchor = istart; + const BYTE* ip0 = istart; + const BYTE* ip1; + const BYTE* ip2; + const BYTE* ip3; + U32 current0; + + U32 rep_offset1 = rep[0]; + U32 rep_offset2 = rep[1]; + U32 offsetSaved = 0; + + size_t hash0; /* hash for ip0 */ + size_t hash1; /* hash for ip1 */ + U32 idx; /* match idx for ip0 */ + U32 mval; /* src value at match idx */ + + U32 offcode; + const BYTE* match0; + size_t mLength; + + /* ip0 and ip1 are always adjacent. The targetLength skipping and + * uncompressibility acceleration is applied to every other position, + * matching the behavior of #1562. step therefore represents the gap + * between pairs of positions, from ip0 to ip2 or ip1 to ip3. */ + size_t step; + const BYTE* nextStep; + const size_t kStepIncr = (1 << (kSearchStrength - 1)); + + DEBUGLOG(5, "ZSTD_compressBlock_fast_generic"); + ip0 += (ip0 == prefixStart); + { U32 const curr = (U32)(ip0 - base); + U32 const windowLow = ZSTD_getLowestPrefixIndex(ms, curr, cParams->windowLog); + U32 const maxRep = curr - windowLow; + if (rep_offset2 > maxRep) offsetSaved = rep_offset2, rep_offset2 = 0; + if (rep_offset1 > maxRep) offsetSaved = rep_offset1, rep_offset1 = 0; + } + + /* start each op */ +_start: /* Requires: ip0 */ + + step = stepSize; + nextStep = ip0 + kStepIncr; + + /* calculate positions, ip0 - anchor == 0, so we skip step calc */ + ip1 = ip0 + 1; + ip2 = ip0 + step; + ip3 = ip2 + 1; + + if (ip3 >= ilimit) { + goto _cleanup; + } + + hash0 = ZSTD_hashPtr(ip0, hlog, mls); + hash1 = ZSTD_hashPtr(ip1, hlog, mls); + + idx = hashTable[hash0]; + + do { + /* load repcode match for ip[2]*/ + const U32 rval = MEM_read32(ip2 - rep_offset1); + + /* write back hash table entry */ + current0 = (U32)(ip0 - base); + hashTable[hash0] = current0; + + /* check repcode at ip[2] */ + if ((MEM_read32(ip2) == rval) & (rep_offset1 > 0)) { + ip0 = ip2; + match0 = ip0 - rep_offset1; + mLength = ip0[-1] == match0[-1]; + ip0 -= mLength; + match0 -= mLength; + offcode = STORE_REPCODE_1; + mLength += 4; + goto _match; + } + + /* load match for ip[0] */ + if (idx >= prefixStartIndex) { + mval = MEM_read32(base + idx); + } else { + mval = MEM_read32(ip0) ^ 1; /* guaranteed to not match. */ + } + + /* check match at ip[0] */ + if (MEM_read32(ip0) == mval) { + /* found a match! */ + goto _offset; + } + + /* lookup ip[1] */ + idx = hashTable[hash1]; + + /* hash ip[2] */ + hash0 = hash1; + hash1 = ZSTD_hashPtr(ip2, hlog, mls); + + /* advance to next positions */ + ip0 = ip1; + ip1 = ip2; + ip2 = ip3; + + /* write back hash table entry */ + current0 = (U32)(ip0 - base); + hashTable[hash0] = current0; + + /* load match for ip[0] */ + if (idx >= prefixStartIndex) { + mval = MEM_read32(base + idx); + } else { + mval = MEM_read32(ip0) ^ 1; /* guaranteed to not match. */ + } + + /* check match at ip[0] */ + if (MEM_read32(ip0) == mval) { + /* found a match! */ + goto _offset; + } + + /* lookup ip[1] */ + idx = hashTable[hash1]; + + /* hash ip[2] */ + hash0 = hash1; + hash1 = ZSTD_hashPtr(ip2, hlog, mls); + + /* advance to next positions */ + ip0 = ip1; + ip1 = ip2; + ip2 = ip0 + step; + ip3 = ip1 + step; + + /* calculate step */ + if (ip2 >= nextStep) { + step++; + PREFETCH_L1(ip1 + 64); + PREFETCH_L1(ip1 + 128); + nextStep += kStepIncr; + } + } while (ip3 < ilimit); + +_cleanup: + /* Note that there are probably still a couple positions we could search. + * However, it seems to be a meaningful performance hit to try to search + * them. So let's not. */ + + /* save reps for next block */ + rep[0] = rep_offset1 ? rep_offset1 : offsetSaved; + rep[1] = rep_offset2 ? rep_offset2 : offsetSaved; + + /* Return the last literals size */ + return (size_t)(iend - anchor); + +_offset: /* Requires: ip0, idx */ + + /* Compute the offset code. */ + match0 = base + idx; + rep_offset2 = rep_offset1; + rep_offset1 = (U32)(ip0-match0); + offcode = STORE_OFFSET(rep_offset1); + mLength = 4; + + /* Count the backwards match length. */ + while (((ip0>anchor) & (match0>prefixStart)) && (ip0[-1] == match0[-1])) { + ip0--; + match0--; + mLength++; + } + +_match: /* Requires: ip0, match0, offcode */ + + /* Count the forward length. */ + mLength += ZSTD_count(ip0 + mLength, match0 + mLength, iend); + + ZSTD_storeSeq(seqStore, (size_t)(ip0 - anchor), anchor, iend, offcode, mLength); + + ip0 += mLength; + anchor = ip0; + + /* write next hash table entry */ + if (ip1 < ip0) { + hashTable[hash1] = (U32)(ip1 - base); + } + + /* Fill table and check for immediate repcode. */ + if (ip0 <= ilimit) { + /* Fill Table */ + assert(base+current0+2 > istart); /* check base overflow */ + hashTable[ZSTD_hashPtr(base+current0+2, hlog, mls)] = current0+2; /* here because current+2 could be > iend-8 */ + hashTable[ZSTD_hashPtr(ip0-2, hlog, mls)] = (U32)(ip0-2-base); + + if (rep_offset2 > 0) { /* rep_offset2==0 means rep_offset2 is invalidated */ + while ( (ip0 <= ilimit) && (MEM_read32(ip0) == MEM_read32(ip0 - rep_offset2)) ) { + /* store sequence */ + size_t const rLength = ZSTD_count(ip0+4, ip0+4-rep_offset2, iend) + 4; + { U32 const tmpOff = rep_offset2; rep_offset2 = rep_offset1; rep_offset1 = tmpOff; } /* swap rep_offset2 <=> rep_offset1 */ + hashTable[ZSTD_hashPtr(ip0, hlog, mls)] = (U32)(ip0-base); + ip0 += rLength; + ZSTD_storeSeq(seqStore, 0 /*litLen*/, anchor, iend, STORE_REPCODE_1, rLength); + anchor = ip0; + continue; /* faster when present (confirmed on gcc-8) ... (?) */ + } } } + + goto _start; +} + +#define ZSTD_GEN_FAST_FN(dictMode, mls, step) \ + static size_t ZSTD_compressBlock_fast_##dictMode##_##mls##_##step( \ + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], \ + void const* src, size_t srcSize) \ + { \ + return ZSTD_compressBlock_fast_##dictMode##_generic(ms, seqStore, rep, src, srcSize, mls, step); \ + } + +ZSTD_GEN_FAST_FN(noDict, 4, 1) +ZSTD_GEN_FAST_FN(noDict, 5, 1) +ZSTD_GEN_FAST_FN(noDict, 6, 1) +ZSTD_GEN_FAST_FN(noDict, 7, 1) + +ZSTD_GEN_FAST_FN(noDict, 4, 0) +ZSTD_GEN_FAST_FN(noDict, 5, 0) +ZSTD_GEN_FAST_FN(noDict, 6, 0) +ZSTD_GEN_FAST_FN(noDict, 7, 0) + +size_t ZSTD_compressBlock_fast( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + U32 const mls = ms->cParams.minMatch; + assert(ms->dictMatchState == NULL); + if (ms->cParams.targetLength > 1) { + switch(mls) + { + default: /* includes case 3 */ + case 4 : + return ZSTD_compressBlock_fast_noDict_4_1(ms, seqStore, rep, src, srcSize); + case 5 : + return ZSTD_compressBlock_fast_noDict_5_1(ms, seqStore, rep, src, srcSize); + case 6 : + return ZSTD_compressBlock_fast_noDict_6_1(ms, seqStore, rep, src, srcSize); + case 7 : + return ZSTD_compressBlock_fast_noDict_7_1(ms, seqStore, rep, src, srcSize); + } + } else { + switch(mls) + { + default: /* includes case 3 */ + case 4 : + return ZSTD_compressBlock_fast_noDict_4_0(ms, seqStore, rep, src, srcSize); + case 5 : + return ZSTD_compressBlock_fast_noDict_5_0(ms, seqStore, rep, src, srcSize); + case 6 : + return ZSTD_compressBlock_fast_noDict_6_0(ms, seqStore, rep, src, srcSize); + case 7 : + return ZSTD_compressBlock_fast_noDict_7_0(ms, seqStore, rep, src, srcSize); + } + + } +} + +FORCE_INLINE_TEMPLATE +size_t ZSTD_compressBlock_fast_dictMatchState_generic( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize, U32 const mls, U32 const hasStep) +{ + const ZSTD_compressionParameters* const cParams = &ms->cParams; + U32* const hashTable = ms->hashTable; + U32 const hlog = cParams->hashLog; + /* support stepSize of 0 */ + U32 const stepSize = cParams->targetLength + !(cParams->targetLength); + const BYTE* const base = ms->window.base; + const BYTE* const istart = (const BYTE*)src; + const BYTE* ip = istart; + const BYTE* anchor = istart; + const U32 prefixStartIndex = ms->window.dictLimit; + const BYTE* const prefixStart = base + prefixStartIndex; + const BYTE* const iend = istart + srcSize; + const BYTE* const ilimit = iend - HASH_READ_SIZE; + U32 offset_1=rep[0], offset_2=rep[1]; + U32 offsetSaved = 0; + + const ZSTD_matchState_t* const dms = ms->dictMatchState; + const ZSTD_compressionParameters* const dictCParams = &dms->cParams ; + const U32* const dictHashTable = dms->hashTable; + const U32 dictStartIndex = dms->window.dictLimit; + const BYTE* const dictBase = dms->window.base; + const BYTE* const dictStart = dictBase + dictStartIndex; + const BYTE* const dictEnd = dms->window.nextSrc; + const U32 dictIndexDelta = prefixStartIndex - (U32)(dictEnd - dictBase); + const U32 dictAndPrefixLength = (U32)(ip - prefixStart + dictEnd - dictStart); + const U32 dictHLog = dictCParams->hashLog; + + /* if a dictionary is still attached, it necessarily means that + * it is within window size. So we just check it. */ + const U32 maxDistance = 1U << cParams->windowLog; + const U32 endIndex = (U32)((size_t)(ip - base) + srcSize); + assert(endIndex - prefixStartIndex <= maxDistance); + (void)maxDistance; (void)endIndex; /* these variables are not used when assert() is disabled */ + + (void)hasStep; /* not currently specialized on whether it's accelerated */ + + /* ensure there will be no underflow + * when translating a dict index into a local index */ + assert(prefixStartIndex >= (U32)(dictEnd - dictBase)); + + /* init */ + DEBUGLOG(5, "ZSTD_compressBlock_fast_dictMatchState_generic"); + ip += (dictAndPrefixLength == 0); + /* dictMatchState repCode checks don't currently handle repCode == 0 + * disabling. */ + assert(offset_1 <= dictAndPrefixLength); + assert(offset_2 <= dictAndPrefixLength); + + /* Main Search Loop */ + while (ip < ilimit) { /* < instead of <=, because repcode check at (ip+1) */ + size_t mLength; + size_t const h = ZSTD_hashPtr(ip, hlog, mls); + U32 const curr = (U32)(ip-base); + U32 const matchIndex = hashTable[h]; + const BYTE* match = base + matchIndex; + const U32 repIndex = curr + 1 - offset_1; + const BYTE* repMatch = (repIndex < prefixStartIndex) ? + dictBase + (repIndex - dictIndexDelta) : + base + repIndex; + hashTable[h] = curr; /* update hash table */ + + if ( ((U32)((prefixStartIndex-1) - repIndex) >= 3) /* intentional underflow : ensure repIndex isn't overlapping dict + prefix */ + && (MEM_read32(repMatch) == MEM_read32(ip+1)) ) { + const BYTE* const repMatchEnd = repIndex < prefixStartIndex ? dictEnd : iend; + mLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, prefixStart) + 4; + ip++; + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, STORE_REPCODE_1, mLength); + } else if ( (matchIndex <= prefixStartIndex) ) { + size_t const dictHash = ZSTD_hashPtr(ip, dictHLog, mls); + U32 const dictMatchIndex = dictHashTable[dictHash]; + const BYTE* dictMatch = dictBase + dictMatchIndex; + if (dictMatchIndex <= dictStartIndex || + MEM_read32(dictMatch) != MEM_read32(ip)) { + assert(stepSize >= 1); + ip += ((ip-anchor) >> kSearchStrength) + stepSize; + continue; + } else { + /* found a dict match */ + U32 const offset = (U32)(curr-dictMatchIndex-dictIndexDelta); + mLength = ZSTD_count_2segments(ip+4, dictMatch+4, iend, dictEnd, prefixStart) + 4; + while (((ip>anchor) & (dictMatch>dictStart)) + && (ip[-1] == dictMatch[-1])) { + ip--; dictMatch--; mLength++; + } /* catch up */ + offset_2 = offset_1; + offset_1 = offset; + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, STORE_OFFSET(offset), mLength); + } + } else if (MEM_read32(match) != MEM_read32(ip)) { + /* it's not a match, and we're not going to check the dictionary */ + assert(stepSize >= 1); + ip += ((ip-anchor) >> kSearchStrength) + stepSize; + continue; + } else { + /* found a regular match */ + U32 const offset = (U32)(ip-match); + mLength = ZSTD_count(ip+4, match+4, iend) + 4; + while (((ip>anchor) & (match>prefixStart)) + && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */ + offset_2 = offset_1; + offset_1 = offset; + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, STORE_OFFSET(offset), mLength); + } + + /* match found */ + ip += mLength; + anchor = ip; + + if (ip <= ilimit) { + /* Fill Table */ + assert(base+curr+2 > istart); /* check base overflow */ + hashTable[ZSTD_hashPtr(base+curr+2, hlog, mls)] = curr+2; /* here because curr+2 could be > iend-8 */ + hashTable[ZSTD_hashPtr(ip-2, hlog, mls)] = (U32)(ip-2-base); + + /* check immediate repcode */ + while (ip <= ilimit) { + U32 const current2 = (U32)(ip-base); + U32 const repIndex2 = current2 - offset_2; + const BYTE* repMatch2 = repIndex2 < prefixStartIndex ? + dictBase - dictIndexDelta + repIndex2 : + base + repIndex2; + if ( ((U32)((prefixStartIndex-1) - (U32)repIndex2) >= 3 /* intentional overflow */) + && (MEM_read32(repMatch2) == MEM_read32(ip)) ) { + const BYTE* const repEnd2 = repIndex2 < prefixStartIndex ? dictEnd : iend; + size_t const repLength2 = ZSTD_count_2segments(ip+4, repMatch2+4, iend, repEnd2, prefixStart) + 4; + U32 tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */ + ZSTD_storeSeq(seqStore, 0, anchor, iend, STORE_REPCODE_1, repLength2); + hashTable[ZSTD_hashPtr(ip, hlog, mls)] = current2; + ip += repLength2; + anchor = ip; + continue; + } + break; + } + } + } + + /* save reps for next block */ + rep[0] = offset_1 ? offset_1 : offsetSaved; + rep[1] = offset_2 ? offset_2 : offsetSaved; + + /* Return the last literals size */ + return (size_t)(iend - anchor); +} + + +ZSTD_GEN_FAST_FN(dictMatchState, 4, 0) +ZSTD_GEN_FAST_FN(dictMatchState, 5, 0) +ZSTD_GEN_FAST_FN(dictMatchState, 6, 0) +ZSTD_GEN_FAST_FN(dictMatchState, 7, 0) + +size_t ZSTD_compressBlock_fast_dictMatchState( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + U32 const mls = ms->cParams.minMatch; + assert(ms->dictMatchState != NULL); + switch(mls) + { + default: /* includes case 3 */ + case 4 : + return ZSTD_compressBlock_fast_dictMatchState_4_0(ms, seqStore, rep, src, srcSize); + case 5 : + return ZSTD_compressBlock_fast_dictMatchState_5_0(ms, seqStore, rep, src, srcSize); + case 6 : + return ZSTD_compressBlock_fast_dictMatchState_6_0(ms, seqStore, rep, src, srcSize); + case 7 : + return ZSTD_compressBlock_fast_dictMatchState_7_0(ms, seqStore, rep, src, srcSize); + } +} + + +static size_t ZSTD_compressBlock_fast_extDict_generic( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize, U32 const mls, U32 const hasStep) +{ + const ZSTD_compressionParameters* const cParams = &ms->cParams; + U32* const hashTable = ms->hashTable; + U32 const hlog = cParams->hashLog; + /* support stepSize of 0 */ + U32 const stepSize = cParams->targetLength + !(cParams->targetLength); + const BYTE* const base = ms->window.base; + const BYTE* const dictBase = ms->window.dictBase; + const BYTE* const istart = (const BYTE*)src; + const BYTE* ip = istart; + const BYTE* anchor = istart; + const U32 endIndex = (U32)((size_t)(istart - base) + srcSize); + const U32 lowLimit = ZSTD_getLowestMatchIndex(ms, endIndex, cParams->windowLog); + const U32 dictStartIndex = lowLimit; + const BYTE* const dictStart = dictBase + dictStartIndex; + const U32 dictLimit = ms->window.dictLimit; + const U32 prefixStartIndex = dictLimit < lowLimit ? lowLimit : dictLimit; + const BYTE* const prefixStart = base + prefixStartIndex; + const BYTE* const dictEnd = dictBase + prefixStartIndex; + const BYTE* const iend = istart + srcSize; + const BYTE* const ilimit = iend - 8; + U32 offset_1=rep[0], offset_2=rep[1]; + + (void)hasStep; /* not currently specialized on whether it's accelerated */ + + DEBUGLOG(5, "ZSTD_compressBlock_fast_extDict_generic (offset_1=%u)", offset_1); + + /* switch to "regular" variant if extDict is invalidated due to maxDistance */ + if (prefixStartIndex == dictStartIndex) + return ZSTD_compressBlock_fast(ms, seqStore, rep, src, srcSize); + + /* Search Loop */ + while (ip < ilimit) { /* < instead of <=, because (ip+1) */ + const size_t h = ZSTD_hashPtr(ip, hlog, mls); + const U32 matchIndex = hashTable[h]; + const BYTE* const matchBase = matchIndex < prefixStartIndex ? dictBase : base; + const BYTE* match = matchBase + matchIndex; + const U32 curr = (U32)(ip-base); + const U32 repIndex = curr + 1 - offset_1; + const BYTE* const repBase = repIndex < prefixStartIndex ? dictBase : base; + const BYTE* const repMatch = repBase + repIndex; + hashTable[h] = curr; /* update hash table */ + DEBUGLOG(7, "offset_1 = %u , curr = %u", offset_1, curr); + + if ( ( ((U32)((prefixStartIndex-1) - repIndex) >= 3) /* intentional underflow */ + & (offset_1 <= curr+1 - dictStartIndex) ) /* note: we are searching at curr+1 */ + && (MEM_read32(repMatch) == MEM_read32(ip+1)) ) { + const BYTE* const repMatchEnd = repIndex < prefixStartIndex ? dictEnd : iend; + size_t const rLength = ZSTD_count_2segments(ip+1 +4, repMatch +4, iend, repMatchEnd, prefixStart) + 4; + ip++; + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, STORE_REPCODE_1, rLength); + ip += rLength; + anchor = ip; + } else { + if ( (matchIndex < dictStartIndex) || + (MEM_read32(match) != MEM_read32(ip)) ) { + assert(stepSize >= 1); + ip += ((ip-anchor) >> kSearchStrength) + stepSize; + continue; + } + { const BYTE* const matchEnd = matchIndex < prefixStartIndex ? dictEnd : iend; + const BYTE* const lowMatchPtr = matchIndex < prefixStartIndex ? dictStart : prefixStart; + U32 const offset = curr - matchIndex; + size_t mLength = ZSTD_count_2segments(ip+4, match+4, iend, matchEnd, prefixStart) + 4; + while (((ip>anchor) & (match>lowMatchPtr)) && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */ + offset_2 = offset_1; offset_1 = offset; /* update offset history */ + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, STORE_OFFSET(offset), mLength); + ip += mLength; + anchor = ip; + } } + + if (ip <= ilimit) { + /* Fill Table */ + hashTable[ZSTD_hashPtr(base+curr+2, hlog, mls)] = curr+2; + hashTable[ZSTD_hashPtr(ip-2, hlog, mls)] = (U32)(ip-2-base); + /* check immediate repcode */ + while (ip <= ilimit) { + U32 const current2 = (U32)(ip-base); + U32 const repIndex2 = current2 - offset_2; + const BYTE* const repMatch2 = repIndex2 < prefixStartIndex ? dictBase + repIndex2 : base + repIndex2; + if ( (((U32)((prefixStartIndex-1) - repIndex2) >= 3) & (offset_2 <= curr - dictStartIndex)) /* intentional overflow */ + && (MEM_read32(repMatch2) == MEM_read32(ip)) ) { + const BYTE* const repEnd2 = repIndex2 < prefixStartIndex ? dictEnd : iend; + size_t const repLength2 = ZSTD_count_2segments(ip+4, repMatch2+4, iend, repEnd2, prefixStart) + 4; + { U32 const tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; } /* swap offset_2 <=> offset_1 */ + ZSTD_storeSeq(seqStore, 0 /*litlen*/, anchor, iend, STORE_REPCODE_1, repLength2); + hashTable[ZSTD_hashPtr(ip, hlog, mls)] = current2; + ip += repLength2; + anchor = ip; + continue; + } + break; + } } } + + /* save reps for next block */ + rep[0] = offset_1; + rep[1] = offset_2; + + /* Return the last literals size */ + return (size_t)(iend - anchor); +} + +ZSTD_GEN_FAST_FN(extDict, 4, 0) +ZSTD_GEN_FAST_FN(extDict, 5, 0) +ZSTD_GEN_FAST_FN(extDict, 6, 0) +ZSTD_GEN_FAST_FN(extDict, 7, 0) + +size_t ZSTD_compressBlock_fast_extDict( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + U32 const mls = ms->cParams.minMatch; + switch(mls) + { + default: /* includes case 3 */ + case 4 : + return ZSTD_compressBlock_fast_extDict_4_0(ms, seqStore, rep, src, srcSize); + case 5 : + return ZSTD_compressBlock_fast_extDict_5_0(ms, seqStore, rep, src, srcSize); + case 6 : + return ZSTD_compressBlock_fast_extDict_6_0(ms, seqStore, rep, src, srcSize); + case 7 : + return ZSTD_compressBlock_fast_extDict_7_0(ms, seqStore, rep, src, srcSize); + } +} diff --git a/src/blosc2/internal-complibs/zstd-1.5.2/compress/zstd_fast.h b/src/blosc2/internal-complibs/zstd-1.5.2/compress/zstd_fast.h new file mode 100644 index 0000000..0d4a0c1 --- /dev/null +++ b/src/blosc2/internal-complibs/zstd-1.5.2/compress/zstd_fast.h @@ -0,0 +1,37 @@ +/* + * Copyright (c) Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_FAST_H +#define ZSTD_FAST_H + +#if defined (__cplusplus) +extern "C" { +#endif + +#include "../common/mem.h" /* U32 */ +#include "zstd_compress_internal.h" + +void ZSTD_fillHashTable(ZSTD_matchState_t* ms, + void const* end, ZSTD_dictTableLoadMethod_e dtlm); +size_t ZSTD_compressBlock_fast( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_fast_dictMatchState( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_fast_extDict( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); + +#if defined (__cplusplus) +} +#endif + +#endif /* ZSTD_FAST_H */ diff --git a/src/blosc2/internal-complibs/zstd-1.5.2/compress/zstd_lazy.c b/src/blosc2/internal-complibs/zstd-1.5.2/compress/zstd_lazy.c new file mode 100644 index 0000000..2e38dcb --- /dev/null +++ b/src/blosc2/internal-complibs/zstd-1.5.2/compress/zstd_lazy.c @@ -0,0 +1,2104 @@ +/* + * Copyright (c) Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#include "zstd_compress_internal.h" +#include "zstd_lazy.h" + + +/*-************************************* +* Binary Tree search +***************************************/ + +static void +ZSTD_updateDUBT(ZSTD_matchState_t* ms, + const BYTE* ip, const BYTE* iend, + U32 mls) +{ + const ZSTD_compressionParameters* const cParams = &ms->cParams; + U32* const hashTable = ms->hashTable; + U32 const hashLog = cParams->hashLog; + + U32* const bt = ms->chainTable; + U32 const btLog = cParams->chainLog - 1; + U32 const btMask = (1 << btLog) - 1; + + const BYTE* const base = ms->window.base; + U32 const target = (U32)(ip - base); + U32 idx = ms->nextToUpdate; + + if (idx != target) + DEBUGLOG(7, "ZSTD_updateDUBT, from %u to %u (dictLimit:%u)", + idx, target, ms->window.dictLimit); + assert(ip + 8 <= iend); /* condition for ZSTD_hashPtr */ + (void)iend; + + assert(idx >= ms->window.dictLimit); /* condition for valid base+idx */ + for ( ; idx < target ; idx++) { + size_t const h = ZSTD_hashPtr(base + idx, hashLog, mls); /* assumption : ip + 8 <= iend */ + U32 const matchIndex = hashTable[h]; + + U32* const nextCandidatePtr = bt + 2*(idx&btMask); + U32* const sortMarkPtr = nextCandidatePtr + 1; + + DEBUGLOG(8, "ZSTD_updateDUBT: insert %u", idx); + hashTable[h] = idx; /* Update Hash Table */ + *nextCandidatePtr = matchIndex; /* update BT like a chain */ + *sortMarkPtr = ZSTD_DUBT_UNSORTED_MARK; + } + ms->nextToUpdate = target; +} + + +/** ZSTD_insertDUBT1() : + * sort one already inserted but unsorted position + * assumption : curr >= btlow == (curr - btmask) + * doesn't fail */ +static void +ZSTD_insertDUBT1(const ZSTD_matchState_t* ms, + U32 curr, const BYTE* inputEnd, + U32 nbCompares, U32 btLow, + const ZSTD_dictMode_e dictMode) +{ + const ZSTD_compressionParameters* const cParams = &ms->cParams; + U32* const bt = ms->chainTable; + U32 const btLog = cParams->chainLog - 1; + U32 const btMask = (1 << btLog) - 1; + size_t commonLengthSmaller=0, commonLengthLarger=0; + const BYTE* const base = ms->window.base; + const BYTE* const dictBase = ms->window.dictBase; + const U32 dictLimit = ms->window.dictLimit; + const BYTE* const ip = (curr>=dictLimit) ? base + curr : dictBase + curr; + const BYTE* const iend = (curr>=dictLimit) ? inputEnd : dictBase + dictLimit; + const BYTE* const dictEnd = dictBase + dictLimit; + const BYTE* const prefixStart = base + dictLimit; + const BYTE* match; + U32* smallerPtr = bt + 2*(curr&btMask); + U32* largerPtr = smallerPtr + 1; + U32 matchIndex = *smallerPtr; /* this candidate is unsorted : next sorted candidate is reached through *smallerPtr, while *largerPtr contains previous unsorted candidate (which is already saved and can be overwritten) */ + U32 dummy32; /* to be nullified at the end */ + U32 const windowValid = ms->window.lowLimit; + U32 const maxDistance = 1U << cParams->windowLog; + U32 const windowLow = (curr - windowValid > maxDistance) ? curr - maxDistance : windowValid; + + + DEBUGLOG(8, "ZSTD_insertDUBT1(%u) (dictLimit=%u, lowLimit=%u)", + curr, dictLimit, windowLow); + assert(curr >= btLow); + assert(ip < iend); /* condition for ZSTD_count */ + + for (; nbCompares && (matchIndex > windowLow); --nbCompares) { + U32* const nextPtr = bt + 2*(matchIndex & btMask); + size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */ + assert(matchIndex < curr); + /* note : all candidates are now supposed sorted, + * but it's still possible to have nextPtr[1] == ZSTD_DUBT_UNSORTED_MARK + * when a real index has the same value as ZSTD_DUBT_UNSORTED_MARK */ + + if ( (dictMode != ZSTD_extDict) + || (matchIndex+matchLength >= dictLimit) /* both in current segment*/ + || (curr < dictLimit) /* both in extDict */) { + const BYTE* const mBase = ( (dictMode != ZSTD_extDict) + || (matchIndex+matchLength >= dictLimit)) ? + base : dictBase; + assert( (matchIndex+matchLength >= dictLimit) /* might be wrong if extDict is incorrectly set to 0 */ + || (curr < dictLimit) ); + match = mBase + matchIndex; + matchLength += ZSTD_count(ip+matchLength, match+matchLength, iend); + } else { + match = dictBase + matchIndex; + matchLength += ZSTD_count_2segments(ip+matchLength, match+matchLength, iend, dictEnd, prefixStart); + if (matchIndex+matchLength >= dictLimit) + match = base + matchIndex; /* preparation for next read of match[matchLength] */ + } + + DEBUGLOG(8, "ZSTD_insertDUBT1: comparing %u with %u : found %u common bytes ", + curr, matchIndex, (U32)matchLength); + + if (ip+matchLength == iend) { /* equal : no way to know if inf or sup */ + break; /* drop , to guarantee consistency ; miss a bit of compression, but other solutions can corrupt tree */ + } + + if (match[matchLength] < ip[matchLength]) { /* necessarily within buffer */ + /* match is smaller than current */ + *smallerPtr = matchIndex; /* update smaller idx */ + commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */ + if (matchIndex <= btLow) { smallerPtr=&dummy32; break; } /* beyond tree size, stop searching */ + DEBUGLOG(8, "ZSTD_insertDUBT1: %u (>btLow=%u) is smaller : next => %u", + matchIndex, btLow, nextPtr[1]); + smallerPtr = nextPtr+1; /* new "candidate" => larger than match, which was smaller than target */ + matchIndex = nextPtr[1]; /* new matchIndex, larger than previous and closer to current */ + } else { + /* match is larger than current */ + *largerPtr = matchIndex; + commonLengthLarger = matchLength; + if (matchIndex <= btLow) { largerPtr=&dummy32; break; } /* beyond tree size, stop searching */ + DEBUGLOG(8, "ZSTD_insertDUBT1: %u (>btLow=%u) is larger => %u", + matchIndex, btLow, nextPtr[0]); + largerPtr = nextPtr; + matchIndex = nextPtr[0]; + } } + + *smallerPtr = *largerPtr = 0; +} + + +static size_t +ZSTD_DUBT_findBetterDictMatch ( + const ZSTD_matchState_t* ms, + const BYTE* const ip, const BYTE* const iend, + size_t* offsetPtr, + size_t bestLength, + U32 nbCompares, + U32 const mls, + const ZSTD_dictMode_e dictMode) +{ + const ZSTD_matchState_t * const dms = ms->dictMatchState; + const ZSTD_compressionParameters* const dmsCParams = &dms->cParams; + const U32 * const dictHashTable = dms->hashTable; + U32 const hashLog = dmsCParams->hashLog; + size_t const h = ZSTD_hashPtr(ip, hashLog, mls); + U32 dictMatchIndex = dictHashTable[h]; + + const BYTE* const base = ms->window.base; + const BYTE* const prefixStart = base + ms->window.dictLimit; + U32 const curr = (U32)(ip-base); + const BYTE* const dictBase = dms->window.base; + const BYTE* const dictEnd = dms->window.nextSrc; + U32 const dictHighLimit = (U32)(dms->window.nextSrc - dms->window.base); + U32 const dictLowLimit = dms->window.lowLimit; + U32 const dictIndexDelta = ms->window.lowLimit - dictHighLimit; + + U32* const dictBt = dms->chainTable; + U32 const btLog = dmsCParams->chainLog - 1; + U32 const btMask = (1 << btLog) - 1; + U32 const btLow = (btMask >= dictHighLimit - dictLowLimit) ? dictLowLimit : dictHighLimit - btMask; + + size_t commonLengthSmaller=0, commonLengthLarger=0; + + (void)dictMode; + assert(dictMode == ZSTD_dictMatchState); + + for (; nbCompares && (dictMatchIndex > dictLowLimit); --nbCompares) { + U32* const nextPtr = dictBt + 2*(dictMatchIndex & btMask); + size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */ + const BYTE* match = dictBase + dictMatchIndex; + matchLength += ZSTD_count_2segments(ip+matchLength, match+matchLength, iend, dictEnd, prefixStart); + if (dictMatchIndex+matchLength >= dictHighLimit) + match = base + dictMatchIndex + dictIndexDelta; /* to prepare for next usage of match[matchLength] */ + + if (matchLength > bestLength) { + U32 matchIndex = dictMatchIndex + dictIndexDelta; + if ( (4*(int)(matchLength-bestLength)) > (int)(ZSTD_highbit32(curr-matchIndex+1) - ZSTD_highbit32((U32)offsetPtr[0]+1)) ) { + DEBUGLOG(9, "ZSTD_DUBT_findBetterDictMatch(%u) : found better match length %u -> %u and offsetCode %u -> %u (dictMatchIndex %u, matchIndex %u)", + curr, (U32)bestLength, (U32)matchLength, (U32)*offsetPtr, STORE_OFFSET(curr - matchIndex), dictMatchIndex, matchIndex); + bestLength = matchLength, *offsetPtr = STORE_OFFSET(curr - matchIndex); + } + if (ip+matchLength == iend) { /* reached end of input : ip[matchLength] is not valid, no way to know if it's larger or smaller than match */ + break; /* drop, to guarantee consistency (miss a little bit of compression) */ + } + } + + if (match[matchLength] < ip[matchLength]) { + if (dictMatchIndex <= btLow) { break; } /* beyond tree size, stop the search */ + commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */ + dictMatchIndex = nextPtr[1]; /* new matchIndex larger than previous (closer to current) */ + } else { + /* match is larger than current */ + if (dictMatchIndex <= btLow) { break; } /* beyond tree size, stop the search */ + commonLengthLarger = matchLength; + dictMatchIndex = nextPtr[0]; + } + } + + if (bestLength >= MINMATCH) { + U32 const mIndex = curr - (U32)STORED_OFFSET(*offsetPtr); (void)mIndex; + DEBUGLOG(8, "ZSTD_DUBT_findBetterDictMatch(%u) : found match of length %u and offsetCode %u (pos %u)", + curr, (U32)bestLength, (U32)*offsetPtr, mIndex); + } + return bestLength; + +} + + +static size_t +ZSTD_DUBT_findBestMatch(ZSTD_matchState_t* ms, + const BYTE* const ip, const BYTE* const iend, + size_t* offsetPtr, + U32 const mls, + const ZSTD_dictMode_e dictMode) +{ + const ZSTD_compressionParameters* const cParams = &ms->cParams; + U32* const hashTable = ms->hashTable; + U32 const hashLog = cParams->hashLog; + size_t const h = ZSTD_hashPtr(ip, hashLog, mls); + U32 matchIndex = hashTable[h]; + + const BYTE* const base = ms->window.base; + U32 const curr = (U32)(ip-base); + U32 const windowLow = ZSTD_getLowestMatchIndex(ms, curr, cParams->windowLog); + + U32* const bt = ms->chainTable; + U32 const btLog = cParams->chainLog - 1; + U32 const btMask = (1 << btLog) - 1; + U32 const btLow = (btMask >= curr) ? 0 : curr - btMask; + U32 const unsortLimit = MAX(btLow, windowLow); + + U32* nextCandidate = bt + 2*(matchIndex&btMask); + U32* unsortedMark = bt + 2*(matchIndex&btMask) + 1; + U32 nbCompares = 1U << cParams->searchLog; + U32 nbCandidates = nbCompares; + U32 previousCandidate = 0; + + DEBUGLOG(7, "ZSTD_DUBT_findBestMatch (%u) ", curr); + assert(ip <= iend-8); /* required for h calculation */ + assert(dictMode != ZSTD_dedicatedDictSearch); + + /* reach end of unsorted candidates list */ + while ( (matchIndex > unsortLimit) + && (*unsortedMark == ZSTD_DUBT_UNSORTED_MARK) + && (nbCandidates > 1) ) { + DEBUGLOG(8, "ZSTD_DUBT_findBestMatch: candidate %u is unsorted", + matchIndex); + *unsortedMark = previousCandidate; /* the unsortedMark becomes a reversed chain, to move up back to original position */ + previousCandidate = matchIndex; + matchIndex = *nextCandidate; + nextCandidate = bt + 2*(matchIndex&btMask); + unsortedMark = bt + 2*(matchIndex&btMask) + 1; + nbCandidates --; + } + + /* nullify last candidate if it's still unsorted + * simplification, detrimental to compression ratio, beneficial for speed */ + if ( (matchIndex > unsortLimit) + && (*unsortedMark==ZSTD_DUBT_UNSORTED_MARK) ) { + DEBUGLOG(7, "ZSTD_DUBT_findBestMatch: nullify last unsorted candidate %u", + matchIndex); + *nextCandidate = *unsortedMark = 0; + } + + /* batch sort stacked candidates */ + matchIndex = previousCandidate; + while (matchIndex) { /* will end on matchIndex == 0 */ + U32* const nextCandidateIdxPtr = bt + 2*(matchIndex&btMask) + 1; + U32 const nextCandidateIdx = *nextCandidateIdxPtr; + ZSTD_insertDUBT1(ms, matchIndex, iend, + nbCandidates, unsortLimit, dictMode); + matchIndex = nextCandidateIdx; + nbCandidates++; + } + + /* find longest match */ + { size_t commonLengthSmaller = 0, commonLengthLarger = 0; + const BYTE* const dictBase = ms->window.dictBase; + const U32 dictLimit = ms->window.dictLimit; + const BYTE* const dictEnd = dictBase + dictLimit; + const BYTE* const prefixStart = base + dictLimit; + U32* smallerPtr = bt + 2*(curr&btMask); + U32* largerPtr = bt + 2*(curr&btMask) + 1; + U32 matchEndIdx = curr + 8 + 1; + U32 dummy32; /* to be nullified at the end */ + size_t bestLength = 0; + + matchIndex = hashTable[h]; + hashTable[h] = curr; /* Update Hash Table */ + + for (; nbCompares && (matchIndex > windowLow); --nbCompares) { + U32* const nextPtr = bt + 2*(matchIndex & btMask); + size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */ + const BYTE* match; + + if ((dictMode != ZSTD_extDict) || (matchIndex+matchLength >= dictLimit)) { + match = base + matchIndex; + matchLength += ZSTD_count(ip+matchLength, match+matchLength, iend); + } else { + match = dictBase + matchIndex; + matchLength += ZSTD_count_2segments(ip+matchLength, match+matchLength, iend, dictEnd, prefixStart); + if (matchIndex+matchLength >= dictLimit) + match = base + matchIndex; /* to prepare for next usage of match[matchLength] */ + } + + if (matchLength > bestLength) { + if (matchLength > matchEndIdx - matchIndex) + matchEndIdx = matchIndex + (U32)matchLength; + if ( (4*(int)(matchLength-bestLength)) > (int)(ZSTD_highbit32(curr-matchIndex+1) - ZSTD_highbit32((U32)offsetPtr[0]+1)) ) + bestLength = matchLength, *offsetPtr = STORE_OFFSET(curr - matchIndex); + if (ip+matchLength == iend) { /* equal : no way to know if inf or sup */ + if (dictMode == ZSTD_dictMatchState) { + nbCompares = 0; /* in addition to avoiding checking any + * further in this loop, make sure we + * skip checking in the dictionary. */ + } + break; /* drop, to guarantee consistency (miss a little bit of compression) */ + } + } + + if (match[matchLength] < ip[matchLength]) { + /* match is smaller than current */ + *smallerPtr = matchIndex; /* update smaller idx */ + commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */ + if (matchIndex <= btLow) { smallerPtr=&dummy32; break; } /* beyond tree size, stop the search */ + smallerPtr = nextPtr+1; /* new "smaller" => larger of match */ + matchIndex = nextPtr[1]; /* new matchIndex larger than previous (closer to current) */ + } else { + /* match is larger than current */ + *largerPtr = matchIndex; + commonLengthLarger = matchLength; + if (matchIndex <= btLow) { largerPtr=&dummy32; break; } /* beyond tree size, stop the search */ + largerPtr = nextPtr; + matchIndex = nextPtr[0]; + } } + + *smallerPtr = *largerPtr = 0; + + assert(nbCompares <= (1U << ZSTD_SEARCHLOG_MAX)); /* Check we haven't underflowed. */ + if (dictMode == ZSTD_dictMatchState && nbCompares) { + bestLength = ZSTD_DUBT_findBetterDictMatch( + ms, ip, iend, + offsetPtr, bestLength, nbCompares, + mls, dictMode); + } + + assert(matchEndIdx > curr+8); /* ensure nextToUpdate is increased */ + ms->nextToUpdate = matchEndIdx - 8; /* skip repetitive patterns */ + if (bestLength >= MINMATCH) { + U32 const mIndex = curr - (U32)STORED_OFFSET(*offsetPtr); (void)mIndex; + DEBUGLOG(8, "ZSTD_DUBT_findBestMatch(%u) : found match of length %u and offsetCode %u (pos %u)", + curr, (U32)bestLength, (U32)*offsetPtr, mIndex); + } + return bestLength; + } +} + + +/** ZSTD_BtFindBestMatch() : Tree updater, providing best match */ +FORCE_INLINE_TEMPLATE size_t +ZSTD_BtFindBestMatch( ZSTD_matchState_t* ms, + const BYTE* const ip, const BYTE* const iLimit, + size_t* offsetPtr, + const U32 mls /* template */, + const ZSTD_dictMode_e dictMode) +{ + DEBUGLOG(7, "ZSTD_BtFindBestMatch"); + if (ip < ms->window.base + ms->nextToUpdate) return 0; /* skipped area */ + ZSTD_updateDUBT(ms, ip, iLimit, mls); + return ZSTD_DUBT_findBestMatch(ms, ip, iLimit, offsetPtr, mls, dictMode); +} + +/*********************************** +* Dedicated dict search +***********************************/ + +void ZSTD_dedicatedDictSearch_lazy_loadDictionary(ZSTD_matchState_t* ms, const BYTE* const ip) +{ + const BYTE* const base = ms->window.base; + U32 const target = (U32)(ip - base); + U32* const hashTable = ms->hashTable; + U32* const chainTable = ms->chainTable; + U32 const chainSize = 1 << ms->cParams.chainLog; + U32 idx = ms->nextToUpdate; + U32 const minChain = chainSize < target - idx ? target - chainSize : idx; + U32 const bucketSize = 1 << ZSTD_LAZY_DDSS_BUCKET_LOG; + U32 const cacheSize = bucketSize - 1; + U32 const chainAttempts = (1 << ms->cParams.searchLog) - cacheSize; + U32 const chainLimit = chainAttempts > 255 ? 255 : chainAttempts; + + /* We know the hashtable is oversized by a factor of `bucketSize`. + * We are going to temporarily pretend `bucketSize == 1`, keeping only a + * single entry. We will use the rest of the space to construct a temporary + * chaintable. + */ + U32 const hashLog = ms->cParams.hashLog - ZSTD_LAZY_DDSS_BUCKET_LOG; + U32* const tmpHashTable = hashTable; + U32* const tmpChainTable = hashTable + ((size_t)1 << hashLog); + U32 const tmpChainSize = (U32)((1 << ZSTD_LAZY_DDSS_BUCKET_LOG) - 1) << hashLog; + U32 const tmpMinChain = tmpChainSize < target ? target - tmpChainSize : idx; + U32 hashIdx; + + assert(ms->cParams.chainLog <= 24); + assert(ms->cParams.hashLog > ms->cParams.chainLog); + assert(idx != 0); + assert(tmpMinChain <= minChain); + + /* fill conventional hash table and conventional chain table */ + for ( ; idx < target; idx++) { + U32 const h = (U32)ZSTD_hashPtr(base + idx, hashLog, ms->cParams.minMatch); + if (idx >= tmpMinChain) { + tmpChainTable[idx - tmpMinChain] = hashTable[h]; + } + tmpHashTable[h] = idx; + } + + /* sort chains into ddss chain table */ + { + U32 chainPos = 0; + for (hashIdx = 0; hashIdx < (1U << hashLog); hashIdx++) { + U32 count; + U32 countBeyondMinChain = 0; + U32 i = tmpHashTable[hashIdx]; + for (count = 0; i >= tmpMinChain && count < cacheSize; count++) { + /* skip through the chain to the first position that won't be + * in the hash cache bucket */ + if (i < minChain) { + countBeyondMinChain++; + } + i = tmpChainTable[i - tmpMinChain]; + } + if (count == cacheSize) { + for (count = 0; count < chainLimit;) { + if (i < minChain) { + if (!i || ++countBeyondMinChain > cacheSize) { + /* only allow pulling `cacheSize` number of entries + * into the cache or chainTable beyond `minChain`, + * to replace the entries pulled out of the + * chainTable into the cache. This lets us reach + * back further without increasing the total number + * of entries in the chainTable, guaranteeing the + * DDSS chain table will fit into the space + * allocated for the regular one. */ + break; + } + } + chainTable[chainPos++] = i; + count++; + if (i < tmpMinChain) { + break; + } + i = tmpChainTable[i - tmpMinChain]; + } + } else { + count = 0; + } + if (count) { + tmpHashTable[hashIdx] = ((chainPos - count) << 8) + count; + } else { + tmpHashTable[hashIdx] = 0; + } + } + assert(chainPos <= chainSize); /* I believe this is guaranteed... */ + } + + /* move chain pointers into the last entry of each hash bucket */ + for (hashIdx = (1 << hashLog); hashIdx; ) { + U32 const bucketIdx = --hashIdx << ZSTD_LAZY_DDSS_BUCKET_LOG; + U32 const chainPackedPointer = tmpHashTable[hashIdx]; + U32 i; + for (i = 0; i < cacheSize; i++) { + hashTable[bucketIdx + i] = 0; + } + hashTable[bucketIdx + bucketSize - 1] = chainPackedPointer; + } + + /* fill the buckets of the hash table */ + for (idx = ms->nextToUpdate; idx < target; idx++) { + U32 const h = (U32)ZSTD_hashPtr(base + idx, hashLog, ms->cParams.minMatch) + << ZSTD_LAZY_DDSS_BUCKET_LOG; + U32 i; + /* Shift hash cache down 1. */ + for (i = cacheSize - 1; i; i--) + hashTable[h + i] = hashTable[h + i - 1]; + hashTable[h] = idx; + } + + ms->nextToUpdate = target; +} + +/* Returns the longest match length found in the dedicated dict search structure. + * If none are longer than the argument ml, then ml will be returned. + */ +FORCE_INLINE_TEMPLATE +size_t ZSTD_dedicatedDictSearch_lazy_search(size_t* offsetPtr, size_t ml, U32 nbAttempts, + const ZSTD_matchState_t* const dms, + const BYTE* const ip, const BYTE* const iLimit, + const BYTE* const prefixStart, const U32 curr, + const U32 dictLimit, const size_t ddsIdx) { + const U32 ddsLowestIndex = dms->window.dictLimit; + const BYTE* const ddsBase = dms->window.base; + const BYTE* const ddsEnd = dms->window.nextSrc; + const U32 ddsSize = (U32)(ddsEnd - ddsBase); + const U32 ddsIndexDelta = dictLimit - ddsSize; + const U32 bucketSize = (1 << ZSTD_LAZY_DDSS_BUCKET_LOG); + const U32 bucketLimit = nbAttempts < bucketSize - 1 ? nbAttempts : bucketSize - 1; + U32 ddsAttempt; + U32 matchIndex; + + for (ddsAttempt = 0; ddsAttempt < bucketSize - 1; ddsAttempt++) { + PREFETCH_L1(ddsBase + dms->hashTable[ddsIdx + ddsAttempt]); + } + + { + U32 const chainPackedPointer = dms->hashTable[ddsIdx + bucketSize - 1]; + U32 const chainIndex = chainPackedPointer >> 8; + + PREFETCH_L1(&dms->chainTable[chainIndex]); + } + + for (ddsAttempt = 0; ddsAttempt < bucketLimit; ddsAttempt++) { + size_t currentMl=0; + const BYTE* match; + matchIndex = dms->hashTable[ddsIdx + ddsAttempt]; + match = ddsBase + matchIndex; + + if (!matchIndex) { + return ml; + } + + /* guaranteed by table construction */ + (void)ddsLowestIndex; + assert(matchIndex >= ddsLowestIndex); + assert(match+4 <= ddsEnd); + if (MEM_read32(match) == MEM_read32(ip)) { + /* assumption : matchIndex <= dictLimit-4 (by table construction) */ + currentMl = ZSTD_count_2segments(ip+4, match+4, iLimit, ddsEnd, prefixStart) + 4; + } + + /* save best solution */ + if (currentMl > ml) { + ml = currentMl; + *offsetPtr = STORE_OFFSET(curr - (matchIndex + ddsIndexDelta)); + if (ip+currentMl == iLimit) { + /* best possible, avoids read overflow on next attempt */ + return ml; + } + } + } + + { + U32 const chainPackedPointer = dms->hashTable[ddsIdx + bucketSize - 1]; + U32 chainIndex = chainPackedPointer >> 8; + U32 const chainLength = chainPackedPointer & 0xFF; + U32 const chainAttempts = nbAttempts - ddsAttempt; + U32 const chainLimit = chainAttempts > chainLength ? chainLength : chainAttempts; + U32 chainAttempt; + + for (chainAttempt = 0 ; chainAttempt < chainLimit; chainAttempt++) { + PREFETCH_L1(ddsBase + dms->chainTable[chainIndex + chainAttempt]); + } + + for (chainAttempt = 0 ; chainAttempt < chainLimit; chainAttempt++, chainIndex++) { + size_t currentMl=0; + const BYTE* match; + matchIndex = dms->chainTable[chainIndex]; + match = ddsBase + matchIndex; + + /* guaranteed by table construction */ + assert(matchIndex >= ddsLowestIndex); + assert(match+4 <= ddsEnd); + if (MEM_read32(match) == MEM_read32(ip)) { + /* assumption : matchIndex <= dictLimit-4 (by table construction) */ + currentMl = ZSTD_count_2segments(ip+4, match+4, iLimit, ddsEnd, prefixStart) + 4; + } + + /* save best solution */ + if (currentMl > ml) { + ml = currentMl; + *offsetPtr = STORE_OFFSET(curr - (matchIndex + ddsIndexDelta)); + if (ip+currentMl == iLimit) break; /* best possible, avoids read overflow on next attempt */ + } + } + } + return ml; +} + + +/* ********************************* +* Hash Chain +***********************************/ +#define NEXT_IN_CHAIN(d, mask) chainTable[(d) & (mask)] + +/* Update chains up to ip (excluded) + Assumption : always within prefix (i.e. not within extDict) */ +FORCE_INLINE_TEMPLATE U32 ZSTD_insertAndFindFirstIndex_internal( + ZSTD_matchState_t* ms, + const ZSTD_compressionParameters* const cParams, + const BYTE* ip, U32 const mls) +{ + U32* const hashTable = ms->hashTable; + const U32 hashLog = cParams->hashLog; + U32* const chainTable = ms->chainTable; + const U32 chainMask = (1 << cParams->chainLog) - 1; + const BYTE* const base = ms->window.base; + const U32 target = (U32)(ip - base); + U32 idx = ms->nextToUpdate; + + while(idx < target) { /* catch up */ + size_t const h = ZSTD_hashPtr(base+idx, hashLog, mls); + NEXT_IN_CHAIN(idx, chainMask) = hashTable[h]; + hashTable[h] = idx; + idx++; + } + + ms->nextToUpdate = target; + return hashTable[ZSTD_hashPtr(ip, hashLog, mls)]; +} + +U32 ZSTD_insertAndFindFirstIndex(ZSTD_matchState_t* ms, const BYTE* ip) { + const ZSTD_compressionParameters* const cParams = &ms->cParams; + return ZSTD_insertAndFindFirstIndex_internal(ms, cParams, ip, ms->cParams.minMatch); +} + +/* inlining is important to hardwire a hot branch (template emulation) */ +FORCE_INLINE_TEMPLATE +size_t ZSTD_HcFindBestMatch( + ZSTD_matchState_t* ms, + const BYTE* const ip, const BYTE* const iLimit, + size_t* offsetPtr, + const U32 mls, const ZSTD_dictMode_e dictMode) +{ + const ZSTD_compressionParameters* const cParams = &ms->cParams; + U32* const chainTable = ms->chainTable; + const U32 chainSize = (1 << cParams->chainLog); + const U32 chainMask = chainSize-1; + const BYTE* const base = ms->window.base; + const BYTE* const dictBase = ms->window.dictBase; + const U32 dictLimit = ms->window.dictLimit; + const BYTE* const prefixStart = base + dictLimit; + const BYTE* const dictEnd = dictBase + dictLimit; + const U32 curr = (U32)(ip-base); + const U32 maxDistance = 1U << cParams->windowLog; + const U32 lowestValid = ms->window.lowLimit; + const U32 withinMaxDistance = (curr - lowestValid > maxDistance) ? curr - maxDistance : lowestValid; + const U32 isDictionary = (ms->loadedDictEnd != 0); + const U32 lowLimit = isDictionary ? lowestValid : withinMaxDistance; + const U32 minChain = curr > chainSize ? curr - chainSize : 0; + U32 nbAttempts = 1U << cParams->searchLog; + size_t ml=4-1; + + const ZSTD_matchState_t* const dms = ms->dictMatchState; + const U32 ddsHashLog = dictMode == ZSTD_dedicatedDictSearch + ? dms->cParams.hashLog - ZSTD_LAZY_DDSS_BUCKET_LOG : 0; + const size_t ddsIdx = dictMode == ZSTD_dedicatedDictSearch + ? ZSTD_hashPtr(ip, ddsHashLog, mls) << ZSTD_LAZY_DDSS_BUCKET_LOG : 0; + + U32 matchIndex; + + if (dictMode == ZSTD_dedicatedDictSearch) { + const U32* entry = &dms->hashTable[ddsIdx]; + PREFETCH_L1(entry); + } + + /* HC4 match finder */ + matchIndex = ZSTD_insertAndFindFirstIndex_internal(ms, cParams, ip, mls); + + for ( ; (matchIndex>=lowLimit) & (nbAttempts>0) ; nbAttempts--) { + size_t currentMl=0; + if ((dictMode != ZSTD_extDict) || matchIndex >= dictLimit) { + const BYTE* const match = base + matchIndex; + assert(matchIndex >= dictLimit); /* ensures this is true if dictMode != ZSTD_extDict */ + if (match[ml] == ip[ml]) /* potentially better */ + currentMl = ZSTD_count(ip, match, iLimit); + } else { + const BYTE* const match = dictBase + matchIndex; + assert(match+4 <= dictEnd); + if (MEM_read32(match) == MEM_read32(ip)) /* assumption : matchIndex <= dictLimit-4 (by table construction) */ + currentMl = ZSTD_count_2segments(ip+4, match+4, iLimit, dictEnd, prefixStart) + 4; + } + + /* save best solution */ + if (currentMl > ml) { + ml = currentMl; + *offsetPtr = STORE_OFFSET(curr - matchIndex); + if (ip+currentMl == iLimit) break; /* best possible, avoids read overflow on next attempt */ + } + + if (matchIndex <= minChain) break; + matchIndex = NEXT_IN_CHAIN(matchIndex, chainMask); + } + + assert(nbAttempts <= (1U << ZSTD_SEARCHLOG_MAX)); /* Check we haven't underflowed. */ + if (dictMode == ZSTD_dedicatedDictSearch) { + ml = ZSTD_dedicatedDictSearch_lazy_search(offsetPtr, ml, nbAttempts, dms, + ip, iLimit, prefixStart, curr, dictLimit, ddsIdx); + } else if (dictMode == ZSTD_dictMatchState) { + const U32* const dmsChainTable = dms->chainTable; + const U32 dmsChainSize = (1 << dms->cParams.chainLog); + const U32 dmsChainMask = dmsChainSize - 1; + const U32 dmsLowestIndex = dms->window.dictLimit; + const BYTE* const dmsBase = dms->window.base; + const BYTE* const dmsEnd = dms->window.nextSrc; + const U32 dmsSize = (U32)(dmsEnd - dmsBase); + const U32 dmsIndexDelta = dictLimit - dmsSize; + const U32 dmsMinChain = dmsSize > dmsChainSize ? dmsSize - dmsChainSize : 0; + + matchIndex = dms->hashTable[ZSTD_hashPtr(ip, dms->cParams.hashLog, mls)]; + + for ( ; (matchIndex>=dmsLowestIndex) & (nbAttempts>0) ; nbAttempts--) { + size_t currentMl=0; + const BYTE* const match = dmsBase + matchIndex; + assert(match+4 <= dmsEnd); + if (MEM_read32(match) == MEM_read32(ip)) /* assumption : matchIndex <= dictLimit-4 (by table construction) */ + currentMl = ZSTD_count_2segments(ip+4, match+4, iLimit, dmsEnd, prefixStart) + 4; + + /* save best solution */ + if (currentMl > ml) { + ml = currentMl; + assert(curr > matchIndex + dmsIndexDelta); + *offsetPtr = STORE_OFFSET(curr - (matchIndex + dmsIndexDelta)); + if (ip+currentMl == iLimit) break; /* best possible, avoids read overflow on next attempt */ + } + + if (matchIndex <= dmsMinChain) break; + + matchIndex = dmsChainTable[matchIndex & dmsChainMask]; + } + } + + return ml; +} + +/* ********************************* +* (SIMD) Row-based matchfinder +***********************************/ +/* Constants for row-based hash */ +#define ZSTD_ROW_HASH_TAG_OFFSET 16 /* byte offset of hashes in the match state's tagTable from the beginning of a row */ +#define ZSTD_ROW_HASH_TAG_BITS 8 /* nb bits to use for the tag */ +#define ZSTD_ROW_HASH_TAG_MASK ((1u << ZSTD_ROW_HASH_TAG_BITS) - 1) +#define ZSTD_ROW_HASH_MAX_ENTRIES 64 /* absolute maximum number of entries per row, for all configurations */ + +#define ZSTD_ROW_HASH_CACHE_MASK (ZSTD_ROW_HASH_CACHE_SIZE - 1) + +typedef U64 ZSTD_VecMask; /* Clarifies when we are interacting with a U64 representing a mask of matches */ + +/* ZSTD_VecMask_next(): + * Starting from the LSB, returns the idx of the next non-zero bit. + * Basically counting the nb of trailing zeroes. + */ +static U32 ZSTD_VecMask_next(ZSTD_VecMask val) { + assert(val != 0); +# if defined(_MSC_VER) && defined(_WIN64) + if (val != 0) { + unsigned long r; + _BitScanForward64(&r, val); + return (U32)(r); + } else { + /* Should not reach this code path */ + __assume(0); + } +# elif (defined(__GNUC__) && ((__GNUC__ > 3) || ((__GNUC__ == 3) && (__GNUC_MINOR__ >= 4)))) + if (sizeof(size_t) == 4) { + U32 mostSignificantWord = (U32)(val >> 32); + U32 leastSignificantWord = (U32)val; + if (leastSignificantWord == 0) { + return 32 + (U32)__builtin_ctz(mostSignificantWord); + } else { + return (U32)__builtin_ctz(leastSignificantWord); + } + } else { + return (U32)__builtin_ctzll(val); + } +# else + /* Software ctz version: http://aggregate.org/MAGIC/#Trailing%20Zero%20Count + * and: https://stackoverflow.com/questions/2709430/count-number-of-bits-in-a-64-bit-long-big-integer + */ + val = ~val & (val - 1ULL); /* Lowest set bit mask */ + val = val - ((val >> 1) & 0x5555555555555555); + val = (val & 0x3333333333333333ULL) + ((val >> 2) & 0x3333333333333333ULL); + return (U32)((((val + (val >> 4)) & 0xF0F0F0F0F0F0F0FULL) * 0x101010101010101ULL) >> 56); +# endif +} + +/* ZSTD_rotateRight_*(): + * Rotates a bitfield to the right by "count" bits. + * https://en.wikipedia.org/w/index.php?title=Circular_shift&oldid=991635599#Implementing_circular_shifts + */ +FORCE_INLINE_TEMPLATE +U64 ZSTD_rotateRight_U64(U64 const value, U32 count) { + assert(count < 64); + count &= 0x3F; /* for fickle pattern recognition */ + return (value >> count) | (U64)(value << ((0U - count) & 0x3F)); +} + +FORCE_INLINE_TEMPLATE +U32 ZSTD_rotateRight_U32(U32 const value, U32 count) { + assert(count < 32); + count &= 0x1F; /* for fickle pattern recognition */ + return (value >> count) | (U32)(value << ((0U - count) & 0x1F)); +} + +FORCE_INLINE_TEMPLATE +U16 ZSTD_rotateRight_U16(U16 const value, U32 count) { + assert(count < 16); + count &= 0x0F; /* for fickle pattern recognition */ + return (value >> count) | (U16)(value << ((0U - count) & 0x0F)); +} + +/* ZSTD_row_nextIndex(): + * Returns the next index to insert at within a tagTable row, and updates the "head" + * value to reflect the update. Essentially cycles backwards from [0, {entries per row}) + */ +FORCE_INLINE_TEMPLATE U32 ZSTD_row_nextIndex(BYTE* const tagRow, U32 const rowMask) { + U32 const next = (*tagRow - 1) & rowMask; + *tagRow = (BYTE)next; + return next; +} + +/* ZSTD_isAligned(): + * Checks that a pointer is aligned to "align" bytes which must be a power of 2. + */ +MEM_STATIC int ZSTD_isAligned(void const* ptr, size_t align) { + assert((align & (align - 1)) == 0); + return (((size_t)ptr) & (align - 1)) == 0; +} + +/* ZSTD_row_prefetch(): + * Performs prefetching for the hashTable and tagTable at a given row. + */ +FORCE_INLINE_TEMPLATE void ZSTD_row_prefetch(U32 const* hashTable, U16 const* tagTable, U32 const relRow, U32 const rowLog) { + PREFETCH_L1(hashTable + relRow); + if (rowLog >= 5) { + PREFETCH_L1(hashTable + relRow + 16); + /* Note: prefetching more of the hash table does not appear to be beneficial for 128-entry rows */ + } + PREFETCH_L1(tagTable + relRow); + if (rowLog == 6) { + PREFETCH_L1(tagTable + relRow + 32); + } + assert(rowLog == 4 || rowLog == 5 || rowLog == 6); + assert(ZSTD_isAligned(hashTable + relRow, 64)); /* prefetched hash row always 64-byte aligned */ + assert(ZSTD_isAligned(tagTable + relRow, (size_t)1 << rowLog)); /* prefetched tagRow sits on correct multiple of bytes (32,64,128) */ +} + +/* ZSTD_row_fillHashCache(): + * Fill up the hash cache starting at idx, prefetching up to ZSTD_ROW_HASH_CACHE_SIZE entries, + * but not beyond iLimit. + */ +FORCE_INLINE_TEMPLATE void ZSTD_row_fillHashCache(ZSTD_matchState_t* ms, const BYTE* base, + U32 const rowLog, U32 const mls, + U32 idx, const BYTE* const iLimit) +{ + U32 const* const hashTable = ms->hashTable; + U16 const* const tagTable = ms->tagTable; + U32 const hashLog = ms->rowHashLog; + U32 const maxElemsToPrefetch = (base + idx) > iLimit ? 0 : (U32)(iLimit - (base + idx) + 1); + U32 const lim = idx + MIN(ZSTD_ROW_HASH_CACHE_SIZE, maxElemsToPrefetch); + + for (; idx < lim; ++idx) { + U32 const hash = (U32)ZSTD_hashPtr(base + idx, hashLog + ZSTD_ROW_HASH_TAG_BITS, mls); + U32 const row = (hash >> ZSTD_ROW_HASH_TAG_BITS) << rowLog; + ZSTD_row_prefetch(hashTable, tagTable, row, rowLog); + ms->hashCache[idx & ZSTD_ROW_HASH_CACHE_MASK] = hash; + } + + DEBUGLOG(6, "ZSTD_row_fillHashCache(): [%u %u %u %u %u %u %u %u]", ms->hashCache[0], ms->hashCache[1], + ms->hashCache[2], ms->hashCache[3], ms->hashCache[4], + ms->hashCache[5], ms->hashCache[6], ms->hashCache[7]); +} + +/* ZSTD_row_nextCachedHash(): + * Returns the hash of base + idx, and replaces the hash in the hash cache with the byte at + * base + idx + ZSTD_ROW_HASH_CACHE_SIZE. Also prefetches the appropriate rows from hashTable and tagTable. + */ +FORCE_INLINE_TEMPLATE U32 ZSTD_row_nextCachedHash(U32* cache, U32 const* hashTable, + U16 const* tagTable, BYTE const* base, + U32 idx, U32 const hashLog, + U32 const rowLog, U32 const mls) +{ + U32 const newHash = (U32)ZSTD_hashPtr(base+idx+ZSTD_ROW_HASH_CACHE_SIZE, hashLog + ZSTD_ROW_HASH_TAG_BITS, mls); + U32 const row = (newHash >> ZSTD_ROW_HASH_TAG_BITS) << rowLog; + ZSTD_row_prefetch(hashTable, tagTable, row, rowLog); + { U32 const hash = cache[idx & ZSTD_ROW_HASH_CACHE_MASK]; + cache[idx & ZSTD_ROW_HASH_CACHE_MASK] = newHash; + return hash; + } +} + +/* ZSTD_row_update_internalImpl(): + * Updates the hash table with positions starting from updateStartIdx until updateEndIdx. + */ +FORCE_INLINE_TEMPLATE void ZSTD_row_update_internalImpl(ZSTD_matchState_t* ms, + U32 updateStartIdx, U32 const updateEndIdx, + U32 const mls, U32 const rowLog, + U32 const rowMask, U32 const useCache) +{ + U32* const hashTable = ms->hashTable; + U16* const tagTable = ms->tagTable; + U32 const hashLog = ms->rowHashLog; + const BYTE* const base = ms->window.base; + + DEBUGLOG(6, "ZSTD_row_update_internalImpl(): updateStartIdx=%u, updateEndIdx=%u", updateStartIdx, updateEndIdx); + for (; updateStartIdx < updateEndIdx; ++updateStartIdx) { + U32 const hash = useCache ? ZSTD_row_nextCachedHash(ms->hashCache, hashTable, tagTable, base, updateStartIdx, hashLog, rowLog, mls) + : (U32)ZSTD_hashPtr(base + updateStartIdx, hashLog + ZSTD_ROW_HASH_TAG_BITS, mls); + U32 const relRow = (hash >> ZSTD_ROW_HASH_TAG_BITS) << rowLog; + U32* const row = hashTable + relRow; + BYTE* tagRow = (BYTE*)(tagTable + relRow); /* Though tagTable is laid out as a table of U16, each tag is only 1 byte. + Explicit cast allows us to get exact desired position within each row */ + U32 const pos = ZSTD_row_nextIndex(tagRow, rowMask); + + assert(hash == ZSTD_hashPtr(base + updateStartIdx, hashLog + ZSTD_ROW_HASH_TAG_BITS, mls)); + ((BYTE*)tagRow)[pos + ZSTD_ROW_HASH_TAG_OFFSET] = hash & ZSTD_ROW_HASH_TAG_MASK; + row[pos] = updateStartIdx; + } +} + +/* ZSTD_row_update_internal(): + * Inserts the byte at ip into the appropriate position in the hash table, and updates ms->nextToUpdate. + * Skips sections of long matches as is necessary. + */ +FORCE_INLINE_TEMPLATE void ZSTD_row_update_internal(ZSTD_matchState_t* ms, const BYTE* ip, + U32 const mls, U32 const rowLog, + U32 const rowMask, U32 const useCache) +{ + U32 idx = ms->nextToUpdate; + const BYTE* const base = ms->window.base; + const U32 target = (U32)(ip - base); + const U32 kSkipThreshold = 384; + const U32 kMaxMatchStartPositionsToUpdate = 96; + const U32 kMaxMatchEndPositionsToUpdate = 32; + + if (useCache) { + /* Only skip positions when using hash cache, i.e. + * if we are loading a dict, don't skip anything. + * If we decide to skip, then we only update a set number + * of positions at the beginning and end of the match. + */ + if (UNLIKELY(target - idx > kSkipThreshold)) { + U32 const bound = idx + kMaxMatchStartPositionsToUpdate; + ZSTD_row_update_internalImpl(ms, idx, bound, mls, rowLog, rowMask, useCache); + idx = target - kMaxMatchEndPositionsToUpdate; + ZSTD_row_fillHashCache(ms, base, rowLog, mls, idx, ip+1); + } + } + assert(target >= idx); + ZSTD_row_update_internalImpl(ms, idx, target, mls, rowLog, rowMask, useCache); + ms->nextToUpdate = target; +} + +/* ZSTD_row_update(): + * External wrapper for ZSTD_row_update_internal(). Used for filling the hashtable during dictionary + * processing. + */ +void ZSTD_row_update(ZSTD_matchState_t* const ms, const BYTE* ip) { + const U32 rowLog = BOUNDED(4, ms->cParams.searchLog, 6); + const U32 rowMask = (1u << rowLog) - 1; + const U32 mls = MIN(ms->cParams.minMatch, 6 /* mls caps out at 6 */); + + DEBUGLOG(5, "ZSTD_row_update(), rowLog=%u", rowLog); + ZSTD_row_update_internal(ms, ip, mls, rowLog, rowMask, 0 /* dont use cache */); +} + +#if defined(ZSTD_ARCH_X86_SSE2) +FORCE_INLINE_TEMPLATE ZSTD_VecMask +ZSTD_row_getSSEMask(int nbChunks, const BYTE* const src, const BYTE tag, const U32 head) +{ + const __m128i comparisonMask = _mm_set1_epi8((char)tag); + int matches[4] = {0}; + int i; + assert(nbChunks == 1 || nbChunks == 2 || nbChunks == 4); + for (i=0; i> chunkSize; + do { + size_t chunk = MEM_readST(&src[i]); + chunk ^= splatChar; + chunk = (((chunk | x80) - x01) | chunk) & x80; + matches <<= chunkSize; + matches |= (chunk * extractMagic) >> shiftAmount; + i -= chunkSize; + } while (i >= 0); + } else { /* big endian: reverse bits during extraction */ + const size_t msb = xFF ^ (xFF >> 1); + const size_t extractMagic = (msb / 0x1FF) | msb; + do { + size_t chunk = MEM_readST(&src[i]); + chunk ^= splatChar; + chunk = (((chunk | x80) - x01) | chunk) & x80; + matches <<= chunkSize; + matches |= ((chunk >> 7) * extractMagic) >> shiftAmount; + i -= chunkSize; + } while (i >= 0); + } + matches = ~matches; + if (rowEntries == 16) { + return ZSTD_rotateRight_U16((U16)matches, head); + } else if (rowEntries == 32) { + return ZSTD_rotateRight_U32((U32)matches, head); + } else { + return ZSTD_rotateRight_U64((U64)matches, head); + } + } +#endif +} + +/* The high-level approach of the SIMD row based match finder is as follows: + * - Figure out where to insert the new entry: + * - Generate a hash from a byte along with an additional 1-byte "short hash". The additional byte is our "tag" + * - The hashTable is effectively split into groups or "rows" of 16 or 32 entries of U32, and the hash determines + * which row to insert into. + * - Determine the correct position within the row to insert the entry into. Each row of 16 or 32 can + * be considered as a circular buffer with a "head" index that resides in the tagTable. + * - Also insert the "tag" into the equivalent row and position in the tagTable. + * - Note: The tagTable has 17 or 33 1-byte entries per row, due to 16 or 32 tags, and 1 "head" entry. + * The 17 or 33 entry rows are spaced out to occur every 32 or 64 bytes, respectively, + * for alignment/performance reasons, leaving some bytes unused. + * - Use SIMD to efficiently compare the tags in the tagTable to the 1-byte "short hash" and + * generate a bitfield that we can cycle through to check the collisions in the hash table. + * - Pick the longest match. + */ +FORCE_INLINE_TEMPLATE +size_t ZSTD_RowFindBestMatch( + ZSTD_matchState_t* ms, + const BYTE* const ip, const BYTE* const iLimit, + size_t* offsetPtr, + const U32 mls, const ZSTD_dictMode_e dictMode, + const U32 rowLog) +{ + U32* const hashTable = ms->hashTable; + U16* const tagTable = ms->tagTable; + U32* const hashCache = ms->hashCache; + const U32 hashLog = ms->rowHashLog; + const ZSTD_compressionParameters* const cParams = &ms->cParams; + const BYTE* const base = ms->window.base; + const BYTE* const dictBase = ms->window.dictBase; + const U32 dictLimit = ms->window.dictLimit; + const BYTE* const prefixStart = base + dictLimit; + const BYTE* const dictEnd = dictBase + dictLimit; + const U32 curr = (U32)(ip-base); + const U32 maxDistance = 1U << cParams->windowLog; + const U32 lowestValid = ms->window.lowLimit; + const U32 withinMaxDistance = (curr - lowestValid > maxDistance) ? curr - maxDistance : lowestValid; + const U32 isDictionary = (ms->loadedDictEnd != 0); + const U32 lowLimit = isDictionary ? lowestValid : withinMaxDistance; + const U32 rowEntries = (1U << rowLog); + const U32 rowMask = rowEntries - 1; + const U32 cappedSearchLog = MIN(cParams->searchLog, rowLog); /* nb of searches is capped at nb entries per row */ + U32 nbAttempts = 1U << cappedSearchLog; + size_t ml=4-1; + + /* DMS/DDS variables that may be referenced laster */ + const ZSTD_matchState_t* const dms = ms->dictMatchState; + + /* Initialize the following variables to satisfy static analyzer */ + size_t ddsIdx = 0; + U32 ddsExtraAttempts = 0; /* cctx hash tables are limited in searches, but allow extra searches into DDS */ + U32 dmsTag = 0; + U32* dmsRow = NULL; + BYTE* dmsTagRow = NULL; + + if (dictMode == ZSTD_dedicatedDictSearch) { + const U32 ddsHashLog = dms->cParams.hashLog - ZSTD_LAZY_DDSS_BUCKET_LOG; + { /* Prefetch DDS hashtable entry */ + ddsIdx = ZSTD_hashPtr(ip, ddsHashLog, mls) << ZSTD_LAZY_DDSS_BUCKET_LOG; + PREFETCH_L1(&dms->hashTable[ddsIdx]); + } + ddsExtraAttempts = cParams->searchLog > rowLog ? 1U << (cParams->searchLog - rowLog) : 0; + } + + if (dictMode == ZSTD_dictMatchState) { + /* Prefetch DMS rows */ + U32* const dmsHashTable = dms->hashTable; + U16* const dmsTagTable = dms->tagTable; + U32 const dmsHash = (U32)ZSTD_hashPtr(ip, dms->rowHashLog + ZSTD_ROW_HASH_TAG_BITS, mls); + U32 const dmsRelRow = (dmsHash >> ZSTD_ROW_HASH_TAG_BITS) << rowLog; + dmsTag = dmsHash & ZSTD_ROW_HASH_TAG_MASK; + dmsTagRow = (BYTE*)(dmsTagTable + dmsRelRow); + dmsRow = dmsHashTable + dmsRelRow; + ZSTD_row_prefetch(dmsHashTable, dmsTagTable, dmsRelRow, rowLog); + } + + /* Update the hashTable and tagTable up to (but not including) ip */ + ZSTD_row_update_internal(ms, ip, mls, rowLog, rowMask, 1 /* useCache */); + { /* Get the hash for ip, compute the appropriate row */ + U32 const hash = ZSTD_row_nextCachedHash(hashCache, hashTable, tagTable, base, curr, hashLog, rowLog, mls); + U32 const relRow = (hash >> ZSTD_ROW_HASH_TAG_BITS) << rowLog; + U32 const tag = hash & ZSTD_ROW_HASH_TAG_MASK; + U32* const row = hashTable + relRow; + BYTE* tagRow = (BYTE*)(tagTable + relRow); + U32 const head = *tagRow & rowMask; + U32 matchBuffer[ZSTD_ROW_HASH_MAX_ENTRIES]; + size_t numMatches = 0; + size_t currMatch = 0; + ZSTD_VecMask matches = ZSTD_row_getMatchMask(tagRow, (BYTE)tag, head, rowEntries); + + /* Cycle through the matches and prefetch */ + for (; (matches > 0) && (nbAttempts > 0); --nbAttempts, matches &= (matches - 1)) { + U32 const matchPos = (head + ZSTD_VecMask_next(matches)) & rowMask; + U32 const matchIndex = row[matchPos]; + assert(numMatches < rowEntries); + if (matchIndex < lowLimit) + break; + if ((dictMode != ZSTD_extDict) || matchIndex >= dictLimit) { + PREFETCH_L1(base + matchIndex); + } else { + PREFETCH_L1(dictBase + matchIndex); + } + matchBuffer[numMatches++] = matchIndex; + } + + /* Speed opt: insert current byte into hashtable too. This allows us to avoid one iteration of the loop + in ZSTD_row_update_internal() at the next search. */ + { + U32 const pos = ZSTD_row_nextIndex(tagRow, rowMask); + tagRow[pos + ZSTD_ROW_HASH_TAG_OFFSET] = (BYTE)tag; + row[pos] = ms->nextToUpdate++; + } + + /* Return the longest match */ + for (; currMatch < numMatches; ++currMatch) { + U32 const matchIndex = matchBuffer[currMatch]; + size_t currentMl=0; + assert(matchIndex < curr); + assert(matchIndex >= lowLimit); + + if ((dictMode != ZSTD_extDict) || matchIndex >= dictLimit) { + const BYTE* const match = base + matchIndex; + assert(matchIndex >= dictLimit); /* ensures this is true if dictMode != ZSTD_extDict */ + if (match[ml] == ip[ml]) /* potentially better */ + currentMl = ZSTD_count(ip, match, iLimit); + } else { + const BYTE* const match = dictBase + matchIndex; + assert(match+4 <= dictEnd); + if (MEM_read32(match) == MEM_read32(ip)) /* assumption : matchIndex <= dictLimit-4 (by table construction) */ + currentMl = ZSTD_count_2segments(ip+4, match+4, iLimit, dictEnd, prefixStart) + 4; + } + + /* Save best solution */ + if (currentMl > ml) { + ml = currentMl; + *offsetPtr = STORE_OFFSET(curr - matchIndex); + if (ip+currentMl == iLimit) break; /* best possible, avoids read overflow on next attempt */ + } + } + } + + assert(nbAttempts <= (1U << ZSTD_SEARCHLOG_MAX)); /* Check we haven't underflowed. */ + if (dictMode == ZSTD_dedicatedDictSearch) { + ml = ZSTD_dedicatedDictSearch_lazy_search(offsetPtr, ml, nbAttempts + ddsExtraAttempts, dms, + ip, iLimit, prefixStart, curr, dictLimit, ddsIdx); + } else if (dictMode == ZSTD_dictMatchState) { + /* TODO: Measure and potentially add prefetching to DMS */ + const U32 dmsLowestIndex = dms->window.dictLimit; + const BYTE* const dmsBase = dms->window.base; + const BYTE* const dmsEnd = dms->window.nextSrc; + const U32 dmsSize = (U32)(dmsEnd - dmsBase); + const U32 dmsIndexDelta = dictLimit - dmsSize; + + { U32 const head = *dmsTagRow & rowMask; + U32 matchBuffer[ZSTD_ROW_HASH_MAX_ENTRIES]; + size_t numMatches = 0; + size_t currMatch = 0; + ZSTD_VecMask matches = ZSTD_row_getMatchMask(dmsTagRow, (BYTE)dmsTag, head, rowEntries); + + for (; (matches > 0) && (nbAttempts > 0); --nbAttempts, matches &= (matches - 1)) { + U32 const matchPos = (head + ZSTD_VecMask_next(matches)) & rowMask; + U32 const matchIndex = dmsRow[matchPos]; + if (matchIndex < dmsLowestIndex) + break; + PREFETCH_L1(dmsBase + matchIndex); + matchBuffer[numMatches++] = matchIndex; + } + + /* Return the longest match */ + for (; currMatch < numMatches; ++currMatch) { + U32 const matchIndex = matchBuffer[currMatch]; + size_t currentMl=0; + assert(matchIndex >= dmsLowestIndex); + assert(matchIndex < curr); + + { const BYTE* const match = dmsBase + matchIndex; + assert(match+4 <= dmsEnd); + if (MEM_read32(match) == MEM_read32(ip)) + currentMl = ZSTD_count_2segments(ip+4, match+4, iLimit, dmsEnd, prefixStart) + 4; + } + + if (currentMl > ml) { + ml = currentMl; + assert(curr > matchIndex + dmsIndexDelta); + *offsetPtr = STORE_OFFSET(curr - (matchIndex + dmsIndexDelta)); + if (ip+currentMl == iLimit) break; + } + } + } + } + return ml; +} + + +typedef size_t (*searchMax_f)( + ZSTD_matchState_t* ms, + const BYTE* ip, const BYTE* iLimit, size_t* offsetPtr); + +/** + * This struct contains the functions necessary for lazy to search. + * Currently, that is only searchMax. However, it is still valuable to have the + * VTable because this makes it easier to add more functions to the VTable later. + * + * TODO: The start of the search function involves loading and calculating a + * bunch of constants from the ZSTD_matchState_t. These computations could be + * done in an initialization function, and saved somewhere in the match state. + * Then we could pass a pointer to the saved state instead of the match state, + * and avoid duplicate computations. + * + * TODO: Move the match re-winding into searchMax. This improves compression + * ratio, and unlocks further simplifications with the next TODO. + * + * TODO: Try moving the repcode search into searchMax. After the re-winding + * and repcode search are in searchMax, there is no more logic in the match + * finder loop that requires knowledge about the dictMode. So we should be + * able to avoid force inlining it, and we can join the extDict loop with + * the single segment loop. It should go in searchMax instead of its own + * function to avoid having multiple virtual function calls per search. + */ +typedef struct { + searchMax_f searchMax; +} ZSTD_LazyVTable; + +#define GEN_ZSTD_BT_VTABLE(dictMode, mls) \ + static size_t ZSTD_BtFindBestMatch_##dictMode##_##mls( \ + ZSTD_matchState_t* ms, \ + const BYTE* ip, const BYTE* const iLimit, \ + size_t* offsetPtr) \ + { \ + assert(MAX(4, MIN(6, ms->cParams.minMatch)) == mls); \ + return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, mls, ZSTD_##dictMode); \ + } \ + static const ZSTD_LazyVTable ZSTD_BtVTable_##dictMode##_##mls = { \ + ZSTD_BtFindBestMatch_##dictMode##_##mls \ + }; + +#define GEN_ZSTD_HC_VTABLE(dictMode, mls) \ + static size_t ZSTD_HcFindBestMatch_##dictMode##_##mls( \ + ZSTD_matchState_t* ms, \ + const BYTE* ip, const BYTE* const iLimit, \ + size_t* offsetPtr) \ + { \ + assert(MAX(4, MIN(6, ms->cParams.minMatch)) == mls); \ + return ZSTD_HcFindBestMatch(ms, ip, iLimit, offsetPtr, mls, ZSTD_##dictMode); \ + } \ + static const ZSTD_LazyVTable ZSTD_HcVTable_##dictMode##_##mls = { \ + ZSTD_HcFindBestMatch_##dictMode##_##mls \ + }; + +#define GEN_ZSTD_ROW_VTABLE(dictMode, mls, rowLog) \ + static size_t ZSTD_RowFindBestMatch_##dictMode##_##mls##_##rowLog( \ + ZSTD_matchState_t* ms, \ + const BYTE* ip, const BYTE* const iLimit, \ + size_t* offsetPtr) \ + { \ + assert(MAX(4, MIN(6, ms->cParams.minMatch)) == mls); \ + assert(MAX(4, MIN(6, ms->cParams.searchLog)) == rowLog); \ + return ZSTD_RowFindBestMatch(ms, ip, iLimit, offsetPtr, mls, ZSTD_##dictMode, rowLog); \ + } \ + static const ZSTD_LazyVTable ZSTD_RowVTable_##dictMode##_##mls##_##rowLog = { \ + ZSTD_RowFindBestMatch_##dictMode##_##mls##_##rowLog \ + }; + +#define ZSTD_FOR_EACH_ROWLOG(X, dictMode, mls) \ + X(dictMode, mls, 4) \ + X(dictMode, mls, 5) \ + X(dictMode, mls, 6) + +#define ZSTD_FOR_EACH_MLS_ROWLOG(X, dictMode) \ + ZSTD_FOR_EACH_ROWLOG(X, dictMode, 4) \ + ZSTD_FOR_EACH_ROWLOG(X, dictMode, 5) \ + ZSTD_FOR_EACH_ROWLOG(X, dictMode, 6) + +#define ZSTD_FOR_EACH_MLS(X, dictMode) \ + X(dictMode, 4) \ + X(dictMode, 5) \ + X(dictMode, 6) + +#define ZSTD_FOR_EACH_DICT_MODE(X, ...) \ + X(__VA_ARGS__, noDict) \ + X(__VA_ARGS__, extDict) \ + X(__VA_ARGS__, dictMatchState) \ + X(__VA_ARGS__, dedicatedDictSearch) + +/* Generate Row VTables for each combination of (dictMode, mls, rowLog) */ +ZSTD_FOR_EACH_DICT_MODE(ZSTD_FOR_EACH_MLS_ROWLOG, GEN_ZSTD_ROW_VTABLE) +/* Generate Binary Tree VTables for each combination of (dictMode, mls) */ +ZSTD_FOR_EACH_DICT_MODE(ZSTD_FOR_EACH_MLS, GEN_ZSTD_BT_VTABLE) +/* Generate Hash Chain VTables for each combination of (dictMode, mls) */ +ZSTD_FOR_EACH_DICT_MODE(ZSTD_FOR_EACH_MLS, GEN_ZSTD_HC_VTABLE) + +#define GEN_ZSTD_BT_VTABLE_ARRAY(dictMode) \ + { \ + &ZSTD_BtVTable_##dictMode##_4, \ + &ZSTD_BtVTable_##dictMode##_5, \ + &ZSTD_BtVTable_##dictMode##_6 \ + } + +#define GEN_ZSTD_HC_VTABLE_ARRAY(dictMode) \ + { \ + &ZSTD_HcVTable_##dictMode##_4, \ + &ZSTD_HcVTable_##dictMode##_5, \ + &ZSTD_HcVTable_##dictMode##_6 \ + } + +#define GEN_ZSTD_ROW_VTABLE_ARRAY_(dictMode, mls) \ + { \ + &ZSTD_RowVTable_##dictMode##_##mls##_4, \ + &ZSTD_RowVTable_##dictMode##_##mls##_5, \ + &ZSTD_RowVTable_##dictMode##_##mls##_6 \ + } + +#define GEN_ZSTD_ROW_VTABLE_ARRAY(dictMode) \ + { \ + GEN_ZSTD_ROW_VTABLE_ARRAY_(dictMode, 4), \ + GEN_ZSTD_ROW_VTABLE_ARRAY_(dictMode, 5), \ + GEN_ZSTD_ROW_VTABLE_ARRAY_(dictMode, 6) \ + } + +#define GEN_ZSTD_VTABLE_ARRAY(X) \ + { \ + X(noDict), \ + X(extDict), \ + X(dictMatchState), \ + X(dedicatedDictSearch) \ + } + +/* ******************************* +* Common parser - lazy strategy +*********************************/ +typedef enum { search_hashChain=0, search_binaryTree=1, search_rowHash=2 } searchMethod_e; + +/** + * This table is indexed first by the four ZSTD_dictMode_e values, and then + * by the two searchMethod_e values. NULLs are placed for configurations + * that should never occur (extDict modes go to the other implementation + * below and there is no DDSS for binary tree search yet). + */ + +static ZSTD_LazyVTable const* +ZSTD_selectLazyVTable(ZSTD_matchState_t const* ms, searchMethod_e searchMethod, ZSTD_dictMode_e dictMode) +{ + /* Fill the Hc/Bt VTable arrays with the right functions for the (dictMode, mls) combination. */ + ZSTD_LazyVTable const* const hcVTables[4][3] = GEN_ZSTD_VTABLE_ARRAY(GEN_ZSTD_HC_VTABLE_ARRAY); + ZSTD_LazyVTable const* const btVTables[4][3] = GEN_ZSTD_VTABLE_ARRAY(GEN_ZSTD_BT_VTABLE_ARRAY); + /* Fill the Row VTable array with the right functions for the (dictMode, mls, rowLog) combination. */ + ZSTD_LazyVTable const* const rowVTables[4][3][3] = GEN_ZSTD_VTABLE_ARRAY(GEN_ZSTD_ROW_VTABLE_ARRAY); + + U32 const mls = MAX(4, MIN(6, ms->cParams.minMatch)); + U32 const rowLog = MAX(4, MIN(6, ms->cParams.searchLog)); + switch (searchMethod) { + case search_hashChain: + return hcVTables[dictMode][mls - 4]; + case search_binaryTree: + return btVTables[dictMode][mls - 4]; + case search_rowHash: + return rowVTables[dictMode][mls - 4][rowLog - 4]; + default: + return NULL; + } +} + +FORCE_INLINE_TEMPLATE size_t +ZSTD_compressBlock_lazy_generic( + ZSTD_matchState_t* ms, seqStore_t* seqStore, + U32 rep[ZSTD_REP_NUM], + const void* src, size_t srcSize, + const searchMethod_e searchMethod, const U32 depth, + ZSTD_dictMode_e const dictMode) +{ + const BYTE* const istart = (const BYTE*)src; + const BYTE* ip = istart; + const BYTE* anchor = istart; + const BYTE* const iend = istart + srcSize; + const BYTE* const ilimit = (searchMethod == search_rowHash) ? iend - 8 - ZSTD_ROW_HASH_CACHE_SIZE : iend - 8; + const BYTE* const base = ms->window.base; + const U32 prefixLowestIndex = ms->window.dictLimit; + const BYTE* const prefixLowest = base + prefixLowestIndex; + + searchMax_f const searchMax = ZSTD_selectLazyVTable(ms, searchMethod, dictMode)->searchMax; + U32 offset_1 = rep[0], offset_2 = rep[1], savedOffset=0; + + const int isDMS = dictMode == ZSTD_dictMatchState; + const int isDDS = dictMode == ZSTD_dedicatedDictSearch; + const int isDxS = isDMS || isDDS; + const ZSTD_matchState_t* const dms = ms->dictMatchState; + const U32 dictLowestIndex = isDxS ? dms->window.dictLimit : 0; + const BYTE* const dictBase = isDxS ? dms->window.base : NULL; + const BYTE* const dictLowest = isDxS ? dictBase + dictLowestIndex : NULL; + const BYTE* const dictEnd = isDxS ? dms->window.nextSrc : NULL; + const U32 dictIndexDelta = isDxS ? + prefixLowestIndex - (U32)(dictEnd - dictBase) : + 0; + const U32 dictAndPrefixLength = (U32)((ip - prefixLowest) + (dictEnd - dictLowest)); + + assert(searchMax != NULL); + + DEBUGLOG(5, "ZSTD_compressBlock_lazy_generic (dictMode=%u) (searchFunc=%u)", (U32)dictMode, (U32)searchMethod); + ip += (dictAndPrefixLength == 0); + if (dictMode == ZSTD_noDict) { + U32 const curr = (U32)(ip - base); + U32 const windowLow = ZSTD_getLowestPrefixIndex(ms, curr, ms->cParams.windowLog); + U32 const maxRep = curr - windowLow; + if (offset_2 > maxRep) savedOffset = offset_2, offset_2 = 0; + if (offset_1 > maxRep) savedOffset = offset_1, offset_1 = 0; + } + if (isDxS) { + /* dictMatchState repCode checks don't currently handle repCode == 0 + * disabling. */ + assert(offset_1 <= dictAndPrefixLength); + assert(offset_2 <= dictAndPrefixLength); + } + + if (searchMethod == search_rowHash) { + const U32 rowLog = MAX(4, MIN(6, ms->cParams.searchLog)); + ZSTD_row_fillHashCache(ms, base, rowLog, + MIN(ms->cParams.minMatch, 6 /* mls caps out at 6 */), + ms->nextToUpdate, ilimit); + } + + /* Match Loop */ +#if defined(__GNUC__) && defined(__x86_64__) + /* I've measured random a 5% speed loss on levels 5 & 6 (greedy) when the + * code alignment is perturbed. To fix the instability align the loop on 32-bytes. + */ + __asm__(".p2align 5"); +#endif + while (ip < ilimit) { + size_t matchLength=0; + size_t offcode=STORE_REPCODE_1; + const BYTE* start=ip+1; + DEBUGLOG(7, "search baseline (depth 0)"); + + /* check repCode */ + if (isDxS) { + const U32 repIndex = (U32)(ip - base) + 1 - offset_1; + const BYTE* repMatch = ((dictMode == ZSTD_dictMatchState || dictMode == ZSTD_dedicatedDictSearch) + && repIndex < prefixLowestIndex) ? + dictBase + (repIndex - dictIndexDelta) : + base + repIndex; + if (((U32)((prefixLowestIndex-1) - repIndex) >= 3 /* intentional underflow */) + && (MEM_read32(repMatch) == MEM_read32(ip+1)) ) { + const BYTE* repMatchEnd = repIndex < prefixLowestIndex ? dictEnd : iend; + matchLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, prefixLowest) + 4; + if (depth==0) goto _storeSequence; + } + } + if ( dictMode == ZSTD_noDict + && ((offset_1 > 0) & (MEM_read32(ip+1-offset_1) == MEM_read32(ip+1)))) { + matchLength = ZSTD_count(ip+1+4, ip+1+4-offset_1, iend) + 4; + if (depth==0) goto _storeSequence; + } + + /* first search (depth 0) */ + { size_t offsetFound = 999999999; + size_t const ml2 = searchMax(ms, ip, iend, &offsetFound); + if (ml2 > matchLength) + matchLength = ml2, start = ip, offcode=offsetFound; + } + + if (matchLength < 4) { + ip += ((ip-anchor) >> kSearchStrength) + 1; /* jump faster over incompressible sections */ + continue; + } + + /* let's try to find a better solution */ + if (depth>=1) + while (ip0) & (MEM_read32(ip) == MEM_read32(ip - offset_1)))) { + size_t const mlRep = ZSTD_count(ip+4, ip+4-offset_1, iend) + 4; + int const gain2 = (int)(mlRep * 3); + int const gain1 = (int)(matchLength*3 - ZSTD_highbit32((U32)STORED_TO_OFFBASE(offcode)) + 1); + if ((mlRep >= 4) && (gain2 > gain1)) + matchLength = mlRep, offcode = STORE_REPCODE_1, start = ip; + } + if (isDxS) { + const U32 repIndex = (U32)(ip - base) - offset_1; + const BYTE* repMatch = repIndex < prefixLowestIndex ? + dictBase + (repIndex - dictIndexDelta) : + base + repIndex; + if (((U32)((prefixLowestIndex-1) - repIndex) >= 3 /* intentional underflow */) + && (MEM_read32(repMatch) == MEM_read32(ip)) ) { + const BYTE* repMatchEnd = repIndex < prefixLowestIndex ? dictEnd : iend; + size_t const mlRep = ZSTD_count_2segments(ip+4, repMatch+4, iend, repMatchEnd, prefixLowest) + 4; + int const gain2 = (int)(mlRep * 3); + int const gain1 = (int)(matchLength*3 - ZSTD_highbit32((U32)STORED_TO_OFFBASE(offcode)) + 1); + if ((mlRep >= 4) && (gain2 > gain1)) + matchLength = mlRep, offcode = STORE_REPCODE_1, start = ip; + } + } + { size_t offset2=999999999; + size_t const ml2 = searchMax(ms, ip, iend, &offset2); + int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)STORED_TO_OFFBASE(offset2))); /* raw approx */ + int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)STORED_TO_OFFBASE(offcode)) + 4); + if ((ml2 >= 4) && (gain2 > gain1)) { + matchLength = ml2, offcode = offset2, start = ip; + continue; /* search a better one */ + } } + + /* let's find an even better one */ + if ((depth==2) && (ip0) & (MEM_read32(ip) == MEM_read32(ip - offset_1)))) { + size_t const mlRep = ZSTD_count(ip+4, ip+4-offset_1, iend) + 4; + int const gain2 = (int)(mlRep * 4); + int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)STORED_TO_OFFBASE(offcode)) + 1); + if ((mlRep >= 4) && (gain2 > gain1)) + matchLength = mlRep, offcode = STORE_REPCODE_1, start = ip; + } + if (isDxS) { + const U32 repIndex = (U32)(ip - base) - offset_1; + const BYTE* repMatch = repIndex < prefixLowestIndex ? + dictBase + (repIndex - dictIndexDelta) : + base + repIndex; + if (((U32)((prefixLowestIndex-1) - repIndex) >= 3 /* intentional underflow */) + && (MEM_read32(repMatch) == MEM_read32(ip)) ) { + const BYTE* repMatchEnd = repIndex < prefixLowestIndex ? dictEnd : iend; + size_t const mlRep = ZSTD_count_2segments(ip+4, repMatch+4, iend, repMatchEnd, prefixLowest) + 4; + int const gain2 = (int)(mlRep * 4); + int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)STORED_TO_OFFBASE(offcode)) + 1); + if ((mlRep >= 4) && (gain2 > gain1)) + matchLength = mlRep, offcode = STORE_REPCODE_1, start = ip; + } + } + { size_t offset2=999999999; + size_t const ml2 = searchMax(ms, ip, iend, &offset2); + int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)STORED_TO_OFFBASE(offset2))); /* raw approx */ + int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)STORED_TO_OFFBASE(offcode)) + 7); + if ((ml2 >= 4) && (gain2 > gain1)) { + matchLength = ml2, offcode = offset2, start = ip; + continue; + } } } + break; /* nothing found : store previous solution */ + } + + /* NOTE: + * Pay attention that `start[-value]` can lead to strange undefined behavior + * notably if `value` is unsigned, resulting in a large positive `-value`. + */ + /* catch up */ + if (STORED_IS_OFFSET(offcode)) { + if (dictMode == ZSTD_noDict) { + while ( ((start > anchor) & (start - STORED_OFFSET(offcode) > prefixLowest)) + && (start[-1] == (start-STORED_OFFSET(offcode))[-1]) ) /* only search for offset within prefix */ + { start--; matchLength++; } + } + if (isDxS) { + U32 const matchIndex = (U32)((size_t)(start-base) - STORED_OFFSET(offcode)); + const BYTE* match = (matchIndex < prefixLowestIndex) ? dictBase + matchIndex - dictIndexDelta : base + matchIndex; + const BYTE* const mStart = (matchIndex < prefixLowestIndex) ? dictLowest : prefixLowest; + while ((start>anchor) && (match>mStart) && (start[-1] == match[-1])) { start--; match--; matchLength++; } /* catch up */ + } + offset_2 = offset_1; offset_1 = (U32)STORED_OFFSET(offcode); + } + /* store sequence */ +_storeSequence: + { size_t const litLength = (size_t)(start - anchor); + ZSTD_storeSeq(seqStore, litLength, anchor, iend, (U32)offcode, matchLength); + anchor = ip = start + matchLength; + } + + /* check immediate repcode */ + if (isDxS) { + while (ip <= ilimit) { + U32 const current2 = (U32)(ip-base); + U32 const repIndex = current2 - offset_2; + const BYTE* repMatch = repIndex < prefixLowestIndex ? + dictBase - dictIndexDelta + repIndex : + base + repIndex; + if ( ((U32)((prefixLowestIndex-1) - (U32)repIndex) >= 3 /* intentional overflow */) + && (MEM_read32(repMatch) == MEM_read32(ip)) ) { + const BYTE* const repEnd2 = repIndex < prefixLowestIndex ? dictEnd : iend; + matchLength = ZSTD_count_2segments(ip+4, repMatch+4, iend, repEnd2, prefixLowest) + 4; + offcode = offset_2; offset_2 = offset_1; offset_1 = (U32)offcode; /* swap offset_2 <=> offset_1 */ + ZSTD_storeSeq(seqStore, 0, anchor, iend, STORE_REPCODE_1, matchLength); + ip += matchLength; + anchor = ip; + continue; + } + break; + } + } + + if (dictMode == ZSTD_noDict) { + while ( ((ip <= ilimit) & (offset_2>0)) + && (MEM_read32(ip) == MEM_read32(ip - offset_2)) ) { + /* store sequence */ + matchLength = ZSTD_count(ip+4, ip+4-offset_2, iend) + 4; + offcode = offset_2; offset_2 = offset_1; offset_1 = (U32)offcode; /* swap repcodes */ + ZSTD_storeSeq(seqStore, 0, anchor, iend, STORE_REPCODE_1, matchLength); + ip += matchLength; + anchor = ip; + continue; /* faster when present ... (?) */ + } } } + + /* Save reps for next block */ + rep[0] = offset_1 ? offset_1 : savedOffset; + rep[1] = offset_2 ? offset_2 : savedOffset; + + /* Return the last literals size */ + return (size_t)(iend - anchor); +} + + +size_t ZSTD_compressBlock_btlazy2( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_binaryTree, 2, ZSTD_noDict); +} + +size_t ZSTD_compressBlock_lazy2( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 2, ZSTD_noDict); +} + +size_t ZSTD_compressBlock_lazy( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 1, ZSTD_noDict); +} + +size_t ZSTD_compressBlock_greedy( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 0, ZSTD_noDict); +} + +size_t ZSTD_compressBlock_btlazy2_dictMatchState( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_binaryTree, 2, ZSTD_dictMatchState); +} + +size_t ZSTD_compressBlock_lazy2_dictMatchState( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 2, ZSTD_dictMatchState); +} + +size_t ZSTD_compressBlock_lazy_dictMatchState( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 1, ZSTD_dictMatchState); +} + +size_t ZSTD_compressBlock_greedy_dictMatchState( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 0, ZSTD_dictMatchState); +} + + +size_t ZSTD_compressBlock_lazy2_dedicatedDictSearch( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 2, ZSTD_dedicatedDictSearch); +} + +size_t ZSTD_compressBlock_lazy_dedicatedDictSearch( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 1, ZSTD_dedicatedDictSearch); +} + +size_t ZSTD_compressBlock_greedy_dedicatedDictSearch( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 0, ZSTD_dedicatedDictSearch); +} + +/* Row-based matchfinder */ +size_t ZSTD_compressBlock_lazy2_row( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 2, ZSTD_noDict); +} + +size_t ZSTD_compressBlock_lazy_row( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 1, ZSTD_noDict); +} + +size_t ZSTD_compressBlock_greedy_row( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 0, ZSTD_noDict); +} + +size_t ZSTD_compressBlock_lazy2_dictMatchState_row( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 2, ZSTD_dictMatchState); +} + +size_t ZSTD_compressBlock_lazy_dictMatchState_row( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 1, ZSTD_dictMatchState); +} + +size_t ZSTD_compressBlock_greedy_dictMatchState_row( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 0, ZSTD_dictMatchState); +} + + +size_t ZSTD_compressBlock_lazy2_dedicatedDictSearch_row( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 2, ZSTD_dedicatedDictSearch); +} + +size_t ZSTD_compressBlock_lazy_dedicatedDictSearch_row( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 1, ZSTD_dedicatedDictSearch); +} + +size_t ZSTD_compressBlock_greedy_dedicatedDictSearch_row( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 0, ZSTD_dedicatedDictSearch); +} + +FORCE_INLINE_TEMPLATE +size_t ZSTD_compressBlock_lazy_extDict_generic( + ZSTD_matchState_t* ms, seqStore_t* seqStore, + U32 rep[ZSTD_REP_NUM], + const void* src, size_t srcSize, + const searchMethod_e searchMethod, const U32 depth) +{ + const BYTE* const istart = (const BYTE*)src; + const BYTE* ip = istart; + const BYTE* anchor = istart; + const BYTE* const iend = istart + srcSize; + const BYTE* const ilimit = searchMethod == search_rowHash ? iend - 8 - ZSTD_ROW_HASH_CACHE_SIZE : iend - 8; + const BYTE* const base = ms->window.base; + const U32 dictLimit = ms->window.dictLimit; + const BYTE* const prefixStart = base + dictLimit; + const BYTE* const dictBase = ms->window.dictBase; + const BYTE* const dictEnd = dictBase + dictLimit; + const BYTE* const dictStart = dictBase + ms->window.lowLimit; + const U32 windowLog = ms->cParams.windowLog; + const U32 rowLog = ms->cParams.searchLog < 5 ? 4 : 5; + + searchMax_f const searchMax = ZSTD_selectLazyVTable(ms, searchMethod, ZSTD_extDict)->searchMax; + U32 offset_1 = rep[0], offset_2 = rep[1]; + + DEBUGLOG(5, "ZSTD_compressBlock_lazy_extDict_generic (searchFunc=%u)", (U32)searchMethod); + + /* init */ + ip += (ip == prefixStart); + if (searchMethod == search_rowHash) { + ZSTD_row_fillHashCache(ms, base, rowLog, + MIN(ms->cParams.minMatch, 6 /* mls caps out at 6 */), + ms->nextToUpdate, ilimit); + } + + /* Match Loop */ +#if defined(__GNUC__) && defined(__x86_64__) + /* I've measured random a 5% speed loss on levels 5 & 6 (greedy) when the + * code alignment is perturbed. To fix the instability align the loop on 32-bytes. + */ + __asm__(".p2align 5"); +#endif + while (ip < ilimit) { + size_t matchLength=0; + size_t offcode=STORE_REPCODE_1; + const BYTE* start=ip+1; + U32 curr = (U32)(ip-base); + + /* check repCode */ + { const U32 windowLow = ZSTD_getLowestMatchIndex(ms, curr+1, windowLog); + const U32 repIndex = (U32)(curr+1 - offset_1); + const BYTE* const repBase = repIndex < dictLimit ? dictBase : base; + const BYTE* const repMatch = repBase + repIndex; + if ( ((U32)((dictLimit-1) - repIndex) >= 3) /* intentional overflow */ + & (offset_1 <= curr+1 - windowLow) ) /* note: we are searching at curr+1 */ + if (MEM_read32(ip+1) == MEM_read32(repMatch)) { + /* repcode detected we should take it */ + const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend; + matchLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repEnd, prefixStart) + 4; + if (depth==0) goto _storeSequence; + } } + + /* first search (depth 0) */ + { size_t offsetFound = 999999999; + size_t const ml2 = searchMax(ms, ip, iend, &offsetFound); + if (ml2 > matchLength) + matchLength = ml2, start = ip, offcode=offsetFound; + } + + if (matchLength < 4) { + ip += ((ip-anchor) >> kSearchStrength) + 1; /* jump faster over incompressible sections */ + continue; + } + + /* let's try to find a better solution */ + if (depth>=1) + while (ip= 3) /* intentional overflow : do not test positions overlapping 2 memory segments */ + & (offset_1 <= curr - windowLow) ) /* equivalent to `curr > repIndex >= windowLow` */ + if (MEM_read32(ip) == MEM_read32(repMatch)) { + /* repcode detected */ + const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend; + size_t const repLength = ZSTD_count_2segments(ip+4, repMatch+4, iend, repEnd, prefixStart) + 4; + int const gain2 = (int)(repLength * 3); + int const gain1 = (int)(matchLength*3 - ZSTD_highbit32((U32)STORED_TO_OFFBASE(offcode)) + 1); + if ((repLength >= 4) && (gain2 > gain1)) + matchLength = repLength, offcode = STORE_REPCODE_1, start = ip; + } } + + /* search match, depth 1 */ + { size_t offset2=999999999; + size_t const ml2 = searchMax(ms, ip, iend, &offset2); + int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)STORED_TO_OFFBASE(offset2))); /* raw approx */ + int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)STORED_TO_OFFBASE(offcode)) + 4); + if ((ml2 >= 4) && (gain2 > gain1)) { + matchLength = ml2, offcode = offset2, start = ip; + continue; /* search a better one */ + } } + + /* let's find an even better one */ + if ((depth==2) && (ip= 3) /* intentional overflow : do not test positions overlapping 2 memory segments */ + & (offset_1 <= curr - windowLow) ) /* equivalent to `curr > repIndex >= windowLow` */ + if (MEM_read32(ip) == MEM_read32(repMatch)) { + /* repcode detected */ + const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend; + size_t const repLength = ZSTD_count_2segments(ip+4, repMatch+4, iend, repEnd, prefixStart) + 4; + int const gain2 = (int)(repLength * 4); + int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)STORED_TO_OFFBASE(offcode)) + 1); + if ((repLength >= 4) && (gain2 > gain1)) + matchLength = repLength, offcode = STORE_REPCODE_1, start = ip; + } } + + /* search match, depth 2 */ + { size_t offset2=999999999; + size_t const ml2 = searchMax(ms, ip, iend, &offset2); + int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)STORED_TO_OFFBASE(offset2))); /* raw approx */ + int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)STORED_TO_OFFBASE(offcode)) + 7); + if ((ml2 >= 4) && (gain2 > gain1)) { + matchLength = ml2, offcode = offset2, start = ip; + continue; + } } } + break; /* nothing found : store previous solution */ + } + + /* catch up */ + if (STORED_IS_OFFSET(offcode)) { + U32 const matchIndex = (U32)((size_t)(start-base) - STORED_OFFSET(offcode)); + const BYTE* match = (matchIndex < dictLimit) ? dictBase + matchIndex : base + matchIndex; + const BYTE* const mStart = (matchIndex < dictLimit) ? dictStart : prefixStart; + while ((start>anchor) && (match>mStart) && (start[-1] == match[-1])) { start--; match--; matchLength++; } /* catch up */ + offset_2 = offset_1; offset_1 = (U32)STORED_OFFSET(offcode); + } + + /* store sequence */ +_storeSequence: + { size_t const litLength = (size_t)(start - anchor); + ZSTD_storeSeq(seqStore, litLength, anchor, iend, (U32)offcode, matchLength); + anchor = ip = start + matchLength; + } + + /* check immediate repcode */ + while (ip <= ilimit) { + const U32 repCurrent = (U32)(ip-base); + const U32 windowLow = ZSTD_getLowestMatchIndex(ms, repCurrent, windowLog); + const U32 repIndex = repCurrent - offset_2; + const BYTE* const repBase = repIndex < dictLimit ? dictBase : base; + const BYTE* const repMatch = repBase + repIndex; + if ( ((U32)((dictLimit-1) - repIndex) >= 3) /* intentional overflow : do not test positions overlapping 2 memory segments */ + & (offset_2 <= repCurrent - windowLow) ) /* equivalent to `curr > repIndex >= windowLow` */ + if (MEM_read32(ip) == MEM_read32(repMatch)) { + /* repcode detected we should take it */ + const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend; + matchLength = ZSTD_count_2segments(ip+4, repMatch+4, iend, repEnd, prefixStart) + 4; + offcode = offset_2; offset_2 = offset_1; offset_1 = (U32)offcode; /* swap offset history */ + ZSTD_storeSeq(seqStore, 0, anchor, iend, STORE_REPCODE_1, matchLength); + ip += matchLength; + anchor = ip; + continue; /* faster when present ... (?) */ + } + break; + } } + + /* Save reps for next block */ + rep[0] = offset_1; + rep[1] = offset_2; + + /* Return the last literals size */ + return (size_t)(iend - anchor); +} + + +size_t ZSTD_compressBlock_greedy_extDict( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 0); +} + +size_t ZSTD_compressBlock_lazy_extDict( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) + +{ + return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 1); +} + +size_t ZSTD_compressBlock_lazy2_extDict( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) + +{ + return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 2); +} + +size_t ZSTD_compressBlock_btlazy2_extDict( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) + +{ + return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, search_binaryTree, 2); +} + +size_t ZSTD_compressBlock_greedy_extDict_row( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 0); +} + +size_t ZSTD_compressBlock_lazy_extDict_row( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) + +{ + return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 1); +} + +size_t ZSTD_compressBlock_lazy2_extDict_row( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) + +{ + return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 2); +} diff --git a/src/blosc2/internal-complibs/zstd-1.5.2/compress/zstd_lazy.h b/src/blosc2/internal-complibs/zstd-1.5.2/compress/zstd_lazy.h new file mode 100644 index 0000000..150f7b3 --- /dev/null +++ b/src/blosc2/internal-complibs/zstd-1.5.2/compress/zstd_lazy.h @@ -0,0 +1,125 @@ +/* + * Copyright (c) Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_LAZY_H +#define ZSTD_LAZY_H + +#if defined (__cplusplus) +extern "C" { +#endif + +#include "zstd_compress_internal.h" + +/** + * Dedicated Dictionary Search Structure bucket log. In the + * ZSTD_dedicatedDictSearch mode, the hashTable has + * 2 ** ZSTD_LAZY_DDSS_BUCKET_LOG entries in each bucket, rather than just + * one. + */ +#define ZSTD_LAZY_DDSS_BUCKET_LOG 2 + +U32 ZSTD_insertAndFindFirstIndex(ZSTD_matchState_t* ms, const BYTE* ip); +void ZSTD_row_update(ZSTD_matchState_t* const ms, const BYTE* ip); + +void ZSTD_dedicatedDictSearch_lazy_loadDictionary(ZSTD_matchState_t* ms, const BYTE* const ip); + +void ZSTD_preserveUnsortedMark (U32* const table, U32 const size, U32 const reducerValue); /*! used in ZSTD_reduceIndex(). preemptively increase value of ZSTD_DUBT_UNSORTED_MARK */ + +size_t ZSTD_compressBlock_btlazy2( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_lazy2( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_lazy( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_greedy( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_lazy2_row( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_lazy_row( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_greedy_row( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); + +size_t ZSTD_compressBlock_btlazy2_dictMatchState( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_lazy2_dictMatchState( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_lazy_dictMatchState( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_greedy_dictMatchState( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_lazy2_dictMatchState_row( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_lazy_dictMatchState_row( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_greedy_dictMatchState_row( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); + +size_t ZSTD_compressBlock_lazy2_dedicatedDictSearch( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_lazy_dedicatedDictSearch( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_greedy_dedicatedDictSearch( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_lazy2_dedicatedDictSearch_row( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_lazy_dedicatedDictSearch_row( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_greedy_dedicatedDictSearch_row( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); + +size_t ZSTD_compressBlock_greedy_extDict( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_lazy_extDict( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_lazy2_extDict( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_greedy_extDict_row( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_lazy_extDict_row( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_lazy2_extDict_row( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_btlazy2_extDict( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); + + +#if defined (__cplusplus) +} +#endif + +#endif /* ZSTD_LAZY_H */ diff --git a/src/blosc2/internal-complibs/zstd-1.5.2/compress/zstd_ldm.c b/src/blosc2/internal-complibs/zstd-1.5.2/compress/zstd_ldm.c new file mode 100644 index 0000000..f662b25 --- /dev/null +++ b/src/blosc2/internal-complibs/zstd-1.5.2/compress/zstd_ldm.c @@ -0,0 +1,724 @@ +/* + * Copyright (c) Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#include "zstd_ldm.h" + +#include "../common/debug.h" +#include "../common/xxhash.h" +#include "zstd_fast.h" /* ZSTD_fillHashTable() */ +#include "zstd_double_fast.h" /* ZSTD_fillDoubleHashTable() */ +#include "zstd_ldm_geartab.h" + +#define LDM_BUCKET_SIZE_LOG 3 +#define LDM_MIN_MATCH_LENGTH 64 +#define LDM_HASH_RLOG 7 + +typedef struct { + U64 rolling; + U64 stopMask; +} ldmRollingHashState_t; + +/** ZSTD_ldm_gear_init(): + * + * Initializes the rolling hash state such that it will honor the + * settings in params. */ +static void ZSTD_ldm_gear_init(ldmRollingHashState_t* state, ldmParams_t const* params) +{ + unsigned maxBitsInMask = MIN(params->minMatchLength, 64); + unsigned hashRateLog = params->hashRateLog; + + state->rolling = ~(U32)0; + + /* The choice of the splitting criterion is subject to two conditions: + * 1. it has to trigger on average every 2^(hashRateLog) bytes; + * 2. ideally, it has to depend on a window of minMatchLength bytes. + * + * In the gear hash algorithm, bit n depends on the last n bytes; + * so in order to obtain a good quality splitting criterion it is + * preferable to use bits with high weight. + * + * To match condition 1 we use a mask with hashRateLog bits set + * and, because of the previous remark, we make sure these bits + * have the highest possible weight while still respecting + * condition 2. + */ + if (hashRateLog > 0 && hashRateLog <= maxBitsInMask) { + state->stopMask = (((U64)1 << hashRateLog) - 1) << (maxBitsInMask - hashRateLog); + } else { + /* In this degenerate case we simply honor the hash rate. */ + state->stopMask = ((U64)1 << hashRateLog) - 1; + } +} + +/** ZSTD_ldm_gear_reset() + * Feeds [data, data + minMatchLength) into the hash without registering any + * splits. This effectively resets the hash state. This is used when skipping + * over data, either at the beginning of a block, or skipping sections. + */ +static void ZSTD_ldm_gear_reset(ldmRollingHashState_t* state, + BYTE const* data, size_t minMatchLength) +{ + U64 hash = state->rolling; + size_t n = 0; + +#define GEAR_ITER_ONCE() do { \ + hash = (hash << 1) + ZSTD_ldm_gearTab[data[n] & 0xff]; \ + n += 1; \ + } while (0) + while (n + 3 < minMatchLength) { + GEAR_ITER_ONCE(); + GEAR_ITER_ONCE(); + GEAR_ITER_ONCE(); + GEAR_ITER_ONCE(); + } + while (n < minMatchLength) { + GEAR_ITER_ONCE(); + } +#undef GEAR_ITER_ONCE +} + +/** ZSTD_ldm_gear_feed(): + * + * Registers in the splits array all the split points found in the first + * size bytes following the data pointer. This function terminates when + * either all the data has been processed or LDM_BATCH_SIZE splits are + * present in the splits array. + * + * Precondition: The splits array must not be full. + * Returns: The number of bytes processed. */ +static size_t ZSTD_ldm_gear_feed(ldmRollingHashState_t* state, + BYTE const* data, size_t size, + size_t* splits, unsigned* numSplits) +{ + size_t n; + U64 hash, mask; + + hash = state->rolling; + mask = state->stopMask; + n = 0; + +#define GEAR_ITER_ONCE() do { \ + hash = (hash << 1) + ZSTD_ldm_gearTab[data[n] & 0xff]; \ + n += 1; \ + if (UNLIKELY((hash & mask) == 0)) { \ + splits[*numSplits] = n; \ + *numSplits += 1; \ + if (*numSplits == LDM_BATCH_SIZE) \ + goto done; \ + } \ + } while (0) + + while (n + 3 < size) { + GEAR_ITER_ONCE(); + GEAR_ITER_ONCE(); + GEAR_ITER_ONCE(); + GEAR_ITER_ONCE(); + } + while (n < size) { + GEAR_ITER_ONCE(); + } + +#undef GEAR_ITER_ONCE + +done: + state->rolling = hash; + return n; +} + +void ZSTD_ldm_adjustParameters(ldmParams_t* params, + ZSTD_compressionParameters const* cParams) +{ + params->windowLog = cParams->windowLog; + ZSTD_STATIC_ASSERT(LDM_BUCKET_SIZE_LOG <= ZSTD_LDM_BUCKETSIZELOG_MAX); + DEBUGLOG(4, "ZSTD_ldm_adjustParameters"); + if (!params->bucketSizeLog) params->bucketSizeLog = LDM_BUCKET_SIZE_LOG; + if (!params->minMatchLength) params->minMatchLength = LDM_MIN_MATCH_LENGTH; + if (params->hashLog == 0) { + params->hashLog = MAX(ZSTD_HASHLOG_MIN, params->windowLog - LDM_HASH_RLOG); + assert(params->hashLog <= ZSTD_HASHLOG_MAX); + } + if (params->hashRateLog == 0) { + params->hashRateLog = params->windowLog < params->hashLog + ? 0 + : params->windowLog - params->hashLog; + } + params->bucketSizeLog = MIN(params->bucketSizeLog, params->hashLog); +} + +size_t ZSTD_ldm_getTableSize(ldmParams_t params) +{ + size_t const ldmHSize = ((size_t)1) << params.hashLog; + size_t const ldmBucketSizeLog = MIN(params.bucketSizeLog, params.hashLog); + size_t const ldmBucketSize = ((size_t)1) << (params.hashLog - ldmBucketSizeLog); + size_t const totalSize = ZSTD_cwksp_alloc_size(ldmBucketSize) + + ZSTD_cwksp_alloc_size(ldmHSize * sizeof(ldmEntry_t)); + return params.enableLdm == ZSTD_ps_enable ? totalSize : 0; +} + +size_t ZSTD_ldm_getMaxNbSeq(ldmParams_t params, size_t maxChunkSize) +{ + return params.enableLdm == ZSTD_ps_enable ? (maxChunkSize / params.minMatchLength) : 0; +} + +/** ZSTD_ldm_getBucket() : + * Returns a pointer to the start of the bucket associated with hash. */ +static ldmEntry_t* ZSTD_ldm_getBucket( + ldmState_t* ldmState, size_t hash, ldmParams_t const ldmParams) +{ + return ldmState->hashTable + (hash << ldmParams.bucketSizeLog); +} + +/** ZSTD_ldm_insertEntry() : + * Insert the entry with corresponding hash into the hash table */ +static void ZSTD_ldm_insertEntry(ldmState_t* ldmState, + size_t const hash, const ldmEntry_t entry, + ldmParams_t const ldmParams) +{ + BYTE* const pOffset = ldmState->bucketOffsets + hash; + unsigned const offset = *pOffset; + + *(ZSTD_ldm_getBucket(ldmState, hash, ldmParams) + offset) = entry; + *pOffset = (BYTE)((offset + 1) & ((1u << ldmParams.bucketSizeLog) - 1)); + +} + +/** ZSTD_ldm_countBackwardsMatch() : + * Returns the number of bytes that match backwards before pIn and pMatch. + * + * We count only bytes where pMatch >= pBase and pIn >= pAnchor. */ +static size_t ZSTD_ldm_countBackwardsMatch( + const BYTE* pIn, const BYTE* pAnchor, + const BYTE* pMatch, const BYTE* pMatchBase) +{ + size_t matchLength = 0; + while (pIn > pAnchor && pMatch > pMatchBase && pIn[-1] == pMatch[-1]) { + pIn--; + pMatch--; + matchLength++; + } + return matchLength; +} + +/** ZSTD_ldm_countBackwardsMatch_2segments() : + * Returns the number of bytes that match backwards from pMatch, + * even with the backwards match spanning 2 different segments. + * + * On reaching `pMatchBase`, start counting from mEnd */ +static size_t ZSTD_ldm_countBackwardsMatch_2segments( + const BYTE* pIn, const BYTE* pAnchor, + const BYTE* pMatch, const BYTE* pMatchBase, + const BYTE* pExtDictStart, const BYTE* pExtDictEnd) +{ + size_t matchLength = ZSTD_ldm_countBackwardsMatch(pIn, pAnchor, pMatch, pMatchBase); + if (pMatch - matchLength != pMatchBase || pMatchBase == pExtDictStart) { + /* If backwards match is entirely in the extDict or prefix, immediately return */ + return matchLength; + } + DEBUGLOG(7, "ZSTD_ldm_countBackwardsMatch_2segments: found 2-parts backwards match (length in prefix==%zu)", matchLength); + matchLength += ZSTD_ldm_countBackwardsMatch(pIn - matchLength, pAnchor, pExtDictEnd, pExtDictStart); + DEBUGLOG(7, "final backwards match length = %zu", matchLength); + return matchLength; +} + +/** ZSTD_ldm_fillFastTables() : + * + * Fills the relevant tables for the ZSTD_fast and ZSTD_dfast strategies. + * This is similar to ZSTD_loadDictionaryContent. + * + * The tables for the other strategies are filled within their + * block compressors. */ +static size_t ZSTD_ldm_fillFastTables(ZSTD_matchState_t* ms, + void const* end) +{ + const BYTE* const iend = (const BYTE*)end; + + switch(ms->cParams.strategy) + { + case ZSTD_fast: + ZSTD_fillHashTable(ms, iend, ZSTD_dtlm_fast); + break; + + case ZSTD_dfast: + ZSTD_fillDoubleHashTable(ms, iend, ZSTD_dtlm_fast); + break; + + case ZSTD_greedy: + case ZSTD_lazy: + case ZSTD_lazy2: + case ZSTD_btlazy2: + case ZSTD_btopt: + case ZSTD_btultra: + case ZSTD_btultra2: + break; + default: + assert(0); /* not possible : not a valid strategy id */ + } + + return 0; +} + +void ZSTD_ldm_fillHashTable( + ldmState_t* ldmState, const BYTE* ip, + const BYTE* iend, ldmParams_t const* params) +{ + U32 const minMatchLength = params->minMatchLength; + U32 const hBits = params->hashLog - params->bucketSizeLog; + BYTE const* const base = ldmState->window.base; + BYTE const* const istart = ip; + ldmRollingHashState_t hashState; + size_t* const splits = ldmState->splitIndices; + unsigned numSplits; + + DEBUGLOG(5, "ZSTD_ldm_fillHashTable"); + + ZSTD_ldm_gear_init(&hashState, params); + while (ip < iend) { + size_t hashed; + unsigned n; + + numSplits = 0; + hashed = ZSTD_ldm_gear_feed(&hashState, ip, iend - ip, splits, &numSplits); + + for (n = 0; n < numSplits; n++) { + if (ip + splits[n] >= istart + minMatchLength) { + BYTE const* const split = ip + splits[n] - minMatchLength; + U64 const xxhash = XXH64(split, minMatchLength, 0); + U32 const hash = (U32)(xxhash & (((U32)1 << hBits) - 1)); + ldmEntry_t entry; + + entry.offset = (U32)(split - base); + entry.checksum = (U32)(xxhash >> 32); + ZSTD_ldm_insertEntry(ldmState, hash, entry, *params); + } + } + + ip += hashed; + } +} + + +/** ZSTD_ldm_limitTableUpdate() : + * + * Sets cctx->nextToUpdate to a position corresponding closer to anchor + * if it is far way + * (after a long match, only update tables a limited amount). */ +static void ZSTD_ldm_limitTableUpdate(ZSTD_matchState_t* ms, const BYTE* anchor) +{ + U32 const curr = (U32)(anchor - ms->window.base); + if (curr > ms->nextToUpdate + 1024) { + ms->nextToUpdate = + curr - MIN(512, curr - ms->nextToUpdate - 1024); + } +} + +static size_t ZSTD_ldm_generateSequences_internal( + ldmState_t* ldmState, rawSeqStore_t* rawSeqStore, + ldmParams_t const* params, void const* src, size_t srcSize) +{ + /* LDM parameters */ + int const extDict = ZSTD_window_hasExtDict(ldmState->window); + U32 const minMatchLength = params->minMatchLength; + U32 const entsPerBucket = 1U << params->bucketSizeLog; + U32 const hBits = params->hashLog - params->bucketSizeLog; + /* Prefix and extDict parameters */ + U32 const dictLimit = ldmState->window.dictLimit; + U32 const lowestIndex = extDict ? ldmState->window.lowLimit : dictLimit; + BYTE const* const base = ldmState->window.base; + BYTE const* const dictBase = extDict ? ldmState->window.dictBase : NULL; + BYTE const* const dictStart = extDict ? dictBase + lowestIndex : NULL; + BYTE const* const dictEnd = extDict ? dictBase + dictLimit : NULL; + BYTE const* const lowPrefixPtr = base + dictLimit; + /* Input bounds */ + BYTE const* const istart = (BYTE const*)src; + BYTE const* const iend = istart + srcSize; + BYTE const* const ilimit = iend - HASH_READ_SIZE; + /* Input positions */ + BYTE const* anchor = istart; + BYTE const* ip = istart; + /* Rolling hash state */ + ldmRollingHashState_t hashState; + /* Arrays for staged-processing */ + size_t* const splits = ldmState->splitIndices; + ldmMatchCandidate_t* const candidates = ldmState->matchCandidates; + unsigned numSplits; + + if (srcSize < minMatchLength) + return iend - anchor; + + /* Initialize the rolling hash state with the first minMatchLength bytes */ + ZSTD_ldm_gear_init(&hashState, params); + ZSTD_ldm_gear_reset(&hashState, ip, minMatchLength); + ip += minMatchLength; + + while (ip < ilimit) { + size_t hashed; + unsigned n; + + numSplits = 0; + hashed = ZSTD_ldm_gear_feed(&hashState, ip, ilimit - ip, + splits, &numSplits); + + for (n = 0; n < numSplits; n++) { + BYTE const* const split = ip + splits[n] - minMatchLength; + U64 const xxhash = XXH64(split, minMatchLength, 0); + U32 const hash = (U32)(xxhash & (((U32)1 << hBits) - 1)); + + candidates[n].split = split; + candidates[n].hash = hash; + candidates[n].checksum = (U32)(xxhash >> 32); + candidates[n].bucket = ZSTD_ldm_getBucket(ldmState, hash, *params); + PREFETCH_L1(candidates[n].bucket); + } + + for (n = 0; n < numSplits; n++) { + size_t forwardMatchLength = 0, backwardMatchLength = 0, + bestMatchLength = 0, mLength; + U32 offset; + BYTE const* const split = candidates[n].split; + U32 const checksum = candidates[n].checksum; + U32 const hash = candidates[n].hash; + ldmEntry_t* const bucket = candidates[n].bucket; + ldmEntry_t const* cur; + ldmEntry_t const* bestEntry = NULL; + ldmEntry_t newEntry; + + newEntry.offset = (U32)(split - base); + newEntry.checksum = checksum; + + /* If a split point would generate a sequence overlapping with + * the previous one, we merely register it in the hash table and + * move on */ + if (split < anchor) { + ZSTD_ldm_insertEntry(ldmState, hash, newEntry, *params); + continue; + } + + for (cur = bucket; cur < bucket + entsPerBucket; cur++) { + size_t curForwardMatchLength, curBackwardMatchLength, + curTotalMatchLength; + if (cur->checksum != checksum || cur->offset <= lowestIndex) { + continue; + } + if (extDict) { + BYTE const* const curMatchBase = + cur->offset < dictLimit ? dictBase : base; + BYTE const* const pMatch = curMatchBase + cur->offset; + BYTE const* const matchEnd = + cur->offset < dictLimit ? dictEnd : iend; + BYTE const* const lowMatchPtr = + cur->offset < dictLimit ? dictStart : lowPrefixPtr; + curForwardMatchLength = + ZSTD_count_2segments(split, pMatch, iend, matchEnd, lowPrefixPtr); + if (curForwardMatchLength < minMatchLength) { + continue; + } + curBackwardMatchLength = ZSTD_ldm_countBackwardsMatch_2segments( + split, anchor, pMatch, lowMatchPtr, dictStart, dictEnd); + } else { /* !extDict */ + BYTE const* const pMatch = base + cur->offset; + curForwardMatchLength = ZSTD_count(split, pMatch, iend); + if (curForwardMatchLength < minMatchLength) { + continue; + } + curBackwardMatchLength = + ZSTD_ldm_countBackwardsMatch(split, anchor, pMatch, lowPrefixPtr); + } + curTotalMatchLength = curForwardMatchLength + curBackwardMatchLength; + + if (curTotalMatchLength > bestMatchLength) { + bestMatchLength = curTotalMatchLength; + forwardMatchLength = curForwardMatchLength; + backwardMatchLength = curBackwardMatchLength; + bestEntry = cur; + } + } + + /* No match found -- insert an entry into the hash table + * and process the next candidate match */ + if (bestEntry == NULL) { + ZSTD_ldm_insertEntry(ldmState, hash, newEntry, *params); + continue; + } + + /* Match found */ + offset = (U32)(split - base) - bestEntry->offset; + mLength = forwardMatchLength + backwardMatchLength; + { + rawSeq* const seq = rawSeqStore->seq + rawSeqStore->size; + + /* Out of sequence storage */ + if (rawSeqStore->size == rawSeqStore->capacity) + return ERROR(dstSize_tooSmall); + seq->litLength = (U32)(split - backwardMatchLength - anchor); + seq->matchLength = (U32)mLength; + seq->offset = offset; + rawSeqStore->size++; + } + + /* Insert the current entry into the hash table --- it must be + * done after the previous block to avoid clobbering bestEntry */ + ZSTD_ldm_insertEntry(ldmState, hash, newEntry, *params); + + anchor = split + forwardMatchLength; + + /* If we find a match that ends after the data that we've hashed + * then we have a repeating, overlapping, pattern. E.g. all zeros. + * If one repetition of the pattern matches our `stopMask` then all + * repetitions will. We don't need to insert them all into out table, + * only the first one. So skip over overlapping matches. + * This is a major speed boost (20x) for compressing a single byte + * repeated, when that byte ends up in the table. + */ + if (anchor > ip + hashed) { + ZSTD_ldm_gear_reset(&hashState, anchor - minMatchLength, minMatchLength); + /* Continue the outer loop at anchor (ip + hashed == anchor). */ + ip = anchor - hashed; + break; + } + } + + ip += hashed; + } + + return iend - anchor; +} + +/*! ZSTD_ldm_reduceTable() : + * reduce table indexes by `reducerValue` */ +static void ZSTD_ldm_reduceTable(ldmEntry_t* const table, U32 const size, + U32 const reducerValue) +{ + U32 u; + for (u = 0; u < size; u++) { + if (table[u].offset < reducerValue) table[u].offset = 0; + else table[u].offset -= reducerValue; + } +} + +size_t ZSTD_ldm_generateSequences( + ldmState_t* ldmState, rawSeqStore_t* sequences, + ldmParams_t const* params, void const* src, size_t srcSize) +{ + U32 const maxDist = 1U << params->windowLog; + BYTE const* const istart = (BYTE const*)src; + BYTE const* const iend = istart + srcSize; + size_t const kMaxChunkSize = 1 << 20; + size_t const nbChunks = (srcSize / kMaxChunkSize) + ((srcSize % kMaxChunkSize) != 0); + size_t chunk; + size_t leftoverSize = 0; + + assert(ZSTD_CHUNKSIZE_MAX >= kMaxChunkSize); + /* Check that ZSTD_window_update() has been called for this chunk prior + * to passing it to this function. + */ + assert(ldmState->window.nextSrc >= (BYTE const*)src + srcSize); + /* The input could be very large (in zstdmt), so it must be broken up into + * chunks to enforce the maximum distance and handle overflow correction. + */ + assert(sequences->pos <= sequences->size); + assert(sequences->size <= sequences->capacity); + for (chunk = 0; chunk < nbChunks && sequences->size < sequences->capacity; ++chunk) { + BYTE const* const chunkStart = istart + chunk * kMaxChunkSize; + size_t const remaining = (size_t)(iend - chunkStart); + BYTE const *const chunkEnd = + (remaining < kMaxChunkSize) ? iend : chunkStart + kMaxChunkSize; + size_t const chunkSize = chunkEnd - chunkStart; + size_t newLeftoverSize; + size_t const prevSize = sequences->size; + + assert(chunkStart < iend); + /* 1. Perform overflow correction if necessary. */ + if (ZSTD_window_needOverflowCorrection(ldmState->window, 0, maxDist, ldmState->loadedDictEnd, chunkStart, chunkEnd)) { + U32 const ldmHSize = 1U << params->hashLog; + U32 const correction = ZSTD_window_correctOverflow( + &ldmState->window, /* cycleLog */ 0, maxDist, chunkStart); + ZSTD_ldm_reduceTable(ldmState->hashTable, ldmHSize, correction); + /* invalidate dictionaries on overflow correction */ + ldmState->loadedDictEnd = 0; + } + /* 2. We enforce the maximum offset allowed. + * + * kMaxChunkSize should be small enough that we don't lose too much of + * the window through early invalidation. + * TODO: * Test the chunk size. + * * Try invalidation after the sequence generation and test the + * the offset against maxDist directly. + * + * NOTE: Because of dictionaries + sequence splitting we MUST make sure + * that any offset used is valid at the END of the sequence, since it may + * be split into two sequences. This condition holds when using + * ZSTD_window_enforceMaxDist(), but if we move to checking offsets + * against maxDist directly, we'll have to carefully handle that case. + */ + ZSTD_window_enforceMaxDist(&ldmState->window, chunkEnd, maxDist, &ldmState->loadedDictEnd, NULL); + /* 3. Generate the sequences for the chunk, and get newLeftoverSize. */ + newLeftoverSize = ZSTD_ldm_generateSequences_internal( + ldmState, sequences, params, chunkStart, chunkSize); + if (ZSTD_isError(newLeftoverSize)) + return newLeftoverSize; + /* 4. We add the leftover literals from previous iterations to the first + * newly generated sequence, or add the `newLeftoverSize` if none are + * generated. + */ + /* Prepend the leftover literals from the last call */ + if (prevSize < sequences->size) { + sequences->seq[prevSize].litLength += (U32)leftoverSize; + leftoverSize = newLeftoverSize; + } else { + assert(newLeftoverSize == chunkSize); + leftoverSize += chunkSize; + } + } + return 0; +} + +void +ZSTD_ldm_skipSequences(rawSeqStore_t* rawSeqStore, size_t srcSize, U32 const minMatch) +{ + while (srcSize > 0 && rawSeqStore->pos < rawSeqStore->size) { + rawSeq* seq = rawSeqStore->seq + rawSeqStore->pos; + if (srcSize <= seq->litLength) { + /* Skip past srcSize literals */ + seq->litLength -= (U32)srcSize; + return; + } + srcSize -= seq->litLength; + seq->litLength = 0; + if (srcSize < seq->matchLength) { + /* Skip past the first srcSize of the match */ + seq->matchLength -= (U32)srcSize; + if (seq->matchLength < minMatch) { + /* The match is too short, omit it */ + if (rawSeqStore->pos + 1 < rawSeqStore->size) { + seq[1].litLength += seq[0].matchLength; + } + rawSeqStore->pos++; + } + return; + } + srcSize -= seq->matchLength; + seq->matchLength = 0; + rawSeqStore->pos++; + } +} + +/** + * If the sequence length is longer than remaining then the sequence is split + * between this block and the next. + * + * Returns the current sequence to handle, or if the rest of the block should + * be literals, it returns a sequence with offset == 0. + */ +static rawSeq maybeSplitSequence(rawSeqStore_t* rawSeqStore, + U32 const remaining, U32 const minMatch) +{ + rawSeq sequence = rawSeqStore->seq[rawSeqStore->pos]; + assert(sequence.offset > 0); + /* Likely: No partial sequence */ + if (remaining >= sequence.litLength + sequence.matchLength) { + rawSeqStore->pos++; + return sequence; + } + /* Cut the sequence short (offset == 0 ==> rest is literals). */ + if (remaining <= sequence.litLength) { + sequence.offset = 0; + } else if (remaining < sequence.litLength + sequence.matchLength) { + sequence.matchLength = remaining - sequence.litLength; + if (sequence.matchLength < minMatch) { + sequence.offset = 0; + } + } + /* Skip past `remaining` bytes for the future sequences. */ + ZSTD_ldm_skipSequences(rawSeqStore, remaining, minMatch); + return sequence; +} + +void ZSTD_ldm_skipRawSeqStoreBytes(rawSeqStore_t* rawSeqStore, size_t nbBytes) { + U32 currPos = (U32)(rawSeqStore->posInSequence + nbBytes); + while (currPos && rawSeqStore->pos < rawSeqStore->size) { + rawSeq currSeq = rawSeqStore->seq[rawSeqStore->pos]; + if (currPos >= currSeq.litLength + currSeq.matchLength) { + currPos -= currSeq.litLength + currSeq.matchLength; + rawSeqStore->pos++; + } else { + rawSeqStore->posInSequence = currPos; + break; + } + } + if (currPos == 0 || rawSeqStore->pos == rawSeqStore->size) { + rawSeqStore->posInSequence = 0; + } +} + +size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore, + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + ZSTD_paramSwitch_e useRowMatchFinder, + void const* src, size_t srcSize) +{ + const ZSTD_compressionParameters* const cParams = &ms->cParams; + unsigned const minMatch = cParams->minMatch; + ZSTD_blockCompressor const blockCompressor = + ZSTD_selectBlockCompressor(cParams->strategy, useRowMatchFinder, ZSTD_matchState_dictMode(ms)); + /* Input bounds */ + BYTE const* const istart = (BYTE const*)src; + BYTE const* const iend = istart + srcSize; + /* Input positions */ + BYTE const* ip = istart; + + DEBUGLOG(5, "ZSTD_ldm_blockCompress: srcSize=%zu", srcSize); + /* If using opt parser, use LDMs only as candidates rather than always accepting them */ + if (cParams->strategy >= ZSTD_btopt) { + size_t lastLLSize; + ms->ldmSeqStore = rawSeqStore; + lastLLSize = blockCompressor(ms, seqStore, rep, src, srcSize); + ZSTD_ldm_skipRawSeqStoreBytes(rawSeqStore, srcSize); + return lastLLSize; + } + + assert(rawSeqStore->pos <= rawSeqStore->size); + assert(rawSeqStore->size <= rawSeqStore->capacity); + /* Loop through each sequence and apply the block compressor to the literals */ + while (rawSeqStore->pos < rawSeqStore->size && ip < iend) { + /* maybeSplitSequence updates rawSeqStore->pos */ + rawSeq const sequence = maybeSplitSequence(rawSeqStore, + (U32)(iend - ip), minMatch); + int i; + /* End signal */ + if (sequence.offset == 0) + break; + + assert(ip + sequence.litLength + sequence.matchLength <= iend); + + /* Fill tables for block compressor */ + ZSTD_ldm_limitTableUpdate(ms, ip); + ZSTD_ldm_fillFastTables(ms, ip); + /* Run the block compressor */ + DEBUGLOG(5, "pos %u : calling block compressor on segment of size %u", (unsigned)(ip-istart), sequence.litLength); + { + size_t const newLitLength = + blockCompressor(ms, seqStore, rep, ip, sequence.litLength); + ip += sequence.litLength; + /* Update the repcodes */ + for (i = ZSTD_REP_NUM - 1; i > 0; i--) + rep[i] = rep[i-1]; + rep[0] = sequence.offset; + /* Store the sequence */ + ZSTD_storeSeq(seqStore, newLitLength, ip - newLitLength, iend, + STORE_OFFSET(sequence.offset), + sequence.matchLength); + ip += sequence.matchLength; + } + } + /* Fill the tables for the block compressor */ + ZSTD_ldm_limitTableUpdate(ms, ip); + ZSTD_ldm_fillFastTables(ms, ip); + /* Compress the last literals */ + return blockCompressor(ms, seqStore, rep, ip, iend - ip); +} diff --git a/src/blosc2/internal-complibs/zstd-1.5.2/compress/zstd_ldm.h b/src/blosc2/internal-complibs/zstd-1.5.2/compress/zstd_ldm.h new file mode 100644 index 0000000..4e68dbf --- /dev/null +++ b/src/blosc2/internal-complibs/zstd-1.5.2/compress/zstd_ldm.h @@ -0,0 +1,117 @@ +/* + * Copyright (c) Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_LDM_H +#define ZSTD_LDM_H + +#if defined (__cplusplus) +extern "C" { +#endif + +#include "zstd_compress_internal.h" /* ldmParams_t, U32 */ +#include "../zstd.h" /* ZSTD_CCtx, size_t */ + +/*-************************************* +* Long distance matching +***************************************/ + +#define ZSTD_LDM_DEFAULT_WINDOW_LOG ZSTD_WINDOWLOG_LIMIT_DEFAULT + +void ZSTD_ldm_fillHashTable( + ldmState_t* state, const BYTE* ip, + const BYTE* iend, ldmParams_t const* params); + +/** + * ZSTD_ldm_generateSequences(): + * + * Generates the sequences using the long distance match finder. + * Generates long range matching sequences in `sequences`, which parse a prefix + * of the source. `sequences` must be large enough to store every sequence, + * which can be checked with `ZSTD_ldm_getMaxNbSeq()`. + * @returns 0 or an error code. + * + * NOTE: The user must have called ZSTD_window_update() for all of the input + * they have, even if they pass it to ZSTD_ldm_generateSequences() in chunks. + * NOTE: This function returns an error if it runs out of space to store + * sequences. + */ +size_t ZSTD_ldm_generateSequences( + ldmState_t* ldms, rawSeqStore_t* sequences, + ldmParams_t const* params, void const* src, size_t srcSize); + +/** + * ZSTD_ldm_blockCompress(): + * + * Compresses a block using the predefined sequences, along with a secondary + * block compressor. The literals section of every sequence is passed to the + * secondary block compressor, and those sequences are interspersed with the + * predefined sequences. Returns the length of the last literals. + * Updates `rawSeqStore.pos` to indicate how many sequences have been consumed. + * `rawSeqStore.seq` may also be updated to split the last sequence between two + * blocks. + * @return The length of the last literals. + * + * NOTE: The source must be at most the maximum block size, but the predefined + * sequences can be any size, and may be longer than the block. In the case that + * they are longer than the block, the last sequences may need to be split into + * two. We handle that case correctly, and update `rawSeqStore` appropriately. + * NOTE: This function does not return any errors. + */ +size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore, + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + ZSTD_paramSwitch_e useRowMatchFinder, + void const* src, size_t srcSize); + +/** + * ZSTD_ldm_skipSequences(): + * + * Skip past `srcSize` bytes worth of sequences in `rawSeqStore`. + * Avoids emitting matches less than `minMatch` bytes. + * Must be called for data that is not passed to ZSTD_ldm_blockCompress(). + */ +void ZSTD_ldm_skipSequences(rawSeqStore_t* rawSeqStore, size_t srcSize, + U32 const minMatch); + +/* ZSTD_ldm_skipRawSeqStoreBytes(): + * Moves forward in rawSeqStore by nbBytes, updating fields 'pos' and 'posInSequence'. + * Not to be used in conjunction with ZSTD_ldm_skipSequences(). + * Must be called for data with is not passed to ZSTD_ldm_blockCompress(). + */ +void ZSTD_ldm_skipRawSeqStoreBytes(rawSeqStore_t* rawSeqStore, size_t nbBytes); + +/** ZSTD_ldm_getTableSize() : + * Estimate the space needed for long distance matching tables or 0 if LDM is + * disabled. + */ +size_t ZSTD_ldm_getTableSize(ldmParams_t params); + +/** ZSTD_ldm_getSeqSpace() : + * Return an upper bound on the number of sequences that can be produced by + * the long distance matcher, or 0 if LDM is disabled. + */ +size_t ZSTD_ldm_getMaxNbSeq(ldmParams_t params, size_t maxChunkSize); + +/** ZSTD_ldm_adjustParameters() : + * If the params->hashRateLog is not set, set it to its default value based on + * windowLog and params->hashLog. + * + * Ensures that params->bucketSizeLog is <= params->hashLog (setting it to + * params->hashLog if it is not). + * + * Ensures that the minMatchLength >= targetLength during optimal parsing. + */ +void ZSTD_ldm_adjustParameters(ldmParams_t* params, + ZSTD_compressionParameters const* cParams); + +#if defined (__cplusplus) +} +#endif + +#endif /* ZSTD_FAST_H */ diff --git a/src/blosc2/internal-complibs/zstd-1.5.2/compress/zstd_ldm_geartab.h b/src/blosc2/internal-complibs/zstd-1.5.2/compress/zstd_ldm_geartab.h new file mode 100644 index 0000000..647f865 --- /dev/null +++ b/src/blosc2/internal-complibs/zstd-1.5.2/compress/zstd_ldm_geartab.h @@ -0,0 +1,106 @@ +/* + * Copyright (c) Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_LDM_GEARTAB_H +#define ZSTD_LDM_GEARTAB_H + +#include "../common/compiler.h" /* UNUSED_ATTR */ +#include "../common/mem.h" /* U64 */ + +static UNUSED_ATTR const U64 ZSTD_ldm_gearTab[256] = { + 0xf5b8f72c5f77775c, 0x84935f266b7ac412, 0xb647ada9ca730ccc, + 0xb065bb4b114fb1de, 0x34584e7e8c3a9fd0, 0x4e97e17c6ae26b05, + 0x3a03d743bc99a604, 0xcecd042422c4044f, 0x76de76c58524259e, + 0x9c8528f65badeaca, 0x86563706e2097529, 0x2902475fa375d889, + 0xafb32a9739a5ebe6, 0xce2714da3883e639, 0x21eaf821722e69e, + 0x37b628620b628, 0x49a8d455d88caf5, 0x8556d711e6958140, + 0x4f7ae74fc605c1f, 0x829f0c3468bd3a20, 0x4ffdc885c625179e, + 0x8473de048a3daf1b, 0x51008822b05646b2, 0x69d75d12b2d1cc5f, + 0x8c9d4a19159154bc, 0xc3cc10f4abbd4003, 0xd06ddc1cecb97391, + 0xbe48e6e7ed80302e, 0x3481db31cee03547, 0xacc3f67cdaa1d210, + 0x65cb771d8c7f96cc, 0x8eb27177055723dd, 0xc789950d44cd94be, + 0x934feadc3700b12b, 0x5e485f11edbdf182, 0x1e2e2a46fd64767a, + 0x2969ca71d82efa7c, 0x9d46e9935ebbba2e, 0xe056b67e05e6822b, + 0x94d73f55739d03a0, 0xcd7010bdb69b5a03, 0x455ef9fcd79b82f4, + 0x869cb54a8749c161, 0x38d1a4fa6185d225, 0xb475166f94bbe9bb, + 0xa4143548720959f1, 0x7aed4780ba6b26ba, 0xd0ce264439e02312, + 0x84366d746078d508, 0xa8ce973c72ed17be, 0x21c323a29a430b01, + 0x9962d617e3af80ee, 0xab0ce91d9c8cf75b, 0x530e8ee6d19a4dbc, + 0x2ef68c0cf53f5d72, 0xc03a681640a85506, 0x496e4e9f9c310967, + 0x78580472b59b14a0, 0x273824c23b388577, 0x66bf923ad45cb553, + 0x47ae1a5a2492ba86, 0x35e304569e229659, 0x4765182a46870b6f, + 0x6cbab625e9099412, 0xddac9a2e598522c1, 0x7172086e666624f2, + 0xdf5003ca503b7837, 0x88c0c1db78563d09, 0x58d51865acfc289d, + 0x177671aec65224f1, 0xfb79d8a241e967d7, 0x2be1e101cad9a49a, + 0x6625682f6e29186b, 0x399553457ac06e50, 0x35dffb4c23abb74, + 0x429db2591f54aade, 0xc52802a8037d1009, 0x6acb27381f0b25f3, + 0xf45e2551ee4f823b, 0x8b0ea2d99580c2f7, 0x3bed519cbcb4e1e1, + 0xff452823dbb010a, 0x9d42ed614f3dd267, 0x5b9313c06257c57b, + 0xa114b8008b5e1442, 0xc1fe311c11c13d4b, 0x66e8763ea34c5568, + 0x8b982af1c262f05d, 0xee8876faaa75fbb7, 0x8a62a4d0d172bb2a, + 0xc13d94a3b7449a97, 0x6dbbba9dc15d037c, 0xc786101f1d92e0f1, + 0xd78681a907a0b79b, 0xf61aaf2962c9abb9, 0x2cfd16fcd3cb7ad9, + 0x868c5b6744624d21, 0x25e650899c74ddd7, 0xba042af4a7c37463, + 0x4eb1a539465a3eca, 0xbe09dbf03b05d5ca, 0x774e5a362b5472ba, + 0x47a1221229d183cd, 0x504b0ca18ef5a2df, 0xdffbdfbde2456eb9, + 0x46cd2b2fbee34634, 0xf2aef8fe819d98c3, 0x357f5276d4599d61, + 0x24a5483879c453e3, 0x88026889192b4b9, 0x28da96671782dbec, + 0x4ef37c40588e9aaa, 0x8837b90651bc9fb3, 0xc164f741d3f0e5d6, + 0xbc135a0a704b70ba, 0x69cd868f7622ada, 0xbc37ba89e0b9c0ab, + 0x47c14a01323552f6, 0x4f00794bacee98bb, 0x7107de7d637a69d5, + 0x88af793bb6f2255e, 0xf3c6466b8799b598, 0xc288c616aa7f3b59, + 0x81ca63cf42fca3fd, 0x88d85ace36a2674b, 0xd056bd3792389e7, + 0xe55c396c4e9dd32d, 0xbefb504571e6c0a6, 0x96ab32115e91e8cc, + 0xbf8acb18de8f38d1, 0x66dae58801672606, 0x833b6017872317fb, + 0xb87c16f2d1c92864, 0xdb766a74e58b669c, 0x89659f85c61417be, + 0xc8daad856011ea0c, 0x76a4b565b6fe7eae, 0xa469d085f6237312, + 0xaaf0365683a3e96c, 0x4dbb746f8424f7b8, 0x638755af4e4acc1, + 0x3d7807f5bde64486, 0x17be6d8f5bbb7639, 0x903f0cd44dc35dc, + 0x67b672eafdf1196c, 0xa676ff93ed4c82f1, 0x521d1004c5053d9d, + 0x37ba9ad09ccc9202, 0x84e54d297aacfb51, 0xa0b4b776a143445, + 0x820d471e20b348e, 0x1874383cb83d46dc, 0x97edeec7a1efe11c, + 0xb330e50b1bdc42aa, 0x1dd91955ce70e032, 0xa514cdb88f2939d5, + 0x2791233fd90db9d3, 0x7b670a4cc50f7a9b, 0x77c07d2a05c6dfa5, + 0xe3778b6646d0a6fa, 0xb39c8eda47b56749, 0x933ed448addbef28, + 0xaf846af6ab7d0bf4, 0xe5af208eb666e49, 0x5e6622f73534cd6a, + 0x297daeca42ef5b6e, 0x862daef3d35539a6, 0xe68722498f8e1ea9, + 0x981c53093dc0d572, 0xfa09b0bfbf86fbf5, 0x30b1e96166219f15, + 0x70e7d466bdc4fb83, 0x5a66736e35f2a8e9, 0xcddb59d2b7c1baef, + 0xd6c7d247d26d8996, 0xea4e39eac8de1ba3, 0x539c8bb19fa3aff2, + 0x9f90e4c5fd508d8, 0xa34e5956fbaf3385, 0x2e2f8e151d3ef375, + 0x173691e9b83faec1, 0xb85a8d56bf016379, 0x8382381267408ae3, + 0xb90f901bbdc0096d, 0x7c6ad32933bcec65, 0x76bb5e2f2c8ad595, + 0x390f851a6cf46d28, 0xc3e6064da1c2da72, 0xc52a0c101cfa5389, + 0xd78eaf84a3fbc530, 0x3781b9e2288b997e, 0x73c2f6dea83d05c4, + 0x4228e364c5b5ed7, 0x9d7a3edf0da43911, 0x8edcfeda24686756, + 0x5e7667a7b7a9b3a1, 0x4c4f389fa143791d, 0xb08bc1023da7cddc, + 0x7ab4be3ae529b1cc, 0x754e6132dbe74ff9, 0x71635442a839df45, + 0x2f6fb1643fbe52de, 0x961e0a42cf7a8177, 0xf3b45d83d89ef2ea, + 0xee3de4cf4a6e3e9b, 0xcd6848542c3295e7, 0xe4cee1664c78662f, + 0x9947548b474c68c4, 0x25d73777a5ed8b0b, 0xc915b1d636b7fc, + 0x21c2ba75d9b0d2da, 0x5f6b5dcf608a64a1, 0xdcf333255ff9570c, + 0x633b922418ced4ee, 0xc136dde0b004b34a, 0x58cc83b05d4b2f5a, + 0x5eb424dda28e42d2, 0x62df47369739cd98, 0xb4e0b42485e4ce17, + 0x16e1f0c1f9a8d1e7, 0x8ec3916707560ebf, 0x62ba6e2df2cc9db3, + 0xcbf9f4ff77d83a16, 0x78d9d7d07d2bbcc4, 0xef554ce1e02c41f4, + 0x8d7581127eccf94d, 0xa9b53336cb3c8a05, 0x38c42c0bf45c4f91, + 0x640893cdf4488863, 0x80ec34bc575ea568, 0x39f324f5b48eaa40, + 0xe9d9ed1f8eff527f, 0x9224fc058cc5a214, 0xbaba00b04cfe7741, + 0x309a9f120fcf52af, 0xa558f3ec65626212, 0x424bec8b7adabe2f, + 0x41622513a6aea433, 0xb88da2d5324ca798, 0xd287733b245528a4, + 0x9a44697e6d68aec3, 0x7b1093be2f49bb28, 0x50bbec632e3d8aad, + 0x6cd90723e1ea8283, 0x897b9e7431b02bf3, 0x219efdcb338a7047, + 0x3b0311f0a27c0656, 0xdb17bf91c0db96e7, 0x8cd4fd6b4e85a5b2, + 0xfab071054ba6409d, 0x40d6fe831fa9dfd9, 0xaf358debad7d791e, + 0xeb8d0e25a65e3e58, 0xbbcbd3df14e08580, 0xcf751f27ecdab2b, + 0x2b4da14f2613d8f4 +}; + +#endif /* ZSTD_LDM_GEARTAB_H */ diff --git a/src/blosc2/internal-complibs/zstd-1.5.2/compress/zstd_opt.c b/src/blosc2/internal-complibs/zstd-1.5.2/compress/zstd_opt.c new file mode 100644 index 0000000..1b1ddad --- /dev/null +++ b/src/blosc2/internal-complibs/zstd-1.5.2/compress/zstd_opt.c @@ -0,0 +1,1446 @@ +/* + * Copyright (c) Przemyslaw Skibinski, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#include "zstd_compress_internal.h" +#include "hist.h" +#include "zstd_opt.h" + + +#define ZSTD_LITFREQ_ADD 2 /* scaling factor for litFreq, so that frequencies adapt faster to new stats */ +#define ZSTD_MAX_PRICE (1<<30) + +#define ZSTD_PREDEF_THRESHOLD 1024 /* if srcSize < ZSTD_PREDEF_THRESHOLD, symbols' cost is assumed static, directly determined by pre-defined distributions */ + + +/*-************************************* +* Price functions for optimal parser +***************************************/ + +#if 0 /* approximation at bit level (for tests) */ +# define BITCOST_ACCURACY 0 +# define BITCOST_MULTIPLIER (1 << BITCOST_ACCURACY) +# define WEIGHT(stat, opt) ((void)opt, ZSTD_bitWeight(stat)) +#elif 0 /* fractional bit accuracy (for tests) */ +# define BITCOST_ACCURACY 8 +# define BITCOST_MULTIPLIER (1 << BITCOST_ACCURACY) +# define WEIGHT(stat,opt) ((void)opt, ZSTD_fracWeight(stat)) +#else /* opt==approx, ultra==accurate */ +# define BITCOST_ACCURACY 8 +# define BITCOST_MULTIPLIER (1 << BITCOST_ACCURACY) +# define WEIGHT(stat,opt) (opt ? ZSTD_fracWeight(stat) : ZSTD_bitWeight(stat)) +#endif + +MEM_STATIC U32 ZSTD_bitWeight(U32 stat) +{ + return (ZSTD_highbit32(stat+1) * BITCOST_MULTIPLIER); +} + +MEM_STATIC U32 ZSTD_fracWeight(U32 rawStat) +{ + U32 const stat = rawStat + 1; + U32 const hb = ZSTD_highbit32(stat); + U32 const BWeight = hb * BITCOST_MULTIPLIER; + U32 const FWeight = (stat << BITCOST_ACCURACY) >> hb; + U32 const weight = BWeight + FWeight; + assert(hb + BITCOST_ACCURACY < 31); + return weight; +} + +#if (DEBUGLEVEL>=2) +/* debugging function, + * @return price in bytes as fractional value + * for debug messages only */ +MEM_STATIC double ZSTD_fCost(U32 price) +{ + return (double)price / (BITCOST_MULTIPLIER*8); +} +#endif + +static int ZSTD_compressedLiterals(optState_t const* const optPtr) +{ + return optPtr->literalCompressionMode != ZSTD_ps_disable; +} + +static void ZSTD_setBasePrices(optState_t* optPtr, int optLevel) +{ + if (ZSTD_compressedLiterals(optPtr)) + optPtr->litSumBasePrice = WEIGHT(optPtr->litSum, optLevel); + optPtr->litLengthSumBasePrice = WEIGHT(optPtr->litLengthSum, optLevel); + optPtr->matchLengthSumBasePrice = WEIGHT(optPtr->matchLengthSum, optLevel); + optPtr->offCodeSumBasePrice = WEIGHT(optPtr->offCodeSum, optLevel); +} + + +static U32 sum_u32(const unsigned table[], size_t nbElts) +{ + size_t n; + U32 total = 0; + for (n=0; n> shift); + sum += table[s]; + } + return sum; +} + +/* ZSTD_scaleStats() : + * reduce all elements in table is sum too large + * return the resulting sum of elements */ +static U32 ZSTD_scaleStats(unsigned* table, U32 lastEltIndex, U32 logTarget) +{ + U32 const prevsum = sum_u32(table, lastEltIndex+1); + U32 const factor = prevsum >> logTarget; + DEBUGLOG(5, "ZSTD_scaleStats (nbElts=%u, target=%u)", (unsigned)lastEltIndex+1, (unsigned)logTarget); + assert(logTarget < 30); + if (factor <= 1) return prevsum; + return ZSTD_downscaleStats(table, lastEltIndex, ZSTD_highbit32(factor)); +} + +/* ZSTD_rescaleFreqs() : + * if first block (detected by optPtr->litLengthSum == 0) : init statistics + * take hints from dictionary if there is one + * and init from zero if there is none, + * using src for literals stats, and baseline stats for sequence symbols + * otherwise downscale existing stats, to be used as seed for next block. + */ +static void +ZSTD_rescaleFreqs(optState_t* const optPtr, + const BYTE* const src, size_t const srcSize, + int const optLevel) +{ + int const compressedLiterals = ZSTD_compressedLiterals(optPtr); + DEBUGLOG(5, "ZSTD_rescaleFreqs (srcSize=%u)", (unsigned)srcSize); + optPtr->priceType = zop_dynamic; + + if (optPtr->litLengthSum == 0) { /* first block : init */ + if (srcSize <= ZSTD_PREDEF_THRESHOLD) { /* heuristic */ + DEBUGLOG(5, "(srcSize <= ZSTD_PREDEF_THRESHOLD) => zop_predef"); + optPtr->priceType = zop_predef; + } + + assert(optPtr->symbolCosts != NULL); + if (optPtr->symbolCosts->huf.repeatMode == HUF_repeat_valid) { + /* huffman table presumed generated by dictionary */ + optPtr->priceType = zop_dynamic; + + if (compressedLiterals) { + unsigned lit; + assert(optPtr->litFreq != NULL); + optPtr->litSum = 0; + for (lit=0; lit<=MaxLit; lit++) { + U32 const scaleLog = 11; /* scale to 2K */ + U32 const bitCost = HUF_getNbBitsFromCTable(optPtr->symbolCosts->huf.CTable, lit); + assert(bitCost <= scaleLog); + optPtr->litFreq[lit] = bitCost ? 1 << (scaleLog-bitCost) : 1 /*minimum to calculate cost*/; + optPtr->litSum += optPtr->litFreq[lit]; + } } + + { unsigned ll; + FSE_CState_t llstate; + FSE_initCState(&llstate, optPtr->symbolCosts->fse.litlengthCTable); + optPtr->litLengthSum = 0; + for (ll=0; ll<=MaxLL; ll++) { + U32 const scaleLog = 10; /* scale to 1K */ + U32 const bitCost = FSE_getMaxNbBits(llstate.symbolTT, ll); + assert(bitCost < scaleLog); + optPtr->litLengthFreq[ll] = bitCost ? 1 << (scaleLog-bitCost) : 1 /*minimum to calculate cost*/; + optPtr->litLengthSum += optPtr->litLengthFreq[ll]; + } } + + { unsigned ml; + FSE_CState_t mlstate; + FSE_initCState(&mlstate, optPtr->symbolCosts->fse.matchlengthCTable); + optPtr->matchLengthSum = 0; + for (ml=0; ml<=MaxML; ml++) { + U32 const scaleLog = 10; + U32 const bitCost = FSE_getMaxNbBits(mlstate.symbolTT, ml); + assert(bitCost < scaleLog); + optPtr->matchLengthFreq[ml] = bitCost ? 1 << (scaleLog-bitCost) : 1 /*minimum to calculate cost*/; + optPtr->matchLengthSum += optPtr->matchLengthFreq[ml]; + } } + + { unsigned of; + FSE_CState_t ofstate; + FSE_initCState(&ofstate, optPtr->symbolCosts->fse.offcodeCTable); + optPtr->offCodeSum = 0; + for (of=0; of<=MaxOff; of++) { + U32 const scaleLog = 10; + U32 const bitCost = FSE_getMaxNbBits(ofstate.symbolTT, of); + assert(bitCost < scaleLog); + optPtr->offCodeFreq[of] = bitCost ? 1 << (scaleLog-bitCost) : 1 /*minimum to calculate cost*/; + optPtr->offCodeSum += optPtr->offCodeFreq[of]; + } } + + } else { /* not a dictionary */ + + assert(optPtr->litFreq != NULL); + if (compressedLiterals) { + unsigned lit = MaxLit; + HIST_count_simple(optPtr->litFreq, &lit, src, srcSize); /* use raw first block to init statistics */ + optPtr->litSum = ZSTD_downscaleStats(optPtr->litFreq, MaxLit, 8); + } + + { unsigned const baseLLfreqs[MaxLL+1] = { + 4, 2, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1 + }; + ZSTD_memcpy(optPtr->litLengthFreq, baseLLfreqs, sizeof(baseLLfreqs)); + optPtr->litLengthSum = sum_u32(baseLLfreqs, MaxLL+1); + } + + { unsigned ml; + for (ml=0; ml<=MaxML; ml++) + optPtr->matchLengthFreq[ml] = 1; + } + optPtr->matchLengthSum = MaxML+1; + + { unsigned const baseOFCfreqs[MaxOff+1] = { + 6, 2, 1, 1, 2, 3, 4, 4, + 4, 3, 2, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1 + }; + ZSTD_memcpy(optPtr->offCodeFreq, baseOFCfreqs, sizeof(baseOFCfreqs)); + optPtr->offCodeSum = sum_u32(baseOFCfreqs, MaxOff+1); + } + + + } + + } else { /* new block : re-use previous statistics, scaled down */ + + if (compressedLiterals) + optPtr->litSum = ZSTD_scaleStats(optPtr->litFreq, MaxLit, 12); + optPtr->litLengthSum = ZSTD_scaleStats(optPtr->litLengthFreq, MaxLL, 11); + optPtr->matchLengthSum = ZSTD_scaleStats(optPtr->matchLengthFreq, MaxML, 11); + optPtr->offCodeSum = ZSTD_scaleStats(optPtr->offCodeFreq, MaxOff, 11); + } + + ZSTD_setBasePrices(optPtr, optLevel); +} + +/* ZSTD_rawLiteralsCost() : + * price of literals (only) in specified segment (which length can be 0). + * does not include price of literalLength symbol */ +static U32 ZSTD_rawLiteralsCost(const BYTE* const literals, U32 const litLength, + const optState_t* const optPtr, + int optLevel) +{ + if (litLength == 0) return 0; + + if (!ZSTD_compressedLiterals(optPtr)) + return (litLength << 3) * BITCOST_MULTIPLIER; /* Uncompressed - 8 bytes per literal. */ + + if (optPtr->priceType == zop_predef) + return (litLength*6) * BITCOST_MULTIPLIER; /* 6 bit per literal - no statistic used */ + + /* dynamic statistics */ + { U32 price = litLength * optPtr->litSumBasePrice; + U32 u; + for (u=0; u < litLength; u++) { + assert(WEIGHT(optPtr->litFreq[literals[u]], optLevel) <= optPtr->litSumBasePrice); /* literal cost should never be negative */ + price -= WEIGHT(optPtr->litFreq[literals[u]], optLevel); + } + return price; + } +} + +/* ZSTD_litLengthPrice() : + * cost of literalLength symbol */ +static U32 ZSTD_litLengthPrice(U32 const litLength, const optState_t* const optPtr, int optLevel) +{ + assert(litLength <= ZSTD_BLOCKSIZE_MAX); + if (optPtr->priceType == zop_predef) + return WEIGHT(litLength, optLevel); + /* We can't compute the litLength price for sizes >= ZSTD_BLOCKSIZE_MAX + * because it isn't representable in the zstd format. So instead just + * call it 1 bit more than ZSTD_BLOCKSIZE_MAX - 1. In this case the block + * would be all literals. + */ + if (litLength == ZSTD_BLOCKSIZE_MAX) + return BITCOST_MULTIPLIER + ZSTD_litLengthPrice(ZSTD_BLOCKSIZE_MAX - 1, optPtr, optLevel); + + /* dynamic statistics */ + { U32 const llCode = ZSTD_LLcode(litLength); + return (LL_bits[llCode] * BITCOST_MULTIPLIER) + + optPtr->litLengthSumBasePrice + - WEIGHT(optPtr->litLengthFreq[llCode], optLevel); + } +} + +/* ZSTD_getMatchPrice() : + * Provides the cost of the match part (offset + matchLength) of a sequence + * Must be combined with ZSTD_fullLiteralsCost() to get the full cost of a sequence. + * @offcode : expects a scale where 0,1,2 are repcodes 1-3, and 3+ are real_offsets+2 + * @optLevel: when <2, favors small offset for decompression speed (improved cache efficiency) + */ +FORCE_INLINE_TEMPLATE U32 +ZSTD_getMatchPrice(U32 const offcode, + U32 const matchLength, + const optState_t* const optPtr, + int const optLevel) +{ + U32 price; + U32 const offCode = ZSTD_highbit32(STORED_TO_OFFBASE(offcode)); + U32 const mlBase = matchLength - MINMATCH; + assert(matchLength >= MINMATCH); + + if (optPtr->priceType == zop_predef) /* fixed scheme, do not use statistics */ + return WEIGHT(mlBase, optLevel) + ((16 + offCode) * BITCOST_MULTIPLIER); + + /* dynamic statistics */ + price = (offCode * BITCOST_MULTIPLIER) + (optPtr->offCodeSumBasePrice - WEIGHT(optPtr->offCodeFreq[offCode], optLevel)); + if ((optLevel<2) /*static*/ && offCode >= 20) + price += (offCode-19)*2 * BITCOST_MULTIPLIER; /* handicap for long distance offsets, favor decompression speed */ + + /* match Length */ + { U32 const mlCode = ZSTD_MLcode(mlBase); + price += (ML_bits[mlCode] * BITCOST_MULTIPLIER) + (optPtr->matchLengthSumBasePrice - WEIGHT(optPtr->matchLengthFreq[mlCode], optLevel)); + } + + price += BITCOST_MULTIPLIER / 5; /* heuristic : make matches a bit more costly to favor less sequences -> faster decompression speed */ + + DEBUGLOG(8, "ZSTD_getMatchPrice(ml:%u) = %u", matchLength, price); + return price; +} + +/* ZSTD_updateStats() : + * assumption : literals + litLengtn <= iend */ +static void ZSTD_updateStats(optState_t* const optPtr, + U32 litLength, const BYTE* literals, + U32 offsetCode, U32 matchLength) +{ + /* literals */ + if (ZSTD_compressedLiterals(optPtr)) { + U32 u; + for (u=0; u < litLength; u++) + optPtr->litFreq[literals[u]] += ZSTD_LITFREQ_ADD; + optPtr->litSum += litLength*ZSTD_LITFREQ_ADD; + } + + /* literal Length */ + { U32 const llCode = ZSTD_LLcode(litLength); + optPtr->litLengthFreq[llCode]++; + optPtr->litLengthSum++; + } + + /* offset code : expected to follow storeSeq() numeric representation */ + { U32 const offCode = ZSTD_highbit32(STORED_TO_OFFBASE(offsetCode)); + assert(offCode <= MaxOff); + optPtr->offCodeFreq[offCode]++; + optPtr->offCodeSum++; + } + + /* match Length */ + { U32 const mlBase = matchLength - MINMATCH; + U32 const mlCode = ZSTD_MLcode(mlBase); + optPtr->matchLengthFreq[mlCode]++; + optPtr->matchLengthSum++; + } +} + + +/* ZSTD_readMINMATCH() : + * function safe only for comparisons + * assumption : memPtr must be at least 4 bytes before end of buffer */ +MEM_STATIC U32 ZSTD_readMINMATCH(const void* memPtr, U32 length) +{ + switch (length) + { + default : + case 4 : return MEM_read32(memPtr); + case 3 : if (MEM_isLittleEndian()) + return MEM_read32(memPtr)<<8; + else + return MEM_read32(memPtr)>>8; + } +} + + +/* Update hashTable3 up to ip (excluded) + Assumption : always within prefix (i.e. not within extDict) */ +static U32 ZSTD_insertAndFindFirstIndexHash3 (const ZSTD_matchState_t* ms, + U32* nextToUpdate3, + const BYTE* const ip) +{ + U32* const hashTable3 = ms->hashTable3; + U32 const hashLog3 = ms->hashLog3; + const BYTE* const base = ms->window.base; + U32 idx = *nextToUpdate3; + U32 const target = (U32)(ip - base); + size_t const hash3 = ZSTD_hash3Ptr(ip, hashLog3); + assert(hashLog3 > 0); + + while(idx < target) { + hashTable3[ZSTD_hash3Ptr(base+idx, hashLog3)] = idx; + idx++; + } + + *nextToUpdate3 = target; + return hashTable3[hash3]; +} + + +/*-************************************* +* Binary Tree search +***************************************/ +/** ZSTD_insertBt1() : add one or multiple positions to tree. + * @param ip assumed <= iend-8 . + * @param target The target of ZSTD_updateTree_internal() - we are filling to this position + * @return : nb of positions added */ +static U32 ZSTD_insertBt1( + const ZSTD_matchState_t* ms, + const BYTE* const ip, const BYTE* const iend, + U32 const target, + U32 const mls, const int extDict) +{ + const ZSTD_compressionParameters* const cParams = &ms->cParams; + U32* const hashTable = ms->hashTable; + U32 const hashLog = cParams->hashLog; + size_t const h = ZSTD_hashPtr(ip, hashLog, mls); + U32* const bt = ms->chainTable; + U32 const btLog = cParams->chainLog - 1; + U32 const btMask = (1 << btLog) - 1; + U32 matchIndex = hashTable[h]; + size_t commonLengthSmaller=0, commonLengthLarger=0; + const BYTE* const base = ms->window.base; + const BYTE* const dictBase = ms->window.dictBase; + const U32 dictLimit = ms->window.dictLimit; + const BYTE* const dictEnd = dictBase + dictLimit; + const BYTE* const prefixStart = base + dictLimit; + const BYTE* match; + const U32 curr = (U32)(ip-base); + const U32 btLow = btMask >= curr ? 0 : curr - btMask; + U32* smallerPtr = bt + 2*(curr&btMask); + U32* largerPtr = smallerPtr + 1; + U32 dummy32; /* to be nullified at the end */ + /* windowLow is based on target because + * we only need positions that will be in the window at the end of the tree update. + */ + U32 const windowLow = ZSTD_getLowestMatchIndex(ms, target, cParams->windowLog); + U32 matchEndIdx = curr+8+1; + size_t bestLength = 8; + U32 nbCompares = 1U << cParams->searchLog; +#ifdef ZSTD_C_PREDICT + U32 predictedSmall = *(bt + 2*((curr-1)&btMask) + 0); + U32 predictedLarge = *(bt + 2*((curr-1)&btMask) + 1); + predictedSmall += (predictedSmall>0); + predictedLarge += (predictedLarge>0); +#endif /* ZSTD_C_PREDICT */ + + DEBUGLOG(8, "ZSTD_insertBt1 (%u)", curr); + + assert(curr <= target); + assert(ip <= iend-8); /* required for h calculation */ + hashTable[h] = curr; /* Update Hash Table */ + + assert(windowLow > 0); + for (; nbCompares && (matchIndex >= windowLow); --nbCompares) { + U32* const nextPtr = bt + 2*(matchIndex & btMask); + size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */ + assert(matchIndex < curr); + +#ifdef ZSTD_C_PREDICT /* note : can create issues when hlog small <= 11 */ + const U32* predictPtr = bt + 2*((matchIndex-1) & btMask); /* written this way, as bt is a roll buffer */ + if (matchIndex == predictedSmall) { + /* no need to check length, result known */ + *smallerPtr = matchIndex; + if (matchIndex <= btLow) { smallerPtr=&dummy32; break; } /* beyond tree size, stop the search */ + smallerPtr = nextPtr+1; /* new "smaller" => larger of match */ + matchIndex = nextPtr[1]; /* new matchIndex larger than previous (closer to current) */ + predictedSmall = predictPtr[1] + (predictPtr[1]>0); + continue; + } + if (matchIndex == predictedLarge) { + *largerPtr = matchIndex; + if (matchIndex <= btLow) { largerPtr=&dummy32; break; } /* beyond tree size, stop the search */ + largerPtr = nextPtr; + matchIndex = nextPtr[0]; + predictedLarge = predictPtr[0] + (predictPtr[0]>0); + continue; + } +#endif + + if (!extDict || (matchIndex+matchLength >= dictLimit)) { + assert(matchIndex+matchLength >= dictLimit); /* might be wrong if actually extDict */ + match = base + matchIndex; + matchLength += ZSTD_count(ip+matchLength, match+matchLength, iend); + } else { + match = dictBase + matchIndex; + matchLength += ZSTD_count_2segments(ip+matchLength, match+matchLength, iend, dictEnd, prefixStart); + if (matchIndex+matchLength >= dictLimit) + match = base + matchIndex; /* to prepare for next usage of match[matchLength] */ + } + + if (matchLength > bestLength) { + bestLength = matchLength; + if (matchLength > matchEndIdx - matchIndex) + matchEndIdx = matchIndex + (U32)matchLength; + } + + if (ip+matchLength == iend) { /* equal : no way to know if inf or sup */ + break; /* drop , to guarantee consistency ; miss a bit of compression, but other solutions can corrupt tree */ + } + + if (match[matchLength] < ip[matchLength]) { /* necessarily within buffer */ + /* match is smaller than current */ + *smallerPtr = matchIndex; /* update smaller idx */ + commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */ + if (matchIndex <= btLow) { smallerPtr=&dummy32; break; } /* beyond tree size, stop searching */ + smallerPtr = nextPtr+1; /* new "candidate" => larger than match, which was smaller than target */ + matchIndex = nextPtr[1]; /* new matchIndex, larger than previous and closer to current */ + } else { + /* match is larger than current */ + *largerPtr = matchIndex; + commonLengthLarger = matchLength; + if (matchIndex <= btLow) { largerPtr=&dummy32; break; } /* beyond tree size, stop searching */ + largerPtr = nextPtr; + matchIndex = nextPtr[0]; + } } + + *smallerPtr = *largerPtr = 0; + { U32 positions = 0; + if (bestLength > 384) positions = MIN(192, (U32)(bestLength - 384)); /* speed optimization */ + assert(matchEndIdx > curr + 8); + return MAX(positions, matchEndIdx - (curr + 8)); + } +} + +FORCE_INLINE_TEMPLATE +void ZSTD_updateTree_internal( + ZSTD_matchState_t* ms, + const BYTE* const ip, const BYTE* const iend, + const U32 mls, const ZSTD_dictMode_e dictMode) +{ + const BYTE* const base = ms->window.base; + U32 const target = (U32)(ip - base); + U32 idx = ms->nextToUpdate; + DEBUGLOG(6, "ZSTD_updateTree_internal, from %u to %u (dictMode:%u)", + idx, target, dictMode); + + while(idx < target) { + U32 const forward = ZSTD_insertBt1(ms, base+idx, iend, target, mls, dictMode == ZSTD_extDict); + assert(idx < (U32)(idx + forward)); + idx += forward; + } + assert((size_t)(ip - base) <= (size_t)(U32)(-1)); + assert((size_t)(iend - base) <= (size_t)(U32)(-1)); + ms->nextToUpdate = target; +} + +void ZSTD_updateTree(ZSTD_matchState_t* ms, const BYTE* ip, const BYTE* iend) { + ZSTD_updateTree_internal(ms, ip, iend, ms->cParams.minMatch, ZSTD_noDict); +} + +FORCE_INLINE_TEMPLATE +U32 ZSTD_insertBtAndGetAllMatches ( + ZSTD_match_t* matches, /* store result (found matches) in this table (presumed large enough) */ + ZSTD_matchState_t* ms, + U32* nextToUpdate3, + const BYTE* const ip, const BYTE* const iLimit, const ZSTD_dictMode_e dictMode, + const U32 rep[ZSTD_REP_NUM], + U32 const ll0, /* tells if associated literal length is 0 or not. This value must be 0 or 1 */ + const U32 lengthToBeat, + U32 const mls /* template */) +{ + const ZSTD_compressionParameters* const cParams = &ms->cParams; + U32 const sufficient_len = MIN(cParams->targetLength, ZSTD_OPT_NUM -1); + const BYTE* const base = ms->window.base; + U32 const curr = (U32)(ip-base); + U32 const hashLog = cParams->hashLog; + U32 const minMatch = (mls==3) ? 3 : 4; + U32* const hashTable = ms->hashTable; + size_t const h = ZSTD_hashPtr(ip, hashLog, mls); + U32 matchIndex = hashTable[h]; + U32* const bt = ms->chainTable; + U32 const btLog = cParams->chainLog - 1; + U32 const btMask= (1U << btLog) - 1; + size_t commonLengthSmaller=0, commonLengthLarger=0; + const BYTE* const dictBase = ms->window.dictBase; + U32 const dictLimit = ms->window.dictLimit; + const BYTE* const dictEnd = dictBase + dictLimit; + const BYTE* const prefixStart = base + dictLimit; + U32 const btLow = (btMask >= curr) ? 0 : curr - btMask; + U32 const windowLow = ZSTD_getLowestMatchIndex(ms, curr, cParams->windowLog); + U32 const matchLow = windowLow ? windowLow : 1; + U32* smallerPtr = bt + 2*(curr&btMask); + U32* largerPtr = bt + 2*(curr&btMask) + 1; + U32 matchEndIdx = curr+8+1; /* farthest referenced position of any match => detects repetitive patterns */ + U32 dummy32; /* to be nullified at the end */ + U32 mnum = 0; + U32 nbCompares = 1U << cParams->searchLog; + + const ZSTD_matchState_t* dms = dictMode == ZSTD_dictMatchState ? ms->dictMatchState : NULL; + const ZSTD_compressionParameters* const dmsCParams = + dictMode == ZSTD_dictMatchState ? &dms->cParams : NULL; + const BYTE* const dmsBase = dictMode == ZSTD_dictMatchState ? dms->window.base : NULL; + const BYTE* const dmsEnd = dictMode == ZSTD_dictMatchState ? dms->window.nextSrc : NULL; + U32 const dmsHighLimit = dictMode == ZSTD_dictMatchState ? (U32)(dmsEnd - dmsBase) : 0; + U32 const dmsLowLimit = dictMode == ZSTD_dictMatchState ? dms->window.lowLimit : 0; + U32 const dmsIndexDelta = dictMode == ZSTD_dictMatchState ? windowLow - dmsHighLimit : 0; + U32 const dmsHashLog = dictMode == ZSTD_dictMatchState ? dmsCParams->hashLog : hashLog; + U32 const dmsBtLog = dictMode == ZSTD_dictMatchState ? dmsCParams->chainLog - 1 : btLog; + U32 const dmsBtMask = dictMode == ZSTD_dictMatchState ? (1U << dmsBtLog) - 1 : 0; + U32 const dmsBtLow = dictMode == ZSTD_dictMatchState && dmsBtMask < dmsHighLimit - dmsLowLimit ? dmsHighLimit - dmsBtMask : dmsLowLimit; + + size_t bestLength = lengthToBeat-1; + DEBUGLOG(8, "ZSTD_insertBtAndGetAllMatches: current=%u", curr); + + /* check repCode */ + assert(ll0 <= 1); /* necessarily 1 or 0 */ + { U32 const lastR = ZSTD_REP_NUM + ll0; + U32 repCode; + for (repCode = ll0; repCode < lastR; repCode++) { + U32 const repOffset = (repCode==ZSTD_REP_NUM) ? (rep[0] - 1) : rep[repCode]; + U32 const repIndex = curr - repOffset; + U32 repLen = 0; + assert(curr >= dictLimit); + if (repOffset-1 /* intentional overflow, discards 0 and -1 */ < curr-dictLimit) { /* equivalent to `curr > repIndex >= dictLimit` */ + /* We must validate the repcode offset because when we're using a dictionary the + * valid offset range shrinks when the dictionary goes out of bounds. + */ + if ((repIndex >= windowLow) & (ZSTD_readMINMATCH(ip, minMatch) == ZSTD_readMINMATCH(ip - repOffset, minMatch))) { + repLen = (U32)ZSTD_count(ip+minMatch, ip+minMatch-repOffset, iLimit) + minMatch; + } + } else { /* repIndex < dictLimit || repIndex >= curr */ + const BYTE* const repMatch = dictMode == ZSTD_dictMatchState ? + dmsBase + repIndex - dmsIndexDelta : + dictBase + repIndex; + assert(curr >= windowLow); + if ( dictMode == ZSTD_extDict + && ( ((repOffset-1) /*intentional overflow*/ < curr - windowLow) /* equivalent to `curr > repIndex >= windowLow` */ + & (((U32)((dictLimit-1) - repIndex) >= 3) ) /* intentional overflow : do not test positions overlapping 2 memory segments */) + && (ZSTD_readMINMATCH(ip, minMatch) == ZSTD_readMINMATCH(repMatch, minMatch)) ) { + repLen = (U32)ZSTD_count_2segments(ip+minMatch, repMatch+minMatch, iLimit, dictEnd, prefixStart) + minMatch; + } + if (dictMode == ZSTD_dictMatchState + && ( ((repOffset-1) /*intentional overflow*/ < curr - (dmsLowLimit + dmsIndexDelta)) /* equivalent to `curr > repIndex >= dmsLowLimit` */ + & ((U32)((dictLimit-1) - repIndex) >= 3) ) /* intentional overflow : do not test positions overlapping 2 memory segments */ + && (ZSTD_readMINMATCH(ip, minMatch) == ZSTD_readMINMATCH(repMatch, minMatch)) ) { + repLen = (U32)ZSTD_count_2segments(ip+minMatch, repMatch+minMatch, iLimit, dmsEnd, prefixStart) + minMatch; + } } + /* save longer solution */ + if (repLen > bestLength) { + DEBUGLOG(8, "found repCode %u (ll0:%u, offset:%u) of length %u", + repCode, ll0, repOffset, repLen); + bestLength = repLen; + matches[mnum].off = STORE_REPCODE(repCode - ll0 + 1); /* expect value between 1 and 3 */ + matches[mnum].len = (U32)repLen; + mnum++; + if ( (repLen > sufficient_len) + | (ip+repLen == iLimit) ) { /* best possible */ + return mnum; + } } } } + + /* HC3 match finder */ + if ((mls == 3) /*static*/ && (bestLength < mls)) { + U32 const matchIndex3 = ZSTD_insertAndFindFirstIndexHash3(ms, nextToUpdate3, ip); + if ((matchIndex3 >= matchLow) + & (curr - matchIndex3 < (1<<18)) /*heuristic : longer distance likely too expensive*/ ) { + size_t mlen; + if ((dictMode == ZSTD_noDict) /*static*/ || (dictMode == ZSTD_dictMatchState) /*static*/ || (matchIndex3 >= dictLimit)) { + const BYTE* const match = base + matchIndex3; + mlen = ZSTD_count(ip, match, iLimit); + } else { + const BYTE* const match = dictBase + matchIndex3; + mlen = ZSTD_count_2segments(ip, match, iLimit, dictEnd, prefixStart); + } + + /* save best solution */ + if (mlen >= mls /* == 3 > bestLength */) { + DEBUGLOG(8, "found small match with hlog3, of length %u", + (U32)mlen); + bestLength = mlen; + assert(curr > matchIndex3); + assert(mnum==0); /* no prior solution */ + matches[0].off = STORE_OFFSET(curr - matchIndex3); + matches[0].len = (U32)mlen; + mnum = 1; + if ( (mlen > sufficient_len) | + (ip+mlen == iLimit) ) { /* best possible length */ + ms->nextToUpdate = curr+1; /* skip insertion */ + return 1; + } } } + /* no dictMatchState lookup: dicts don't have a populated HC3 table */ + } /* if (mls == 3) */ + + hashTable[h] = curr; /* Update Hash Table */ + + for (; nbCompares && (matchIndex >= matchLow); --nbCompares) { + U32* const nextPtr = bt + 2*(matchIndex & btMask); + const BYTE* match; + size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */ + assert(curr > matchIndex); + + if ((dictMode == ZSTD_noDict) || (dictMode == ZSTD_dictMatchState) || (matchIndex+matchLength >= dictLimit)) { + assert(matchIndex+matchLength >= dictLimit); /* ensure the condition is correct when !extDict */ + match = base + matchIndex; + if (matchIndex >= dictLimit) assert(memcmp(match, ip, matchLength) == 0); /* ensure early section of match is equal as expected */ + matchLength += ZSTD_count(ip+matchLength, match+matchLength, iLimit); + } else { + match = dictBase + matchIndex; + assert(memcmp(match, ip, matchLength) == 0); /* ensure early section of match is equal as expected */ + matchLength += ZSTD_count_2segments(ip+matchLength, match+matchLength, iLimit, dictEnd, prefixStart); + if (matchIndex+matchLength >= dictLimit) + match = base + matchIndex; /* prepare for match[matchLength] read */ + } + + if (matchLength > bestLength) { + DEBUGLOG(8, "found match of length %u at distance %u (offCode=%u)", + (U32)matchLength, curr - matchIndex, STORE_OFFSET(curr - matchIndex)); + assert(matchEndIdx > matchIndex); + if (matchLength > matchEndIdx - matchIndex) + matchEndIdx = matchIndex + (U32)matchLength; + bestLength = matchLength; + matches[mnum].off = STORE_OFFSET(curr - matchIndex); + matches[mnum].len = (U32)matchLength; + mnum++; + if ( (matchLength > ZSTD_OPT_NUM) + | (ip+matchLength == iLimit) /* equal : no way to know if inf or sup */) { + if (dictMode == ZSTD_dictMatchState) nbCompares = 0; /* break should also skip searching dms */ + break; /* drop, to preserve bt consistency (miss a little bit of compression) */ + } } + + if (match[matchLength] < ip[matchLength]) { + /* match smaller than current */ + *smallerPtr = matchIndex; /* update smaller idx */ + commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */ + if (matchIndex <= btLow) { smallerPtr=&dummy32; break; } /* beyond tree size, stop the search */ + smallerPtr = nextPtr+1; /* new candidate => larger than match, which was smaller than current */ + matchIndex = nextPtr[1]; /* new matchIndex, larger than previous, closer to current */ + } else { + *largerPtr = matchIndex; + commonLengthLarger = matchLength; + if (matchIndex <= btLow) { largerPtr=&dummy32; break; } /* beyond tree size, stop the search */ + largerPtr = nextPtr; + matchIndex = nextPtr[0]; + } } + + *smallerPtr = *largerPtr = 0; + + assert(nbCompares <= (1U << ZSTD_SEARCHLOG_MAX)); /* Check we haven't underflowed. */ + if (dictMode == ZSTD_dictMatchState && nbCompares) { + size_t const dmsH = ZSTD_hashPtr(ip, dmsHashLog, mls); + U32 dictMatchIndex = dms->hashTable[dmsH]; + const U32* const dmsBt = dms->chainTable; + commonLengthSmaller = commonLengthLarger = 0; + for (; nbCompares && (dictMatchIndex > dmsLowLimit); --nbCompares) { + const U32* const nextPtr = dmsBt + 2*(dictMatchIndex & dmsBtMask); + size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */ + const BYTE* match = dmsBase + dictMatchIndex; + matchLength += ZSTD_count_2segments(ip+matchLength, match+matchLength, iLimit, dmsEnd, prefixStart); + if (dictMatchIndex+matchLength >= dmsHighLimit) + match = base + dictMatchIndex + dmsIndexDelta; /* to prepare for next usage of match[matchLength] */ + + if (matchLength > bestLength) { + matchIndex = dictMatchIndex + dmsIndexDelta; + DEBUGLOG(8, "found dms match of length %u at distance %u (offCode=%u)", + (U32)matchLength, curr - matchIndex, STORE_OFFSET(curr - matchIndex)); + if (matchLength > matchEndIdx - matchIndex) + matchEndIdx = matchIndex + (U32)matchLength; + bestLength = matchLength; + matches[mnum].off = STORE_OFFSET(curr - matchIndex); + matches[mnum].len = (U32)matchLength; + mnum++; + if ( (matchLength > ZSTD_OPT_NUM) + | (ip+matchLength == iLimit) /* equal : no way to know if inf or sup */) { + break; /* drop, to guarantee consistency (miss a little bit of compression) */ + } } + + if (dictMatchIndex <= dmsBtLow) { break; } /* beyond tree size, stop the search */ + if (match[matchLength] < ip[matchLength]) { + commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */ + dictMatchIndex = nextPtr[1]; /* new matchIndex larger than previous (closer to current) */ + } else { + /* match is larger than current */ + commonLengthLarger = matchLength; + dictMatchIndex = nextPtr[0]; + } } } /* if (dictMode == ZSTD_dictMatchState) */ + + assert(matchEndIdx > curr+8); + ms->nextToUpdate = matchEndIdx - 8; /* skip repetitive patterns */ + return mnum; +} + +typedef U32 (*ZSTD_getAllMatchesFn)( + ZSTD_match_t*, + ZSTD_matchState_t*, + U32*, + const BYTE*, + const BYTE*, + const U32 rep[ZSTD_REP_NUM], + U32 const ll0, + U32 const lengthToBeat); + +FORCE_INLINE_TEMPLATE U32 ZSTD_btGetAllMatches_internal( + ZSTD_match_t* matches, + ZSTD_matchState_t* ms, + U32* nextToUpdate3, + const BYTE* ip, + const BYTE* const iHighLimit, + const U32 rep[ZSTD_REP_NUM], + U32 const ll0, + U32 const lengthToBeat, + const ZSTD_dictMode_e dictMode, + const U32 mls) +{ + assert(BOUNDED(3, ms->cParams.minMatch, 6) == mls); + DEBUGLOG(8, "ZSTD_BtGetAllMatches(dictMode=%d, mls=%u)", (int)dictMode, mls); + if (ip < ms->window.base + ms->nextToUpdate) + return 0; /* skipped area */ + ZSTD_updateTree_internal(ms, ip, iHighLimit, mls, dictMode); + return ZSTD_insertBtAndGetAllMatches(matches, ms, nextToUpdate3, ip, iHighLimit, dictMode, rep, ll0, lengthToBeat, mls); +} + +#define ZSTD_BT_GET_ALL_MATCHES_FN(dictMode, mls) ZSTD_btGetAllMatches_##dictMode##_##mls + +#define GEN_ZSTD_BT_GET_ALL_MATCHES_(dictMode, mls) \ + static U32 ZSTD_BT_GET_ALL_MATCHES_FN(dictMode, mls)( \ + ZSTD_match_t* matches, \ + ZSTD_matchState_t* ms, \ + U32* nextToUpdate3, \ + const BYTE* ip, \ + const BYTE* const iHighLimit, \ + const U32 rep[ZSTD_REP_NUM], \ + U32 const ll0, \ + U32 const lengthToBeat) \ + { \ + return ZSTD_btGetAllMatches_internal( \ + matches, ms, nextToUpdate3, ip, iHighLimit, \ + rep, ll0, lengthToBeat, ZSTD_##dictMode, mls); \ + } + +#define GEN_ZSTD_BT_GET_ALL_MATCHES(dictMode) \ + GEN_ZSTD_BT_GET_ALL_MATCHES_(dictMode, 3) \ + GEN_ZSTD_BT_GET_ALL_MATCHES_(dictMode, 4) \ + GEN_ZSTD_BT_GET_ALL_MATCHES_(dictMode, 5) \ + GEN_ZSTD_BT_GET_ALL_MATCHES_(dictMode, 6) + +GEN_ZSTD_BT_GET_ALL_MATCHES(noDict) +GEN_ZSTD_BT_GET_ALL_MATCHES(extDict) +GEN_ZSTD_BT_GET_ALL_MATCHES(dictMatchState) + +#define ZSTD_BT_GET_ALL_MATCHES_ARRAY(dictMode) \ + { \ + ZSTD_BT_GET_ALL_MATCHES_FN(dictMode, 3), \ + ZSTD_BT_GET_ALL_MATCHES_FN(dictMode, 4), \ + ZSTD_BT_GET_ALL_MATCHES_FN(dictMode, 5), \ + ZSTD_BT_GET_ALL_MATCHES_FN(dictMode, 6) \ + } + +static ZSTD_getAllMatchesFn +ZSTD_selectBtGetAllMatches(ZSTD_matchState_t const* ms, ZSTD_dictMode_e const dictMode) +{ + ZSTD_getAllMatchesFn const getAllMatchesFns[3][4] = { + ZSTD_BT_GET_ALL_MATCHES_ARRAY(noDict), + ZSTD_BT_GET_ALL_MATCHES_ARRAY(extDict), + ZSTD_BT_GET_ALL_MATCHES_ARRAY(dictMatchState) + }; + U32 const mls = BOUNDED(3, ms->cParams.minMatch, 6); + assert((U32)dictMode < 3); + assert(mls - 3 < 4); + return getAllMatchesFns[(int)dictMode][mls - 3]; +} + +/************************* +* LDM helper functions * +*************************/ + +/* Struct containing info needed to make decision about ldm inclusion */ +typedef struct { + rawSeqStore_t seqStore; /* External match candidates store for this block */ + U32 startPosInBlock; /* Start position of the current match candidate */ + U32 endPosInBlock; /* End position of the current match candidate */ + U32 offset; /* Offset of the match candidate */ +} ZSTD_optLdm_t; + +/* ZSTD_optLdm_skipRawSeqStoreBytes(): + * Moves forward in @rawSeqStore by @nbBytes, + * which will update the fields 'pos' and 'posInSequence'. + */ +static void ZSTD_optLdm_skipRawSeqStoreBytes(rawSeqStore_t* rawSeqStore, size_t nbBytes) +{ + U32 currPos = (U32)(rawSeqStore->posInSequence + nbBytes); + while (currPos && rawSeqStore->pos < rawSeqStore->size) { + rawSeq currSeq = rawSeqStore->seq[rawSeqStore->pos]; + if (currPos >= currSeq.litLength + currSeq.matchLength) { + currPos -= currSeq.litLength + currSeq.matchLength; + rawSeqStore->pos++; + } else { + rawSeqStore->posInSequence = currPos; + break; + } + } + if (currPos == 0 || rawSeqStore->pos == rawSeqStore->size) { + rawSeqStore->posInSequence = 0; + } +} + +/* ZSTD_opt_getNextMatchAndUpdateSeqStore(): + * Calculates the beginning and end of the next match in the current block. + * Updates 'pos' and 'posInSequence' of the ldmSeqStore. + */ +static void +ZSTD_opt_getNextMatchAndUpdateSeqStore(ZSTD_optLdm_t* optLdm, U32 currPosInBlock, + U32 blockBytesRemaining) +{ + rawSeq currSeq; + U32 currBlockEndPos; + U32 literalsBytesRemaining; + U32 matchBytesRemaining; + + /* Setting match end position to MAX to ensure we never use an LDM during this block */ + if (optLdm->seqStore.size == 0 || optLdm->seqStore.pos >= optLdm->seqStore.size) { + optLdm->startPosInBlock = UINT_MAX; + optLdm->endPosInBlock = UINT_MAX; + return; + } + /* Calculate appropriate bytes left in matchLength and litLength + * after adjusting based on ldmSeqStore->posInSequence */ + currSeq = optLdm->seqStore.seq[optLdm->seqStore.pos]; + assert(optLdm->seqStore.posInSequence <= currSeq.litLength + currSeq.matchLength); + currBlockEndPos = currPosInBlock + blockBytesRemaining; + literalsBytesRemaining = (optLdm->seqStore.posInSequence < currSeq.litLength) ? + currSeq.litLength - (U32)optLdm->seqStore.posInSequence : + 0; + matchBytesRemaining = (literalsBytesRemaining == 0) ? + currSeq.matchLength - ((U32)optLdm->seqStore.posInSequence - currSeq.litLength) : + currSeq.matchLength; + + /* If there are more literal bytes than bytes remaining in block, no ldm is possible */ + if (literalsBytesRemaining >= blockBytesRemaining) { + optLdm->startPosInBlock = UINT_MAX; + optLdm->endPosInBlock = UINT_MAX; + ZSTD_optLdm_skipRawSeqStoreBytes(&optLdm->seqStore, blockBytesRemaining); + return; + } + + /* Matches may be < MINMATCH by this process. In that case, we will reject them + when we are deciding whether or not to add the ldm */ + optLdm->startPosInBlock = currPosInBlock + literalsBytesRemaining; + optLdm->endPosInBlock = optLdm->startPosInBlock + matchBytesRemaining; + optLdm->offset = currSeq.offset; + + if (optLdm->endPosInBlock > currBlockEndPos) { + /* Match ends after the block ends, we can't use the whole match */ + optLdm->endPosInBlock = currBlockEndPos; + ZSTD_optLdm_skipRawSeqStoreBytes(&optLdm->seqStore, currBlockEndPos - currPosInBlock); + } else { + /* Consume nb of bytes equal to size of sequence left */ + ZSTD_optLdm_skipRawSeqStoreBytes(&optLdm->seqStore, literalsBytesRemaining + matchBytesRemaining); + } +} + +/* ZSTD_optLdm_maybeAddMatch(): + * Adds a match if it's long enough, + * based on it's 'matchStartPosInBlock' and 'matchEndPosInBlock', + * into 'matches'. Maintains the correct ordering of 'matches'. + */ +static void ZSTD_optLdm_maybeAddMatch(ZSTD_match_t* matches, U32* nbMatches, + const ZSTD_optLdm_t* optLdm, U32 currPosInBlock) +{ + U32 const posDiff = currPosInBlock - optLdm->startPosInBlock; + /* Note: ZSTD_match_t actually contains offCode and matchLength (before subtracting MINMATCH) */ + U32 const candidateMatchLength = optLdm->endPosInBlock - optLdm->startPosInBlock - posDiff; + + /* Ensure that current block position is not outside of the match */ + if (currPosInBlock < optLdm->startPosInBlock + || currPosInBlock >= optLdm->endPosInBlock + || candidateMatchLength < MINMATCH) { + return; + } + + if (*nbMatches == 0 || ((candidateMatchLength > matches[*nbMatches-1].len) && *nbMatches < ZSTD_OPT_NUM)) { + U32 const candidateOffCode = STORE_OFFSET(optLdm->offset); + DEBUGLOG(6, "ZSTD_optLdm_maybeAddMatch(): Adding ldm candidate match (offCode: %u matchLength %u) at block position=%u", + candidateOffCode, candidateMatchLength, currPosInBlock); + matches[*nbMatches].len = candidateMatchLength; + matches[*nbMatches].off = candidateOffCode; + (*nbMatches)++; + } +} + +/* ZSTD_optLdm_processMatchCandidate(): + * Wrapper function to update ldm seq store and call ldm functions as necessary. + */ +static void +ZSTD_optLdm_processMatchCandidate(ZSTD_optLdm_t* optLdm, + ZSTD_match_t* matches, U32* nbMatches, + U32 currPosInBlock, U32 remainingBytes) +{ + if (optLdm->seqStore.size == 0 || optLdm->seqStore.pos >= optLdm->seqStore.size) { + return; + } + + if (currPosInBlock >= optLdm->endPosInBlock) { + if (currPosInBlock > optLdm->endPosInBlock) { + /* The position at which ZSTD_optLdm_processMatchCandidate() is called is not necessarily + * at the end of a match from the ldm seq store, and will often be some bytes + * over beyond matchEndPosInBlock. As such, we need to correct for these "overshoots" + */ + U32 const posOvershoot = currPosInBlock - optLdm->endPosInBlock; + ZSTD_optLdm_skipRawSeqStoreBytes(&optLdm->seqStore, posOvershoot); + } + ZSTD_opt_getNextMatchAndUpdateSeqStore(optLdm, currPosInBlock, remainingBytes); + } + ZSTD_optLdm_maybeAddMatch(matches, nbMatches, optLdm, currPosInBlock); +} + + +/*-******************************* +* Optimal parser +*********************************/ + +static U32 ZSTD_totalLen(ZSTD_optimal_t sol) +{ + return sol.litlen + sol.mlen; +} + +#if 0 /* debug */ + +static void +listStats(const U32* table, int lastEltID) +{ + int const nbElts = lastEltID + 1; + int enb; + for (enb=0; enb < nbElts; enb++) { + (void)table; + /* RAWLOG(2, "%3i:%3i, ", enb, table[enb]); */ + RAWLOG(2, "%4i,", table[enb]); + } + RAWLOG(2, " \n"); +} + +#endif + +FORCE_INLINE_TEMPLATE size_t +ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms, + seqStore_t* seqStore, + U32 rep[ZSTD_REP_NUM], + const void* src, size_t srcSize, + const int optLevel, + const ZSTD_dictMode_e dictMode) +{ + optState_t* const optStatePtr = &ms->opt; + const BYTE* const istart = (const BYTE*)src; + const BYTE* ip = istart; + const BYTE* anchor = istart; + const BYTE* const iend = istart + srcSize; + const BYTE* const ilimit = iend - 8; + const BYTE* const base = ms->window.base; + const BYTE* const prefixStart = base + ms->window.dictLimit; + const ZSTD_compressionParameters* const cParams = &ms->cParams; + + ZSTD_getAllMatchesFn getAllMatches = ZSTD_selectBtGetAllMatches(ms, dictMode); + + U32 const sufficient_len = MIN(cParams->targetLength, ZSTD_OPT_NUM -1); + U32 const minMatch = (cParams->minMatch == 3) ? 3 : 4; + U32 nextToUpdate3 = ms->nextToUpdate; + + ZSTD_optimal_t* const opt = optStatePtr->priceTable; + ZSTD_match_t* const matches = optStatePtr->matchTable; + ZSTD_optimal_t lastSequence; + ZSTD_optLdm_t optLdm; + + optLdm.seqStore = ms->ldmSeqStore ? *ms->ldmSeqStore : kNullRawSeqStore; + optLdm.endPosInBlock = optLdm.startPosInBlock = optLdm.offset = 0; + ZSTD_opt_getNextMatchAndUpdateSeqStore(&optLdm, (U32)(ip-istart), (U32)(iend-ip)); + + /* init */ + DEBUGLOG(5, "ZSTD_compressBlock_opt_generic: current=%u, prefix=%u, nextToUpdate=%u", + (U32)(ip - base), ms->window.dictLimit, ms->nextToUpdate); + assert(optLevel <= 2); + ZSTD_rescaleFreqs(optStatePtr, (const BYTE*)src, srcSize, optLevel); + ip += (ip==prefixStart); + + /* Match Loop */ + while (ip < ilimit) { + U32 cur, last_pos = 0; + + /* find first match */ + { U32 const litlen = (U32)(ip - anchor); + U32 const ll0 = !litlen; + U32 nbMatches = getAllMatches(matches, ms, &nextToUpdate3, ip, iend, rep, ll0, minMatch); + ZSTD_optLdm_processMatchCandidate(&optLdm, matches, &nbMatches, + (U32)(ip-istart), (U32)(iend - ip)); + if (!nbMatches) { ip++; continue; } + + /* initialize opt[0] */ + { U32 i ; for (i=0; i immediate encoding */ + { U32 const maxML = matches[nbMatches-1].len; + U32 const maxOffcode = matches[nbMatches-1].off; + DEBUGLOG(6, "found %u matches of maxLength=%u and maxOffCode=%u at cPos=%u => start new series", + nbMatches, maxML, maxOffcode, (U32)(ip-prefixStart)); + + if (maxML > sufficient_len) { + lastSequence.litlen = litlen; + lastSequence.mlen = maxML; + lastSequence.off = maxOffcode; + DEBUGLOG(6, "large match (%u>%u), immediate encoding", + maxML, sufficient_len); + cur = 0; + last_pos = ZSTD_totalLen(lastSequence); + goto _shortestPath; + } } + + /* set prices for first matches starting position == 0 */ + assert(opt[0].price >= 0); + { U32 const literalsPrice = (U32)opt[0].price + ZSTD_litLengthPrice(0, optStatePtr, optLevel); + U32 pos; + U32 matchNb; + for (pos = 1; pos < minMatch; pos++) { + opt[pos].price = ZSTD_MAX_PRICE; /* mlen, litlen and price will be fixed during forward scanning */ + } + for (matchNb = 0; matchNb < nbMatches; matchNb++) { + U32 const offcode = matches[matchNb].off; + U32 const end = matches[matchNb].len; + for ( ; pos <= end ; pos++ ) { + U32 const matchPrice = ZSTD_getMatchPrice(offcode, pos, optStatePtr, optLevel); + U32 const sequencePrice = literalsPrice + matchPrice; + DEBUGLOG(7, "rPos:%u => set initial price : %.2f", + pos, ZSTD_fCost(sequencePrice)); + opt[pos].mlen = pos; + opt[pos].off = offcode; + opt[pos].litlen = litlen; + opt[pos].price = (int)sequencePrice; + } } + last_pos = pos-1; + } + } + + /* check further positions */ + for (cur = 1; cur <= last_pos; cur++) { + const BYTE* const inr = ip + cur; + assert(cur < ZSTD_OPT_NUM); + DEBUGLOG(7, "cPos:%zi==rPos:%u", inr-istart, cur) + + /* Fix current position with one literal if cheaper */ + { U32 const litlen = (opt[cur-1].mlen == 0) ? opt[cur-1].litlen + 1 : 1; + int const price = opt[cur-1].price + + (int)ZSTD_rawLiteralsCost(ip+cur-1, 1, optStatePtr, optLevel) + + (int)ZSTD_litLengthPrice(litlen, optStatePtr, optLevel) + - (int)ZSTD_litLengthPrice(litlen-1, optStatePtr, optLevel); + assert(price < 1000000000); /* overflow check */ + if (price <= opt[cur].price) { + DEBUGLOG(7, "cPos:%zi==rPos:%u : better price (%.2f<=%.2f) using literal (ll==%u) (hist:%u,%u,%u)", + inr-istart, cur, ZSTD_fCost(price), ZSTD_fCost(opt[cur].price), litlen, + opt[cur-1].rep[0], opt[cur-1].rep[1], opt[cur-1].rep[2]); + opt[cur].mlen = 0; + opt[cur].off = 0; + opt[cur].litlen = litlen; + opt[cur].price = price; + } else { + DEBUGLOG(7, "cPos:%zi==rPos:%u : literal would cost more (%.2f>%.2f) (hist:%u,%u,%u)", + inr-istart, cur, ZSTD_fCost(price), ZSTD_fCost(opt[cur].price), + opt[cur].rep[0], opt[cur].rep[1], opt[cur].rep[2]); + } + } + + /* Set the repcodes of the current position. We must do it here + * because we rely on the repcodes of the 2nd to last sequence being + * correct to set the next chunks repcodes during the backward + * traversal. + */ + ZSTD_STATIC_ASSERT(sizeof(opt[cur].rep) == sizeof(repcodes_t)); + assert(cur >= opt[cur].mlen); + if (opt[cur].mlen != 0) { + U32 const prev = cur - opt[cur].mlen; + repcodes_t const newReps = ZSTD_newRep(opt[prev].rep, opt[cur].off, opt[cur].litlen==0); + ZSTD_memcpy(opt[cur].rep, &newReps, sizeof(repcodes_t)); + } else { + ZSTD_memcpy(opt[cur].rep, opt[cur - 1].rep, sizeof(repcodes_t)); + } + + /* last match must start at a minimum distance of 8 from oend */ + if (inr > ilimit) continue; + + if (cur == last_pos) break; + + if ( (optLevel==0) /*static_test*/ + && (opt[cur+1].price <= opt[cur].price + (BITCOST_MULTIPLIER/2)) ) { + DEBUGLOG(7, "move to next rPos:%u : price is <=", cur+1); + continue; /* skip unpromising positions; about ~+6% speed, -0.01 ratio */ + } + + assert(opt[cur].price >= 0); + { U32 const ll0 = (opt[cur].mlen != 0); + U32 const litlen = (opt[cur].mlen == 0) ? opt[cur].litlen : 0; + U32 const previousPrice = (U32)opt[cur].price; + U32 const basePrice = previousPrice + ZSTD_litLengthPrice(0, optStatePtr, optLevel); + U32 nbMatches = getAllMatches(matches, ms, &nextToUpdate3, inr, iend, opt[cur].rep, ll0, minMatch); + U32 matchNb; + + ZSTD_optLdm_processMatchCandidate(&optLdm, matches, &nbMatches, + (U32)(inr-istart), (U32)(iend-inr)); + + if (!nbMatches) { + DEBUGLOG(7, "rPos:%u : no match found", cur); + continue; + } + + { U32 const maxML = matches[nbMatches-1].len; + DEBUGLOG(7, "cPos:%zi==rPos:%u, found %u matches, of maxLength=%u", + inr-istart, cur, nbMatches, maxML); + + if ( (maxML > sufficient_len) + || (cur + maxML >= ZSTD_OPT_NUM) ) { + lastSequence.mlen = maxML; + lastSequence.off = matches[nbMatches-1].off; + lastSequence.litlen = litlen; + cur -= (opt[cur].mlen==0) ? opt[cur].litlen : 0; /* last sequence is actually only literals, fix cur to last match - note : may underflow, in which case, it's first sequence, and it's okay */ + last_pos = cur + ZSTD_totalLen(lastSequence); + if (cur > ZSTD_OPT_NUM) cur = 0; /* underflow => first match */ + goto _shortestPath; + } } + + /* set prices using matches found at position == cur */ + for (matchNb = 0; matchNb < nbMatches; matchNb++) { + U32 const offset = matches[matchNb].off; + U32 const lastML = matches[matchNb].len; + U32 const startML = (matchNb>0) ? matches[matchNb-1].len+1 : minMatch; + U32 mlen; + + DEBUGLOG(7, "testing match %u => offCode=%4u, mlen=%2u, llen=%2u", + matchNb, matches[matchNb].off, lastML, litlen); + + for (mlen = lastML; mlen >= startML; mlen--) { /* scan downward */ + U32 const pos = cur + mlen; + int const price = (int)basePrice + (int)ZSTD_getMatchPrice(offset, mlen, optStatePtr, optLevel); + + if ((pos > last_pos) || (price < opt[pos].price)) { + DEBUGLOG(7, "rPos:%u (ml=%2u) => new better price (%.2f<%.2f)", + pos, mlen, ZSTD_fCost(price), ZSTD_fCost(opt[pos].price)); + while (last_pos < pos) { opt[last_pos+1].price = ZSTD_MAX_PRICE; last_pos++; } /* fill empty positions */ + opt[pos].mlen = mlen; + opt[pos].off = offset; + opt[pos].litlen = litlen; + opt[pos].price = price; + } else { + DEBUGLOG(7, "rPos:%u (ml=%2u) => new price is worse (%.2f>=%.2f)", + pos, mlen, ZSTD_fCost(price), ZSTD_fCost(opt[pos].price)); + if (optLevel==0) break; /* early update abort; gets ~+10% speed for about -0.01 ratio loss */ + } + } } } + } /* for (cur = 1; cur <= last_pos; cur++) */ + + lastSequence = opt[last_pos]; + cur = last_pos > ZSTD_totalLen(lastSequence) ? last_pos - ZSTD_totalLen(lastSequence) : 0; /* single sequence, and it starts before `ip` */ + assert(cur < ZSTD_OPT_NUM); /* control overflow*/ + +_shortestPath: /* cur, last_pos, best_mlen, best_off have to be set */ + assert(opt[0].mlen == 0); + + /* Set the next chunk's repcodes based on the repcodes of the beginning + * of the last match, and the last sequence. This avoids us having to + * update them while traversing the sequences. + */ + if (lastSequence.mlen != 0) { + repcodes_t const reps = ZSTD_newRep(opt[cur].rep, lastSequence.off, lastSequence.litlen==0); + ZSTD_memcpy(rep, &reps, sizeof(reps)); + } else { + ZSTD_memcpy(rep, opt[cur].rep, sizeof(repcodes_t)); + } + + { U32 const storeEnd = cur + 1; + U32 storeStart = storeEnd; + U32 seqPos = cur; + + DEBUGLOG(6, "start reverse traversal (last_pos:%u, cur:%u)", + last_pos, cur); (void)last_pos; + assert(storeEnd < ZSTD_OPT_NUM); + DEBUGLOG(6, "last sequence copied into pos=%u (llen=%u,mlen=%u,ofc=%u)", + storeEnd, lastSequence.litlen, lastSequence.mlen, lastSequence.off); + opt[storeEnd] = lastSequence; + while (seqPos > 0) { + U32 const backDist = ZSTD_totalLen(opt[seqPos]); + storeStart--; + DEBUGLOG(6, "sequence from rPos=%u copied into pos=%u (llen=%u,mlen=%u,ofc=%u)", + seqPos, storeStart, opt[seqPos].litlen, opt[seqPos].mlen, opt[seqPos].off); + opt[storeStart] = opt[seqPos]; + seqPos = (seqPos > backDist) ? seqPos - backDist : 0; + } + + /* save sequences */ + DEBUGLOG(6, "sending selected sequences into seqStore") + { U32 storePos; + for (storePos=storeStart; storePos <= storeEnd; storePos++) { + U32 const llen = opt[storePos].litlen; + U32 const mlen = opt[storePos].mlen; + U32 const offCode = opt[storePos].off; + U32 const advance = llen + mlen; + DEBUGLOG(6, "considering seq starting at %zi, llen=%u, mlen=%u", + anchor - istart, (unsigned)llen, (unsigned)mlen); + + if (mlen==0) { /* only literals => must be last "sequence", actually starting a new stream of sequences */ + assert(storePos == storeEnd); /* must be last sequence */ + ip = anchor + llen; /* last "sequence" is a bunch of literals => don't progress anchor */ + continue; /* will finish */ + } + + assert(anchor + llen <= iend); + ZSTD_updateStats(optStatePtr, llen, anchor, offCode, mlen); + ZSTD_storeSeq(seqStore, llen, anchor, iend, offCode, mlen); + anchor += advance; + ip = anchor; + } } + ZSTD_setBasePrices(optStatePtr, optLevel); + } + } /* while (ip < ilimit) */ + + /* Return the last literals size */ + return (size_t)(iend - anchor); +} + +static size_t ZSTD_compressBlock_opt0( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + const void* src, size_t srcSize, const ZSTD_dictMode_e dictMode) +{ + return ZSTD_compressBlock_opt_generic(ms, seqStore, rep, src, srcSize, 0 /* optLevel */, dictMode); +} + +static size_t ZSTD_compressBlock_opt2( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + const void* src, size_t srcSize, const ZSTD_dictMode_e dictMode) +{ + return ZSTD_compressBlock_opt_generic(ms, seqStore, rep, src, srcSize, 2 /* optLevel */, dictMode); +} + +size_t ZSTD_compressBlock_btopt( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + const void* src, size_t srcSize) +{ + DEBUGLOG(5, "ZSTD_compressBlock_btopt"); + return ZSTD_compressBlock_opt0(ms, seqStore, rep, src, srcSize, ZSTD_noDict); +} + + + + +/* ZSTD_initStats_ultra(): + * make a first compression pass, just to seed stats with more accurate starting values. + * only works on first block, with no dictionary and no ldm. + * this function cannot error, hence its contract must be respected. + */ +static void +ZSTD_initStats_ultra(ZSTD_matchState_t* ms, + seqStore_t* seqStore, + U32 rep[ZSTD_REP_NUM], + const void* src, size_t srcSize) +{ + U32 tmpRep[ZSTD_REP_NUM]; /* updated rep codes will sink here */ + ZSTD_memcpy(tmpRep, rep, sizeof(tmpRep)); + + DEBUGLOG(4, "ZSTD_initStats_ultra (srcSize=%zu)", srcSize); + assert(ms->opt.litLengthSum == 0); /* first block */ + assert(seqStore->sequences == seqStore->sequencesStart); /* no ldm */ + assert(ms->window.dictLimit == ms->window.lowLimit); /* no dictionary */ + assert(ms->window.dictLimit - ms->nextToUpdate <= 1); /* no prefix (note: intentional overflow, defined as 2-complement) */ + + ZSTD_compressBlock_opt2(ms, seqStore, tmpRep, src, srcSize, ZSTD_noDict); /* generate stats into ms->opt*/ + + /* invalidate first scan from history */ + ZSTD_resetSeqStore(seqStore); + ms->window.base -= srcSize; + ms->window.dictLimit += (U32)srcSize; + ms->window.lowLimit = ms->window.dictLimit; + ms->nextToUpdate = ms->window.dictLimit; + +} + +size_t ZSTD_compressBlock_btultra( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + const void* src, size_t srcSize) +{ + DEBUGLOG(5, "ZSTD_compressBlock_btultra (srcSize=%zu)", srcSize); + return ZSTD_compressBlock_opt2(ms, seqStore, rep, src, srcSize, ZSTD_noDict); +} + +size_t ZSTD_compressBlock_btultra2( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + const void* src, size_t srcSize) +{ + U32 const curr = (U32)((const BYTE*)src - ms->window.base); + DEBUGLOG(5, "ZSTD_compressBlock_btultra2 (srcSize=%zu)", srcSize); + + /* 2-pass strategy: + * this strategy makes a first pass over first block to collect statistics + * and seed next round's statistics with it. + * After 1st pass, function forgets everything, and starts a new block. + * Consequently, this can only work if no data has been previously loaded in tables, + * aka, no dictionary, no prefix, no ldm preprocessing. + * The compression ratio gain is generally small (~0.5% on first block), + * the cost is 2x cpu time on first block. */ + assert(srcSize <= ZSTD_BLOCKSIZE_MAX); + if ( (ms->opt.litLengthSum==0) /* first block */ + && (seqStore->sequences == seqStore->sequencesStart) /* no ldm */ + && (ms->window.dictLimit == ms->window.lowLimit) /* no dictionary */ + && (curr == ms->window.dictLimit) /* start of frame, nothing already loaded nor skipped */ + && (srcSize > ZSTD_PREDEF_THRESHOLD) + ) { + ZSTD_initStats_ultra(ms, seqStore, rep, src, srcSize); + } + + return ZSTD_compressBlock_opt2(ms, seqStore, rep, src, srcSize, ZSTD_noDict); +} + +size_t ZSTD_compressBlock_btopt_dictMatchState( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + const void* src, size_t srcSize) +{ + return ZSTD_compressBlock_opt0(ms, seqStore, rep, src, srcSize, ZSTD_dictMatchState); +} + +size_t ZSTD_compressBlock_btultra_dictMatchState( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + const void* src, size_t srcSize) +{ + return ZSTD_compressBlock_opt2(ms, seqStore, rep, src, srcSize, ZSTD_dictMatchState); +} + +size_t ZSTD_compressBlock_btopt_extDict( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + const void* src, size_t srcSize) +{ + return ZSTD_compressBlock_opt0(ms, seqStore, rep, src, srcSize, ZSTD_extDict); +} + +size_t ZSTD_compressBlock_btultra_extDict( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + const void* src, size_t srcSize) +{ + return ZSTD_compressBlock_opt2(ms, seqStore, rep, src, srcSize, ZSTD_extDict); +} + +/* note : no btultra2 variant for extDict nor dictMatchState, + * because btultra2 is not meant to work with dictionaries + * and is only specific for the first block (no prefix) */ diff --git a/src/blosc2/internal-complibs/zstd-1.5.2/compress/zstd_opt.h b/src/blosc2/internal-complibs/zstd-1.5.2/compress/zstd_opt.h new file mode 100644 index 0000000..627255f --- /dev/null +++ b/src/blosc2/internal-complibs/zstd-1.5.2/compress/zstd_opt.h @@ -0,0 +1,56 @@ +/* + * Copyright (c) Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_OPT_H +#define ZSTD_OPT_H + +#if defined (__cplusplus) +extern "C" { +#endif + +#include "zstd_compress_internal.h" + +/* used in ZSTD_loadDictionaryContent() */ +void ZSTD_updateTree(ZSTD_matchState_t* ms, const BYTE* ip, const BYTE* iend); + +size_t ZSTD_compressBlock_btopt( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_btultra( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_btultra2( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); + + +size_t ZSTD_compressBlock_btopt_dictMatchState( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_btultra_dictMatchState( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); + +size_t ZSTD_compressBlock_btopt_extDict( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_btultra_extDict( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); + + /* note : no btultra2 variant for extDict nor dictMatchState, + * because btultra2 is not meant to work with dictionaries + * and is only specific for the first block (no prefix) */ + +#if defined (__cplusplus) +} +#endif + +#endif /* ZSTD_OPT_H */ diff --git a/src/blosc2/internal-complibs/zstd-1.5.2/compress/zstdmt_compress.c b/src/blosc2/internal-complibs/zstd-1.5.2/compress/zstdmt_compress.c new file mode 100644 index 0000000..6bc14b0 --- /dev/null +++ b/src/blosc2/internal-complibs/zstd-1.5.2/compress/zstdmt_compress.c @@ -0,0 +1,1859 @@ +/* + * Copyright (c) Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + + +/* ====== Compiler specifics ====== */ +#if defined(_MSC_VER) +# pragma warning(disable : 4204) /* disable: C4204: non-constant aggregate initializer */ +#endif + + +/* ====== Constants ====== */ +#define ZSTDMT_OVERLAPLOG_DEFAULT 0 + + +/* ====== Dependencies ====== */ +#include "../common/zstd_deps.h" /* ZSTD_memcpy, ZSTD_memset, INT_MAX, UINT_MAX */ +#include "../common/mem.h" /* MEM_STATIC */ +#include "../common/pool.h" /* threadpool */ +#include "../common/threading.h" /* mutex */ +#include "zstd_compress_internal.h" /* MIN, ERROR, ZSTD_*, ZSTD_highbit32 */ +#include "zstd_ldm.h" +#include "zstdmt_compress.h" + +/* Guards code to support resizing the SeqPool. + * We will want to resize the SeqPool to save memory in the future. + * Until then, comment the code out since it is unused. + */ +#define ZSTD_RESIZE_SEQPOOL 0 + +/* ====== Debug ====== */ +#if defined(DEBUGLEVEL) && (DEBUGLEVEL>=2) \ + && !defined(_MSC_VER) \ + && !defined(__MINGW32__) + +# include +# include +# include + +# define DEBUG_PRINTHEX(l,p,n) { \ + unsigned debug_u; \ + for (debug_u=0; debug_u<(n); debug_u++) \ + RAWLOG(l, "%02X ", ((const unsigned char*)(p))[debug_u]); \ + RAWLOG(l, " \n"); \ +} + +static unsigned long long GetCurrentClockTimeMicroseconds(void) +{ + static clock_t _ticksPerSecond = 0; + if (_ticksPerSecond <= 0) _ticksPerSecond = sysconf(_SC_CLK_TCK); + + { struct tms junk; clock_t newTicks = (clock_t) times(&junk); + return ((((unsigned long long)newTicks)*(1000000))/_ticksPerSecond); +} } + +#define MUTEX_WAIT_TIME_DLEVEL 6 +#define ZSTD_PTHREAD_MUTEX_LOCK(mutex) { \ + if (DEBUGLEVEL >= MUTEX_WAIT_TIME_DLEVEL) { \ + unsigned long long const beforeTime = GetCurrentClockTimeMicroseconds(); \ + ZSTD_pthread_mutex_lock(mutex); \ + { unsigned long long const afterTime = GetCurrentClockTimeMicroseconds(); \ + unsigned long long const elapsedTime = (afterTime-beforeTime); \ + if (elapsedTime > 1000) { /* or whatever threshold you like; I'm using 1 millisecond here */ \ + DEBUGLOG(MUTEX_WAIT_TIME_DLEVEL, "Thread took %llu microseconds to acquire mutex %s \n", \ + elapsedTime, #mutex); \ + } } \ + } else { \ + ZSTD_pthread_mutex_lock(mutex); \ + } \ +} + +#else + +# define ZSTD_PTHREAD_MUTEX_LOCK(m) ZSTD_pthread_mutex_lock(m) +# define DEBUG_PRINTHEX(l,p,n) {} + +#endif + + +/* ===== Buffer Pool ===== */ +/* a single Buffer Pool can be invoked from multiple threads in parallel */ + +typedef struct buffer_s { + void* start; + size_t capacity; +} buffer_t; + +static const buffer_t g_nullBuffer = { NULL, 0 }; + +typedef struct ZSTDMT_bufferPool_s { + ZSTD_pthread_mutex_t poolMutex; + size_t bufferSize; + unsigned totalBuffers; + unsigned nbBuffers; + ZSTD_customMem cMem; + buffer_t bTable[1]; /* variable size */ +} ZSTDMT_bufferPool; + +static ZSTDMT_bufferPool* ZSTDMT_createBufferPool(unsigned maxNbBuffers, ZSTD_customMem cMem) +{ + ZSTDMT_bufferPool* const bufPool = (ZSTDMT_bufferPool*)ZSTD_customCalloc( + sizeof(ZSTDMT_bufferPool) + (maxNbBuffers-1) * sizeof(buffer_t), cMem); + if (bufPool==NULL) return NULL; + if (ZSTD_pthread_mutex_init(&bufPool->poolMutex, NULL)) { + ZSTD_customFree(bufPool, cMem); + return NULL; + } + bufPool->bufferSize = 64 KB; + bufPool->totalBuffers = maxNbBuffers; + bufPool->nbBuffers = 0; + bufPool->cMem = cMem; + return bufPool; +} + +static void ZSTDMT_freeBufferPool(ZSTDMT_bufferPool* bufPool) +{ + unsigned u; + DEBUGLOG(3, "ZSTDMT_freeBufferPool (address:%08X)", (U32)(size_t)bufPool); + if (!bufPool) return; /* compatibility with free on NULL */ + for (u=0; utotalBuffers; u++) { + DEBUGLOG(4, "free buffer %2u (address:%08X)", u, (U32)(size_t)bufPool->bTable[u].start); + ZSTD_customFree(bufPool->bTable[u].start, bufPool->cMem); + } + ZSTD_pthread_mutex_destroy(&bufPool->poolMutex); + ZSTD_customFree(bufPool, bufPool->cMem); +} + +/* only works at initialization, not during compression */ +static size_t ZSTDMT_sizeof_bufferPool(ZSTDMT_bufferPool* bufPool) +{ + size_t const poolSize = sizeof(*bufPool) + + (bufPool->totalBuffers - 1) * sizeof(buffer_t); + unsigned u; + size_t totalBufferSize = 0; + ZSTD_pthread_mutex_lock(&bufPool->poolMutex); + for (u=0; utotalBuffers; u++) + totalBufferSize += bufPool->bTable[u].capacity; + ZSTD_pthread_mutex_unlock(&bufPool->poolMutex); + + return poolSize + totalBufferSize; +} + +/* ZSTDMT_setBufferSize() : + * all future buffers provided by this buffer pool will have _at least_ this size + * note : it's better for all buffers to have same size, + * as they become freely interchangeable, reducing malloc/free usages and memory fragmentation */ +static void ZSTDMT_setBufferSize(ZSTDMT_bufferPool* const bufPool, size_t const bSize) +{ + ZSTD_pthread_mutex_lock(&bufPool->poolMutex); + DEBUGLOG(4, "ZSTDMT_setBufferSize: bSize = %u", (U32)bSize); + bufPool->bufferSize = bSize; + ZSTD_pthread_mutex_unlock(&bufPool->poolMutex); +} + + +static ZSTDMT_bufferPool* ZSTDMT_expandBufferPool(ZSTDMT_bufferPool* srcBufPool, unsigned maxNbBuffers) +{ + if (srcBufPool==NULL) return NULL; + if (srcBufPool->totalBuffers >= maxNbBuffers) /* good enough */ + return srcBufPool; + /* need a larger buffer pool */ + { ZSTD_customMem const cMem = srcBufPool->cMem; + size_t const bSize = srcBufPool->bufferSize; /* forward parameters */ + ZSTDMT_bufferPool* newBufPool; + ZSTDMT_freeBufferPool(srcBufPool); + newBufPool = ZSTDMT_createBufferPool(maxNbBuffers, cMem); + if (newBufPool==NULL) return newBufPool; + ZSTDMT_setBufferSize(newBufPool, bSize); + return newBufPool; + } +} + +/** ZSTDMT_getBuffer() : + * assumption : bufPool must be valid + * @return : a buffer, with start pointer and size + * note: allocation may fail, in this case, start==NULL and size==0 */ +static buffer_t ZSTDMT_getBuffer(ZSTDMT_bufferPool* bufPool) +{ + size_t const bSize = bufPool->bufferSize; + DEBUGLOG(5, "ZSTDMT_getBuffer: bSize = %u", (U32)bufPool->bufferSize); + ZSTD_pthread_mutex_lock(&bufPool->poolMutex); + if (bufPool->nbBuffers) { /* try to use an existing buffer */ + buffer_t const buf = bufPool->bTable[--(bufPool->nbBuffers)]; + size_t const availBufferSize = buf.capacity; + bufPool->bTable[bufPool->nbBuffers] = g_nullBuffer; + if ((availBufferSize >= bSize) & ((availBufferSize>>3) <= bSize)) { + /* large enough, but not too much */ + DEBUGLOG(5, "ZSTDMT_getBuffer: provide buffer %u of size %u", + bufPool->nbBuffers, (U32)buf.capacity); + ZSTD_pthread_mutex_unlock(&bufPool->poolMutex); + return buf; + } + /* size conditions not respected : scratch this buffer, create new one */ + DEBUGLOG(5, "ZSTDMT_getBuffer: existing buffer does not meet size conditions => freeing"); + ZSTD_customFree(buf.start, bufPool->cMem); + } + ZSTD_pthread_mutex_unlock(&bufPool->poolMutex); + /* create new buffer */ + DEBUGLOG(5, "ZSTDMT_getBuffer: create a new buffer"); + { buffer_t buffer; + void* const start = ZSTD_customMalloc(bSize, bufPool->cMem); + buffer.start = start; /* note : start can be NULL if malloc fails ! */ + buffer.capacity = (start==NULL) ? 0 : bSize; + if (start==NULL) { + DEBUGLOG(5, "ZSTDMT_getBuffer: buffer allocation failure !!"); + } else { + DEBUGLOG(5, "ZSTDMT_getBuffer: created buffer of size %u", (U32)bSize); + } + return buffer; + } +} + +#if ZSTD_RESIZE_SEQPOOL +/** ZSTDMT_resizeBuffer() : + * assumption : bufPool must be valid + * @return : a buffer that is at least the buffer pool buffer size. + * If a reallocation happens, the data in the input buffer is copied. + */ +static buffer_t ZSTDMT_resizeBuffer(ZSTDMT_bufferPool* bufPool, buffer_t buffer) +{ + size_t const bSize = bufPool->bufferSize; + if (buffer.capacity < bSize) { + void* const start = ZSTD_customMalloc(bSize, bufPool->cMem); + buffer_t newBuffer; + newBuffer.start = start; + newBuffer.capacity = start == NULL ? 0 : bSize; + if (start != NULL) { + assert(newBuffer.capacity >= buffer.capacity); + ZSTD_memcpy(newBuffer.start, buffer.start, buffer.capacity); + DEBUGLOG(5, "ZSTDMT_resizeBuffer: created buffer of size %u", (U32)bSize); + return newBuffer; + } + DEBUGLOG(5, "ZSTDMT_resizeBuffer: buffer allocation failure !!"); + } + return buffer; +} +#endif + +/* store buffer for later re-use, up to pool capacity */ +static void ZSTDMT_releaseBuffer(ZSTDMT_bufferPool* bufPool, buffer_t buf) +{ + DEBUGLOG(5, "ZSTDMT_releaseBuffer"); + if (buf.start == NULL) return; /* compatible with release on NULL */ + ZSTD_pthread_mutex_lock(&bufPool->poolMutex); + if (bufPool->nbBuffers < bufPool->totalBuffers) { + bufPool->bTable[bufPool->nbBuffers++] = buf; /* stored for later use */ + DEBUGLOG(5, "ZSTDMT_releaseBuffer: stored buffer of size %u in slot %u", + (U32)buf.capacity, (U32)(bufPool->nbBuffers-1)); + ZSTD_pthread_mutex_unlock(&bufPool->poolMutex); + return; + } + ZSTD_pthread_mutex_unlock(&bufPool->poolMutex); + /* Reached bufferPool capacity (should not happen) */ + DEBUGLOG(5, "ZSTDMT_releaseBuffer: pool capacity reached => freeing "); + ZSTD_customFree(buf.start, bufPool->cMem); +} + +/* We need 2 output buffers per worker since each dstBuff must be flushed after it is released. + * The 3 additional buffers are as follows: + * 1 buffer for input loading + * 1 buffer for "next input" when submitting current one + * 1 buffer stuck in queue */ +#define BUF_POOL_MAX_NB_BUFFERS(nbWorkers) 2*nbWorkers + 3 + +/* After a worker releases its rawSeqStore, it is immediately ready for reuse. + * So we only need one seq buffer per worker. */ +#define SEQ_POOL_MAX_NB_BUFFERS(nbWorkers) nbWorkers + +/* ===== Seq Pool Wrapper ====== */ + +typedef ZSTDMT_bufferPool ZSTDMT_seqPool; + +static size_t ZSTDMT_sizeof_seqPool(ZSTDMT_seqPool* seqPool) +{ + return ZSTDMT_sizeof_bufferPool(seqPool); +} + +static rawSeqStore_t bufferToSeq(buffer_t buffer) +{ + rawSeqStore_t seq = kNullRawSeqStore; + seq.seq = (rawSeq*)buffer.start; + seq.capacity = buffer.capacity / sizeof(rawSeq); + return seq; +} + +static buffer_t seqToBuffer(rawSeqStore_t seq) +{ + buffer_t buffer; + buffer.start = seq.seq; + buffer.capacity = seq.capacity * sizeof(rawSeq); + return buffer; +} + +static rawSeqStore_t ZSTDMT_getSeq(ZSTDMT_seqPool* seqPool) +{ + if (seqPool->bufferSize == 0) { + return kNullRawSeqStore; + } + return bufferToSeq(ZSTDMT_getBuffer(seqPool)); +} + +#if ZSTD_RESIZE_SEQPOOL +static rawSeqStore_t ZSTDMT_resizeSeq(ZSTDMT_seqPool* seqPool, rawSeqStore_t seq) +{ + return bufferToSeq(ZSTDMT_resizeBuffer(seqPool, seqToBuffer(seq))); +} +#endif + +static void ZSTDMT_releaseSeq(ZSTDMT_seqPool* seqPool, rawSeqStore_t seq) +{ + ZSTDMT_releaseBuffer(seqPool, seqToBuffer(seq)); +} + +static void ZSTDMT_setNbSeq(ZSTDMT_seqPool* const seqPool, size_t const nbSeq) +{ + ZSTDMT_setBufferSize(seqPool, nbSeq * sizeof(rawSeq)); +} + +static ZSTDMT_seqPool* ZSTDMT_createSeqPool(unsigned nbWorkers, ZSTD_customMem cMem) +{ + ZSTDMT_seqPool* const seqPool = ZSTDMT_createBufferPool(SEQ_POOL_MAX_NB_BUFFERS(nbWorkers), cMem); + if (seqPool == NULL) return NULL; + ZSTDMT_setNbSeq(seqPool, 0); + return seqPool; +} + +static void ZSTDMT_freeSeqPool(ZSTDMT_seqPool* seqPool) +{ + ZSTDMT_freeBufferPool(seqPool); +} + +static ZSTDMT_seqPool* ZSTDMT_expandSeqPool(ZSTDMT_seqPool* pool, U32 nbWorkers) +{ + return ZSTDMT_expandBufferPool(pool, SEQ_POOL_MAX_NB_BUFFERS(nbWorkers)); +} + + +/* ===== CCtx Pool ===== */ +/* a single CCtx Pool can be invoked from multiple threads in parallel */ + +typedef struct { + ZSTD_pthread_mutex_t poolMutex; + int totalCCtx; + int availCCtx; + ZSTD_customMem cMem; + ZSTD_CCtx* cctx[1]; /* variable size */ +} ZSTDMT_CCtxPool; + +/* note : all CCtx borrowed from the pool should be released back to the pool _before_ freeing the pool */ +static void ZSTDMT_freeCCtxPool(ZSTDMT_CCtxPool* pool) +{ + int cid; + for (cid=0; cidtotalCCtx; cid++) + ZSTD_freeCCtx(pool->cctx[cid]); /* note : compatible with free on NULL */ + ZSTD_pthread_mutex_destroy(&pool->poolMutex); + ZSTD_customFree(pool, pool->cMem); +} + +/* ZSTDMT_createCCtxPool() : + * implies nbWorkers >= 1 , checked by caller ZSTDMT_createCCtx() */ +static ZSTDMT_CCtxPool* ZSTDMT_createCCtxPool(int nbWorkers, + ZSTD_customMem cMem) +{ + ZSTDMT_CCtxPool* const cctxPool = (ZSTDMT_CCtxPool*) ZSTD_customCalloc( + sizeof(ZSTDMT_CCtxPool) + (nbWorkers-1)*sizeof(ZSTD_CCtx*), cMem); + assert(nbWorkers > 0); + if (!cctxPool) return NULL; + if (ZSTD_pthread_mutex_init(&cctxPool->poolMutex, NULL)) { + ZSTD_customFree(cctxPool, cMem); + return NULL; + } + cctxPool->cMem = cMem; + cctxPool->totalCCtx = nbWorkers; + cctxPool->availCCtx = 1; /* at least one cctx for single-thread mode */ + cctxPool->cctx[0] = ZSTD_createCCtx_advanced(cMem); + if (!cctxPool->cctx[0]) { ZSTDMT_freeCCtxPool(cctxPool); return NULL; } + DEBUGLOG(3, "cctxPool created, with %u workers", nbWorkers); + return cctxPool; +} + +static ZSTDMT_CCtxPool* ZSTDMT_expandCCtxPool(ZSTDMT_CCtxPool* srcPool, + int nbWorkers) +{ + if (srcPool==NULL) return NULL; + if (nbWorkers <= srcPool->totalCCtx) return srcPool; /* good enough */ + /* need a larger cctx pool */ + { ZSTD_customMem const cMem = srcPool->cMem; + ZSTDMT_freeCCtxPool(srcPool); + return ZSTDMT_createCCtxPool(nbWorkers, cMem); + } +} + +/* only works during initialization phase, not during compression */ +static size_t ZSTDMT_sizeof_CCtxPool(ZSTDMT_CCtxPool* cctxPool) +{ + ZSTD_pthread_mutex_lock(&cctxPool->poolMutex); + { unsigned const nbWorkers = cctxPool->totalCCtx; + size_t const poolSize = sizeof(*cctxPool) + + (nbWorkers-1) * sizeof(ZSTD_CCtx*); + unsigned u; + size_t totalCCtxSize = 0; + for (u=0; ucctx[u]); + } + ZSTD_pthread_mutex_unlock(&cctxPool->poolMutex); + assert(nbWorkers > 0); + return poolSize + totalCCtxSize; + } +} + +static ZSTD_CCtx* ZSTDMT_getCCtx(ZSTDMT_CCtxPool* cctxPool) +{ + DEBUGLOG(5, "ZSTDMT_getCCtx"); + ZSTD_pthread_mutex_lock(&cctxPool->poolMutex); + if (cctxPool->availCCtx) { + cctxPool->availCCtx--; + { ZSTD_CCtx* const cctx = cctxPool->cctx[cctxPool->availCCtx]; + ZSTD_pthread_mutex_unlock(&cctxPool->poolMutex); + return cctx; + } } + ZSTD_pthread_mutex_unlock(&cctxPool->poolMutex); + DEBUGLOG(5, "create one more CCtx"); + return ZSTD_createCCtx_advanced(cctxPool->cMem); /* note : can be NULL, when creation fails ! */ +} + +static void ZSTDMT_releaseCCtx(ZSTDMT_CCtxPool* pool, ZSTD_CCtx* cctx) +{ + if (cctx==NULL) return; /* compatibility with release on NULL */ + ZSTD_pthread_mutex_lock(&pool->poolMutex); + if (pool->availCCtx < pool->totalCCtx) + pool->cctx[pool->availCCtx++] = cctx; + else { + /* pool overflow : should not happen, since totalCCtx==nbWorkers */ + DEBUGLOG(4, "CCtx pool overflow : free cctx"); + ZSTD_freeCCtx(cctx); + } + ZSTD_pthread_mutex_unlock(&pool->poolMutex); +} + +/* ==== Serial State ==== */ + +typedef struct { + void const* start; + size_t size; +} range_t; + +typedef struct { + /* All variables in the struct are protected by mutex. */ + ZSTD_pthread_mutex_t mutex; + ZSTD_pthread_cond_t cond; + ZSTD_CCtx_params params; + ldmState_t ldmState; + XXH64_state_t xxhState; + unsigned nextJobID; + /* Protects ldmWindow. + * Must be acquired after the main mutex when acquiring both. + */ + ZSTD_pthread_mutex_t ldmWindowMutex; + ZSTD_pthread_cond_t ldmWindowCond; /* Signaled when ldmWindow is updated */ + ZSTD_window_t ldmWindow; /* A thread-safe copy of ldmState.window */ +} serialState_t; + +static int +ZSTDMT_serialState_reset(serialState_t* serialState, + ZSTDMT_seqPool* seqPool, + ZSTD_CCtx_params params, + size_t jobSize, + const void* dict, size_t const dictSize, + ZSTD_dictContentType_e dictContentType) +{ + /* Adjust parameters */ + if (params.ldmParams.enableLdm == ZSTD_ps_enable) { + DEBUGLOG(4, "LDM window size = %u KB", (1U << params.cParams.windowLog) >> 10); + ZSTD_ldm_adjustParameters(¶ms.ldmParams, ¶ms.cParams); + assert(params.ldmParams.hashLog >= params.ldmParams.bucketSizeLog); + assert(params.ldmParams.hashRateLog < 32); + } else { + ZSTD_memset(¶ms.ldmParams, 0, sizeof(params.ldmParams)); + } + serialState->nextJobID = 0; + if (params.fParams.checksumFlag) + XXH64_reset(&serialState->xxhState, 0); + if (params.ldmParams.enableLdm == ZSTD_ps_enable) { + ZSTD_customMem cMem = params.customMem; + unsigned const hashLog = params.ldmParams.hashLog; + size_t const hashSize = ((size_t)1 << hashLog) * sizeof(ldmEntry_t); + unsigned const bucketLog = + params.ldmParams.hashLog - params.ldmParams.bucketSizeLog; + unsigned const prevBucketLog = + serialState->params.ldmParams.hashLog - + serialState->params.ldmParams.bucketSizeLog; + size_t const numBuckets = (size_t)1 << bucketLog; + /* Size the seq pool tables */ + ZSTDMT_setNbSeq(seqPool, ZSTD_ldm_getMaxNbSeq(params.ldmParams, jobSize)); + /* Reset the window */ + ZSTD_window_init(&serialState->ldmState.window); + /* Resize tables and output space if necessary. */ + if (serialState->ldmState.hashTable == NULL || serialState->params.ldmParams.hashLog < hashLog) { + ZSTD_customFree(serialState->ldmState.hashTable, cMem); + serialState->ldmState.hashTable = (ldmEntry_t*)ZSTD_customMalloc(hashSize, cMem); + } + if (serialState->ldmState.bucketOffsets == NULL || prevBucketLog < bucketLog) { + ZSTD_customFree(serialState->ldmState.bucketOffsets, cMem); + serialState->ldmState.bucketOffsets = (BYTE*)ZSTD_customMalloc(numBuckets, cMem); + } + if (!serialState->ldmState.hashTable || !serialState->ldmState.bucketOffsets) + return 1; + /* Zero the tables */ + ZSTD_memset(serialState->ldmState.hashTable, 0, hashSize); + ZSTD_memset(serialState->ldmState.bucketOffsets, 0, numBuckets); + + /* Update window state and fill hash table with dict */ + serialState->ldmState.loadedDictEnd = 0; + if (dictSize > 0) { + if (dictContentType == ZSTD_dct_rawContent) { + BYTE const* const dictEnd = (const BYTE*)dict + dictSize; + ZSTD_window_update(&serialState->ldmState.window, dict, dictSize, /* forceNonContiguous */ 0); + ZSTD_ldm_fillHashTable(&serialState->ldmState, (const BYTE*)dict, dictEnd, ¶ms.ldmParams); + serialState->ldmState.loadedDictEnd = params.forceWindow ? 0 : (U32)(dictEnd - serialState->ldmState.window.base); + } else { + /* don't even load anything */ + } + } + + /* Initialize serialState's copy of ldmWindow. */ + serialState->ldmWindow = serialState->ldmState.window; + } + + serialState->params = params; + serialState->params.jobSize = (U32)jobSize; + return 0; +} + +static int ZSTDMT_serialState_init(serialState_t* serialState) +{ + int initError = 0; + ZSTD_memset(serialState, 0, sizeof(*serialState)); + initError |= ZSTD_pthread_mutex_init(&serialState->mutex, NULL); + initError |= ZSTD_pthread_cond_init(&serialState->cond, NULL); + initError |= ZSTD_pthread_mutex_init(&serialState->ldmWindowMutex, NULL); + initError |= ZSTD_pthread_cond_init(&serialState->ldmWindowCond, NULL); + return initError; +} + +static void ZSTDMT_serialState_free(serialState_t* serialState) +{ + ZSTD_customMem cMem = serialState->params.customMem; + ZSTD_pthread_mutex_destroy(&serialState->mutex); + ZSTD_pthread_cond_destroy(&serialState->cond); + ZSTD_pthread_mutex_destroy(&serialState->ldmWindowMutex); + ZSTD_pthread_cond_destroy(&serialState->ldmWindowCond); + ZSTD_customFree(serialState->ldmState.hashTable, cMem); + ZSTD_customFree(serialState->ldmState.bucketOffsets, cMem); +} + +static void ZSTDMT_serialState_update(serialState_t* serialState, + ZSTD_CCtx* jobCCtx, rawSeqStore_t seqStore, + range_t src, unsigned jobID) +{ + /* Wait for our turn */ + ZSTD_PTHREAD_MUTEX_LOCK(&serialState->mutex); + while (serialState->nextJobID < jobID) { + DEBUGLOG(5, "wait for serialState->cond"); + ZSTD_pthread_cond_wait(&serialState->cond, &serialState->mutex); + } + /* A future job may error and skip our job */ + if (serialState->nextJobID == jobID) { + /* It is now our turn, do any processing necessary */ + if (serialState->params.ldmParams.enableLdm == ZSTD_ps_enable) { + size_t error; + assert(seqStore.seq != NULL && seqStore.pos == 0 && + seqStore.size == 0 && seqStore.capacity > 0); + assert(src.size <= serialState->params.jobSize); + ZSTD_window_update(&serialState->ldmState.window, src.start, src.size, /* forceNonContiguous */ 0); + error = ZSTD_ldm_generateSequences( + &serialState->ldmState, &seqStore, + &serialState->params.ldmParams, src.start, src.size); + /* We provide a large enough buffer to never fail. */ + assert(!ZSTD_isError(error)); (void)error; + /* Update ldmWindow to match the ldmState.window and signal the main + * thread if it is waiting for a buffer. + */ + ZSTD_PTHREAD_MUTEX_LOCK(&serialState->ldmWindowMutex); + serialState->ldmWindow = serialState->ldmState.window; + ZSTD_pthread_cond_signal(&serialState->ldmWindowCond); + ZSTD_pthread_mutex_unlock(&serialState->ldmWindowMutex); + } + if (serialState->params.fParams.checksumFlag && src.size > 0) + XXH64_update(&serialState->xxhState, src.start, src.size); + } + /* Now it is the next jobs turn */ + serialState->nextJobID++; + ZSTD_pthread_cond_broadcast(&serialState->cond); + ZSTD_pthread_mutex_unlock(&serialState->mutex); + + if (seqStore.size > 0) { + size_t const err = ZSTD_referenceExternalSequences( + jobCCtx, seqStore.seq, seqStore.size); + assert(serialState->params.ldmParams.enableLdm == ZSTD_ps_enable); + assert(!ZSTD_isError(err)); + (void)err; + } +} + +static void ZSTDMT_serialState_ensureFinished(serialState_t* serialState, + unsigned jobID, size_t cSize) +{ + ZSTD_PTHREAD_MUTEX_LOCK(&serialState->mutex); + if (serialState->nextJobID <= jobID) { + assert(ZSTD_isError(cSize)); (void)cSize; + DEBUGLOG(5, "Skipping past job %u because of error", jobID); + serialState->nextJobID = jobID + 1; + ZSTD_pthread_cond_broadcast(&serialState->cond); + + ZSTD_PTHREAD_MUTEX_LOCK(&serialState->ldmWindowMutex); + ZSTD_window_clear(&serialState->ldmWindow); + ZSTD_pthread_cond_signal(&serialState->ldmWindowCond); + ZSTD_pthread_mutex_unlock(&serialState->ldmWindowMutex); + } + ZSTD_pthread_mutex_unlock(&serialState->mutex); + +} + + +/* ------------------------------------------ */ +/* ===== Worker thread ===== */ +/* ------------------------------------------ */ + +static const range_t kNullRange = { NULL, 0 }; + +typedef struct { + size_t consumed; /* SHARED - set0 by mtctx, then modified by worker AND read by mtctx */ + size_t cSize; /* SHARED - set0 by mtctx, then modified by worker AND read by mtctx, then set0 by mtctx */ + ZSTD_pthread_mutex_t job_mutex; /* Thread-safe - used by mtctx and worker */ + ZSTD_pthread_cond_t job_cond; /* Thread-safe - used by mtctx and worker */ + ZSTDMT_CCtxPool* cctxPool; /* Thread-safe - used by mtctx and (all) workers */ + ZSTDMT_bufferPool* bufPool; /* Thread-safe - used by mtctx and (all) workers */ + ZSTDMT_seqPool* seqPool; /* Thread-safe - used by mtctx and (all) workers */ + serialState_t* serial; /* Thread-safe - used by mtctx and (all) workers */ + buffer_t dstBuff; /* set by worker (or mtctx), then read by worker & mtctx, then modified by mtctx => no barrier */ + range_t prefix; /* set by mtctx, then read by worker & mtctx => no barrier */ + range_t src; /* set by mtctx, then read by worker & mtctx => no barrier */ + unsigned jobID; /* set by mtctx, then read by worker => no barrier */ + unsigned firstJob; /* set by mtctx, then read by worker => no barrier */ + unsigned lastJob; /* set by mtctx, then read by worker => no barrier */ + ZSTD_CCtx_params params; /* set by mtctx, then read by worker => no barrier */ + const ZSTD_CDict* cdict; /* set by mtctx, then read by worker => no barrier */ + unsigned long long fullFrameSize; /* set by mtctx, then read by worker => no barrier */ + size_t dstFlushed; /* used only by mtctx */ + unsigned frameChecksumNeeded; /* used only by mtctx */ +} ZSTDMT_jobDescription; + +#define JOB_ERROR(e) { \ + ZSTD_PTHREAD_MUTEX_LOCK(&job->job_mutex); \ + job->cSize = e; \ + ZSTD_pthread_mutex_unlock(&job->job_mutex); \ + goto _endJob; \ +} + +/* ZSTDMT_compressionJob() is a POOL_function type */ +static void ZSTDMT_compressionJob(void* jobDescription) +{ + ZSTDMT_jobDescription* const job = (ZSTDMT_jobDescription*)jobDescription; + ZSTD_CCtx_params jobParams = job->params; /* do not modify job->params ! copy it, modify the copy */ + ZSTD_CCtx* const cctx = ZSTDMT_getCCtx(job->cctxPool); + rawSeqStore_t rawSeqStore = ZSTDMT_getSeq(job->seqPool); + buffer_t dstBuff = job->dstBuff; + size_t lastCBlockSize = 0; + + /* resources */ + if (cctx==NULL) JOB_ERROR(ERROR(memory_allocation)); + if (dstBuff.start == NULL) { /* streaming job : doesn't provide a dstBuffer */ + dstBuff = ZSTDMT_getBuffer(job->bufPool); + if (dstBuff.start==NULL) JOB_ERROR(ERROR(memory_allocation)); + job->dstBuff = dstBuff; /* this value can be read in ZSTDMT_flush, when it copies the whole job */ + } + if (jobParams.ldmParams.enableLdm == ZSTD_ps_enable && rawSeqStore.seq == NULL) + JOB_ERROR(ERROR(memory_allocation)); + + /* Don't compute the checksum for chunks, since we compute it externally, + * but write it in the header. + */ + if (job->jobID != 0) jobParams.fParams.checksumFlag = 0; + /* Don't run LDM for the chunks, since we handle it externally */ + jobParams.ldmParams.enableLdm = ZSTD_ps_disable; + /* Correct nbWorkers to 0. */ + jobParams.nbWorkers = 0; + + + /* init */ + if (job->cdict) { + size_t const initError = ZSTD_compressBegin_advanced_internal(cctx, NULL, 0, ZSTD_dct_auto, ZSTD_dtlm_fast, job->cdict, &jobParams, job->fullFrameSize); + assert(job->firstJob); /* only allowed for first job */ + if (ZSTD_isError(initError)) JOB_ERROR(initError); + } else { /* srcStart points at reloaded section */ + U64 const pledgedSrcSize = job->firstJob ? job->fullFrameSize : job->src.size; + { size_t const forceWindowError = ZSTD_CCtxParams_setParameter(&jobParams, ZSTD_c_forceMaxWindow, !job->firstJob); + if (ZSTD_isError(forceWindowError)) JOB_ERROR(forceWindowError); + } + if (!job->firstJob) { + size_t const err = ZSTD_CCtxParams_setParameter(&jobParams, ZSTD_c_deterministicRefPrefix, 0); + if (ZSTD_isError(err)) JOB_ERROR(err); + } + { size_t const initError = ZSTD_compressBegin_advanced_internal(cctx, + job->prefix.start, job->prefix.size, ZSTD_dct_rawContent, /* load dictionary in "content-only" mode (no header analysis) */ + ZSTD_dtlm_fast, + NULL, /*cdict*/ + &jobParams, pledgedSrcSize); + if (ZSTD_isError(initError)) JOB_ERROR(initError); + } } + + /* Perform serial step as early as possible, but after CCtx initialization */ + ZSTDMT_serialState_update(job->serial, cctx, rawSeqStore, job->src, job->jobID); + + if (!job->firstJob) { /* flush and overwrite frame header when it's not first job */ + size_t const hSize = ZSTD_compressContinue(cctx, dstBuff.start, dstBuff.capacity, job->src.start, 0); + if (ZSTD_isError(hSize)) JOB_ERROR(hSize); + DEBUGLOG(5, "ZSTDMT_compressionJob: flush and overwrite %u bytes of frame header (not first job)", (U32)hSize); + ZSTD_invalidateRepCodes(cctx); + } + + /* compress */ + { size_t const chunkSize = 4*ZSTD_BLOCKSIZE_MAX; + int const nbChunks = (int)((job->src.size + (chunkSize-1)) / chunkSize); + const BYTE* ip = (const BYTE*) job->src.start; + BYTE* const ostart = (BYTE*)dstBuff.start; + BYTE* op = ostart; + BYTE* oend = op + dstBuff.capacity; + int chunkNb; + if (sizeof(size_t) > sizeof(int)) assert(job->src.size < ((size_t)INT_MAX) * chunkSize); /* check overflow */ + DEBUGLOG(5, "ZSTDMT_compressionJob: compress %u bytes in %i blocks", (U32)job->src.size, nbChunks); + assert(job->cSize == 0); + for (chunkNb = 1; chunkNb < nbChunks; chunkNb++) { + size_t const cSize = ZSTD_compressContinue(cctx, op, oend-op, ip, chunkSize); + if (ZSTD_isError(cSize)) JOB_ERROR(cSize); + ip += chunkSize; + op += cSize; assert(op < oend); + /* stats */ + ZSTD_PTHREAD_MUTEX_LOCK(&job->job_mutex); + job->cSize += cSize; + job->consumed = chunkSize * chunkNb; + DEBUGLOG(5, "ZSTDMT_compressionJob: compress new block : cSize==%u bytes (total: %u)", + (U32)cSize, (U32)job->cSize); + ZSTD_pthread_cond_signal(&job->job_cond); /* warns some more data is ready to be flushed */ + ZSTD_pthread_mutex_unlock(&job->job_mutex); + } + /* last block */ + assert(chunkSize > 0); + assert((chunkSize & (chunkSize - 1)) == 0); /* chunkSize must be power of 2 for mask==(chunkSize-1) to work */ + if ((nbChunks > 0) | job->lastJob /*must output a "last block" flag*/ ) { + size_t const lastBlockSize1 = job->src.size & (chunkSize-1); + size_t const lastBlockSize = ((lastBlockSize1==0) & (job->src.size>=chunkSize)) ? chunkSize : lastBlockSize1; + size_t const cSize = (job->lastJob) ? + ZSTD_compressEnd (cctx, op, oend-op, ip, lastBlockSize) : + ZSTD_compressContinue(cctx, op, oend-op, ip, lastBlockSize); + if (ZSTD_isError(cSize)) JOB_ERROR(cSize); + lastCBlockSize = cSize; + } } + if (!job->firstJob) { + /* Double check that we don't have an ext-dict, because then our + * repcode invalidation doesn't work. + */ + assert(!ZSTD_window_hasExtDict(cctx->blockState.matchState.window)); + } + ZSTD_CCtx_trace(cctx, 0); + +_endJob: + ZSTDMT_serialState_ensureFinished(job->serial, job->jobID, job->cSize); + if (job->prefix.size > 0) + DEBUGLOG(5, "Finished with prefix: %zx", (size_t)job->prefix.start); + DEBUGLOG(5, "Finished with source: %zx", (size_t)job->src.start); + /* release resources */ + ZSTDMT_releaseSeq(job->seqPool, rawSeqStore); + ZSTDMT_releaseCCtx(job->cctxPool, cctx); + /* report */ + ZSTD_PTHREAD_MUTEX_LOCK(&job->job_mutex); + if (ZSTD_isError(job->cSize)) assert(lastCBlockSize == 0); + job->cSize += lastCBlockSize; + job->consumed = job->src.size; /* when job->consumed == job->src.size , compression job is presumed completed */ + ZSTD_pthread_cond_signal(&job->job_cond); + ZSTD_pthread_mutex_unlock(&job->job_mutex); +} + + +/* ------------------------------------------ */ +/* ===== Multi-threaded compression ===== */ +/* ------------------------------------------ */ + +typedef struct { + range_t prefix; /* read-only non-owned prefix buffer */ + buffer_t buffer; + size_t filled; +} inBuff_t; + +typedef struct { + BYTE* buffer; /* The round input buffer. All jobs get references + * to pieces of the buffer. ZSTDMT_tryGetInputRange() + * handles handing out job input buffers, and makes + * sure it doesn't overlap with any pieces still in use. + */ + size_t capacity; /* The capacity of buffer. */ + size_t pos; /* The position of the current inBuff in the round + * buffer. Updated past the end if the inBuff once + * the inBuff is sent to the worker thread. + * pos <= capacity. + */ +} roundBuff_t; + +static const roundBuff_t kNullRoundBuff = {NULL, 0, 0}; + +#define RSYNC_LENGTH 32 +/* Don't create chunks smaller than the zstd block size. + * This stops us from regressing compression ratio too much, + * and ensures our output fits in ZSTD_compressBound(). + * + * If this is shrunk < ZSTD_BLOCKSIZELOG_MIN then + * ZSTD_COMPRESSBOUND() will need to be updated. + */ +#define RSYNC_MIN_BLOCK_LOG ZSTD_BLOCKSIZELOG_MAX +#define RSYNC_MIN_BLOCK_SIZE (1< one job is already prepared, but pool has shortage of workers. Don't create a new job. */ + inBuff_t inBuff; + roundBuff_t roundBuff; + serialState_t serial; + rsyncState_t rsync; + unsigned jobIDMask; + unsigned doneJobID; + unsigned nextJobID; + unsigned frameEnded; + unsigned allJobsCompleted; + unsigned long long frameContentSize; + unsigned long long consumed; + unsigned long long produced; + ZSTD_customMem cMem; + ZSTD_CDict* cdictLocal; + const ZSTD_CDict* cdict; + unsigned providedFactory: 1; +}; + +static void ZSTDMT_freeJobsTable(ZSTDMT_jobDescription* jobTable, U32 nbJobs, ZSTD_customMem cMem) +{ + U32 jobNb; + if (jobTable == NULL) return; + for (jobNb=0; jobNb mtctx->jobIDMask+1) { /* need more job capacity */ + ZSTDMT_freeJobsTable(mtctx->jobs, mtctx->jobIDMask+1, mtctx->cMem); + mtctx->jobIDMask = 0; + mtctx->jobs = ZSTDMT_createJobsTable(&nbJobs, mtctx->cMem); + if (mtctx->jobs==NULL) return ERROR(memory_allocation); + assert((nbJobs != 0) && ((nbJobs & (nbJobs - 1)) == 0)); /* ensure nbJobs is a power of 2 */ + mtctx->jobIDMask = nbJobs - 1; + } + return 0; +} + + +/* ZSTDMT_CCtxParam_setNbWorkers(): + * Internal use only */ +static size_t ZSTDMT_CCtxParam_setNbWorkers(ZSTD_CCtx_params* params, unsigned nbWorkers) +{ + return ZSTD_CCtxParams_setParameter(params, ZSTD_c_nbWorkers, (int)nbWorkers); +} + +MEM_STATIC ZSTDMT_CCtx* ZSTDMT_createCCtx_advanced_internal(unsigned nbWorkers, ZSTD_customMem cMem, ZSTD_threadPool* pool) +{ + ZSTDMT_CCtx* mtctx; + U32 nbJobs = nbWorkers + 2; + int initError; + DEBUGLOG(3, "ZSTDMT_createCCtx_advanced (nbWorkers = %u)", nbWorkers); + + if (nbWorkers < 1) return NULL; + nbWorkers = MIN(nbWorkers , ZSTDMT_NBWORKERS_MAX); + if ((cMem.customAlloc!=NULL) ^ (cMem.customFree!=NULL)) + /* invalid custom allocator */ + return NULL; + + mtctx = (ZSTDMT_CCtx*) ZSTD_customCalloc(sizeof(ZSTDMT_CCtx), cMem); + if (!mtctx) return NULL; + ZSTDMT_CCtxParam_setNbWorkers(&mtctx->params, nbWorkers); + mtctx->cMem = cMem; + mtctx->allJobsCompleted = 1; + if (pool != NULL) { + mtctx->factory = pool; + mtctx->providedFactory = 1; + } + else { + mtctx->factory = POOL_create_advanced(nbWorkers, 0, cMem); + mtctx->providedFactory = 0; + } + mtctx->jobs = ZSTDMT_createJobsTable(&nbJobs, cMem); + assert(nbJobs > 0); assert((nbJobs & (nbJobs - 1)) == 0); /* ensure nbJobs is a power of 2 */ + mtctx->jobIDMask = nbJobs - 1; + mtctx->bufPool = ZSTDMT_createBufferPool(BUF_POOL_MAX_NB_BUFFERS(nbWorkers), cMem); + mtctx->cctxPool = ZSTDMT_createCCtxPool(nbWorkers, cMem); + mtctx->seqPool = ZSTDMT_createSeqPool(nbWorkers, cMem); + initError = ZSTDMT_serialState_init(&mtctx->serial); + mtctx->roundBuff = kNullRoundBuff; + if (!mtctx->factory | !mtctx->jobs | !mtctx->bufPool | !mtctx->cctxPool | !mtctx->seqPool | initError) { + ZSTDMT_freeCCtx(mtctx); + return NULL; + } + DEBUGLOG(3, "mt_cctx created, for %u threads", nbWorkers); + return mtctx; +} + +ZSTDMT_CCtx* ZSTDMT_createCCtx_advanced(unsigned nbWorkers, ZSTD_customMem cMem, ZSTD_threadPool* pool) +{ +#ifdef ZSTD_MULTITHREAD + return ZSTDMT_createCCtx_advanced_internal(nbWorkers, cMem, pool); +#else + (void)nbWorkers; + (void)cMem; + (void)pool; + return NULL; +#endif +} + + +/* ZSTDMT_releaseAllJobResources() : + * note : ensure all workers are killed first ! */ +static void ZSTDMT_releaseAllJobResources(ZSTDMT_CCtx* mtctx) +{ + unsigned jobID; + DEBUGLOG(3, "ZSTDMT_releaseAllJobResources"); + for (jobID=0; jobID <= mtctx->jobIDMask; jobID++) { + /* Copy the mutex/cond out */ + ZSTD_pthread_mutex_t const mutex = mtctx->jobs[jobID].job_mutex; + ZSTD_pthread_cond_t const cond = mtctx->jobs[jobID].job_cond; + + DEBUGLOG(4, "job%02u: release dst address %08X", jobID, (U32)(size_t)mtctx->jobs[jobID].dstBuff.start); + ZSTDMT_releaseBuffer(mtctx->bufPool, mtctx->jobs[jobID].dstBuff); + + /* Clear the job description, but keep the mutex/cond */ + ZSTD_memset(&mtctx->jobs[jobID], 0, sizeof(mtctx->jobs[jobID])); + mtctx->jobs[jobID].job_mutex = mutex; + mtctx->jobs[jobID].job_cond = cond; + } + mtctx->inBuff.buffer = g_nullBuffer; + mtctx->inBuff.filled = 0; + mtctx->allJobsCompleted = 1; +} + +static void ZSTDMT_waitForAllJobsCompleted(ZSTDMT_CCtx* mtctx) +{ + DEBUGLOG(4, "ZSTDMT_waitForAllJobsCompleted"); + while (mtctx->doneJobID < mtctx->nextJobID) { + unsigned const jobID = mtctx->doneJobID & mtctx->jobIDMask; + ZSTD_PTHREAD_MUTEX_LOCK(&mtctx->jobs[jobID].job_mutex); + while (mtctx->jobs[jobID].consumed < mtctx->jobs[jobID].src.size) { + DEBUGLOG(4, "waiting for jobCompleted signal from job %u", mtctx->doneJobID); /* we want to block when waiting for data to flush */ + ZSTD_pthread_cond_wait(&mtctx->jobs[jobID].job_cond, &mtctx->jobs[jobID].job_mutex); + } + ZSTD_pthread_mutex_unlock(&mtctx->jobs[jobID].job_mutex); + mtctx->doneJobID++; + } +} + +size_t ZSTDMT_freeCCtx(ZSTDMT_CCtx* mtctx) +{ + if (mtctx==NULL) return 0; /* compatible with free on NULL */ + if (!mtctx->providedFactory) + POOL_free(mtctx->factory); /* stop and free worker threads */ + ZSTDMT_releaseAllJobResources(mtctx); /* release job resources into pools first */ + ZSTDMT_freeJobsTable(mtctx->jobs, mtctx->jobIDMask+1, mtctx->cMem); + ZSTDMT_freeBufferPool(mtctx->bufPool); + ZSTDMT_freeCCtxPool(mtctx->cctxPool); + ZSTDMT_freeSeqPool(mtctx->seqPool); + ZSTDMT_serialState_free(&mtctx->serial); + ZSTD_freeCDict(mtctx->cdictLocal); + if (mtctx->roundBuff.buffer) + ZSTD_customFree(mtctx->roundBuff.buffer, mtctx->cMem); + ZSTD_customFree(mtctx, mtctx->cMem); + return 0; +} + +size_t ZSTDMT_sizeof_CCtx(ZSTDMT_CCtx* mtctx) +{ + if (mtctx == NULL) return 0; /* supports sizeof NULL */ + return sizeof(*mtctx) + + POOL_sizeof(mtctx->factory) + + ZSTDMT_sizeof_bufferPool(mtctx->bufPool) + + (mtctx->jobIDMask+1) * sizeof(ZSTDMT_jobDescription) + + ZSTDMT_sizeof_CCtxPool(mtctx->cctxPool) + + ZSTDMT_sizeof_seqPool(mtctx->seqPool) + + ZSTD_sizeof_CDict(mtctx->cdictLocal) + + mtctx->roundBuff.capacity; +} + + +/* ZSTDMT_resize() : + * @return : error code if fails, 0 on success */ +static size_t ZSTDMT_resize(ZSTDMT_CCtx* mtctx, unsigned nbWorkers) +{ + if (POOL_resize(mtctx->factory, nbWorkers)) return ERROR(memory_allocation); + FORWARD_IF_ERROR( ZSTDMT_expandJobsTable(mtctx, nbWorkers) , ""); + mtctx->bufPool = ZSTDMT_expandBufferPool(mtctx->bufPool, BUF_POOL_MAX_NB_BUFFERS(nbWorkers)); + if (mtctx->bufPool == NULL) return ERROR(memory_allocation); + mtctx->cctxPool = ZSTDMT_expandCCtxPool(mtctx->cctxPool, nbWorkers); + if (mtctx->cctxPool == NULL) return ERROR(memory_allocation); + mtctx->seqPool = ZSTDMT_expandSeqPool(mtctx->seqPool, nbWorkers); + if (mtctx->seqPool == NULL) return ERROR(memory_allocation); + ZSTDMT_CCtxParam_setNbWorkers(&mtctx->params, nbWorkers); + return 0; +} + + +/*! ZSTDMT_updateCParams_whileCompressing() : + * Updates a selected set of compression parameters, remaining compatible with currently active frame. + * New parameters will be applied to next compression job. */ +void ZSTDMT_updateCParams_whileCompressing(ZSTDMT_CCtx* mtctx, const ZSTD_CCtx_params* cctxParams) +{ + U32 const saved_wlog = mtctx->params.cParams.windowLog; /* Do not modify windowLog while compressing */ + int const compressionLevel = cctxParams->compressionLevel; + DEBUGLOG(5, "ZSTDMT_updateCParams_whileCompressing (level:%i)", + compressionLevel); + mtctx->params.compressionLevel = compressionLevel; + { ZSTD_compressionParameters cParams = ZSTD_getCParamsFromCCtxParams(cctxParams, ZSTD_CONTENTSIZE_UNKNOWN, 0, ZSTD_cpm_noAttachDict); + cParams.windowLog = saved_wlog; + mtctx->params.cParams = cParams; + } +} + +/* ZSTDMT_getFrameProgression(): + * tells how much data has been consumed (input) and produced (output) for current frame. + * able to count progression inside worker threads. + * Note : mutex will be acquired during statistics collection inside workers. */ +ZSTD_frameProgression ZSTDMT_getFrameProgression(ZSTDMT_CCtx* mtctx) +{ + ZSTD_frameProgression fps; + DEBUGLOG(5, "ZSTDMT_getFrameProgression"); + fps.ingested = mtctx->consumed + mtctx->inBuff.filled; + fps.consumed = mtctx->consumed; + fps.produced = fps.flushed = mtctx->produced; + fps.currentJobID = mtctx->nextJobID; + fps.nbActiveWorkers = 0; + { unsigned jobNb; + unsigned lastJobNb = mtctx->nextJobID + mtctx->jobReady; assert(mtctx->jobReady <= 1); + DEBUGLOG(6, "ZSTDMT_getFrameProgression: jobs: from %u to <%u (jobReady:%u)", + mtctx->doneJobID, lastJobNb, mtctx->jobReady) + for (jobNb = mtctx->doneJobID ; jobNb < lastJobNb ; jobNb++) { + unsigned const wJobID = jobNb & mtctx->jobIDMask; + ZSTDMT_jobDescription* jobPtr = &mtctx->jobs[wJobID]; + ZSTD_pthread_mutex_lock(&jobPtr->job_mutex); + { size_t const cResult = jobPtr->cSize; + size_t const produced = ZSTD_isError(cResult) ? 0 : cResult; + size_t const flushed = ZSTD_isError(cResult) ? 0 : jobPtr->dstFlushed; + assert(flushed <= produced); + fps.ingested += jobPtr->src.size; + fps.consumed += jobPtr->consumed; + fps.produced += produced; + fps.flushed += flushed; + fps.nbActiveWorkers += (jobPtr->consumed < jobPtr->src.size); + } + ZSTD_pthread_mutex_unlock(&mtctx->jobs[wJobID].job_mutex); + } + } + return fps; +} + + +size_t ZSTDMT_toFlushNow(ZSTDMT_CCtx* mtctx) +{ + size_t toFlush; + unsigned const jobID = mtctx->doneJobID; + assert(jobID <= mtctx->nextJobID); + if (jobID == mtctx->nextJobID) return 0; /* no active job => nothing to flush */ + + /* look into oldest non-fully-flushed job */ + { unsigned const wJobID = jobID & mtctx->jobIDMask; + ZSTDMT_jobDescription* const jobPtr = &mtctx->jobs[wJobID]; + ZSTD_pthread_mutex_lock(&jobPtr->job_mutex); + { size_t const cResult = jobPtr->cSize; + size_t const produced = ZSTD_isError(cResult) ? 0 : cResult; + size_t const flushed = ZSTD_isError(cResult) ? 0 : jobPtr->dstFlushed; + assert(flushed <= produced); + assert(jobPtr->consumed <= jobPtr->src.size); + toFlush = produced - flushed; + /* if toFlush==0, nothing is available to flush. + * However, jobID is expected to still be active: + * if jobID was already completed and fully flushed, + * ZSTDMT_flushProduced() should have already moved onto next job. + * Therefore, some input has not yet been consumed. */ + if (toFlush==0) { + assert(jobPtr->consumed < jobPtr->src.size); + } + } + ZSTD_pthread_mutex_unlock(&mtctx->jobs[wJobID].job_mutex); + } + + return toFlush; +} + + +/* ------------------------------------------ */ +/* ===== Multi-threaded compression ===== */ +/* ------------------------------------------ */ + +static unsigned ZSTDMT_computeTargetJobLog(const ZSTD_CCtx_params* params) +{ + unsigned jobLog; + if (params->ldmParams.enableLdm == ZSTD_ps_enable) { + /* In Long Range Mode, the windowLog is typically oversized. + * In which case, it's preferable to determine the jobSize + * based on cycleLog instead. */ + jobLog = MAX(21, ZSTD_cycleLog(params->cParams.chainLog, params->cParams.strategy) + 3); + } else { + jobLog = MAX(20, params->cParams.windowLog + 2); + } + return MIN(jobLog, (unsigned)ZSTDMT_JOBLOG_MAX); +} + +static int ZSTDMT_overlapLog_default(ZSTD_strategy strat) +{ + switch(strat) + { + case ZSTD_btultra2: + return 9; + case ZSTD_btultra: + case ZSTD_btopt: + return 8; + case ZSTD_btlazy2: + case ZSTD_lazy2: + return 7; + case ZSTD_lazy: + case ZSTD_greedy: + case ZSTD_dfast: + case ZSTD_fast: + default:; + } + return 6; +} + +static int ZSTDMT_overlapLog(int ovlog, ZSTD_strategy strat) +{ + assert(0 <= ovlog && ovlog <= 9); + if (ovlog == 0) return ZSTDMT_overlapLog_default(strat); + return ovlog; +} + +static size_t ZSTDMT_computeOverlapSize(const ZSTD_CCtx_params* params) +{ + int const overlapRLog = 9 - ZSTDMT_overlapLog(params->overlapLog, params->cParams.strategy); + int ovLog = (overlapRLog >= 8) ? 0 : (params->cParams.windowLog - overlapRLog); + assert(0 <= overlapRLog && overlapRLog <= 8); + if (params->ldmParams.enableLdm == ZSTD_ps_enable) { + /* In Long Range Mode, the windowLog is typically oversized. + * In which case, it's preferable to determine the jobSize + * based on chainLog instead. + * Then, ovLog becomes a fraction of the jobSize, rather than windowSize */ + ovLog = MIN(params->cParams.windowLog, ZSTDMT_computeTargetJobLog(params) - 2) + - overlapRLog; + } + assert(0 <= ovLog && ovLog <= ZSTD_WINDOWLOG_MAX); + DEBUGLOG(4, "overlapLog : %i", params->overlapLog); + DEBUGLOG(4, "overlap size : %i", 1 << ovLog); + return (ovLog==0) ? 0 : (size_t)1 << ovLog; +} + +/* ====================================== */ +/* ======= Streaming API ======= */ +/* ====================================== */ + +size_t ZSTDMT_initCStream_internal( + ZSTDMT_CCtx* mtctx, + const void* dict, size_t dictSize, ZSTD_dictContentType_e dictContentType, + const ZSTD_CDict* cdict, ZSTD_CCtx_params params, + unsigned long long pledgedSrcSize) +{ + DEBUGLOG(4, "ZSTDMT_initCStream_internal (pledgedSrcSize=%u, nbWorkers=%u, cctxPool=%u)", + (U32)pledgedSrcSize, params.nbWorkers, mtctx->cctxPool->totalCCtx); + + /* params supposed partially fully validated at this point */ + assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams))); + assert(!((dict) && (cdict))); /* either dict or cdict, not both */ + + /* init */ + if (params.nbWorkers != mtctx->params.nbWorkers) + FORWARD_IF_ERROR( ZSTDMT_resize(mtctx, params.nbWorkers) , ""); + + if (params.jobSize != 0 && params.jobSize < ZSTDMT_JOBSIZE_MIN) params.jobSize = ZSTDMT_JOBSIZE_MIN; + if (params.jobSize > (size_t)ZSTDMT_JOBSIZE_MAX) params.jobSize = (size_t)ZSTDMT_JOBSIZE_MAX; + + DEBUGLOG(4, "ZSTDMT_initCStream_internal: %u workers", params.nbWorkers); + + if (mtctx->allJobsCompleted == 0) { /* previous compression not correctly finished */ + ZSTDMT_waitForAllJobsCompleted(mtctx); + ZSTDMT_releaseAllJobResources(mtctx); + mtctx->allJobsCompleted = 1; + } + + mtctx->params = params; + mtctx->frameContentSize = pledgedSrcSize; + if (dict) { + ZSTD_freeCDict(mtctx->cdictLocal); + mtctx->cdictLocal = ZSTD_createCDict_advanced(dict, dictSize, + ZSTD_dlm_byCopy, dictContentType, /* note : a loadPrefix becomes an internal CDict */ + params.cParams, mtctx->cMem); + mtctx->cdict = mtctx->cdictLocal; + if (mtctx->cdictLocal == NULL) return ERROR(memory_allocation); + } else { + ZSTD_freeCDict(mtctx->cdictLocal); + mtctx->cdictLocal = NULL; + mtctx->cdict = cdict; + } + + mtctx->targetPrefixSize = ZSTDMT_computeOverlapSize(¶ms); + DEBUGLOG(4, "overlapLog=%i => %u KB", params.overlapLog, (U32)(mtctx->targetPrefixSize>>10)); + mtctx->targetSectionSize = params.jobSize; + if (mtctx->targetSectionSize == 0) { + mtctx->targetSectionSize = 1ULL << ZSTDMT_computeTargetJobLog(¶ms); + } + assert(mtctx->targetSectionSize <= (size_t)ZSTDMT_JOBSIZE_MAX); + + if (params.rsyncable) { + /* Aim for the targetsectionSize as the average job size. */ + U32 const jobSizeKB = (U32)(mtctx->targetSectionSize >> 10); + U32 const rsyncBits = (assert(jobSizeKB >= 1), ZSTD_highbit32(jobSizeKB) + 10); + /* We refuse to create jobs < RSYNC_MIN_BLOCK_SIZE bytes, so make sure our + * expected job size is at least 4x larger. */ + assert(rsyncBits >= RSYNC_MIN_BLOCK_LOG + 2); + DEBUGLOG(4, "rsyncLog = %u", rsyncBits); + mtctx->rsync.hash = 0; + mtctx->rsync.hitMask = (1ULL << rsyncBits) - 1; + mtctx->rsync.primePower = ZSTD_rollingHash_primePower(RSYNC_LENGTH); + } + if (mtctx->targetSectionSize < mtctx->targetPrefixSize) mtctx->targetSectionSize = mtctx->targetPrefixSize; /* job size must be >= overlap size */ + DEBUGLOG(4, "Job Size : %u KB (note : set to %u)", (U32)(mtctx->targetSectionSize>>10), (U32)params.jobSize); + DEBUGLOG(4, "inBuff Size : %u KB", (U32)(mtctx->targetSectionSize>>10)); + ZSTDMT_setBufferSize(mtctx->bufPool, ZSTD_compressBound(mtctx->targetSectionSize)); + { + /* If ldm is enabled we need windowSize space. */ + size_t const windowSize = mtctx->params.ldmParams.enableLdm == ZSTD_ps_enable ? (1U << mtctx->params.cParams.windowLog) : 0; + /* Two buffers of slack, plus extra space for the overlap + * This is the minimum slack that LDM works with. One extra because + * flush might waste up to targetSectionSize-1 bytes. Another extra + * for the overlap (if > 0), then one to fill which doesn't overlap + * with the LDM window. + */ + size_t const nbSlackBuffers = 2 + (mtctx->targetPrefixSize > 0); + size_t const slackSize = mtctx->targetSectionSize * nbSlackBuffers; + /* Compute the total size, and always have enough slack */ + size_t const nbWorkers = MAX(mtctx->params.nbWorkers, 1); + size_t const sectionsSize = mtctx->targetSectionSize * nbWorkers; + size_t const capacity = MAX(windowSize, sectionsSize) + slackSize; + if (mtctx->roundBuff.capacity < capacity) { + if (mtctx->roundBuff.buffer) + ZSTD_customFree(mtctx->roundBuff.buffer, mtctx->cMem); + mtctx->roundBuff.buffer = (BYTE*)ZSTD_customMalloc(capacity, mtctx->cMem); + if (mtctx->roundBuff.buffer == NULL) { + mtctx->roundBuff.capacity = 0; + return ERROR(memory_allocation); + } + mtctx->roundBuff.capacity = capacity; + } + } + DEBUGLOG(4, "roundBuff capacity : %u KB", (U32)(mtctx->roundBuff.capacity>>10)); + mtctx->roundBuff.pos = 0; + mtctx->inBuff.buffer = g_nullBuffer; + mtctx->inBuff.filled = 0; + mtctx->inBuff.prefix = kNullRange; + mtctx->doneJobID = 0; + mtctx->nextJobID = 0; + mtctx->frameEnded = 0; + mtctx->allJobsCompleted = 0; + mtctx->consumed = 0; + mtctx->produced = 0; + if (ZSTDMT_serialState_reset(&mtctx->serial, mtctx->seqPool, params, mtctx->targetSectionSize, + dict, dictSize, dictContentType)) + return ERROR(memory_allocation); + return 0; +} + + +/* ZSTDMT_writeLastEmptyBlock() + * Write a single empty block with an end-of-frame to finish a frame. + * Job must be created from streaming variant. + * This function is always successful if expected conditions are fulfilled. + */ +static void ZSTDMT_writeLastEmptyBlock(ZSTDMT_jobDescription* job) +{ + assert(job->lastJob == 1); + assert(job->src.size == 0); /* last job is empty -> will be simplified into a last empty block */ + assert(job->firstJob == 0); /* cannot be first job, as it also needs to create frame header */ + assert(job->dstBuff.start == NULL); /* invoked from streaming variant only (otherwise, dstBuff might be user's output) */ + job->dstBuff = ZSTDMT_getBuffer(job->bufPool); + if (job->dstBuff.start == NULL) { + job->cSize = ERROR(memory_allocation); + return; + } + assert(job->dstBuff.capacity >= ZSTD_blockHeaderSize); /* no buffer should ever be that small */ + job->src = kNullRange; + job->cSize = ZSTD_writeLastEmptyBlock(job->dstBuff.start, job->dstBuff.capacity); + assert(!ZSTD_isError(job->cSize)); + assert(job->consumed == 0); +} + +static size_t ZSTDMT_createCompressionJob(ZSTDMT_CCtx* mtctx, size_t srcSize, ZSTD_EndDirective endOp) +{ + unsigned const jobID = mtctx->nextJobID & mtctx->jobIDMask; + int const endFrame = (endOp == ZSTD_e_end); + + if (mtctx->nextJobID > mtctx->doneJobID + mtctx->jobIDMask) { + DEBUGLOG(5, "ZSTDMT_createCompressionJob: will not create new job : table is full"); + assert((mtctx->nextJobID & mtctx->jobIDMask) == (mtctx->doneJobID & mtctx->jobIDMask)); + return 0; + } + + if (!mtctx->jobReady) { + BYTE const* src = (BYTE const*)mtctx->inBuff.buffer.start; + DEBUGLOG(5, "ZSTDMT_createCompressionJob: preparing job %u to compress %u bytes with %u preload ", + mtctx->nextJobID, (U32)srcSize, (U32)mtctx->inBuff.prefix.size); + mtctx->jobs[jobID].src.start = src; + mtctx->jobs[jobID].src.size = srcSize; + assert(mtctx->inBuff.filled >= srcSize); + mtctx->jobs[jobID].prefix = mtctx->inBuff.prefix; + mtctx->jobs[jobID].consumed = 0; + mtctx->jobs[jobID].cSize = 0; + mtctx->jobs[jobID].params = mtctx->params; + mtctx->jobs[jobID].cdict = mtctx->nextJobID==0 ? mtctx->cdict : NULL; + mtctx->jobs[jobID].fullFrameSize = mtctx->frameContentSize; + mtctx->jobs[jobID].dstBuff = g_nullBuffer; + mtctx->jobs[jobID].cctxPool = mtctx->cctxPool; + mtctx->jobs[jobID].bufPool = mtctx->bufPool; + mtctx->jobs[jobID].seqPool = mtctx->seqPool; + mtctx->jobs[jobID].serial = &mtctx->serial; + mtctx->jobs[jobID].jobID = mtctx->nextJobID; + mtctx->jobs[jobID].firstJob = (mtctx->nextJobID==0); + mtctx->jobs[jobID].lastJob = endFrame; + mtctx->jobs[jobID].frameChecksumNeeded = mtctx->params.fParams.checksumFlag && endFrame && (mtctx->nextJobID>0); + mtctx->jobs[jobID].dstFlushed = 0; + + /* Update the round buffer pos and clear the input buffer to be reset */ + mtctx->roundBuff.pos += srcSize; + mtctx->inBuff.buffer = g_nullBuffer; + mtctx->inBuff.filled = 0; + /* Set the prefix */ + if (!endFrame) { + size_t const newPrefixSize = MIN(srcSize, mtctx->targetPrefixSize); + mtctx->inBuff.prefix.start = src + srcSize - newPrefixSize; + mtctx->inBuff.prefix.size = newPrefixSize; + } else { /* endFrame==1 => no need for another input buffer */ + mtctx->inBuff.prefix = kNullRange; + mtctx->frameEnded = endFrame; + if (mtctx->nextJobID == 0) { + /* single job exception : checksum is already calculated directly within worker thread */ + mtctx->params.fParams.checksumFlag = 0; + } } + + if ( (srcSize == 0) + && (mtctx->nextJobID>0)/*single job must also write frame header*/ ) { + DEBUGLOG(5, "ZSTDMT_createCompressionJob: creating a last empty block to end frame"); + assert(endOp == ZSTD_e_end); /* only possible case : need to end the frame with an empty last block */ + ZSTDMT_writeLastEmptyBlock(mtctx->jobs + jobID); + mtctx->nextJobID++; + return 0; + } + } + + DEBUGLOG(5, "ZSTDMT_createCompressionJob: posting job %u : %u bytes (end:%u, jobNb == %u (mod:%u))", + mtctx->nextJobID, + (U32)mtctx->jobs[jobID].src.size, + mtctx->jobs[jobID].lastJob, + mtctx->nextJobID, + jobID); + if (POOL_tryAdd(mtctx->factory, ZSTDMT_compressionJob, &mtctx->jobs[jobID])) { + mtctx->nextJobID++; + mtctx->jobReady = 0; + } else { + DEBUGLOG(5, "ZSTDMT_createCompressionJob: no worker available for job %u", mtctx->nextJobID); + mtctx->jobReady = 1; + } + return 0; +} + + +/*! ZSTDMT_flushProduced() : + * flush whatever data has been produced but not yet flushed in current job. + * move to next job if current one is fully flushed. + * `output` : `pos` will be updated with amount of data flushed . + * `blockToFlush` : if >0, the function will block and wait if there is no data available to flush . + * @return : amount of data remaining within internal buffer, 0 if no more, 1 if unknown but > 0, or an error code */ +static size_t ZSTDMT_flushProduced(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output, unsigned blockToFlush, ZSTD_EndDirective end) +{ + unsigned const wJobID = mtctx->doneJobID & mtctx->jobIDMask; + DEBUGLOG(5, "ZSTDMT_flushProduced (blocking:%u , job %u <= %u)", + blockToFlush, mtctx->doneJobID, mtctx->nextJobID); + assert(output->size >= output->pos); + + ZSTD_PTHREAD_MUTEX_LOCK(&mtctx->jobs[wJobID].job_mutex); + if ( blockToFlush + && (mtctx->doneJobID < mtctx->nextJobID) ) { + assert(mtctx->jobs[wJobID].dstFlushed <= mtctx->jobs[wJobID].cSize); + while (mtctx->jobs[wJobID].dstFlushed == mtctx->jobs[wJobID].cSize) { /* nothing to flush */ + if (mtctx->jobs[wJobID].consumed == mtctx->jobs[wJobID].src.size) { + DEBUGLOG(5, "job %u is completely consumed (%u == %u) => don't wait for cond, there will be none", + mtctx->doneJobID, (U32)mtctx->jobs[wJobID].consumed, (U32)mtctx->jobs[wJobID].src.size); + break; + } + DEBUGLOG(5, "waiting for something to flush from job %u (currently flushed: %u bytes)", + mtctx->doneJobID, (U32)mtctx->jobs[wJobID].dstFlushed); + ZSTD_pthread_cond_wait(&mtctx->jobs[wJobID].job_cond, &mtctx->jobs[wJobID].job_mutex); /* block when nothing to flush but some to come */ + } } + + /* try to flush something */ + { size_t cSize = mtctx->jobs[wJobID].cSize; /* shared */ + size_t const srcConsumed = mtctx->jobs[wJobID].consumed; /* shared */ + size_t const srcSize = mtctx->jobs[wJobID].src.size; /* read-only, could be done after mutex lock, but no-declaration-after-statement */ + ZSTD_pthread_mutex_unlock(&mtctx->jobs[wJobID].job_mutex); + if (ZSTD_isError(cSize)) { + DEBUGLOG(5, "ZSTDMT_flushProduced: job %u : compression error detected : %s", + mtctx->doneJobID, ZSTD_getErrorName(cSize)); + ZSTDMT_waitForAllJobsCompleted(mtctx); + ZSTDMT_releaseAllJobResources(mtctx); + return cSize; + } + /* add frame checksum if necessary (can only happen once) */ + assert(srcConsumed <= srcSize); + if ( (srcConsumed == srcSize) /* job completed -> worker no longer active */ + && mtctx->jobs[wJobID].frameChecksumNeeded ) { + U32 const checksum = (U32)XXH64_digest(&mtctx->serial.xxhState); + DEBUGLOG(4, "ZSTDMT_flushProduced: writing checksum : %08X \n", checksum); + MEM_writeLE32((char*)mtctx->jobs[wJobID].dstBuff.start + mtctx->jobs[wJobID].cSize, checksum); + cSize += 4; + mtctx->jobs[wJobID].cSize += 4; /* can write this shared value, as worker is no longer active */ + mtctx->jobs[wJobID].frameChecksumNeeded = 0; + } + + if (cSize > 0) { /* compression is ongoing or completed */ + size_t const toFlush = MIN(cSize - mtctx->jobs[wJobID].dstFlushed, output->size - output->pos); + DEBUGLOG(5, "ZSTDMT_flushProduced: Flushing %u bytes from job %u (completion:%u/%u, generated:%u)", + (U32)toFlush, mtctx->doneJobID, (U32)srcConsumed, (U32)srcSize, (U32)cSize); + assert(mtctx->doneJobID < mtctx->nextJobID); + assert(cSize >= mtctx->jobs[wJobID].dstFlushed); + assert(mtctx->jobs[wJobID].dstBuff.start != NULL); + if (toFlush > 0) { + ZSTD_memcpy((char*)output->dst + output->pos, + (const char*)mtctx->jobs[wJobID].dstBuff.start + mtctx->jobs[wJobID].dstFlushed, + toFlush); + } + output->pos += toFlush; + mtctx->jobs[wJobID].dstFlushed += toFlush; /* can write : this value is only used by mtctx */ + + if ( (srcConsumed == srcSize) /* job is completed */ + && (mtctx->jobs[wJobID].dstFlushed == cSize) ) { /* output buffer fully flushed => free this job position */ + DEBUGLOG(5, "Job %u completed (%u bytes), moving to next one", + mtctx->doneJobID, (U32)mtctx->jobs[wJobID].dstFlushed); + ZSTDMT_releaseBuffer(mtctx->bufPool, mtctx->jobs[wJobID].dstBuff); + DEBUGLOG(5, "dstBuffer released"); + mtctx->jobs[wJobID].dstBuff = g_nullBuffer; + mtctx->jobs[wJobID].cSize = 0; /* ensure this job slot is considered "not started" in future check */ + mtctx->consumed += srcSize; + mtctx->produced += cSize; + mtctx->doneJobID++; + } } + + /* return value : how many bytes left in buffer ; fake it to 1 when unknown but >0 */ + if (cSize > mtctx->jobs[wJobID].dstFlushed) return (cSize - mtctx->jobs[wJobID].dstFlushed); + if (srcSize > srcConsumed) return 1; /* current job not completely compressed */ + } + if (mtctx->doneJobID < mtctx->nextJobID) return 1; /* some more jobs ongoing */ + if (mtctx->jobReady) return 1; /* one job is ready to push, just not yet in the list */ + if (mtctx->inBuff.filled > 0) return 1; /* input is not empty, and still needs to be converted into a job */ + mtctx->allJobsCompleted = mtctx->frameEnded; /* all jobs are entirely flushed => if this one is last one, frame is completed */ + if (end == ZSTD_e_end) return !mtctx->frameEnded; /* for ZSTD_e_end, question becomes : is frame completed ? instead of : are internal buffers fully flushed ? */ + return 0; /* internal buffers fully flushed */ +} + +/** + * Returns the range of data used by the earliest job that is not yet complete. + * If the data of the first job is broken up into two segments, we cover both + * sections. + */ +static range_t ZSTDMT_getInputDataInUse(ZSTDMT_CCtx* mtctx) +{ + unsigned const firstJobID = mtctx->doneJobID; + unsigned const lastJobID = mtctx->nextJobID; + unsigned jobID; + + for (jobID = firstJobID; jobID < lastJobID; ++jobID) { + unsigned const wJobID = jobID & mtctx->jobIDMask; + size_t consumed; + + ZSTD_PTHREAD_MUTEX_LOCK(&mtctx->jobs[wJobID].job_mutex); + consumed = mtctx->jobs[wJobID].consumed; + ZSTD_pthread_mutex_unlock(&mtctx->jobs[wJobID].job_mutex); + + if (consumed < mtctx->jobs[wJobID].src.size) { + range_t range = mtctx->jobs[wJobID].prefix; + if (range.size == 0) { + /* Empty prefix */ + range = mtctx->jobs[wJobID].src; + } + /* Job source in multiple segments not supported yet */ + assert(range.start <= mtctx->jobs[wJobID].src.start); + return range; + } + } + return kNullRange; +} + +/** + * Returns non-zero iff buffer and range overlap. + */ +static int ZSTDMT_isOverlapped(buffer_t buffer, range_t range) +{ + BYTE const* const bufferStart = (BYTE const*)buffer.start; + BYTE const* const rangeStart = (BYTE const*)range.start; + + if (rangeStart == NULL || bufferStart == NULL) + return 0; + + { + BYTE const* const bufferEnd = bufferStart + buffer.capacity; + BYTE const* const rangeEnd = rangeStart + range.size; + + /* Empty ranges cannot overlap */ + if (bufferStart == bufferEnd || rangeStart == rangeEnd) + return 0; + + return bufferStart < rangeEnd && rangeStart < bufferEnd; + } +} + +static int ZSTDMT_doesOverlapWindow(buffer_t buffer, ZSTD_window_t window) +{ + range_t extDict; + range_t prefix; + + DEBUGLOG(5, "ZSTDMT_doesOverlapWindow"); + extDict.start = window.dictBase + window.lowLimit; + extDict.size = window.dictLimit - window.lowLimit; + + prefix.start = window.base + window.dictLimit; + prefix.size = window.nextSrc - (window.base + window.dictLimit); + DEBUGLOG(5, "extDict [0x%zx, 0x%zx)", + (size_t)extDict.start, + (size_t)extDict.start + extDict.size); + DEBUGLOG(5, "prefix [0x%zx, 0x%zx)", + (size_t)prefix.start, + (size_t)prefix.start + prefix.size); + + return ZSTDMT_isOverlapped(buffer, extDict) + || ZSTDMT_isOverlapped(buffer, prefix); +} + +static void ZSTDMT_waitForLdmComplete(ZSTDMT_CCtx* mtctx, buffer_t buffer) +{ + if (mtctx->params.ldmParams.enableLdm == ZSTD_ps_enable) { + ZSTD_pthread_mutex_t* mutex = &mtctx->serial.ldmWindowMutex; + DEBUGLOG(5, "ZSTDMT_waitForLdmComplete"); + DEBUGLOG(5, "source [0x%zx, 0x%zx)", + (size_t)buffer.start, + (size_t)buffer.start + buffer.capacity); + ZSTD_PTHREAD_MUTEX_LOCK(mutex); + while (ZSTDMT_doesOverlapWindow(buffer, mtctx->serial.ldmWindow)) { + DEBUGLOG(5, "Waiting for LDM to finish..."); + ZSTD_pthread_cond_wait(&mtctx->serial.ldmWindowCond, mutex); + } + DEBUGLOG(6, "Done waiting for LDM to finish"); + ZSTD_pthread_mutex_unlock(mutex); + } +} + +/** + * Attempts to set the inBuff to the next section to fill. + * If any part of the new section is still in use we give up. + * Returns non-zero if the buffer is filled. + */ +static int ZSTDMT_tryGetInputRange(ZSTDMT_CCtx* mtctx) +{ + range_t const inUse = ZSTDMT_getInputDataInUse(mtctx); + size_t const spaceLeft = mtctx->roundBuff.capacity - mtctx->roundBuff.pos; + size_t const target = mtctx->targetSectionSize; + buffer_t buffer; + + DEBUGLOG(5, "ZSTDMT_tryGetInputRange"); + assert(mtctx->inBuff.buffer.start == NULL); + assert(mtctx->roundBuff.capacity >= target); + + if (spaceLeft < target) { + /* ZSTD_invalidateRepCodes() doesn't work for extDict variants. + * Simply copy the prefix to the beginning in that case. + */ + BYTE* const start = (BYTE*)mtctx->roundBuff.buffer; + size_t const prefixSize = mtctx->inBuff.prefix.size; + + buffer.start = start; + buffer.capacity = prefixSize; + if (ZSTDMT_isOverlapped(buffer, inUse)) { + DEBUGLOG(5, "Waiting for buffer..."); + return 0; + } + ZSTDMT_waitForLdmComplete(mtctx, buffer); + ZSTD_memmove(start, mtctx->inBuff.prefix.start, prefixSize); + mtctx->inBuff.prefix.start = start; + mtctx->roundBuff.pos = prefixSize; + } + buffer.start = mtctx->roundBuff.buffer + mtctx->roundBuff.pos; + buffer.capacity = target; + + if (ZSTDMT_isOverlapped(buffer, inUse)) { + DEBUGLOG(5, "Waiting for buffer..."); + return 0; + } + assert(!ZSTDMT_isOverlapped(buffer, mtctx->inBuff.prefix)); + + ZSTDMT_waitForLdmComplete(mtctx, buffer); + + DEBUGLOG(5, "Using prefix range [%zx, %zx)", + (size_t)mtctx->inBuff.prefix.start, + (size_t)mtctx->inBuff.prefix.start + mtctx->inBuff.prefix.size); + DEBUGLOG(5, "Using source range [%zx, %zx)", + (size_t)buffer.start, + (size_t)buffer.start + buffer.capacity); + + + mtctx->inBuff.buffer = buffer; + mtctx->inBuff.filled = 0; + assert(mtctx->roundBuff.pos + buffer.capacity <= mtctx->roundBuff.capacity); + return 1; +} + +typedef struct { + size_t toLoad; /* The number of bytes to load from the input. */ + int flush; /* Boolean declaring if we must flush because we found a synchronization point. */ +} syncPoint_t; + +/** + * Searches through the input for a synchronization point. If one is found, we + * will instruct the caller to flush, and return the number of bytes to load. + * Otherwise, we will load as many bytes as possible and instruct the caller + * to continue as normal. + */ +static syncPoint_t +findSynchronizationPoint(ZSTDMT_CCtx const* mtctx, ZSTD_inBuffer const input) +{ + BYTE const* const istart = (BYTE const*)input.src + input.pos; + U64 const primePower = mtctx->rsync.primePower; + U64 const hitMask = mtctx->rsync.hitMask; + + syncPoint_t syncPoint; + U64 hash; + BYTE const* prev; + size_t pos; + + syncPoint.toLoad = MIN(input.size - input.pos, mtctx->targetSectionSize - mtctx->inBuff.filled); + syncPoint.flush = 0; + if (!mtctx->params.rsyncable) + /* Rsync is disabled. */ + return syncPoint; + if (mtctx->inBuff.filled + input.size - input.pos < RSYNC_MIN_BLOCK_SIZE) + /* We don't emit synchronization points if it would produce too small blocks. + * We don't have enough input to find a synchronization point, so don't look. + */ + return syncPoint; + if (mtctx->inBuff.filled + syncPoint.toLoad < RSYNC_LENGTH) + /* Not enough to compute the hash. + * We will miss any synchronization points in this RSYNC_LENGTH byte + * window. However, since it depends only in the internal buffers, if the + * state is already synchronized, we will remain synchronized. + * Additionally, the probability that we miss a synchronization point is + * low: RSYNC_LENGTH / targetSectionSize. + */ + return syncPoint; + /* Initialize the loop variables. */ + if (mtctx->inBuff.filled < RSYNC_MIN_BLOCK_SIZE) { + /* We don't need to scan the first RSYNC_MIN_BLOCK_SIZE positions + * because they can't possibly be a sync point. So we can start + * part way through the input buffer. + */ + pos = RSYNC_MIN_BLOCK_SIZE - mtctx->inBuff.filled; + if (pos >= RSYNC_LENGTH) { + prev = istart + pos - RSYNC_LENGTH; + hash = ZSTD_rollingHash_compute(prev, RSYNC_LENGTH); + } else { + assert(mtctx->inBuff.filled >= RSYNC_LENGTH); + prev = (BYTE const*)mtctx->inBuff.buffer.start + mtctx->inBuff.filled - RSYNC_LENGTH; + hash = ZSTD_rollingHash_compute(prev + pos, (RSYNC_LENGTH - pos)); + hash = ZSTD_rollingHash_append(hash, istart, pos); + } + } else { + /* We have enough bytes buffered to initialize the hash, + * and are have processed enough bytes to find a sync point. + * Start scanning at the beginning of the input. + */ + assert(mtctx->inBuff.filled >= RSYNC_MIN_BLOCK_SIZE); + assert(RSYNC_MIN_BLOCK_SIZE >= RSYNC_LENGTH); + pos = 0; + prev = (BYTE const*)mtctx->inBuff.buffer.start + mtctx->inBuff.filled - RSYNC_LENGTH; + hash = ZSTD_rollingHash_compute(prev, RSYNC_LENGTH); + if ((hash & hitMask) == hitMask) { + /* We're already at a sync point so don't load any more until + * we're able to flush this sync point. + * This likely happened because the job table was full so we + * couldn't add our job. + */ + syncPoint.toLoad = 0; + syncPoint.flush = 1; + return syncPoint; + } + } + /* Starting with the hash of the previous RSYNC_LENGTH bytes, roll + * through the input. If we hit a synchronization point, then cut the + * job off, and tell the compressor to flush the job. Otherwise, load + * all the bytes and continue as normal. + * If we go too long without a synchronization point (targetSectionSize) + * then a block will be emitted anyways, but this is okay, since if we + * are already synchronized we will remain synchronized. + */ + for (; pos < syncPoint.toLoad; ++pos) { + BYTE const toRemove = pos < RSYNC_LENGTH ? prev[pos] : istart[pos - RSYNC_LENGTH]; + assert(pos < RSYNC_LENGTH || ZSTD_rollingHash_compute(istart + pos - RSYNC_LENGTH, RSYNC_LENGTH) == hash); + hash = ZSTD_rollingHash_rotate(hash, toRemove, istart[pos], primePower); + assert(mtctx->inBuff.filled + pos >= RSYNC_MIN_BLOCK_SIZE); + if ((hash & hitMask) == hitMask) { + syncPoint.toLoad = pos + 1; + syncPoint.flush = 1; + break; + } + } + return syncPoint; +} + +size_t ZSTDMT_nextInputSizeHint(const ZSTDMT_CCtx* mtctx) +{ + size_t hintInSize = mtctx->targetSectionSize - mtctx->inBuff.filled; + if (hintInSize==0) hintInSize = mtctx->targetSectionSize; + return hintInSize; +} + +/** ZSTDMT_compressStream_generic() : + * internal use only - exposed to be invoked from zstd_compress.c + * assumption : output and input are valid (pos <= size) + * @return : minimum amount of data remaining to flush, 0 if none */ +size_t ZSTDMT_compressStream_generic(ZSTDMT_CCtx* mtctx, + ZSTD_outBuffer* output, + ZSTD_inBuffer* input, + ZSTD_EndDirective endOp) +{ + unsigned forwardInputProgress = 0; + DEBUGLOG(5, "ZSTDMT_compressStream_generic (endOp=%u, srcSize=%u)", + (U32)endOp, (U32)(input->size - input->pos)); + assert(output->pos <= output->size); + assert(input->pos <= input->size); + + if ((mtctx->frameEnded) && (endOp==ZSTD_e_continue)) { + /* current frame being ended. Only flush/end are allowed */ + return ERROR(stage_wrong); + } + + /* fill input buffer */ + if ( (!mtctx->jobReady) + && (input->size > input->pos) ) { /* support NULL input */ + if (mtctx->inBuff.buffer.start == NULL) { + assert(mtctx->inBuff.filled == 0); /* Can't fill an empty buffer */ + if (!ZSTDMT_tryGetInputRange(mtctx)) { + /* It is only possible for this operation to fail if there are + * still compression jobs ongoing. + */ + DEBUGLOG(5, "ZSTDMT_tryGetInputRange failed"); + assert(mtctx->doneJobID != mtctx->nextJobID); + } else + DEBUGLOG(5, "ZSTDMT_tryGetInputRange completed successfully : mtctx->inBuff.buffer.start = %p", mtctx->inBuff.buffer.start); + } + if (mtctx->inBuff.buffer.start != NULL) { + syncPoint_t const syncPoint = findSynchronizationPoint(mtctx, *input); + if (syncPoint.flush && endOp == ZSTD_e_continue) { + endOp = ZSTD_e_flush; + } + assert(mtctx->inBuff.buffer.capacity >= mtctx->targetSectionSize); + DEBUGLOG(5, "ZSTDMT_compressStream_generic: adding %u bytes on top of %u to buffer of size %u", + (U32)syncPoint.toLoad, (U32)mtctx->inBuff.filled, (U32)mtctx->targetSectionSize); + ZSTD_memcpy((char*)mtctx->inBuff.buffer.start + mtctx->inBuff.filled, (const char*)input->src + input->pos, syncPoint.toLoad); + input->pos += syncPoint.toLoad; + mtctx->inBuff.filled += syncPoint.toLoad; + forwardInputProgress = syncPoint.toLoad>0; + } + } + if ((input->pos < input->size) && (endOp == ZSTD_e_end)) { + /* Can't end yet because the input is not fully consumed. + * We are in one of these cases: + * - mtctx->inBuff is NULL & empty: we couldn't get an input buffer so don't create a new job. + * - We filled the input buffer: flush this job but don't end the frame. + * - We hit a synchronization point: flush this job but don't end the frame. + */ + assert(mtctx->inBuff.filled == 0 || mtctx->inBuff.filled == mtctx->targetSectionSize || mtctx->params.rsyncable); + endOp = ZSTD_e_flush; + } + + if ( (mtctx->jobReady) + || (mtctx->inBuff.filled >= mtctx->targetSectionSize) /* filled enough : let's compress */ + || ((endOp != ZSTD_e_continue) && (mtctx->inBuff.filled > 0)) /* something to flush : let's go */ + || ((endOp == ZSTD_e_end) && (!mtctx->frameEnded)) ) { /* must finish the frame with a zero-size block */ + size_t const jobSize = mtctx->inBuff.filled; + assert(mtctx->inBuff.filled <= mtctx->targetSectionSize); + FORWARD_IF_ERROR( ZSTDMT_createCompressionJob(mtctx, jobSize, endOp) , ""); + } + + /* check for potential compressed data ready to be flushed */ + { size_t const remainingToFlush = ZSTDMT_flushProduced(mtctx, output, !forwardInputProgress, endOp); /* block if there was no forward input progress */ + if (input->pos < input->size) return MAX(remainingToFlush, 1); /* input not consumed : do not end flush yet */ + DEBUGLOG(5, "end of ZSTDMT_compressStream_generic: remainingToFlush = %u", (U32)remainingToFlush); + return remainingToFlush; + } +} diff --git a/src/blosc2/internal-complibs/zstd-1.5.2/compress/zstdmt_compress.h b/src/blosc2/internal-complibs/zstd-1.5.2/compress/zstdmt_compress.h new file mode 100644 index 0000000..271eb1a --- /dev/null +++ b/src/blosc2/internal-complibs/zstd-1.5.2/compress/zstdmt_compress.h @@ -0,0 +1,113 @@ +/* + * Copyright (c) Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + + #ifndef ZSTDMT_COMPRESS_H + #define ZSTDMT_COMPRESS_H + + #if defined (__cplusplus) + extern "C" { + #endif + + +/* Note : This is an internal API. + * These APIs used to be exposed with ZSTDLIB_API, + * because it used to be the only way to invoke MT compression. + * Now, you must use ZSTD_compress2 and ZSTD_compressStream2() instead. + * + * This API requires ZSTD_MULTITHREAD to be defined during compilation, + * otherwise ZSTDMT_createCCtx*() will fail. + */ + +/* === Dependencies === */ +#include "../common/zstd_deps.h" /* size_t */ +#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_parameters */ +#include "../zstd.h" /* ZSTD_inBuffer, ZSTD_outBuffer, ZSTDLIB_API */ + + +/* === Constants === */ +#ifndef ZSTDMT_NBWORKERS_MAX /* a different value can be selected at compile time */ +# define ZSTDMT_NBWORKERS_MAX ((sizeof(void*)==4) /*32-bit*/ ? 64 : 256) +#endif +#ifndef ZSTDMT_JOBSIZE_MIN /* a different value can be selected at compile time */ +# define ZSTDMT_JOBSIZE_MIN (512 KB) +#endif +#define ZSTDMT_JOBLOG_MAX (MEM_32bits() ? 29 : 30) +#define ZSTDMT_JOBSIZE_MAX (MEM_32bits() ? (512 MB) : (1024 MB)) + + +/* ======================================================== + * === Private interface, for use by ZSTD_compress.c === + * === Not exposed in libzstd. Never invoke directly === + * ======================================================== */ + +/* === Memory management === */ +typedef struct ZSTDMT_CCtx_s ZSTDMT_CCtx; +/* Requires ZSTD_MULTITHREAD to be defined during compilation, otherwise it will return NULL. */ +ZSTDMT_CCtx* ZSTDMT_createCCtx_advanced(unsigned nbWorkers, + ZSTD_customMem cMem, + ZSTD_threadPool *pool); +size_t ZSTDMT_freeCCtx(ZSTDMT_CCtx* mtctx); + +size_t ZSTDMT_sizeof_CCtx(ZSTDMT_CCtx* mtctx); + +/* === Streaming functions === */ + +size_t ZSTDMT_nextInputSizeHint(const ZSTDMT_CCtx* mtctx); + +/*! ZSTDMT_initCStream_internal() : + * Private use only. Init streaming operation. + * expects params to be valid. + * must receive dict, or cdict, or none, but not both. + * mtctx can be freshly constructed or reused from a prior compression. + * If mtctx is reused, memory allocations from the prior compression may not be freed, + * even if they are not needed for the current compression. + * @return : 0, or an error code */ +size_t ZSTDMT_initCStream_internal(ZSTDMT_CCtx* mtctx, + const void* dict, size_t dictSize, ZSTD_dictContentType_e dictContentType, + const ZSTD_CDict* cdict, + ZSTD_CCtx_params params, unsigned long long pledgedSrcSize); + +/*! ZSTDMT_compressStream_generic() : + * Combines ZSTDMT_compressStream() with optional ZSTDMT_flushStream() or ZSTDMT_endStream() + * depending on flush directive. + * @return : minimum amount of data still to be flushed + * 0 if fully flushed + * or an error code + * note : needs to be init using any ZSTD_initCStream*() variant */ +size_t ZSTDMT_compressStream_generic(ZSTDMT_CCtx* mtctx, + ZSTD_outBuffer* output, + ZSTD_inBuffer* input, + ZSTD_EndDirective endOp); + + /*! ZSTDMT_toFlushNow() + * Tell how many bytes are ready to be flushed immediately. + * Probe the oldest active job (not yet entirely flushed) and check its output buffer. + * If return 0, it means there is no active job, + * or, it means oldest job is still active, but everything produced has been flushed so far, + * therefore flushing is limited by speed of oldest job. */ +size_t ZSTDMT_toFlushNow(ZSTDMT_CCtx* mtctx); + +/*! ZSTDMT_updateCParams_whileCompressing() : + * Updates only a selected set of compression parameters, to remain compatible with current frame. + * New parameters will be applied to next compression job. */ +void ZSTDMT_updateCParams_whileCompressing(ZSTDMT_CCtx* mtctx, const ZSTD_CCtx_params* cctxParams); + +/*! ZSTDMT_getFrameProgression(): + * tells how much data has been consumed (input) and produced (output) for current frame. + * able to count progression inside worker threads. + */ +ZSTD_frameProgression ZSTDMT_getFrameProgression(ZSTDMT_CCtx* mtctx); + + +#if defined (__cplusplus) +} +#endif + +#endif /* ZSTDMT_COMPRESS_H */ diff --git a/src/blosc2/internal-complibs/zstd-1.5.2/decompress/huf_decompress.c b/src/blosc2/internal-complibs/zstd-1.5.2/decompress/huf_decompress.c new file mode 100644 index 0000000..2027188 --- /dev/null +++ b/src/blosc2/internal-complibs/zstd-1.5.2/decompress/huf_decompress.c @@ -0,0 +1,1889 @@ +/* ****************************************************************** + * huff0 huffman decoder, + * part of Finite State Entropy library + * Copyright (c) Yann Collet, Facebook, Inc. + * + * You can contact the author at : + * - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. +****************************************************************** */ + +/* ************************************************************** +* Dependencies +****************************************************************/ +#include "../common/zstd_deps.h" /* ZSTD_memcpy, ZSTD_memset */ +#include "../common/compiler.h" +#include "../common/bitstream.h" /* BIT_* */ +#include "../common/fse.h" /* to compress headers */ +#define HUF_STATIC_LINKING_ONLY +#include "../common/huf.h" +#include "../common/error_private.h" +#include "../common/zstd_internal.h" + +/* ************************************************************** +* Constants +****************************************************************/ + +#define HUF_DECODER_FAST_TABLELOG 11 + +/* ************************************************************** +* Macros +****************************************************************/ + +/* These two optional macros force the use one way or another of the two + * Huffman decompression implementations. You can't force in both directions + * at the same time. + */ +#if defined(HUF_FORCE_DECOMPRESS_X1) && \ + defined(HUF_FORCE_DECOMPRESS_X2) +#error "Cannot force the use of the X1 and X2 decoders at the same time!" +#endif + +#if ZSTD_ENABLE_ASM_X86_64_BMI2 && DYNAMIC_BMI2 +# define HUF_ASM_X86_64_BMI2_ATTRS BMI2_TARGET_ATTRIBUTE +#else +# define HUF_ASM_X86_64_BMI2_ATTRS +#endif + +#ifdef __cplusplus +# define HUF_EXTERN_C extern "C" +#else +# define HUF_EXTERN_C +#endif +#define HUF_ASM_DECL HUF_EXTERN_C + +#if DYNAMIC_BMI2 || (ZSTD_ENABLE_ASM_X86_64_BMI2 && defined(__BMI2__)) +# define HUF_NEED_BMI2_FUNCTION 1 +#else +# define HUF_NEED_BMI2_FUNCTION 0 +#endif + +#if !(ZSTD_ENABLE_ASM_X86_64_BMI2 && defined(__BMI2__)) +# define HUF_NEED_DEFAULT_FUNCTION 1 +#else +# define HUF_NEED_DEFAULT_FUNCTION 0 +#endif + +/* ************************************************************** +* Error Management +****************************************************************/ +#define HUF_isError ERR_isError + + +/* ************************************************************** +* Byte alignment for workSpace management +****************************************************************/ +#define HUF_ALIGN(x, a) HUF_ALIGN_MASK((x), (a) - 1) +#define HUF_ALIGN_MASK(x, mask) (((x) + (mask)) & ~(mask)) + + +/* ************************************************************** +* BMI2 Variant Wrappers +****************************************************************/ +#if DYNAMIC_BMI2 + +#define HUF_DGEN(fn) \ + \ + static size_t fn##_default( \ + void* dst, size_t dstSize, \ + const void* cSrc, size_t cSrcSize, \ + const HUF_DTable* DTable) \ + { \ + return fn##_body(dst, dstSize, cSrc, cSrcSize, DTable); \ + } \ + \ + static BMI2_TARGET_ATTRIBUTE size_t fn##_bmi2( \ + void* dst, size_t dstSize, \ + const void* cSrc, size_t cSrcSize, \ + const HUF_DTable* DTable) \ + { \ + return fn##_body(dst, dstSize, cSrc, cSrcSize, DTable); \ + } \ + \ + static size_t fn(void* dst, size_t dstSize, void const* cSrc, \ + size_t cSrcSize, HUF_DTable const* DTable, int bmi2) \ + { \ + if (bmi2) { \ + return fn##_bmi2(dst, dstSize, cSrc, cSrcSize, DTable); \ + } \ + return fn##_default(dst, dstSize, cSrc, cSrcSize, DTable); \ + } + +#else + +#define HUF_DGEN(fn) \ + static size_t fn(void* dst, size_t dstSize, void const* cSrc, \ + size_t cSrcSize, HUF_DTable const* DTable, int bmi2) \ + { \ + (void)bmi2; \ + return fn##_body(dst, dstSize, cSrc, cSrcSize, DTable); \ + } + +#endif + + +/*-***************************/ +/* generic DTableDesc */ +/*-***************************/ +typedef struct { BYTE maxTableLog; BYTE tableType; BYTE tableLog; BYTE reserved; } DTableDesc; + +static DTableDesc HUF_getDTableDesc(const HUF_DTable* table) +{ + DTableDesc dtd; + ZSTD_memcpy(&dtd, table, sizeof(dtd)); + return dtd; +} + +#if ZSTD_ENABLE_ASM_X86_64_BMI2 + +static size_t HUF_initDStream(BYTE const* ip) { + BYTE const lastByte = ip[7]; + size_t const bitsConsumed = lastByte ? 8 - BIT_highbit32(lastByte) : 0; + size_t const value = MEM_readLEST(ip) | 1; + assert(bitsConsumed <= 8); + return value << bitsConsumed; +} +typedef struct { + BYTE const* ip[4]; + BYTE* op[4]; + U64 bits[4]; + void const* dt; + BYTE const* ilimit; + BYTE* oend; + BYTE const* iend[4]; +} HUF_DecompressAsmArgs; + +/** + * Initializes args for the asm decoding loop. + * @returns 0 on success + * 1 if the fallback implementation should be used. + * Or an error code on failure. + */ +static size_t HUF_DecompressAsmArgs_init(HUF_DecompressAsmArgs* args, void* dst, size_t dstSize, void const* src, size_t srcSize, const HUF_DTable* DTable) +{ + void const* dt = DTable + 1; + U32 const dtLog = HUF_getDTableDesc(DTable).tableLog; + + const BYTE* const ilimit = (const BYTE*)src + 6 + 8; + + BYTE* const oend = (BYTE*)dst + dstSize; + + /* The following condition is false on x32 platform, + * but HUF_asm is not compatible with this ABI */ + if (!(MEM_isLittleEndian() && !MEM_32bits())) return 1; + + /* strict minimum : jump table + 1 byte per stream */ + if (srcSize < 10) + return ERROR(corruption_detected); + + /* Must have at least 8 bytes per stream because we don't handle initializing smaller bit containers. + * If table log is not correct at this point, fallback to the old decoder. + * On small inputs we don't have enough data to trigger the fast loop, so use the old decoder. + */ + if (dtLog != HUF_DECODER_FAST_TABLELOG) + return 1; + + /* Read the jump table. */ + { + const BYTE* const istart = (const BYTE*)src; + size_t const length1 = MEM_readLE16(istart); + size_t const length2 = MEM_readLE16(istart+2); + size_t const length3 = MEM_readLE16(istart+4); + size_t const length4 = srcSize - (length1 + length2 + length3 + 6); + args->iend[0] = istart + 6; /* jumpTable */ + args->iend[1] = args->iend[0] + length1; + args->iend[2] = args->iend[1] + length2; + args->iend[3] = args->iend[2] + length3; + + /* HUF_initDStream() requires this, and this small of an input + * won't benefit from the ASM loop anyways. + * length1 must be >= 16 so that ip[0] >= ilimit before the loop + * starts. + */ + if (length1 < 16 || length2 < 8 || length3 < 8 || length4 < 8) + return 1; + if (length4 > srcSize) return ERROR(corruption_detected); /* overflow */ + } + /* ip[] contains the position that is currently loaded into bits[]. */ + args->ip[0] = args->iend[1] - sizeof(U64); + args->ip[1] = args->iend[2] - sizeof(U64); + args->ip[2] = args->iend[3] - sizeof(U64); + args->ip[3] = (BYTE const*)src + srcSize - sizeof(U64); + + /* op[] contains the output pointers. */ + args->op[0] = (BYTE*)dst; + args->op[1] = args->op[0] + (dstSize+3)/4; + args->op[2] = args->op[1] + (dstSize+3)/4; + args->op[3] = args->op[2] + (dstSize+3)/4; + + /* No point to call the ASM loop for tiny outputs. */ + if (args->op[3] >= oend) + return 1; + + /* bits[] is the bit container. + * It is read from the MSB down to the LSB. + * It is shifted left as it is read, and zeros are + * shifted in. After the lowest valid bit a 1 is + * set, so that CountTrailingZeros(bits[]) can be used + * to count how many bits we've consumed. + */ + args->bits[0] = HUF_initDStream(args->ip[0]); + args->bits[1] = HUF_initDStream(args->ip[1]); + args->bits[2] = HUF_initDStream(args->ip[2]); + args->bits[3] = HUF_initDStream(args->ip[3]); + + /* If ip[] >= ilimit, it is guaranteed to be safe to + * reload bits[]. It may be beyond its section, but is + * guaranteed to be valid (>= istart). + */ + args->ilimit = ilimit; + + args->oend = oend; + args->dt = dt; + + return 0; +} + +static size_t HUF_initRemainingDStream(BIT_DStream_t* bit, HUF_DecompressAsmArgs const* args, int stream, BYTE* segmentEnd) +{ + /* Validate that we haven't overwritten. */ + if (args->op[stream] > segmentEnd) + return ERROR(corruption_detected); + /* Validate that we haven't read beyond iend[]. + * Note that ip[] may be < iend[] because the MSB is + * the next bit to read, and we may have consumed 100% + * of the stream, so down to iend[i] - 8 is valid. + */ + if (args->ip[stream] < args->iend[stream] - 8) + return ERROR(corruption_detected); + + /* Construct the BIT_DStream_t. */ + bit->bitContainer = MEM_readLE64(args->ip[stream]); + bit->bitsConsumed = ZSTD_countTrailingZeros((size_t)args->bits[stream]); + bit->start = (const char*)args->iend[0]; + bit->limitPtr = bit->start + sizeof(size_t); + bit->ptr = (const char*)args->ip[stream]; + + return 0; +} +#endif + + +#ifndef HUF_FORCE_DECOMPRESS_X2 + +/*-***************************/ +/* single-symbol decoding */ +/*-***************************/ +typedef struct { BYTE nbBits; BYTE byte; } HUF_DEltX1; /* single-symbol decoding */ + +/** + * Packs 4 HUF_DEltX1 structs into a U64. This is used to lay down 4 entries at + * a time. + */ +static U64 HUF_DEltX1_set4(BYTE symbol, BYTE nbBits) { + U64 D4; + if (MEM_isLittleEndian()) { + D4 = (symbol << 8) + nbBits; + } else { + D4 = symbol + (nbBits << 8); + } + D4 *= 0x0001000100010001ULL; + return D4; +} + +/** + * Increase the tableLog to targetTableLog and rescales the stats. + * If tableLog > targetTableLog this is a no-op. + * @returns New tableLog + */ +static U32 HUF_rescaleStats(BYTE* huffWeight, U32* rankVal, U32 nbSymbols, U32 tableLog, U32 targetTableLog) +{ + if (tableLog > targetTableLog) + return tableLog; + if (tableLog < targetTableLog) { + U32 const scale = targetTableLog - tableLog; + U32 s; + /* Increase the weight for all non-zero probability symbols by scale. */ + for (s = 0; s < nbSymbols; ++s) { + huffWeight[s] += (BYTE)((huffWeight[s] == 0) ? 0 : scale); + } + /* Update rankVal to reflect the new weights. + * All weights except 0 get moved to weight + scale. + * Weights [1, scale] are empty. + */ + for (s = targetTableLog; s > scale; --s) { + rankVal[s] = rankVal[s - scale]; + } + for (s = scale; s > 0; --s) { + rankVal[s] = 0; + } + } + return targetTableLog; +} + +typedef struct { + U32 rankVal[HUF_TABLELOG_ABSOLUTEMAX + 1]; + U32 rankStart[HUF_TABLELOG_ABSOLUTEMAX + 1]; + U32 statsWksp[HUF_READ_STATS_WORKSPACE_SIZE_U32]; + BYTE symbols[HUF_SYMBOLVALUE_MAX + 1]; + BYTE huffWeight[HUF_SYMBOLVALUE_MAX + 1]; +} HUF_ReadDTableX1_Workspace; + + +size_t HUF_readDTableX1_wksp(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize) +{ + return HUF_readDTableX1_wksp_bmi2(DTable, src, srcSize, workSpace, wkspSize, /* bmi2 */ 0); +} + +size_t HUF_readDTableX1_wksp_bmi2(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize, int bmi2) +{ + U32 tableLog = 0; + U32 nbSymbols = 0; + size_t iSize; + void* const dtPtr = DTable + 1; + HUF_DEltX1* const dt = (HUF_DEltX1*)dtPtr; + HUF_ReadDTableX1_Workspace* wksp = (HUF_ReadDTableX1_Workspace*)workSpace; + + DEBUG_STATIC_ASSERT(HUF_DECOMPRESS_WORKSPACE_SIZE >= sizeof(*wksp)); + if (sizeof(*wksp) > wkspSize) return ERROR(tableLog_tooLarge); + + DEBUG_STATIC_ASSERT(sizeof(DTableDesc) == sizeof(HUF_DTable)); + /* ZSTD_memset(huffWeight, 0, sizeof(huffWeight)); */ /* is not necessary, even though some analyzer complain ... */ + + iSize = HUF_readStats_wksp(wksp->huffWeight, HUF_SYMBOLVALUE_MAX + 1, wksp->rankVal, &nbSymbols, &tableLog, src, srcSize, wksp->statsWksp, sizeof(wksp->statsWksp), bmi2); + if (HUF_isError(iSize)) return iSize; + + + /* Table header */ + { DTableDesc dtd = HUF_getDTableDesc(DTable); + U32 const maxTableLog = dtd.maxTableLog + 1; + U32 const targetTableLog = MIN(maxTableLog, HUF_DECODER_FAST_TABLELOG); + tableLog = HUF_rescaleStats(wksp->huffWeight, wksp->rankVal, nbSymbols, tableLog, targetTableLog); + if (tableLog > (U32)(dtd.maxTableLog+1)) return ERROR(tableLog_tooLarge); /* DTable too small, Huffman tree cannot fit in */ + dtd.tableType = 0; + dtd.tableLog = (BYTE)tableLog; + ZSTD_memcpy(DTable, &dtd, sizeof(dtd)); + } + + /* Compute symbols and rankStart given rankVal: + * + * rankVal already contains the number of values of each weight. + * + * symbols contains the symbols ordered by weight. First are the rankVal[0] + * weight 0 symbols, followed by the rankVal[1] weight 1 symbols, and so on. + * symbols[0] is filled (but unused) to avoid a branch. + * + * rankStart contains the offset where each rank belongs in the DTable. + * rankStart[0] is not filled because there are no entries in the table for + * weight 0. + */ + { + int n; + int nextRankStart = 0; + int const unroll = 4; + int const nLimit = (int)nbSymbols - unroll + 1; + for (n=0; n<(int)tableLog+1; n++) { + U32 const curr = nextRankStart; + nextRankStart += wksp->rankVal[n]; + wksp->rankStart[n] = curr; + } + for (n=0; n < nLimit; n += unroll) { + int u; + for (u=0; u < unroll; ++u) { + size_t const w = wksp->huffWeight[n+u]; + wksp->symbols[wksp->rankStart[w]++] = (BYTE)(n+u); + } + } + for (; n < (int)nbSymbols; ++n) { + size_t const w = wksp->huffWeight[n]; + wksp->symbols[wksp->rankStart[w]++] = (BYTE)n; + } + } + + /* fill DTable + * We fill all entries of each weight in order. + * That way length is a constant for each iteration of the outer loop. + * We can switch based on the length to a different inner loop which is + * optimized for that particular case. + */ + { + U32 w; + int symbol=wksp->rankVal[0]; + int rankStart=0; + for (w=1; wrankVal[w]; + int const length = (1 << w) >> 1; + int uStart = rankStart; + BYTE const nbBits = (BYTE)(tableLog + 1 - w); + int s; + int u; + switch (length) { + case 1: + for (s=0; ssymbols[symbol + s]; + D.nbBits = nbBits; + dt[uStart] = D; + uStart += 1; + } + break; + case 2: + for (s=0; ssymbols[symbol + s]; + D.nbBits = nbBits; + dt[uStart+0] = D; + dt[uStart+1] = D; + uStart += 2; + } + break; + case 4: + for (s=0; ssymbols[symbol + s], nbBits); + MEM_write64(dt + uStart, D4); + uStart += 4; + } + break; + case 8: + for (s=0; ssymbols[symbol + s], nbBits); + MEM_write64(dt + uStart, D4); + MEM_write64(dt + uStart + 4, D4); + uStart += 8; + } + break; + default: + for (s=0; ssymbols[symbol + s], nbBits); + for (u=0; u < length; u += 16) { + MEM_write64(dt + uStart + u + 0, D4); + MEM_write64(dt + uStart + u + 4, D4); + MEM_write64(dt + uStart + u + 8, D4); + MEM_write64(dt + uStart + u + 12, D4); + } + assert(u == length); + uStart += length; + } + break; + } + symbol += symbolCount; + rankStart += symbolCount * length; + } + } + return iSize; +} + +FORCE_INLINE_TEMPLATE BYTE +HUF_decodeSymbolX1(BIT_DStream_t* Dstream, const HUF_DEltX1* dt, const U32 dtLog) +{ + size_t const val = BIT_lookBitsFast(Dstream, dtLog); /* note : dtLog >= 1 */ + BYTE const c = dt[val].byte; + BIT_skipBits(Dstream, dt[val].nbBits); + return c; +} + +#define HUF_DECODE_SYMBOLX1_0(ptr, DStreamPtr) \ + *ptr++ = HUF_decodeSymbolX1(DStreamPtr, dt, dtLog) + +#define HUF_DECODE_SYMBOLX1_1(ptr, DStreamPtr) \ + if (MEM_64bits() || (HUF_TABLELOG_MAX<=12)) \ + HUF_DECODE_SYMBOLX1_0(ptr, DStreamPtr) + +#define HUF_DECODE_SYMBOLX1_2(ptr, DStreamPtr) \ + if (MEM_64bits()) \ + HUF_DECODE_SYMBOLX1_0(ptr, DStreamPtr) + +HINT_INLINE size_t +HUF_decodeStreamX1(BYTE* p, BIT_DStream_t* const bitDPtr, BYTE* const pEnd, const HUF_DEltX1* const dt, const U32 dtLog) +{ + BYTE* const pStart = p; + + /* up to 4 symbols at a time */ + if ((pEnd - p) > 3) { + while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p < pEnd-3)) { + HUF_DECODE_SYMBOLX1_2(p, bitDPtr); + HUF_DECODE_SYMBOLX1_1(p, bitDPtr); + HUF_DECODE_SYMBOLX1_2(p, bitDPtr); + HUF_DECODE_SYMBOLX1_0(p, bitDPtr); + } + } else { + BIT_reloadDStream(bitDPtr); + } + + /* [0-3] symbols remaining */ + if (MEM_32bits()) + while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p < pEnd)) + HUF_DECODE_SYMBOLX1_0(p, bitDPtr); + + /* no more data to retrieve from bitstream, no need to reload */ + while (p < pEnd) + HUF_DECODE_SYMBOLX1_0(p, bitDPtr); + + return pEnd-pStart; +} + +FORCE_INLINE_TEMPLATE size_t +HUF_decompress1X1_usingDTable_internal_body( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable) +{ + BYTE* op = (BYTE*)dst; + BYTE* const oend = op + dstSize; + const void* dtPtr = DTable + 1; + const HUF_DEltX1* const dt = (const HUF_DEltX1*)dtPtr; + BIT_DStream_t bitD; + DTableDesc const dtd = HUF_getDTableDesc(DTable); + U32 const dtLog = dtd.tableLog; + + CHECK_F( BIT_initDStream(&bitD, cSrc, cSrcSize) ); + + HUF_decodeStreamX1(op, &bitD, oend, dt, dtLog); + + if (!BIT_endOfDStream(&bitD)) return ERROR(corruption_detected); + + return dstSize; +} + +FORCE_INLINE_TEMPLATE size_t +HUF_decompress4X1_usingDTable_internal_body( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable) +{ + /* Check */ + if (cSrcSize < 10) return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */ + + { const BYTE* const istart = (const BYTE*) cSrc; + BYTE* const ostart = (BYTE*) dst; + BYTE* const oend = ostart + dstSize; + BYTE* const olimit = oend - 3; + const void* const dtPtr = DTable + 1; + const HUF_DEltX1* const dt = (const HUF_DEltX1*)dtPtr; + + /* Init */ + BIT_DStream_t bitD1; + BIT_DStream_t bitD2; + BIT_DStream_t bitD3; + BIT_DStream_t bitD4; + size_t const length1 = MEM_readLE16(istart); + size_t const length2 = MEM_readLE16(istart+2); + size_t const length3 = MEM_readLE16(istart+4); + size_t const length4 = cSrcSize - (length1 + length2 + length3 + 6); + const BYTE* const istart1 = istart + 6; /* jumpTable */ + const BYTE* const istart2 = istart1 + length1; + const BYTE* const istart3 = istart2 + length2; + const BYTE* const istart4 = istart3 + length3; + const size_t segmentSize = (dstSize+3) / 4; + BYTE* const opStart2 = ostart + segmentSize; + BYTE* const opStart3 = opStart2 + segmentSize; + BYTE* const opStart4 = opStart3 + segmentSize; + BYTE* op1 = ostart; + BYTE* op2 = opStart2; + BYTE* op3 = opStart3; + BYTE* op4 = opStart4; + DTableDesc const dtd = HUF_getDTableDesc(DTable); + U32 const dtLog = dtd.tableLog; + U32 endSignal = 1; + + if (length4 > cSrcSize) return ERROR(corruption_detected); /* overflow */ + if (opStart4 > oend) return ERROR(corruption_detected); /* overflow */ + CHECK_F( BIT_initDStream(&bitD1, istart1, length1) ); + CHECK_F( BIT_initDStream(&bitD2, istart2, length2) ); + CHECK_F( BIT_initDStream(&bitD3, istart3, length3) ); + CHECK_F( BIT_initDStream(&bitD4, istart4, length4) ); + + /* up to 16 symbols per loop (4 symbols per stream) in 64-bit mode */ + if ((size_t)(oend - op4) >= sizeof(size_t)) { + for ( ; (endSignal) & (op4 < olimit) ; ) { + HUF_DECODE_SYMBOLX1_2(op1, &bitD1); + HUF_DECODE_SYMBOLX1_2(op2, &bitD2); + HUF_DECODE_SYMBOLX1_2(op3, &bitD3); + HUF_DECODE_SYMBOLX1_2(op4, &bitD4); + HUF_DECODE_SYMBOLX1_1(op1, &bitD1); + HUF_DECODE_SYMBOLX1_1(op2, &bitD2); + HUF_DECODE_SYMBOLX1_1(op3, &bitD3); + HUF_DECODE_SYMBOLX1_1(op4, &bitD4); + HUF_DECODE_SYMBOLX1_2(op1, &bitD1); + HUF_DECODE_SYMBOLX1_2(op2, &bitD2); + HUF_DECODE_SYMBOLX1_2(op3, &bitD3); + HUF_DECODE_SYMBOLX1_2(op4, &bitD4); + HUF_DECODE_SYMBOLX1_0(op1, &bitD1); + HUF_DECODE_SYMBOLX1_0(op2, &bitD2); + HUF_DECODE_SYMBOLX1_0(op3, &bitD3); + HUF_DECODE_SYMBOLX1_0(op4, &bitD4); + endSignal &= BIT_reloadDStreamFast(&bitD1) == BIT_DStream_unfinished; + endSignal &= BIT_reloadDStreamFast(&bitD2) == BIT_DStream_unfinished; + endSignal &= BIT_reloadDStreamFast(&bitD3) == BIT_DStream_unfinished; + endSignal &= BIT_reloadDStreamFast(&bitD4) == BIT_DStream_unfinished; + } + } + + /* check corruption */ + /* note : should not be necessary : op# advance in lock step, and we control op4. + * but curiously, binary generated by gcc 7.2 & 7.3 with -mbmi2 runs faster when >=1 test is present */ + if (op1 > opStart2) return ERROR(corruption_detected); + if (op2 > opStart3) return ERROR(corruption_detected); + if (op3 > opStart4) return ERROR(corruption_detected); + /* note : op4 supposed already verified within main loop */ + + /* finish bitStreams one by one */ + HUF_decodeStreamX1(op1, &bitD1, opStart2, dt, dtLog); + HUF_decodeStreamX1(op2, &bitD2, opStart3, dt, dtLog); + HUF_decodeStreamX1(op3, &bitD3, opStart4, dt, dtLog); + HUF_decodeStreamX1(op4, &bitD4, oend, dt, dtLog); + + /* check */ + { U32 const endCheck = BIT_endOfDStream(&bitD1) & BIT_endOfDStream(&bitD2) & BIT_endOfDStream(&bitD3) & BIT_endOfDStream(&bitD4); + if (!endCheck) return ERROR(corruption_detected); } + + /* decoded size */ + return dstSize; + } +} + +#if HUF_NEED_BMI2_FUNCTION +static BMI2_TARGET_ATTRIBUTE +size_t HUF_decompress4X1_usingDTable_internal_bmi2(void* dst, size_t dstSize, void const* cSrc, + size_t cSrcSize, HUF_DTable const* DTable) { + return HUF_decompress4X1_usingDTable_internal_body(dst, dstSize, cSrc, cSrcSize, DTable); +} +#endif + +#if HUF_NEED_DEFAULT_FUNCTION +static +size_t HUF_decompress4X1_usingDTable_internal_default(void* dst, size_t dstSize, void const* cSrc, + size_t cSrcSize, HUF_DTable const* DTable) { + return HUF_decompress4X1_usingDTable_internal_body(dst, dstSize, cSrc, cSrcSize, DTable); +} +#endif + +#if ZSTD_ENABLE_ASM_X86_64_BMI2 + +HUF_ASM_DECL void HUF_decompress4X1_usingDTable_internal_bmi2_asm_loop(HUF_DecompressAsmArgs* args) ZSTDLIB_HIDDEN; + +static HUF_ASM_X86_64_BMI2_ATTRS +size_t +HUF_decompress4X1_usingDTable_internal_bmi2_asm( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable) +{ + void const* dt = DTable + 1; + const BYTE* const iend = (const BYTE*)cSrc + 6; + BYTE* const oend = (BYTE*)dst + dstSize; + HUF_DecompressAsmArgs args; + { + size_t const ret = HUF_DecompressAsmArgs_init(&args, dst, dstSize, cSrc, cSrcSize, DTable); + FORWARD_IF_ERROR(ret, "Failed to init asm args"); + if (ret != 0) + return HUF_decompress4X1_usingDTable_internal_bmi2(dst, dstSize, cSrc, cSrcSize, DTable); + } + + assert(args.ip[0] >= args.ilimit); + HUF_decompress4X1_usingDTable_internal_bmi2_asm_loop(&args); + + /* Our loop guarantees that ip[] >= ilimit and that we haven't + * overwritten any op[]. + */ + assert(args.ip[0] >= iend); + assert(args.ip[1] >= iend); + assert(args.ip[2] >= iend); + assert(args.ip[3] >= iend); + assert(args.op[3] <= oend); + (void)iend; + + /* finish bit streams one by one. */ + { + size_t const segmentSize = (dstSize+3) / 4; + BYTE* segmentEnd = (BYTE*)dst; + int i; + for (i = 0; i < 4; ++i) { + BIT_DStream_t bit; + if (segmentSize <= (size_t)(oend - segmentEnd)) + segmentEnd += segmentSize; + else + segmentEnd = oend; + FORWARD_IF_ERROR(HUF_initRemainingDStream(&bit, &args, i, segmentEnd), "corruption"); + /* Decompress and validate that we've produced exactly the expected length. */ + args.op[i] += HUF_decodeStreamX1(args.op[i], &bit, segmentEnd, (HUF_DEltX1 const*)dt, HUF_DECODER_FAST_TABLELOG); + if (args.op[i] != segmentEnd) return ERROR(corruption_detected); + } + } + + /* decoded size */ + return dstSize; +} +#endif /* ZSTD_ENABLE_ASM_X86_64_BMI2 */ + +typedef size_t (*HUF_decompress_usingDTable_t)(void *dst, size_t dstSize, + const void *cSrc, + size_t cSrcSize, + const HUF_DTable *DTable); + +HUF_DGEN(HUF_decompress1X1_usingDTable_internal) + +static size_t HUF_decompress4X1_usingDTable_internal(void* dst, size_t dstSize, void const* cSrc, + size_t cSrcSize, HUF_DTable const* DTable, int bmi2) +{ +#if DYNAMIC_BMI2 + if (bmi2) { +# if ZSTD_ENABLE_ASM_X86_64_BMI2 + return HUF_decompress4X1_usingDTable_internal_bmi2_asm(dst, dstSize, cSrc, cSrcSize, DTable); +# else + return HUF_decompress4X1_usingDTable_internal_bmi2(dst, dstSize, cSrc, cSrcSize, DTable); +# endif + } +#else + (void)bmi2; +#endif + +#if ZSTD_ENABLE_ASM_X86_64_BMI2 && defined(__BMI2__) + return HUF_decompress4X1_usingDTable_internal_bmi2_asm(dst, dstSize, cSrc, cSrcSize, DTable); +#else + return HUF_decompress4X1_usingDTable_internal_default(dst, dstSize, cSrc, cSrcSize, DTable); +#endif +} + + +size_t HUF_decompress1X1_usingDTable( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable) +{ + DTableDesc dtd = HUF_getDTableDesc(DTable); + if (dtd.tableType != 0) return ERROR(GENERIC); + return HUF_decompress1X1_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0); +} + +size_t HUF_decompress1X1_DCtx_wksp(HUF_DTable* DCtx, void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + void* workSpace, size_t wkspSize) +{ + const BYTE* ip = (const BYTE*) cSrc; + + size_t const hSize = HUF_readDTableX1_wksp(DCtx, cSrc, cSrcSize, workSpace, wkspSize); + if (HUF_isError(hSize)) return hSize; + if (hSize >= cSrcSize) return ERROR(srcSize_wrong); + ip += hSize; cSrcSize -= hSize; + + return HUF_decompress1X1_usingDTable_internal(dst, dstSize, ip, cSrcSize, DCtx, /* bmi2 */ 0); +} + + +size_t HUF_decompress4X1_usingDTable( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable) +{ + DTableDesc dtd = HUF_getDTableDesc(DTable); + if (dtd.tableType != 0) return ERROR(GENERIC); + return HUF_decompress4X1_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0); +} + +static size_t HUF_decompress4X1_DCtx_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + void* workSpace, size_t wkspSize, int bmi2) +{ + const BYTE* ip = (const BYTE*) cSrc; + + size_t const hSize = HUF_readDTableX1_wksp_bmi2(dctx, cSrc, cSrcSize, workSpace, wkspSize, bmi2); + if (HUF_isError(hSize)) return hSize; + if (hSize >= cSrcSize) return ERROR(srcSize_wrong); + ip += hSize; cSrcSize -= hSize; + + return HUF_decompress4X1_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx, bmi2); +} + +size_t HUF_decompress4X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + void* workSpace, size_t wkspSize) +{ + return HUF_decompress4X1_DCtx_wksp_bmi2(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, 0); +} + + +#endif /* HUF_FORCE_DECOMPRESS_X2 */ + + +#ifndef HUF_FORCE_DECOMPRESS_X1 + +/* *************************/ +/* double-symbols decoding */ +/* *************************/ + +typedef struct { U16 sequence; BYTE nbBits; BYTE length; } HUF_DEltX2; /* double-symbols decoding */ +typedef struct { BYTE symbol; } sortedSymbol_t; +typedef U32 rankValCol_t[HUF_TABLELOG_MAX + 1]; +typedef rankValCol_t rankVal_t[HUF_TABLELOG_MAX]; + +/** + * Constructs a HUF_DEltX2 in a U32. + */ +static U32 HUF_buildDEltX2U32(U32 symbol, U32 nbBits, U32 baseSeq, int level) +{ + U32 seq; + DEBUG_STATIC_ASSERT(offsetof(HUF_DEltX2, sequence) == 0); + DEBUG_STATIC_ASSERT(offsetof(HUF_DEltX2, nbBits) == 2); + DEBUG_STATIC_ASSERT(offsetof(HUF_DEltX2, length) == 3); + DEBUG_STATIC_ASSERT(sizeof(HUF_DEltX2) == sizeof(U32)); + if (MEM_isLittleEndian()) { + seq = level == 1 ? symbol : (baseSeq + (symbol << 8)); + return seq + (nbBits << 16) + ((U32)level << 24); + } else { + seq = level == 1 ? (symbol << 8) : ((baseSeq << 8) + symbol); + return (seq << 16) + (nbBits << 8) + (U32)level; + } +} + +/** + * Constructs a HUF_DEltX2. + */ +static HUF_DEltX2 HUF_buildDEltX2(U32 symbol, U32 nbBits, U32 baseSeq, int level) +{ + HUF_DEltX2 DElt; + U32 const val = HUF_buildDEltX2U32(symbol, nbBits, baseSeq, level); + DEBUG_STATIC_ASSERT(sizeof(DElt) == sizeof(val)); + ZSTD_memcpy(&DElt, &val, sizeof(val)); + return DElt; +} + +/** + * Constructs 2 HUF_DEltX2s and packs them into a U64. + */ +static U64 HUF_buildDEltX2U64(U32 symbol, U32 nbBits, U16 baseSeq, int level) +{ + U32 DElt = HUF_buildDEltX2U32(symbol, nbBits, baseSeq, level); + return (U64)DElt + ((U64)DElt << 32); +} + +/** + * Fills the DTable rank with all the symbols from [begin, end) that are each + * nbBits long. + * + * @param DTableRank The start of the rank in the DTable. + * @param begin The first symbol to fill (inclusive). + * @param end The last symbol to fill (exclusive). + * @param nbBits Each symbol is nbBits long. + * @param tableLog The table log. + * @param baseSeq If level == 1 { 0 } else { the first level symbol } + * @param level The level in the table. Must be 1 or 2. + */ +static void HUF_fillDTableX2ForWeight( + HUF_DEltX2* DTableRank, + sortedSymbol_t const* begin, sortedSymbol_t const* end, + U32 nbBits, U32 tableLog, + U16 baseSeq, int const level) +{ + U32 const length = 1U << ((tableLog - nbBits) & 0x1F /* quiet static-analyzer */); + const sortedSymbol_t* ptr; + assert(level >= 1 && level <= 2); + switch (length) { + case 1: + for (ptr = begin; ptr != end; ++ptr) { + HUF_DEltX2 const DElt = HUF_buildDEltX2(ptr->symbol, nbBits, baseSeq, level); + *DTableRank++ = DElt; + } + break; + case 2: + for (ptr = begin; ptr != end; ++ptr) { + HUF_DEltX2 const DElt = HUF_buildDEltX2(ptr->symbol, nbBits, baseSeq, level); + DTableRank[0] = DElt; + DTableRank[1] = DElt; + DTableRank += 2; + } + break; + case 4: + for (ptr = begin; ptr != end; ++ptr) { + U64 const DEltX2 = HUF_buildDEltX2U64(ptr->symbol, nbBits, baseSeq, level); + ZSTD_memcpy(DTableRank + 0, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTableRank + 2, &DEltX2, sizeof(DEltX2)); + DTableRank += 4; + } + break; + case 8: + for (ptr = begin; ptr != end; ++ptr) { + U64 const DEltX2 = HUF_buildDEltX2U64(ptr->symbol, nbBits, baseSeq, level); + ZSTD_memcpy(DTableRank + 0, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTableRank + 2, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTableRank + 4, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTableRank + 6, &DEltX2, sizeof(DEltX2)); + DTableRank += 8; + } + break; + default: + for (ptr = begin; ptr != end; ++ptr) { + U64 const DEltX2 = HUF_buildDEltX2U64(ptr->symbol, nbBits, baseSeq, level); + HUF_DEltX2* const DTableRankEnd = DTableRank + length; + for (; DTableRank != DTableRankEnd; DTableRank += 8) { + ZSTD_memcpy(DTableRank + 0, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTableRank + 2, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTableRank + 4, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTableRank + 6, &DEltX2, sizeof(DEltX2)); + } + } + break; + } +} + +/* HUF_fillDTableX2Level2() : + * `rankValOrigin` must be a table of at least (HUF_TABLELOG_MAX + 1) U32 */ +static void HUF_fillDTableX2Level2(HUF_DEltX2* DTable, U32 targetLog, const U32 consumedBits, + const U32* rankVal, const int minWeight, const int maxWeight1, + const sortedSymbol_t* sortedSymbols, U32 const* rankStart, + U32 nbBitsBaseline, U16 baseSeq) +{ + /* Fill skipped values (all positions up to rankVal[minWeight]). + * These are positions only get a single symbol because the combined weight + * is too large. + */ + if (minWeight>1) { + U32 const length = 1U << ((targetLog - consumedBits) & 0x1F /* quiet static-analyzer */); + U64 const DEltX2 = HUF_buildDEltX2U64(baseSeq, consumedBits, /* baseSeq */ 0, /* level */ 1); + int const skipSize = rankVal[minWeight]; + assert(length > 1); + assert((U32)skipSize < length); + switch (length) { + case 2: + assert(skipSize == 1); + ZSTD_memcpy(DTable, &DEltX2, sizeof(DEltX2)); + break; + case 4: + assert(skipSize <= 4); + ZSTD_memcpy(DTable + 0, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTable + 2, &DEltX2, sizeof(DEltX2)); + break; + default: + { + int i; + for (i = 0; i < skipSize; i += 8) { + ZSTD_memcpy(DTable + i + 0, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTable + i + 2, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTable + i + 4, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTable + i + 6, &DEltX2, sizeof(DEltX2)); + } + } + } + } + + /* Fill each of the second level symbols by weight. */ + { + int w; + for (w = minWeight; w < maxWeight1; ++w) { + int const begin = rankStart[w]; + int const end = rankStart[w+1]; + U32 const nbBits = nbBitsBaseline - w; + U32 const totalBits = nbBits + consumedBits; + HUF_fillDTableX2ForWeight( + DTable + rankVal[w], + sortedSymbols + begin, sortedSymbols + end, + totalBits, targetLog, + baseSeq, /* level */ 2); + } + } +} + +static void HUF_fillDTableX2(HUF_DEltX2* DTable, const U32 targetLog, + const sortedSymbol_t* sortedList, + const U32* rankStart, rankVal_t rankValOrigin, const U32 maxWeight, + const U32 nbBitsBaseline) +{ + U32* const rankVal = rankValOrigin[0]; + const int scaleLog = nbBitsBaseline - targetLog; /* note : targetLog >= srcLog, hence scaleLog <= 1 */ + const U32 minBits = nbBitsBaseline - maxWeight; + int w; + int const wEnd = (int)maxWeight + 1; + + /* Fill DTable in order of weight. */ + for (w = 1; w < wEnd; ++w) { + int const begin = (int)rankStart[w]; + int const end = (int)rankStart[w+1]; + U32 const nbBits = nbBitsBaseline - w; + + if (targetLog-nbBits >= minBits) { + /* Enough room for a second symbol. */ + int start = rankVal[w]; + U32 const length = 1U << ((targetLog - nbBits) & 0x1F /* quiet static-analyzer */); + int minWeight = nbBits + scaleLog; + int s; + if (minWeight < 1) minWeight = 1; + /* Fill the DTable for every symbol of weight w. + * These symbols get at least 1 second symbol. + */ + for (s = begin; s != end; ++s) { + HUF_fillDTableX2Level2( + DTable + start, targetLog, nbBits, + rankValOrigin[nbBits], minWeight, wEnd, + sortedList, rankStart, + nbBitsBaseline, sortedList[s].symbol); + start += length; + } + } else { + /* Only a single symbol. */ + HUF_fillDTableX2ForWeight( + DTable + rankVal[w], + sortedList + begin, sortedList + end, + nbBits, targetLog, + /* baseSeq */ 0, /* level */ 1); + } + } +} + +typedef struct { + rankValCol_t rankVal[HUF_TABLELOG_MAX]; + U32 rankStats[HUF_TABLELOG_MAX + 1]; + U32 rankStart0[HUF_TABLELOG_MAX + 3]; + sortedSymbol_t sortedSymbol[HUF_SYMBOLVALUE_MAX + 1]; + BYTE weightList[HUF_SYMBOLVALUE_MAX + 1]; + U32 calleeWksp[HUF_READ_STATS_WORKSPACE_SIZE_U32]; +} HUF_ReadDTableX2_Workspace; + +size_t HUF_readDTableX2_wksp(HUF_DTable* DTable, + const void* src, size_t srcSize, + void* workSpace, size_t wkspSize) +{ + return HUF_readDTableX2_wksp_bmi2(DTable, src, srcSize, workSpace, wkspSize, /* bmi2 */ 0); +} + +size_t HUF_readDTableX2_wksp_bmi2(HUF_DTable* DTable, + const void* src, size_t srcSize, + void* workSpace, size_t wkspSize, int bmi2) +{ + U32 tableLog, maxW, nbSymbols; + DTableDesc dtd = HUF_getDTableDesc(DTable); + U32 maxTableLog = dtd.maxTableLog; + size_t iSize; + void* dtPtr = DTable+1; /* force compiler to avoid strict-aliasing */ + HUF_DEltX2* const dt = (HUF_DEltX2*)dtPtr; + U32 *rankStart; + + HUF_ReadDTableX2_Workspace* const wksp = (HUF_ReadDTableX2_Workspace*)workSpace; + + if (sizeof(*wksp) > wkspSize) return ERROR(GENERIC); + + rankStart = wksp->rankStart0 + 1; + ZSTD_memset(wksp->rankStats, 0, sizeof(wksp->rankStats)); + ZSTD_memset(wksp->rankStart0, 0, sizeof(wksp->rankStart0)); + + DEBUG_STATIC_ASSERT(sizeof(HUF_DEltX2) == sizeof(HUF_DTable)); /* if compiler fails here, assertion is wrong */ + if (maxTableLog > HUF_TABLELOG_MAX) return ERROR(tableLog_tooLarge); + /* ZSTD_memset(weightList, 0, sizeof(weightList)); */ /* is not necessary, even though some analyzer complain ... */ + + iSize = HUF_readStats_wksp(wksp->weightList, HUF_SYMBOLVALUE_MAX + 1, wksp->rankStats, &nbSymbols, &tableLog, src, srcSize, wksp->calleeWksp, sizeof(wksp->calleeWksp), bmi2); + if (HUF_isError(iSize)) return iSize; + + /* check result */ + if (tableLog > maxTableLog) return ERROR(tableLog_tooLarge); /* DTable can't fit code depth */ + if (tableLog <= HUF_DECODER_FAST_TABLELOG && maxTableLog > HUF_DECODER_FAST_TABLELOG) maxTableLog = HUF_DECODER_FAST_TABLELOG; + + /* find maxWeight */ + for (maxW = tableLog; wksp->rankStats[maxW]==0; maxW--) {} /* necessarily finds a solution before 0 */ + + /* Get start index of each weight */ + { U32 w, nextRankStart = 0; + for (w=1; wrankStats[w]; + rankStart[w] = curr; + } + rankStart[0] = nextRankStart; /* put all 0w symbols at the end of sorted list*/ + rankStart[maxW+1] = nextRankStart; + } + + /* sort symbols by weight */ + { U32 s; + for (s=0; sweightList[s]; + U32 const r = rankStart[w]++; + wksp->sortedSymbol[r].symbol = (BYTE)s; + } + rankStart[0] = 0; /* forget 0w symbols; this is beginning of weight(1) */ + } + + /* Build rankVal */ + { U32* const rankVal0 = wksp->rankVal[0]; + { int const rescale = (maxTableLog-tableLog) - 1; /* tableLog <= maxTableLog */ + U32 nextRankVal = 0; + U32 w; + for (w=1; wrankStats[w] << (w+rescale); + rankVal0[w] = curr; + } } + { U32 const minBits = tableLog+1 - maxW; + U32 consumed; + for (consumed = minBits; consumed < maxTableLog - minBits + 1; consumed++) { + U32* const rankValPtr = wksp->rankVal[consumed]; + U32 w; + for (w = 1; w < maxW+1; w++) { + rankValPtr[w] = rankVal0[w] >> consumed; + } } } } + + HUF_fillDTableX2(dt, maxTableLog, + wksp->sortedSymbol, + wksp->rankStart0, wksp->rankVal, maxW, + tableLog+1); + + dtd.tableLog = (BYTE)maxTableLog; + dtd.tableType = 1; + ZSTD_memcpy(DTable, &dtd, sizeof(dtd)); + return iSize; +} + + +FORCE_INLINE_TEMPLATE U32 +HUF_decodeSymbolX2(void* op, BIT_DStream_t* DStream, const HUF_DEltX2* dt, const U32 dtLog) +{ + size_t const val = BIT_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */ + ZSTD_memcpy(op, &dt[val].sequence, 2); + BIT_skipBits(DStream, dt[val].nbBits); + return dt[val].length; +} + +FORCE_INLINE_TEMPLATE U32 +HUF_decodeLastSymbolX2(void* op, BIT_DStream_t* DStream, const HUF_DEltX2* dt, const U32 dtLog) +{ + size_t const val = BIT_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */ + ZSTD_memcpy(op, &dt[val].sequence, 1); + if (dt[val].length==1) { + BIT_skipBits(DStream, dt[val].nbBits); + } else { + if (DStream->bitsConsumed < (sizeof(DStream->bitContainer)*8)) { + BIT_skipBits(DStream, dt[val].nbBits); + if (DStream->bitsConsumed > (sizeof(DStream->bitContainer)*8)) + /* ugly hack; works only because it's the last symbol. Note : can't easily extract nbBits from just this symbol */ + DStream->bitsConsumed = (sizeof(DStream->bitContainer)*8); + } + } + return 1; +} + +#define HUF_DECODE_SYMBOLX2_0(ptr, DStreamPtr) \ + ptr += HUF_decodeSymbolX2(ptr, DStreamPtr, dt, dtLog) + +#define HUF_DECODE_SYMBOLX2_1(ptr, DStreamPtr) \ + if (MEM_64bits() || (HUF_TABLELOG_MAX<=12)) \ + ptr += HUF_decodeSymbolX2(ptr, DStreamPtr, dt, dtLog) + +#define HUF_DECODE_SYMBOLX2_2(ptr, DStreamPtr) \ + if (MEM_64bits()) \ + ptr += HUF_decodeSymbolX2(ptr, DStreamPtr, dt, dtLog) + +HINT_INLINE size_t +HUF_decodeStreamX2(BYTE* p, BIT_DStream_t* bitDPtr, BYTE* const pEnd, + const HUF_DEltX2* const dt, const U32 dtLog) +{ + BYTE* const pStart = p; + + /* up to 8 symbols at a time */ + if ((size_t)(pEnd - p) >= sizeof(bitDPtr->bitContainer)) { + if (dtLog <= 11 && MEM_64bits()) { + /* up to 10 symbols at a time */ + while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p < pEnd-9)) { + HUF_DECODE_SYMBOLX2_0(p, bitDPtr); + HUF_DECODE_SYMBOLX2_0(p, bitDPtr); + HUF_DECODE_SYMBOLX2_0(p, bitDPtr); + HUF_DECODE_SYMBOLX2_0(p, bitDPtr); + HUF_DECODE_SYMBOLX2_0(p, bitDPtr); + } + } else { + /* up to 8 symbols at a time */ + while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p < pEnd-(sizeof(bitDPtr->bitContainer)-1))) { + HUF_DECODE_SYMBOLX2_2(p, bitDPtr); + HUF_DECODE_SYMBOLX2_1(p, bitDPtr); + HUF_DECODE_SYMBOLX2_2(p, bitDPtr); + HUF_DECODE_SYMBOLX2_0(p, bitDPtr); + } + } + } else { + BIT_reloadDStream(bitDPtr); + } + + /* closer to end : up to 2 symbols at a time */ + if ((size_t)(pEnd - p) >= 2) { + while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p <= pEnd-2)) + HUF_DECODE_SYMBOLX2_0(p, bitDPtr); + + while (p <= pEnd-2) + HUF_DECODE_SYMBOLX2_0(p, bitDPtr); /* no need to reload : reached the end of DStream */ + } + + if (p < pEnd) + p += HUF_decodeLastSymbolX2(p, bitDPtr, dt, dtLog); + + return p-pStart; +} + +FORCE_INLINE_TEMPLATE size_t +HUF_decompress1X2_usingDTable_internal_body( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable) +{ + BIT_DStream_t bitD; + + /* Init */ + CHECK_F( BIT_initDStream(&bitD, cSrc, cSrcSize) ); + + /* decode */ + { BYTE* const ostart = (BYTE*) dst; + BYTE* const oend = ostart + dstSize; + const void* const dtPtr = DTable+1; /* force compiler to not use strict-aliasing */ + const HUF_DEltX2* const dt = (const HUF_DEltX2*)dtPtr; + DTableDesc const dtd = HUF_getDTableDesc(DTable); + HUF_decodeStreamX2(ostart, &bitD, oend, dt, dtd.tableLog); + } + + /* check */ + if (!BIT_endOfDStream(&bitD)) return ERROR(corruption_detected); + + /* decoded size */ + return dstSize; +} +FORCE_INLINE_TEMPLATE size_t +HUF_decompress4X2_usingDTable_internal_body( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable) +{ + if (cSrcSize < 10) return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */ + + { const BYTE* const istart = (const BYTE*) cSrc; + BYTE* const ostart = (BYTE*) dst; + BYTE* const oend = ostart + dstSize; + BYTE* const olimit = oend - (sizeof(size_t)-1); + const void* const dtPtr = DTable+1; + const HUF_DEltX2* const dt = (const HUF_DEltX2*)dtPtr; + + /* Init */ + BIT_DStream_t bitD1; + BIT_DStream_t bitD2; + BIT_DStream_t bitD3; + BIT_DStream_t bitD4; + size_t const length1 = MEM_readLE16(istart); + size_t const length2 = MEM_readLE16(istart+2); + size_t const length3 = MEM_readLE16(istart+4); + size_t const length4 = cSrcSize - (length1 + length2 + length3 + 6); + const BYTE* const istart1 = istart + 6; /* jumpTable */ + const BYTE* const istart2 = istart1 + length1; + const BYTE* const istart3 = istart2 + length2; + const BYTE* const istart4 = istart3 + length3; + size_t const segmentSize = (dstSize+3) / 4; + BYTE* const opStart2 = ostart + segmentSize; + BYTE* const opStart3 = opStart2 + segmentSize; + BYTE* const opStart4 = opStart3 + segmentSize; + BYTE* op1 = ostart; + BYTE* op2 = opStart2; + BYTE* op3 = opStart3; + BYTE* op4 = opStart4; + U32 endSignal = 1; + DTableDesc const dtd = HUF_getDTableDesc(DTable); + U32 const dtLog = dtd.tableLog; + + if (length4 > cSrcSize) return ERROR(corruption_detected); /* overflow */ + if (opStart4 > oend) return ERROR(corruption_detected); /* overflow */ + CHECK_F( BIT_initDStream(&bitD1, istart1, length1) ); + CHECK_F( BIT_initDStream(&bitD2, istart2, length2) ); + CHECK_F( BIT_initDStream(&bitD3, istart3, length3) ); + CHECK_F( BIT_initDStream(&bitD4, istart4, length4) ); + + /* 16-32 symbols per loop (4-8 symbols per stream) */ + if ((size_t)(oend - op4) >= sizeof(size_t)) { + for ( ; (endSignal) & (op4 < olimit); ) { +#if defined(__clang__) && (defined(__x86_64__) || defined(__i386__)) + HUF_DECODE_SYMBOLX2_2(op1, &bitD1); + HUF_DECODE_SYMBOLX2_1(op1, &bitD1); + HUF_DECODE_SYMBOLX2_2(op1, &bitD1); + HUF_DECODE_SYMBOLX2_0(op1, &bitD1); + HUF_DECODE_SYMBOLX2_2(op2, &bitD2); + HUF_DECODE_SYMBOLX2_1(op2, &bitD2); + HUF_DECODE_SYMBOLX2_2(op2, &bitD2); + HUF_DECODE_SYMBOLX2_0(op2, &bitD2); + endSignal &= BIT_reloadDStreamFast(&bitD1) == BIT_DStream_unfinished; + endSignal &= BIT_reloadDStreamFast(&bitD2) == BIT_DStream_unfinished; + HUF_DECODE_SYMBOLX2_2(op3, &bitD3); + HUF_DECODE_SYMBOLX2_1(op3, &bitD3); + HUF_DECODE_SYMBOLX2_2(op3, &bitD3); + HUF_DECODE_SYMBOLX2_0(op3, &bitD3); + HUF_DECODE_SYMBOLX2_2(op4, &bitD4); + HUF_DECODE_SYMBOLX2_1(op4, &bitD4); + HUF_DECODE_SYMBOLX2_2(op4, &bitD4); + HUF_DECODE_SYMBOLX2_0(op4, &bitD4); + endSignal &= BIT_reloadDStreamFast(&bitD3) == BIT_DStream_unfinished; + endSignal &= BIT_reloadDStreamFast(&bitD4) == BIT_DStream_unfinished; +#else + HUF_DECODE_SYMBOLX2_2(op1, &bitD1); + HUF_DECODE_SYMBOLX2_2(op2, &bitD2); + HUF_DECODE_SYMBOLX2_2(op3, &bitD3); + HUF_DECODE_SYMBOLX2_2(op4, &bitD4); + HUF_DECODE_SYMBOLX2_1(op1, &bitD1); + HUF_DECODE_SYMBOLX2_1(op2, &bitD2); + HUF_DECODE_SYMBOLX2_1(op3, &bitD3); + HUF_DECODE_SYMBOLX2_1(op4, &bitD4); + HUF_DECODE_SYMBOLX2_2(op1, &bitD1); + HUF_DECODE_SYMBOLX2_2(op2, &bitD2); + HUF_DECODE_SYMBOLX2_2(op3, &bitD3); + HUF_DECODE_SYMBOLX2_2(op4, &bitD4); + HUF_DECODE_SYMBOLX2_0(op1, &bitD1); + HUF_DECODE_SYMBOLX2_0(op2, &bitD2); + HUF_DECODE_SYMBOLX2_0(op3, &bitD3); + HUF_DECODE_SYMBOLX2_0(op4, &bitD4); + endSignal = (U32)LIKELY((U32) + (BIT_reloadDStreamFast(&bitD1) == BIT_DStream_unfinished) + & (BIT_reloadDStreamFast(&bitD2) == BIT_DStream_unfinished) + & (BIT_reloadDStreamFast(&bitD3) == BIT_DStream_unfinished) + & (BIT_reloadDStreamFast(&bitD4) == BIT_DStream_unfinished)); +#endif + } + } + + /* check corruption */ + if (op1 > opStart2) return ERROR(corruption_detected); + if (op2 > opStart3) return ERROR(corruption_detected); + if (op3 > opStart4) return ERROR(corruption_detected); + /* note : op4 already verified within main loop */ + + /* finish bitStreams one by one */ + HUF_decodeStreamX2(op1, &bitD1, opStart2, dt, dtLog); + HUF_decodeStreamX2(op2, &bitD2, opStart3, dt, dtLog); + HUF_decodeStreamX2(op3, &bitD3, opStart4, dt, dtLog); + HUF_decodeStreamX2(op4, &bitD4, oend, dt, dtLog); + + /* check */ + { U32 const endCheck = BIT_endOfDStream(&bitD1) & BIT_endOfDStream(&bitD2) & BIT_endOfDStream(&bitD3) & BIT_endOfDStream(&bitD4); + if (!endCheck) return ERROR(corruption_detected); } + + /* decoded size */ + return dstSize; + } +} + +#if HUF_NEED_BMI2_FUNCTION +static BMI2_TARGET_ATTRIBUTE +size_t HUF_decompress4X2_usingDTable_internal_bmi2(void* dst, size_t dstSize, void const* cSrc, + size_t cSrcSize, HUF_DTable const* DTable) { + return HUF_decompress4X2_usingDTable_internal_body(dst, dstSize, cSrc, cSrcSize, DTable); +} +#endif + +#if HUF_NEED_DEFAULT_FUNCTION +static +size_t HUF_decompress4X2_usingDTable_internal_default(void* dst, size_t dstSize, void const* cSrc, + size_t cSrcSize, HUF_DTable const* DTable) { + return HUF_decompress4X2_usingDTable_internal_body(dst, dstSize, cSrc, cSrcSize, DTable); +} +#endif + +#if ZSTD_ENABLE_ASM_X86_64_BMI2 + +HUF_ASM_DECL void HUF_decompress4X2_usingDTable_internal_bmi2_asm_loop(HUF_DecompressAsmArgs* args) ZSTDLIB_HIDDEN; + +static HUF_ASM_X86_64_BMI2_ATTRS size_t +HUF_decompress4X2_usingDTable_internal_bmi2_asm( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable) { + void const* dt = DTable + 1; + const BYTE* const iend = (const BYTE*)cSrc + 6; + BYTE* const oend = (BYTE*)dst + dstSize; + HUF_DecompressAsmArgs args; + { + size_t const ret = HUF_DecompressAsmArgs_init(&args, dst, dstSize, cSrc, cSrcSize, DTable); + FORWARD_IF_ERROR(ret, "Failed to init asm args"); + if (ret != 0) + return HUF_decompress4X2_usingDTable_internal_bmi2(dst, dstSize, cSrc, cSrcSize, DTable); + } + + assert(args.ip[0] >= args.ilimit); + HUF_decompress4X2_usingDTable_internal_bmi2_asm_loop(&args); + + /* note : op4 already verified within main loop */ + assert(args.ip[0] >= iend); + assert(args.ip[1] >= iend); + assert(args.ip[2] >= iend); + assert(args.ip[3] >= iend); + assert(args.op[3] <= oend); + (void)iend; + + /* finish bitStreams one by one */ + { + size_t const segmentSize = (dstSize+3) / 4; + BYTE* segmentEnd = (BYTE*)dst; + int i; + for (i = 0; i < 4; ++i) { + BIT_DStream_t bit; + if (segmentSize <= (size_t)(oend - segmentEnd)) + segmentEnd += segmentSize; + else + segmentEnd = oend; + FORWARD_IF_ERROR(HUF_initRemainingDStream(&bit, &args, i, segmentEnd), "corruption"); + args.op[i] += HUF_decodeStreamX2(args.op[i], &bit, segmentEnd, (HUF_DEltX2 const*)dt, HUF_DECODER_FAST_TABLELOG); + if (args.op[i] != segmentEnd) + return ERROR(corruption_detected); + } + } + + /* decoded size */ + return dstSize; +} +#endif /* ZSTD_ENABLE_ASM_X86_64_BMI2 */ + +static size_t HUF_decompress4X2_usingDTable_internal(void* dst, size_t dstSize, void const* cSrc, + size_t cSrcSize, HUF_DTable const* DTable, int bmi2) +{ +#if DYNAMIC_BMI2 + if (bmi2) { +# if ZSTD_ENABLE_ASM_X86_64_BMI2 + return HUF_decompress4X2_usingDTable_internal_bmi2_asm(dst, dstSize, cSrc, cSrcSize, DTable); +# else + return HUF_decompress4X2_usingDTable_internal_bmi2(dst, dstSize, cSrc, cSrcSize, DTable); +# endif + } +#else + (void)bmi2; +#endif + +#if ZSTD_ENABLE_ASM_X86_64_BMI2 && defined(__BMI2__) + return HUF_decompress4X2_usingDTable_internal_bmi2_asm(dst, dstSize, cSrc, cSrcSize, DTable); +#else + return HUF_decompress4X2_usingDTable_internal_default(dst, dstSize, cSrc, cSrcSize, DTable); +#endif +} + +HUF_DGEN(HUF_decompress1X2_usingDTable_internal) + +size_t HUF_decompress1X2_usingDTable( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable) +{ + DTableDesc dtd = HUF_getDTableDesc(DTable); + if (dtd.tableType != 1) return ERROR(GENERIC); + return HUF_decompress1X2_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0); +} + +size_t HUF_decompress1X2_DCtx_wksp(HUF_DTable* DCtx, void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + void* workSpace, size_t wkspSize) +{ + const BYTE* ip = (const BYTE*) cSrc; + + size_t const hSize = HUF_readDTableX2_wksp(DCtx, cSrc, cSrcSize, + workSpace, wkspSize); + if (HUF_isError(hSize)) return hSize; + if (hSize >= cSrcSize) return ERROR(srcSize_wrong); + ip += hSize; cSrcSize -= hSize; + + return HUF_decompress1X2_usingDTable_internal(dst, dstSize, ip, cSrcSize, DCtx, /* bmi2 */ 0); +} + + +size_t HUF_decompress4X2_usingDTable( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable) +{ + DTableDesc dtd = HUF_getDTableDesc(DTable); + if (dtd.tableType != 1) return ERROR(GENERIC); + return HUF_decompress4X2_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0); +} + +static size_t HUF_decompress4X2_DCtx_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + void* workSpace, size_t wkspSize, int bmi2) +{ + const BYTE* ip = (const BYTE*) cSrc; + + size_t hSize = HUF_readDTableX2_wksp(dctx, cSrc, cSrcSize, + workSpace, wkspSize); + if (HUF_isError(hSize)) return hSize; + if (hSize >= cSrcSize) return ERROR(srcSize_wrong); + ip += hSize; cSrcSize -= hSize; + + return HUF_decompress4X2_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx, bmi2); +} + +size_t HUF_decompress4X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + void* workSpace, size_t wkspSize) +{ + return HUF_decompress4X2_DCtx_wksp_bmi2(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, /* bmi2 */ 0); +} + + +#endif /* HUF_FORCE_DECOMPRESS_X1 */ + + +/* ***********************************/ +/* Universal decompression selectors */ +/* ***********************************/ + +size_t HUF_decompress1X_usingDTable(void* dst, size_t maxDstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable) +{ + DTableDesc const dtd = HUF_getDTableDesc(DTable); +#if defined(HUF_FORCE_DECOMPRESS_X1) + (void)dtd; + assert(dtd.tableType == 0); + return HUF_decompress1X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0); +#elif defined(HUF_FORCE_DECOMPRESS_X2) + (void)dtd; + assert(dtd.tableType == 1); + return HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0); +#else + return dtd.tableType ? HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0) : + HUF_decompress1X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0); +#endif +} + +size_t HUF_decompress4X_usingDTable(void* dst, size_t maxDstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable) +{ + DTableDesc const dtd = HUF_getDTableDesc(DTable); +#if defined(HUF_FORCE_DECOMPRESS_X1) + (void)dtd; + assert(dtd.tableType == 0); + return HUF_decompress4X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0); +#elif defined(HUF_FORCE_DECOMPRESS_X2) + (void)dtd; + assert(dtd.tableType == 1); + return HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0); +#else + return dtd.tableType ? HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0) : + HUF_decompress4X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0); +#endif +} + + +#if !defined(HUF_FORCE_DECOMPRESS_X1) && !defined(HUF_FORCE_DECOMPRESS_X2) +typedef struct { U32 tableTime; U32 decode256Time; } algo_time_t; +static const algo_time_t algoTime[16 /* Quantization */][2 /* single, double */] = +{ + /* single, double, quad */ + {{0,0}, {1,1}}, /* Q==0 : impossible */ + {{0,0}, {1,1}}, /* Q==1 : impossible */ + {{ 150,216}, { 381,119}}, /* Q == 2 : 12-18% */ + {{ 170,205}, { 514,112}}, /* Q == 3 : 18-25% */ + {{ 177,199}, { 539,110}}, /* Q == 4 : 25-32% */ + {{ 197,194}, { 644,107}}, /* Q == 5 : 32-38% */ + {{ 221,192}, { 735,107}}, /* Q == 6 : 38-44% */ + {{ 256,189}, { 881,106}}, /* Q == 7 : 44-50% */ + {{ 359,188}, {1167,109}}, /* Q == 8 : 50-56% */ + {{ 582,187}, {1570,114}}, /* Q == 9 : 56-62% */ + {{ 688,187}, {1712,122}}, /* Q ==10 : 62-69% */ + {{ 825,186}, {1965,136}}, /* Q ==11 : 69-75% */ + {{ 976,185}, {2131,150}}, /* Q ==12 : 75-81% */ + {{1180,186}, {2070,175}}, /* Q ==13 : 81-87% */ + {{1377,185}, {1731,202}}, /* Q ==14 : 87-93% */ + {{1412,185}, {1695,202}}, /* Q ==15 : 93-99% */ +}; +#endif + +/** HUF_selectDecoder() : + * Tells which decoder is likely to decode faster, + * based on a set of pre-computed metrics. + * @return : 0==HUF_decompress4X1, 1==HUF_decompress4X2 . + * Assumption : 0 < dstSize <= 128 KB */ +U32 HUF_selectDecoder (size_t dstSize, size_t cSrcSize) +{ + assert(dstSize > 0); + assert(dstSize <= 128*1024); +#if defined(HUF_FORCE_DECOMPRESS_X1) + (void)dstSize; + (void)cSrcSize; + return 0; +#elif defined(HUF_FORCE_DECOMPRESS_X2) + (void)dstSize; + (void)cSrcSize; + return 1; +#else + /* decoder timing evaluation */ + { U32 const Q = (cSrcSize >= dstSize) ? 15 : (U32)(cSrcSize * 16 / dstSize); /* Q < 16 */ + U32 const D256 = (U32)(dstSize >> 8); + U32 const DTime0 = algoTime[Q][0].tableTime + (algoTime[Q][0].decode256Time * D256); + U32 DTime1 = algoTime[Q][1].tableTime + (algoTime[Q][1].decode256Time * D256); + DTime1 += DTime1 >> 5; /* small advantage to algorithm using less memory, to reduce cache eviction */ + return DTime1 < DTime0; + } +#endif +} + + +size_t HUF_decompress4X_hufOnly_wksp(HUF_DTable* dctx, void* dst, + size_t dstSize, const void* cSrc, + size_t cSrcSize, void* workSpace, + size_t wkspSize) +{ + /* validation checks */ + if (dstSize == 0) return ERROR(dstSize_tooSmall); + if (cSrcSize == 0) return ERROR(corruption_detected); + + { U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize); +#if defined(HUF_FORCE_DECOMPRESS_X1) + (void)algoNb; + assert(algoNb == 0); + return HUF_decompress4X1_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize); +#elif defined(HUF_FORCE_DECOMPRESS_X2) + (void)algoNb; + assert(algoNb == 1); + return HUF_decompress4X2_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize); +#else + return algoNb ? HUF_decompress4X2_DCtx_wksp(dctx, dst, dstSize, cSrc, + cSrcSize, workSpace, wkspSize): + HUF_decompress4X1_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize); +#endif + } +} + +size_t HUF_decompress1X_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + void* workSpace, size_t wkspSize) +{ + /* validation checks */ + if (dstSize == 0) return ERROR(dstSize_tooSmall); + if (cSrcSize > dstSize) return ERROR(corruption_detected); /* invalid */ + if (cSrcSize == dstSize) { ZSTD_memcpy(dst, cSrc, dstSize); return dstSize; } /* not compressed */ + if (cSrcSize == 1) { ZSTD_memset(dst, *(const BYTE*)cSrc, dstSize); return dstSize; } /* RLE */ + + { U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize); +#if defined(HUF_FORCE_DECOMPRESS_X1) + (void)algoNb; + assert(algoNb == 0); + return HUF_decompress1X1_DCtx_wksp(dctx, dst, dstSize, cSrc, + cSrcSize, workSpace, wkspSize); +#elif defined(HUF_FORCE_DECOMPRESS_X2) + (void)algoNb; + assert(algoNb == 1); + return HUF_decompress1X2_DCtx_wksp(dctx, dst, dstSize, cSrc, + cSrcSize, workSpace, wkspSize); +#else + return algoNb ? HUF_decompress1X2_DCtx_wksp(dctx, dst, dstSize, cSrc, + cSrcSize, workSpace, wkspSize): + HUF_decompress1X1_DCtx_wksp(dctx, dst, dstSize, cSrc, + cSrcSize, workSpace, wkspSize); +#endif + } +} + + +size_t HUF_decompress1X_usingDTable_bmi2(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int bmi2) +{ + DTableDesc const dtd = HUF_getDTableDesc(DTable); +#if defined(HUF_FORCE_DECOMPRESS_X1) + (void)dtd; + assert(dtd.tableType == 0); + return HUF_decompress1X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2); +#elif defined(HUF_FORCE_DECOMPRESS_X2) + (void)dtd; + assert(dtd.tableType == 1); + return HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2); +#else + return dtd.tableType ? HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2) : + HUF_decompress1X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2); +#endif +} + +#ifndef HUF_FORCE_DECOMPRESS_X2 +size_t HUF_decompress1X1_DCtx_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int bmi2) +{ + const BYTE* ip = (const BYTE*) cSrc; + + size_t const hSize = HUF_readDTableX1_wksp_bmi2(dctx, cSrc, cSrcSize, workSpace, wkspSize, bmi2); + if (HUF_isError(hSize)) return hSize; + if (hSize >= cSrcSize) return ERROR(srcSize_wrong); + ip += hSize; cSrcSize -= hSize; + + return HUF_decompress1X1_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx, bmi2); +} +#endif + +size_t HUF_decompress4X_usingDTable_bmi2(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int bmi2) +{ + DTableDesc const dtd = HUF_getDTableDesc(DTable); +#if defined(HUF_FORCE_DECOMPRESS_X1) + (void)dtd; + assert(dtd.tableType == 0); + return HUF_decompress4X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2); +#elif defined(HUF_FORCE_DECOMPRESS_X2) + (void)dtd; + assert(dtd.tableType == 1); + return HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2); +#else + return dtd.tableType ? HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2) : + HUF_decompress4X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2); +#endif +} + +size_t HUF_decompress4X_hufOnly_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int bmi2) +{ + /* validation checks */ + if (dstSize == 0) return ERROR(dstSize_tooSmall); + if (cSrcSize == 0) return ERROR(corruption_detected); + + { U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize); +#if defined(HUF_FORCE_DECOMPRESS_X1) + (void)algoNb; + assert(algoNb == 0); + return HUF_decompress4X1_DCtx_wksp_bmi2(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, bmi2); +#elif defined(HUF_FORCE_DECOMPRESS_X2) + (void)algoNb; + assert(algoNb == 1); + return HUF_decompress4X2_DCtx_wksp_bmi2(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, bmi2); +#else + return algoNb ? HUF_decompress4X2_DCtx_wksp_bmi2(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, bmi2) : + HUF_decompress4X1_DCtx_wksp_bmi2(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, bmi2); +#endif + } +} + +#ifndef ZSTD_NO_UNUSED_FUNCTIONS +#ifndef HUF_FORCE_DECOMPRESS_X2 +size_t HUF_readDTableX1(HUF_DTable* DTable, const void* src, size_t srcSize) +{ + U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32]; + return HUF_readDTableX1_wksp(DTable, src, srcSize, + workSpace, sizeof(workSpace)); +} + +size_t HUF_decompress1X1_DCtx(HUF_DTable* DCtx, void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize) +{ + U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32]; + return HUF_decompress1X1_DCtx_wksp(DCtx, dst, dstSize, cSrc, cSrcSize, + workSpace, sizeof(workSpace)); +} + +size_t HUF_decompress1X1 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) +{ + HUF_CREATE_STATIC_DTABLEX1(DTable, HUF_TABLELOG_MAX); + return HUF_decompress1X1_DCtx (DTable, dst, dstSize, cSrc, cSrcSize); +} +#endif + +#ifndef HUF_FORCE_DECOMPRESS_X1 +size_t HUF_readDTableX2(HUF_DTable* DTable, const void* src, size_t srcSize) +{ + U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32]; + return HUF_readDTableX2_wksp(DTable, src, srcSize, + workSpace, sizeof(workSpace)); +} + +size_t HUF_decompress1X2_DCtx(HUF_DTable* DCtx, void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize) +{ + U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32]; + return HUF_decompress1X2_DCtx_wksp(DCtx, dst, dstSize, cSrc, cSrcSize, + workSpace, sizeof(workSpace)); +} + +size_t HUF_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) +{ + HUF_CREATE_STATIC_DTABLEX2(DTable, HUF_TABLELOG_MAX); + return HUF_decompress1X2_DCtx(DTable, dst, dstSize, cSrc, cSrcSize); +} +#endif + +#ifndef HUF_FORCE_DECOMPRESS_X2 +size_t HUF_decompress4X1_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) +{ + U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32]; + return HUF_decompress4X1_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, + workSpace, sizeof(workSpace)); +} +size_t HUF_decompress4X1 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) +{ + HUF_CREATE_STATIC_DTABLEX1(DTable, HUF_TABLELOG_MAX); + return HUF_decompress4X1_DCtx(DTable, dst, dstSize, cSrc, cSrcSize); +} +#endif + +#ifndef HUF_FORCE_DECOMPRESS_X1 +size_t HUF_decompress4X2_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize) +{ + U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32]; + return HUF_decompress4X2_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, + workSpace, sizeof(workSpace)); +} + +size_t HUF_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) +{ + HUF_CREATE_STATIC_DTABLEX2(DTable, HUF_TABLELOG_MAX); + return HUF_decompress4X2_DCtx(DTable, dst, dstSize, cSrc, cSrcSize); +} +#endif + +typedef size_t (*decompressionAlgo)(void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); + +size_t HUF_decompress (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) +{ +#if !defined(HUF_FORCE_DECOMPRESS_X1) && !defined(HUF_FORCE_DECOMPRESS_X2) + static const decompressionAlgo decompress[2] = { HUF_decompress4X1, HUF_decompress4X2 }; +#endif + + /* validation checks */ + if (dstSize == 0) return ERROR(dstSize_tooSmall); + if (cSrcSize > dstSize) return ERROR(corruption_detected); /* invalid */ + if (cSrcSize == dstSize) { ZSTD_memcpy(dst, cSrc, dstSize); return dstSize; } /* not compressed */ + if (cSrcSize == 1) { ZSTD_memset(dst, *(const BYTE*)cSrc, dstSize); return dstSize; } /* RLE */ + + { U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize); +#if defined(HUF_FORCE_DECOMPRESS_X1) + (void)algoNb; + assert(algoNb == 0); + return HUF_decompress4X1(dst, dstSize, cSrc, cSrcSize); +#elif defined(HUF_FORCE_DECOMPRESS_X2) + (void)algoNb; + assert(algoNb == 1); + return HUF_decompress4X2(dst, dstSize, cSrc, cSrcSize); +#else + return decompress[algoNb](dst, dstSize, cSrc, cSrcSize); +#endif + } +} + +size_t HUF_decompress4X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) +{ + /* validation checks */ + if (dstSize == 0) return ERROR(dstSize_tooSmall); + if (cSrcSize > dstSize) return ERROR(corruption_detected); /* invalid */ + if (cSrcSize == dstSize) { ZSTD_memcpy(dst, cSrc, dstSize); return dstSize; } /* not compressed */ + if (cSrcSize == 1) { ZSTD_memset(dst, *(const BYTE*)cSrc, dstSize); return dstSize; } /* RLE */ + + { U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize); +#if defined(HUF_FORCE_DECOMPRESS_X1) + (void)algoNb; + assert(algoNb == 0); + return HUF_decompress4X1_DCtx(dctx, dst, dstSize, cSrc, cSrcSize); +#elif defined(HUF_FORCE_DECOMPRESS_X2) + (void)algoNb; + assert(algoNb == 1); + return HUF_decompress4X2_DCtx(dctx, dst, dstSize, cSrc, cSrcSize); +#else + return algoNb ? HUF_decompress4X2_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) : + HUF_decompress4X1_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) ; +#endif + } +} + +size_t HUF_decompress4X_hufOnly(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) +{ + U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32]; + return HUF_decompress4X_hufOnly_wksp(dctx, dst, dstSize, cSrc, cSrcSize, + workSpace, sizeof(workSpace)); +} + +size_t HUF_decompress1X_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize) +{ + U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32]; + return HUF_decompress1X_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, + workSpace, sizeof(workSpace)); +} +#endif diff --git a/src/blosc2/internal-complibs/zstd-1.5.2/decompress/huf_decompress_amd64.S b/src/blosc2/internal-complibs/zstd-1.5.2/decompress/huf_decompress_amd64.S new file mode 100644 index 0000000..49589cb --- /dev/null +++ b/src/blosc2/internal-complibs/zstd-1.5.2/decompress/huf_decompress_amd64.S @@ -0,0 +1,585 @@ +/* + * Copyright (c) Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#include "../common/portability_macros.h" + +/* Stack marking + * ref: https://wiki.gentoo.org/wiki/Hardened/GNU_stack_quickstart + */ +#if defined(__ELF__) && defined(__GNUC__) +.section .note.GNU-stack,"",%progbits +#endif + +#if ZSTD_ENABLE_ASM_X86_64_BMI2 + +/* Calling convention: + * + * %rdi contains the first argument: HUF_DecompressAsmArgs*. + * %rbp isn't maintained (no frame pointer). + * %rsp contains the stack pointer that grows down. + * No red-zone is assumed, only addresses >= %rsp are used. + * All register contents are preserved. + * + * TODO: Support Windows calling convention. + */ + +ZSTD_HIDE_ASM_FUNCTION(HUF_decompress4X1_usingDTable_internal_bmi2_asm_loop) +ZSTD_HIDE_ASM_FUNCTION(HUF_decompress4X2_usingDTable_internal_bmi2_asm_loop) +ZSTD_HIDE_ASM_FUNCTION(_HUF_decompress4X2_usingDTable_internal_bmi2_asm_loop) +ZSTD_HIDE_ASM_FUNCTION(_HUF_decompress4X1_usingDTable_internal_bmi2_asm_loop) +.global HUF_decompress4X1_usingDTable_internal_bmi2_asm_loop +.global HUF_decompress4X2_usingDTable_internal_bmi2_asm_loop +.global _HUF_decompress4X1_usingDTable_internal_bmi2_asm_loop +.global _HUF_decompress4X2_usingDTable_internal_bmi2_asm_loop +.text + +/* Sets up register mappings for clarity. + * op[], bits[], dtable & ip[0] each get their own register. + * ip[1,2,3] & olimit alias var[]. + * %rax is a scratch register. + */ + +#define op0 rsi +#define op1 rbx +#define op2 rcx +#define op3 rdi + +#define ip0 r8 +#define ip1 r9 +#define ip2 r10 +#define ip3 r11 + +#define bits0 rbp +#define bits1 rdx +#define bits2 r12 +#define bits3 r13 +#define dtable r14 +#define olimit r15 + +/* var[] aliases ip[1,2,3] & olimit + * ip[1,2,3] are saved every iteration. + * olimit is only used in compute_olimit. + */ +#define var0 r15 +#define var1 r9 +#define var2 r10 +#define var3 r11 + +/* 32-bit var registers */ +#define vard0 r15d +#define vard1 r9d +#define vard2 r10d +#define vard3 r11d + +/* Calls X(N) for each stream 0, 1, 2, 3. */ +#define FOR_EACH_STREAM(X) \ + X(0); \ + X(1); \ + X(2); \ + X(3) + +/* Calls X(N, idx) for each stream 0, 1, 2, 3. */ +#define FOR_EACH_STREAM_WITH_INDEX(X, idx) \ + X(0, idx); \ + X(1, idx); \ + X(2, idx); \ + X(3, idx) + +/* Define both _HUF_* & HUF_* symbols because MacOS + * C symbols are prefixed with '_' & Linux symbols aren't. + */ +_HUF_decompress4X1_usingDTable_internal_bmi2_asm_loop: +HUF_decompress4X1_usingDTable_internal_bmi2_asm_loop: + /* Save all registers - even if they are callee saved for simplicity. */ + push %rax + push %rbx + push %rcx + push %rdx + push %rbp + push %rsi + push %rdi + push %r8 + push %r9 + push %r10 + push %r11 + push %r12 + push %r13 + push %r14 + push %r15 + + /* Read HUF_DecompressAsmArgs* args from %rax */ + movq %rdi, %rax + movq 0(%rax), %ip0 + movq 8(%rax), %ip1 + movq 16(%rax), %ip2 + movq 24(%rax), %ip3 + movq 32(%rax), %op0 + movq 40(%rax), %op1 + movq 48(%rax), %op2 + movq 56(%rax), %op3 + movq 64(%rax), %bits0 + movq 72(%rax), %bits1 + movq 80(%rax), %bits2 + movq 88(%rax), %bits3 + movq 96(%rax), %dtable + push %rax /* argument */ + push 104(%rax) /* ilimit */ + push 112(%rax) /* oend */ + push %olimit /* olimit space */ + + subq $24, %rsp + +.L_4X1_compute_olimit: + /* Computes how many iterations we can do safely + * %r15, %rax may be clobbered + * rbx, rdx must be saved + * op3 & ip0 mustn't be clobbered + */ + movq %rbx, 0(%rsp) + movq %rdx, 8(%rsp) + + movq 32(%rsp), %rax /* rax = oend */ + subq %op3, %rax /* rax = oend - op3 */ + + /* r15 = (oend - op3) / 5 */ + movabsq $-3689348814741910323, %rdx + mulq %rdx + movq %rdx, %r15 + shrq $2, %r15 + + movq %ip0, %rax /* rax = ip0 */ + movq 40(%rsp), %rdx /* rdx = ilimit */ + subq %rdx, %rax /* rax = ip0 - ilimit */ + movq %rax, %rbx /* rbx = ip0 - ilimit */ + + /* rdx = (ip0 - ilimit) / 7 */ + movabsq $2635249153387078803, %rdx + mulq %rdx + subq %rdx, %rbx + shrq %rbx + addq %rbx, %rdx + shrq $2, %rdx + + /* r15 = min(%rdx, %r15) */ + cmpq %rdx, %r15 + cmova %rdx, %r15 + + /* r15 = r15 * 5 */ + leaq (%r15, %r15, 4), %r15 + + /* olimit = op3 + r15 */ + addq %op3, %olimit + + movq 8(%rsp), %rdx + movq 0(%rsp), %rbx + + /* If (op3 + 20 > olimit) */ + movq %op3, %rax /* rax = op3 */ + addq $20, %rax /* rax = op3 + 20 */ + cmpq %rax, %olimit /* op3 + 20 > olimit */ + jb .L_4X1_exit + + /* If (ip1 < ip0) go to exit */ + cmpq %ip0, %ip1 + jb .L_4X1_exit + + /* If (ip2 < ip1) go to exit */ + cmpq %ip1, %ip2 + jb .L_4X1_exit + + /* If (ip3 < ip2) go to exit */ + cmpq %ip2, %ip3 + jb .L_4X1_exit + +/* Reads top 11 bits from bits[n] + * Loads dt[bits[n]] into var[n] + */ +#define GET_NEXT_DELT(n) \ + movq $53, %var##n; \ + shrxq %var##n, %bits##n, %var##n; \ + movzwl (%dtable,%var##n,2),%vard##n + +/* var[n] must contain the DTable entry computed with GET_NEXT_DELT + * Moves var[n] to %rax + * bits[n] <<= var[n] & 63 + * op[n][idx] = %rax >> 8 + * %ah is a way to access bits [8, 16) of %rax + */ +#define DECODE_FROM_DELT(n, idx) \ + movq %var##n, %rax; \ + shlxq %var##n, %bits##n, %bits##n; \ + movb %ah, idx(%op##n) + +/* Assumes GET_NEXT_DELT has been called. + * Calls DECODE_FROM_DELT then GET_NEXT_DELT + */ +#define DECODE_AND_GET_NEXT(n, idx) \ + DECODE_FROM_DELT(n, idx); \ + GET_NEXT_DELT(n) \ + +/* // ctz & nbBytes is stored in bits[n] + * // nbBits is stored in %rax + * ctz = CTZ[bits[n]] + * nbBits = ctz & 7 + * nbBytes = ctz >> 3 + * op[n] += 5 + * ip[n] -= nbBytes + * // Note: x86-64 is little-endian ==> no bswap + * bits[n] = MEM_readST(ip[n]) | 1 + * bits[n] <<= nbBits + */ +#define RELOAD_BITS(n) \ + bsfq %bits##n, %bits##n; \ + movq %bits##n, %rax; \ + andq $7, %rax; \ + shrq $3, %bits##n; \ + leaq 5(%op##n), %op##n; \ + subq %bits##n, %ip##n; \ + movq (%ip##n), %bits##n; \ + orq $1, %bits##n; \ + shlx %rax, %bits##n, %bits##n + + /* Store clobbered variables on the stack */ + movq %olimit, 24(%rsp) + movq %ip1, 0(%rsp) + movq %ip2, 8(%rsp) + movq %ip3, 16(%rsp) + + /* Call GET_NEXT_DELT for each stream */ + FOR_EACH_STREAM(GET_NEXT_DELT) + + .p2align 6 + +.L_4X1_loop_body: + /* Decode 5 symbols in each of the 4 streams (20 total) + * Must have called GET_NEXT_DELT for each stream + */ + FOR_EACH_STREAM_WITH_INDEX(DECODE_AND_GET_NEXT, 0) + FOR_EACH_STREAM_WITH_INDEX(DECODE_AND_GET_NEXT, 1) + FOR_EACH_STREAM_WITH_INDEX(DECODE_AND_GET_NEXT, 2) + FOR_EACH_STREAM_WITH_INDEX(DECODE_AND_GET_NEXT, 3) + FOR_EACH_STREAM_WITH_INDEX(DECODE_FROM_DELT, 4) + + /* Load ip[1,2,3] from stack (var[] aliases them) + * ip[] is needed for RELOAD_BITS + * Each will be stored back to the stack after RELOAD + */ + movq 0(%rsp), %ip1 + movq 8(%rsp), %ip2 + movq 16(%rsp), %ip3 + + /* Reload each stream & fetch the next table entry + * to prepare for the next iteration + */ + RELOAD_BITS(0) + GET_NEXT_DELT(0) + + RELOAD_BITS(1) + movq %ip1, 0(%rsp) + GET_NEXT_DELT(1) + + RELOAD_BITS(2) + movq %ip2, 8(%rsp) + GET_NEXT_DELT(2) + + RELOAD_BITS(3) + movq %ip3, 16(%rsp) + GET_NEXT_DELT(3) + + /* If op3 < olimit: continue the loop */ + cmp %op3, 24(%rsp) + ja .L_4X1_loop_body + + /* Reload ip[1,2,3] from stack */ + movq 0(%rsp), %ip1 + movq 8(%rsp), %ip2 + movq 16(%rsp), %ip3 + + /* Re-compute olimit */ + jmp .L_4X1_compute_olimit + +#undef GET_NEXT_DELT +#undef DECODE_FROM_DELT +#undef DECODE +#undef RELOAD_BITS +.L_4X1_exit: + addq $24, %rsp + + /* Restore stack (oend & olimit) */ + pop %rax /* olimit */ + pop %rax /* oend */ + pop %rax /* ilimit */ + pop %rax /* arg */ + + /* Save ip / op / bits */ + movq %ip0, 0(%rax) + movq %ip1, 8(%rax) + movq %ip2, 16(%rax) + movq %ip3, 24(%rax) + movq %op0, 32(%rax) + movq %op1, 40(%rax) + movq %op2, 48(%rax) + movq %op3, 56(%rax) + movq %bits0, 64(%rax) + movq %bits1, 72(%rax) + movq %bits2, 80(%rax) + movq %bits3, 88(%rax) + + /* Restore registers */ + pop %r15 + pop %r14 + pop %r13 + pop %r12 + pop %r11 + pop %r10 + pop %r9 + pop %r8 + pop %rdi + pop %rsi + pop %rbp + pop %rdx + pop %rcx + pop %rbx + pop %rax + ret + +_HUF_decompress4X2_usingDTable_internal_bmi2_asm_loop: +HUF_decompress4X2_usingDTable_internal_bmi2_asm_loop: + /* Save all registers - even if they are callee saved for simplicity. */ + push %rax + push %rbx + push %rcx + push %rdx + push %rbp + push %rsi + push %rdi + push %r8 + push %r9 + push %r10 + push %r11 + push %r12 + push %r13 + push %r14 + push %r15 + + movq %rdi, %rax + movq 0(%rax), %ip0 + movq 8(%rax), %ip1 + movq 16(%rax), %ip2 + movq 24(%rax), %ip3 + movq 32(%rax), %op0 + movq 40(%rax), %op1 + movq 48(%rax), %op2 + movq 56(%rax), %op3 + movq 64(%rax), %bits0 + movq 72(%rax), %bits1 + movq 80(%rax), %bits2 + movq 88(%rax), %bits3 + movq 96(%rax), %dtable + push %rax /* argument */ + push %rax /* olimit */ + push 104(%rax) /* ilimit */ + + movq 112(%rax), %rax + push %rax /* oend3 */ + + movq %op3, %rax + push %rax /* oend2 */ + + movq %op2, %rax + push %rax /* oend1 */ + + movq %op1, %rax + push %rax /* oend0 */ + + /* Scratch space */ + subq $8, %rsp + +.L_4X2_compute_olimit: + /* Computes how many iterations we can do safely + * %r15, %rax may be clobbered + * rdx must be saved + * op[1,2,3,4] & ip0 mustn't be clobbered + */ + movq %rdx, 0(%rsp) + + /* We can consume up to 7 input bytes each iteration. */ + movq %ip0, %rax /* rax = ip0 */ + movq 40(%rsp), %rdx /* rdx = ilimit */ + subq %rdx, %rax /* rax = ip0 - ilimit */ + movq %rax, %r15 /* r15 = ip0 - ilimit */ + + /* rdx = rax / 7 */ + movabsq $2635249153387078803, %rdx + mulq %rdx + subq %rdx, %r15 + shrq %r15 + addq %r15, %rdx + shrq $2, %rdx + + /* r15 = (ip0 - ilimit) / 7 */ + movq %rdx, %r15 + + movabsq $-3689348814741910323, %rdx + movq 8(%rsp), %rax /* rax = oend0 */ + subq %op0, %rax /* rax = oend0 - op0 */ + mulq %rdx + shrq $3, %rdx /* rdx = rax / 10 */ + + /* r15 = min(%rdx, %r15) */ + cmpq %rdx, %r15 + cmova %rdx, %r15 + + movabsq $-3689348814741910323, %rdx + movq 16(%rsp), %rax /* rax = oend1 */ + subq %op1, %rax /* rax = oend1 - op1 */ + mulq %rdx + shrq $3, %rdx /* rdx = rax / 10 */ + + /* r15 = min(%rdx, %r15) */ + cmpq %rdx, %r15 + cmova %rdx, %r15 + + movabsq $-3689348814741910323, %rdx + movq 24(%rsp), %rax /* rax = oend2 */ + subq %op2, %rax /* rax = oend2 - op2 */ + mulq %rdx + shrq $3, %rdx /* rdx = rax / 10 */ + + /* r15 = min(%rdx, %r15) */ + cmpq %rdx, %r15 + cmova %rdx, %r15 + + movabsq $-3689348814741910323, %rdx + movq 32(%rsp), %rax /* rax = oend3 */ + subq %op3, %rax /* rax = oend3 - op3 */ + mulq %rdx + shrq $3, %rdx /* rdx = rax / 10 */ + + /* r15 = min(%rdx, %r15) */ + cmpq %rdx, %r15 + cmova %rdx, %r15 + + /* olimit = op3 + 5 * r15 */ + movq %r15, %rax + leaq (%op3, %rax, 4), %olimit + addq %rax, %olimit + + movq 0(%rsp), %rdx + + /* If (op3 + 10 > olimit) */ + movq %op3, %rax /* rax = op3 */ + addq $10, %rax /* rax = op3 + 10 */ + cmpq %rax, %olimit /* op3 + 10 > olimit */ + jb .L_4X2_exit + + /* If (ip1 < ip0) go to exit */ + cmpq %ip0, %ip1 + jb .L_4X2_exit + + /* If (ip2 < ip1) go to exit */ + cmpq %ip1, %ip2 + jb .L_4X2_exit + + /* If (ip3 < ip2) go to exit */ + cmpq %ip2, %ip3 + jb .L_4X2_exit + +#define DECODE(n, idx) \ + movq %bits##n, %rax; \ + shrq $53, %rax; \ + movzwl 0(%dtable,%rax,4),%r8d; \ + movzbl 2(%dtable,%rax,4),%r15d; \ + movzbl 3(%dtable,%rax,4),%eax; \ + movw %r8w, (%op##n); \ + shlxq %r15, %bits##n, %bits##n; \ + addq %rax, %op##n + +#define RELOAD_BITS(n) \ + bsfq %bits##n, %bits##n; \ + movq %bits##n, %rax; \ + shrq $3, %bits##n; \ + andq $7, %rax; \ + subq %bits##n, %ip##n; \ + movq (%ip##n), %bits##n; \ + orq $1, %bits##n; \ + shlxq %rax, %bits##n, %bits##n + + + movq %olimit, 48(%rsp) + + .p2align 6 + +.L_4X2_loop_body: + /* We clobber r8, so store it on the stack */ + movq %r8, 0(%rsp) + + /* Decode 5 symbols from each of the 4 streams (20 symbols total). */ + FOR_EACH_STREAM_WITH_INDEX(DECODE, 0) + FOR_EACH_STREAM_WITH_INDEX(DECODE, 1) + FOR_EACH_STREAM_WITH_INDEX(DECODE, 2) + FOR_EACH_STREAM_WITH_INDEX(DECODE, 3) + FOR_EACH_STREAM_WITH_INDEX(DECODE, 4) + + /* Reload r8 */ + movq 0(%rsp), %r8 + + FOR_EACH_STREAM(RELOAD_BITS) + + cmp %op3, 48(%rsp) + ja .L_4X2_loop_body + jmp .L_4X2_compute_olimit + +#undef DECODE +#undef RELOAD_BITS +.L_4X2_exit: + addq $8, %rsp + /* Restore stack (oend & olimit) */ + pop %rax /* oend0 */ + pop %rax /* oend1 */ + pop %rax /* oend2 */ + pop %rax /* oend3 */ + pop %rax /* ilimit */ + pop %rax /* olimit */ + pop %rax /* arg */ + + /* Save ip / op / bits */ + movq %ip0, 0(%rax) + movq %ip1, 8(%rax) + movq %ip2, 16(%rax) + movq %ip3, 24(%rax) + movq %op0, 32(%rax) + movq %op1, 40(%rax) + movq %op2, 48(%rax) + movq %op3, 56(%rax) + movq %bits0, 64(%rax) + movq %bits1, 72(%rax) + movq %bits2, 80(%rax) + movq %bits3, 88(%rax) + + /* Restore registers */ + pop %r15 + pop %r14 + pop %r13 + pop %r12 + pop %r11 + pop %r10 + pop %r9 + pop %r8 + pop %rdi + pop %rsi + pop %rbp + pop %rdx + pop %rcx + pop %rbx + pop %rax + ret + +#endif diff --git a/src/blosc2/internal-complibs/zstd-1.5.2/decompress/zstd_ddict.c b/src/blosc2/internal-complibs/zstd-1.5.2/decompress/zstd_ddict.c new file mode 100644 index 0000000..ce33547 --- /dev/null +++ b/src/blosc2/internal-complibs/zstd-1.5.2/decompress/zstd_ddict.c @@ -0,0 +1,244 @@ +/* + * Copyright (c) Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/* zstd_ddict.c : + * concentrates all logic that needs to know the internals of ZSTD_DDict object */ + +/*-******************************************************* +* Dependencies +*********************************************************/ +#include "../common/zstd_deps.h" /* ZSTD_memcpy, ZSTD_memmove, ZSTD_memset */ +#include "../common/cpu.h" /* bmi2 */ +#include "../common/mem.h" /* low level memory routines */ +#define FSE_STATIC_LINKING_ONLY +#include "../common/fse.h" +#define HUF_STATIC_LINKING_ONLY +#include "../common/huf.h" +#include "zstd_decompress_internal.h" +#include "zstd_ddict.h" + +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) +# include "../legacy/zstd_legacy.h" +#endif + + + +/*-******************************************************* +* Types +*********************************************************/ +struct ZSTD_DDict_s { + void* dictBuffer; + const void* dictContent; + size_t dictSize; + ZSTD_entropyDTables_t entropy; + U32 dictID; + U32 entropyPresent; + ZSTD_customMem cMem; +}; /* typedef'd to ZSTD_DDict within "zstd.h" */ + +const void* ZSTD_DDict_dictContent(const ZSTD_DDict* ddict) +{ + assert(ddict != NULL); + return ddict->dictContent; +} + +size_t ZSTD_DDict_dictSize(const ZSTD_DDict* ddict) +{ + assert(ddict != NULL); + return ddict->dictSize; +} + +void ZSTD_copyDDictParameters(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict) +{ + DEBUGLOG(4, "ZSTD_copyDDictParameters"); + assert(dctx != NULL); + assert(ddict != NULL); + dctx->dictID = ddict->dictID; + dctx->prefixStart = ddict->dictContent; + dctx->virtualStart = ddict->dictContent; + dctx->dictEnd = (const BYTE*)ddict->dictContent + ddict->dictSize; + dctx->previousDstEnd = dctx->dictEnd; +#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION + dctx->dictContentBeginForFuzzing = dctx->prefixStart; + dctx->dictContentEndForFuzzing = dctx->previousDstEnd; +#endif + if (ddict->entropyPresent) { + dctx->litEntropy = 1; + dctx->fseEntropy = 1; + dctx->LLTptr = ddict->entropy.LLTable; + dctx->MLTptr = ddict->entropy.MLTable; + dctx->OFTptr = ddict->entropy.OFTable; + dctx->HUFptr = ddict->entropy.hufTable; + dctx->entropy.rep[0] = ddict->entropy.rep[0]; + dctx->entropy.rep[1] = ddict->entropy.rep[1]; + dctx->entropy.rep[2] = ddict->entropy.rep[2]; + } else { + dctx->litEntropy = 0; + dctx->fseEntropy = 0; + } +} + + +static size_t +ZSTD_loadEntropy_intoDDict(ZSTD_DDict* ddict, + ZSTD_dictContentType_e dictContentType) +{ + ddict->dictID = 0; + ddict->entropyPresent = 0; + if (dictContentType == ZSTD_dct_rawContent) return 0; + + if (ddict->dictSize < 8) { + if (dictContentType == ZSTD_dct_fullDict) + return ERROR(dictionary_corrupted); /* only accept specified dictionaries */ + return 0; /* pure content mode */ + } + { U32 const magic = MEM_readLE32(ddict->dictContent); + if (magic != ZSTD_MAGIC_DICTIONARY) { + if (dictContentType == ZSTD_dct_fullDict) + return ERROR(dictionary_corrupted); /* only accept specified dictionaries */ + return 0; /* pure content mode */ + } + } + ddict->dictID = MEM_readLE32((const char*)ddict->dictContent + ZSTD_FRAMEIDSIZE); + + /* load entropy tables */ + RETURN_ERROR_IF(ZSTD_isError(ZSTD_loadDEntropy( + &ddict->entropy, ddict->dictContent, ddict->dictSize)), + dictionary_corrupted, ""); + ddict->entropyPresent = 1; + return 0; +} + + +static size_t ZSTD_initDDict_internal(ZSTD_DDict* ddict, + const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType) +{ + if ((dictLoadMethod == ZSTD_dlm_byRef) || (!dict) || (!dictSize)) { + ddict->dictBuffer = NULL; + ddict->dictContent = dict; + if (!dict) dictSize = 0; + } else { + void* const internalBuffer = ZSTD_customMalloc(dictSize, ddict->cMem); + ddict->dictBuffer = internalBuffer; + ddict->dictContent = internalBuffer; + if (!internalBuffer) return ERROR(memory_allocation); + ZSTD_memcpy(internalBuffer, dict, dictSize); + } + ddict->dictSize = dictSize; + ddict->entropy.hufTable[0] = (HUF_DTable)((HufLog)*0x1000001); /* cover both little and big endian */ + + /* parse dictionary content */ + FORWARD_IF_ERROR( ZSTD_loadEntropy_intoDDict(ddict, dictContentType) , ""); + + return 0; +} + +ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType, + ZSTD_customMem customMem) +{ + if ((!customMem.customAlloc) ^ (!customMem.customFree)) return NULL; + + { ZSTD_DDict* const ddict = (ZSTD_DDict*) ZSTD_customMalloc(sizeof(ZSTD_DDict), customMem); + if (ddict == NULL) return NULL; + ddict->cMem = customMem; + { size_t const initResult = ZSTD_initDDict_internal(ddict, + dict, dictSize, + dictLoadMethod, dictContentType); + if (ZSTD_isError(initResult)) { + ZSTD_freeDDict(ddict); + return NULL; + } } + return ddict; + } +} + +/*! ZSTD_createDDict() : +* Create a digested dictionary, to start decompression without startup delay. +* `dict` content is copied inside DDict. +* Consequently, `dict` can be released after `ZSTD_DDict` creation */ +ZSTD_DDict* ZSTD_createDDict(const void* dict, size_t dictSize) +{ + ZSTD_customMem const allocator = { NULL, NULL, NULL }; + return ZSTD_createDDict_advanced(dict, dictSize, ZSTD_dlm_byCopy, ZSTD_dct_auto, allocator); +} + +/*! ZSTD_createDDict_byReference() : + * Create a digested dictionary, to start decompression without startup delay. + * Dictionary content is simply referenced, it will be accessed during decompression. + * Warning : dictBuffer must outlive DDict (DDict must be freed before dictBuffer) */ +ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize) +{ + ZSTD_customMem const allocator = { NULL, NULL, NULL }; + return ZSTD_createDDict_advanced(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto, allocator); +} + + +const ZSTD_DDict* ZSTD_initStaticDDict( + void* sBuffer, size_t sBufferSize, + const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType) +{ + size_t const neededSpace = sizeof(ZSTD_DDict) + + (dictLoadMethod == ZSTD_dlm_byRef ? 0 : dictSize); + ZSTD_DDict* const ddict = (ZSTD_DDict*)sBuffer; + assert(sBuffer != NULL); + assert(dict != NULL); + if ((size_t)sBuffer & 7) return NULL; /* 8-aligned */ + if (sBufferSize < neededSpace) return NULL; + if (dictLoadMethod == ZSTD_dlm_byCopy) { + ZSTD_memcpy(ddict+1, dict, dictSize); /* local copy */ + dict = ddict+1; + } + if (ZSTD_isError( ZSTD_initDDict_internal(ddict, + dict, dictSize, + ZSTD_dlm_byRef, dictContentType) )) + return NULL; + return ddict; +} + + +size_t ZSTD_freeDDict(ZSTD_DDict* ddict) +{ + if (ddict==NULL) return 0; /* support free on NULL */ + { ZSTD_customMem const cMem = ddict->cMem; + ZSTD_customFree(ddict->dictBuffer, cMem); + ZSTD_customFree(ddict, cMem); + return 0; + } +} + +/*! ZSTD_estimateDDictSize() : + * Estimate amount of memory that will be needed to create a dictionary for decompression. + * Note : dictionary created by reference using ZSTD_dlm_byRef are smaller */ +size_t ZSTD_estimateDDictSize(size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod) +{ + return sizeof(ZSTD_DDict) + (dictLoadMethod == ZSTD_dlm_byRef ? 0 : dictSize); +} + +size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict) +{ + if (ddict==NULL) return 0; /* support sizeof on NULL */ + return sizeof(*ddict) + (ddict->dictBuffer ? ddict->dictSize : 0) ; +} + +/*! ZSTD_getDictID_fromDDict() : + * Provides the dictID of the dictionary loaded into `ddict`. + * If @return == 0, the dictionary is not conformant to Zstandard specification, or empty. + * Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */ +unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict) +{ + if (ddict==NULL) return 0; + return ZSTD_getDictID_fromDict(ddict->dictContent, ddict->dictSize); +} diff --git a/src/blosc2/internal-complibs/zstd-1.5.2/decompress/zstd_ddict.h b/src/blosc2/internal-complibs/zstd-1.5.2/decompress/zstd_ddict.h new file mode 100644 index 0000000..bd03268 --- /dev/null +++ b/src/blosc2/internal-complibs/zstd-1.5.2/decompress/zstd_ddict.h @@ -0,0 +1,44 @@ +/* + * Copyright (c) Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + + +#ifndef ZSTD_DDICT_H +#define ZSTD_DDICT_H + +/*-******************************************************* + * Dependencies + *********************************************************/ +#include "../common/zstd_deps.h" /* size_t */ +#include "../zstd.h" /* ZSTD_DDict, and several public functions */ + + +/*-******************************************************* + * Interface + *********************************************************/ + +/* note: several prototypes are already published in `zstd.h` : + * ZSTD_createDDict() + * ZSTD_createDDict_byReference() + * ZSTD_createDDict_advanced() + * ZSTD_freeDDict() + * ZSTD_initStaticDDict() + * ZSTD_sizeof_DDict() + * ZSTD_estimateDDictSize() + * ZSTD_getDictID_fromDict() + */ + +const void* ZSTD_DDict_dictContent(const ZSTD_DDict* ddict); +size_t ZSTD_DDict_dictSize(const ZSTD_DDict* ddict); + +void ZSTD_copyDDictParameters(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict); + + + +#endif /* ZSTD_DDICT_H */ diff --git a/src/blosc2/internal-complibs/zstd-1.5.2/decompress/zstd_decompress.c b/src/blosc2/internal-complibs/zstd-1.5.2/decompress/zstd_decompress.c new file mode 100644 index 0000000..0031e98 --- /dev/null +++ b/src/blosc2/internal-complibs/zstd-1.5.2/decompress/zstd_decompress.c @@ -0,0 +1,2230 @@ +/* + * Copyright (c) Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + + +/* *************************************************************** +* Tuning parameters +*****************************************************************/ +/*! + * HEAPMODE : + * Select how default decompression function ZSTD_decompress() allocates its context, + * on stack (0), or into heap (1, default; requires malloc()). + * Note that functions with explicit context such as ZSTD_decompressDCtx() are unaffected. + */ +#ifndef ZSTD_HEAPMODE +# define ZSTD_HEAPMODE 1 +#endif + +/*! +* LEGACY_SUPPORT : +* if set to 1+, ZSTD_decompress() can decode older formats (v0.1+) +*/ +#ifndef ZSTD_LEGACY_SUPPORT +# define ZSTD_LEGACY_SUPPORT 0 +#endif + +/*! + * MAXWINDOWSIZE_DEFAULT : + * maximum window size accepted by DStream __by default__. + * Frames requiring more memory will be rejected. + * It's possible to set a different limit using ZSTD_DCtx_setMaxWindowSize(). + */ +#ifndef ZSTD_MAXWINDOWSIZE_DEFAULT +# define ZSTD_MAXWINDOWSIZE_DEFAULT (((U32)1 << ZSTD_WINDOWLOG_LIMIT_DEFAULT) + 1) +#endif + +/*! + * NO_FORWARD_PROGRESS_MAX : + * maximum allowed nb of calls to ZSTD_decompressStream() + * without any forward progress + * (defined as: no byte read from input, and no byte flushed to output) + * before triggering an error. + */ +#ifndef ZSTD_NO_FORWARD_PROGRESS_MAX +# define ZSTD_NO_FORWARD_PROGRESS_MAX 16 +#endif + + +/*-******************************************************* +* Dependencies +*********************************************************/ +#include "../common/zstd_deps.h" /* ZSTD_memcpy, ZSTD_memmove, ZSTD_memset */ +#include "../common/mem.h" /* low level memory routines */ +#define FSE_STATIC_LINKING_ONLY +#include "../common/fse.h" +#define HUF_STATIC_LINKING_ONLY +#include "../common/huf.h" +#include "../common/xxhash.h" /* XXH64_reset, XXH64_update, XXH64_digest, XXH64 */ +#include "../common/zstd_internal.h" /* blockProperties_t */ +#include "zstd_decompress_internal.h" /* ZSTD_DCtx */ +#include "zstd_ddict.h" /* ZSTD_DDictDictContent */ +#include "zstd_decompress_block.h" /* ZSTD_decompressBlock_internal */ + +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) +# include "../legacy/zstd_legacy.h" +#endif + + + +/************************************* + * Multiple DDicts Hashset internals * + *************************************/ + +#define DDICT_HASHSET_MAX_LOAD_FACTOR_COUNT_MULT 4 +#define DDICT_HASHSET_MAX_LOAD_FACTOR_SIZE_MULT 3 /* These two constants represent SIZE_MULT/COUNT_MULT load factor without using a float. + * Currently, that means a 0.75 load factor. + * So, if count * COUNT_MULT / size * SIZE_MULT != 0, then we've exceeded + * the load factor of the ddict hash set. + */ + +#define DDICT_HASHSET_TABLE_BASE_SIZE 64 +#define DDICT_HASHSET_RESIZE_FACTOR 2 + +/* Hash function to determine starting position of dict insertion within the table + * Returns an index between [0, hashSet->ddictPtrTableSize] + */ +static size_t ZSTD_DDictHashSet_getIndex(const ZSTD_DDictHashSet* hashSet, U32 dictID) { + const U64 hash = XXH64(&dictID, sizeof(U32), 0); + /* DDict ptr table size is a multiple of 2, use size - 1 as mask to get index within [0, hashSet->ddictPtrTableSize) */ + return hash & (hashSet->ddictPtrTableSize - 1); +} + +/* Adds DDict to a hashset without resizing it. + * If inserting a DDict with a dictID that already exists in the set, replaces the one in the set. + * Returns 0 if successful, or a zstd error code if something went wrong. + */ +static size_t ZSTD_DDictHashSet_emplaceDDict(ZSTD_DDictHashSet* hashSet, const ZSTD_DDict* ddict) { + const U32 dictID = ZSTD_getDictID_fromDDict(ddict); + size_t idx = ZSTD_DDictHashSet_getIndex(hashSet, dictID); + const size_t idxRangeMask = hashSet->ddictPtrTableSize - 1; + RETURN_ERROR_IF(hashSet->ddictPtrCount == hashSet->ddictPtrTableSize, GENERIC, "Hash set is full!"); + DEBUGLOG(4, "Hashed index: for dictID: %u is %zu", dictID, idx); + while (hashSet->ddictPtrTable[idx] != NULL) { + /* Replace existing ddict if inserting ddict with same dictID */ + if (ZSTD_getDictID_fromDDict(hashSet->ddictPtrTable[idx]) == dictID) { + DEBUGLOG(4, "DictID already exists, replacing rather than adding"); + hashSet->ddictPtrTable[idx] = ddict; + return 0; + } + idx &= idxRangeMask; + idx++; + } + DEBUGLOG(4, "Final idx after probing for dictID %u is: %zu", dictID, idx); + hashSet->ddictPtrTable[idx] = ddict; + hashSet->ddictPtrCount++; + return 0; +} + +/* Expands hash table by factor of DDICT_HASHSET_RESIZE_FACTOR and + * rehashes all values, allocates new table, frees old table. + * Returns 0 on success, otherwise a zstd error code. + */ +static size_t ZSTD_DDictHashSet_expand(ZSTD_DDictHashSet* hashSet, ZSTD_customMem customMem) { + size_t newTableSize = hashSet->ddictPtrTableSize * DDICT_HASHSET_RESIZE_FACTOR; + const ZSTD_DDict** newTable = (const ZSTD_DDict**)ZSTD_customCalloc(sizeof(ZSTD_DDict*) * newTableSize, customMem); + const ZSTD_DDict** oldTable = hashSet->ddictPtrTable; + size_t oldTableSize = hashSet->ddictPtrTableSize; + size_t i; + + DEBUGLOG(4, "Expanding DDict hash table! Old size: %zu new size: %zu", oldTableSize, newTableSize); + RETURN_ERROR_IF(!newTable, memory_allocation, "Expanded hashset allocation failed!"); + hashSet->ddictPtrTable = newTable; + hashSet->ddictPtrTableSize = newTableSize; + hashSet->ddictPtrCount = 0; + for (i = 0; i < oldTableSize; ++i) { + if (oldTable[i] != NULL) { + FORWARD_IF_ERROR(ZSTD_DDictHashSet_emplaceDDict(hashSet, oldTable[i]), ""); + } + } + ZSTD_customFree((void*)oldTable, customMem); + DEBUGLOG(4, "Finished re-hash"); + return 0; +} + +/* Fetches a DDict with the given dictID + * Returns the ZSTD_DDict* with the requested dictID. If it doesn't exist, then returns NULL. + */ +static const ZSTD_DDict* ZSTD_DDictHashSet_getDDict(ZSTD_DDictHashSet* hashSet, U32 dictID) { + size_t idx = ZSTD_DDictHashSet_getIndex(hashSet, dictID); + const size_t idxRangeMask = hashSet->ddictPtrTableSize - 1; + DEBUGLOG(4, "Hashed index: for dictID: %u is %zu", dictID, idx); + for (;;) { + size_t currDictID = ZSTD_getDictID_fromDDict(hashSet->ddictPtrTable[idx]); + if (currDictID == dictID || currDictID == 0) { + /* currDictID == 0 implies a NULL ddict entry */ + break; + } else { + idx &= idxRangeMask; /* Goes to start of table when we reach the end */ + idx++; + } + } + DEBUGLOG(4, "Final idx after probing for dictID %u is: %zu", dictID, idx); + return hashSet->ddictPtrTable[idx]; +} + +/* Allocates space for and returns a ddict hash set + * The hash set's ZSTD_DDict* table has all values automatically set to NULL to begin with. + * Returns NULL if allocation failed. + */ +static ZSTD_DDictHashSet* ZSTD_createDDictHashSet(ZSTD_customMem customMem) { + ZSTD_DDictHashSet* ret = (ZSTD_DDictHashSet*)ZSTD_customMalloc(sizeof(ZSTD_DDictHashSet), customMem); + DEBUGLOG(4, "Allocating new hash set"); + if (!ret) + return NULL; + ret->ddictPtrTable = (const ZSTD_DDict**)ZSTD_customCalloc(DDICT_HASHSET_TABLE_BASE_SIZE * sizeof(ZSTD_DDict*), customMem); + if (!ret->ddictPtrTable) { + ZSTD_customFree(ret, customMem); + return NULL; + } + ret->ddictPtrTableSize = DDICT_HASHSET_TABLE_BASE_SIZE; + ret->ddictPtrCount = 0; + return ret; +} + +/* Frees the table of ZSTD_DDict* within a hashset, then frees the hashset itself. + * Note: The ZSTD_DDict* within the table are NOT freed. + */ +static void ZSTD_freeDDictHashSet(ZSTD_DDictHashSet* hashSet, ZSTD_customMem customMem) { + DEBUGLOG(4, "Freeing ddict hash set"); + if (hashSet && hashSet->ddictPtrTable) { + ZSTD_customFree((void*)hashSet->ddictPtrTable, customMem); + } + if (hashSet) { + ZSTD_customFree(hashSet, customMem); + } +} + +/* Public function: Adds a DDict into the ZSTD_DDictHashSet, possibly triggering a resize of the hash set. + * Returns 0 on success, or a ZSTD error. + */ +static size_t ZSTD_DDictHashSet_addDDict(ZSTD_DDictHashSet* hashSet, const ZSTD_DDict* ddict, ZSTD_customMem customMem) { + DEBUGLOG(4, "Adding dict ID: %u to hashset with - Count: %zu Tablesize: %zu", ZSTD_getDictID_fromDDict(ddict), hashSet->ddictPtrCount, hashSet->ddictPtrTableSize); + if (hashSet->ddictPtrCount * DDICT_HASHSET_MAX_LOAD_FACTOR_COUNT_MULT / hashSet->ddictPtrTableSize * DDICT_HASHSET_MAX_LOAD_FACTOR_SIZE_MULT != 0) { + FORWARD_IF_ERROR(ZSTD_DDictHashSet_expand(hashSet, customMem), ""); + } + FORWARD_IF_ERROR(ZSTD_DDictHashSet_emplaceDDict(hashSet, ddict), ""); + return 0; +} + +/*-************************************************************* +* Context management +***************************************************************/ +size_t ZSTD_sizeof_DCtx (const ZSTD_DCtx* dctx) +{ + if (dctx==NULL) return 0; /* support sizeof NULL */ + return sizeof(*dctx) + + ZSTD_sizeof_DDict(dctx->ddictLocal) + + dctx->inBuffSize + dctx->outBuffSize; +} + +size_t ZSTD_estimateDCtxSize(void) { return sizeof(ZSTD_DCtx); } + + +static size_t ZSTD_startingInputLength(ZSTD_format_e format) +{ + size_t const startingInputLength = ZSTD_FRAMEHEADERSIZE_PREFIX(format); + /* only supports formats ZSTD_f_zstd1 and ZSTD_f_zstd1_magicless */ + assert( (format == ZSTD_f_zstd1) || (format == ZSTD_f_zstd1_magicless) ); + return startingInputLength; +} + +static void ZSTD_DCtx_resetParameters(ZSTD_DCtx* dctx) +{ + assert(dctx->streamStage == zdss_init); + dctx->format = ZSTD_f_zstd1; + dctx->maxWindowSize = ZSTD_MAXWINDOWSIZE_DEFAULT; + dctx->outBufferMode = ZSTD_bm_buffered; + dctx->forceIgnoreChecksum = ZSTD_d_validateChecksum; + dctx->refMultipleDDicts = ZSTD_rmd_refSingleDDict; +} + +static void ZSTD_initDCtx_internal(ZSTD_DCtx* dctx) +{ + dctx->staticSize = 0; + dctx->ddict = NULL; + dctx->ddictLocal = NULL; + dctx->dictEnd = NULL; + dctx->ddictIsCold = 0; + dctx->dictUses = ZSTD_dont_use; + dctx->inBuff = NULL; + dctx->inBuffSize = 0; + dctx->outBuffSize = 0; + dctx->streamStage = zdss_init; +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) + dctx->legacyContext = NULL; + dctx->previousLegacyVersion = 0; +#endif + dctx->noForwardProgress = 0; + dctx->oversizedDuration = 0; +#if DYNAMIC_BMI2 + dctx->bmi2 = ZSTD_cpuSupportsBmi2(); +#endif + dctx->ddictSet = NULL; + ZSTD_DCtx_resetParameters(dctx); +#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION + dctx->dictContentEndForFuzzing = NULL; +#endif +} + +ZSTD_DCtx* ZSTD_initStaticDCtx(void *workspace, size_t workspaceSize) +{ + ZSTD_DCtx* const dctx = (ZSTD_DCtx*) workspace; + + if ((size_t)workspace & 7) return NULL; /* 8-aligned */ + if (workspaceSize < sizeof(ZSTD_DCtx)) return NULL; /* minimum size */ + + ZSTD_initDCtx_internal(dctx); + dctx->staticSize = workspaceSize; + dctx->inBuff = (char*)(dctx+1); + return dctx; +} + +static ZSTD_DCtx* ZSTD_createDCtx_internal(ZSTD_customMem customMem) { + if ((!customMem.customAlloc) ^ (!customMem.customFree)) return NULL; + + { ZSTD_DCtx* const dctx = (ZSTD_DCtx*)ZSTD_customMalloc(sizeof(*dctx), customMem); + if (!dctx) return NULL; + dctx->customMem = customMem; + ZSTD_initDCtx_internal(dctx); + return dctx; + } +} + +ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem) +{ + return ZSTD_createDCtx_internal(customMem); +} + +ZSTD_DCtx* ZSTD_createDCtx(void) +{ + DEBUGLOG(3, "ZSTD_createDCtx"); + return ZSTD_createDCtx_internal(ZSTD_defaultCMem); +} + +static void ZSTD_clearDict(ZSTD_DCtx* dctx) +{ + ZSTD_freeDDict(dctx->ddictLocal); + dctx->ddictLocal = NULL; + dctx->ddict = NULL; + dctx->dictUses = ZSTD_dont_use; +} + +size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx) +{ + if (dctx==NULL) return 0; /* support free on NULL */ + RETURN_ERROR_IF(dctx->staticSize, memory_allocation, "not compatible with static DCtx"); + { ZSTD_customMem const cMem = dctx->customMem; + ZSTD_clearDict(dctx); + ZSTD_customFree(dctx->inBuff, cMem); + dctx->inBuff = NULL; +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1) + if (dctx->legacyContext) + ZSTD_freeLegacyStreamContext(dctx->legacyContext, dctx->previousLegacyVersion); +#endif + if (dctx->ddictSet) { + ZSTD_freeDDictHashSet(dctx->ddictSet, cMem); + dctx->ddictSet = NULL; + } + ZSTD_customFree(dctx, cMem); + return 0; + } +} + +/* no longer useful */ +void ZSTD_copyDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx) +{ + size_t const toCopy = (size_t)((char*)(&dstDCtx->inBuff) - (char*)dstDCtx); + ZSTD_memcpy(dstDCtx, srcDCtx, toCopy); /* no need to copy workspace */ +} + +/* Given a dctx with a digested frame params, re-selects the correct ZSTD_DDict based on + * the requested dict ID from the frame. If there exists a reference to the correct ZSTD_DDict, then + * accordingly sets the ddict to be used to decompress the frame. + * + * If no DDict is found, then no action is taken, and the ZSTD_DCtx::ddict remains as-is. + * + * ZSTD_d_refMultipleDDicts must be enabled for this function to be called. + */ +static void ZSTD_DCtx_selectFrameDDict(ZSTD_DCtx* dctx) { + assert(dctx->refMultipleDDicts && dctx->ddictSet); + DEBUGLOG(4, "Adjusting DDict based on requested dict ID from frame"); + if (dctx->ddict) { + const ZSTD_DDict* frameDDict = ZSTD_DDictHashSet_getDDict(dctx->ddictSet, dctx->fParams.dictID); + if (frameDDict) { + DEBUGLOG(4, "DDict found!"); + ZSTD_clearDict(dctx); + dctx->dictID = dctx->fParams.dictID; + dctx->ddict = frameDDict; + dctx->dictUses = ZSTD_use_indefinitely; + } + } +} + + +/*-************************************************************* + * Frame header decoding + ***************************************************************/ + +/*! ZSTD_isFrame() : + * Tells if the content of `buffer` starts with a valid Frame Identifier. + * Note : Frame Identifier is 4 bytes. If `size < 4`, @return will always be 0. + * Note 2 : Legacy Frame Identifiers are considered valid only if Legacy Support is enabled. + * Note 3 : Skippable Frame Identifiers are considered valid. */ +unsigned ZSTD_isFrame(const void* buffer, size_t size) +{ + if (size < ZSTD_FRAMEIDSIZE) return 0; + { U32 const magic = MEM_readLE32(buffer); + if (magic == ZSTD_MAGICNUMBER) return 1; + if ((magic & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) return 1; + } +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1) + if (ZSTD_isLegacy(buffer, size)) return 1; +#endif + return 0; +} + +/*! ZSTD_isSkippableFrame() : + * Tells if the content of `buffer` starts with a valid Frame Identifier for a skippable frame. + * Note : Frame Identifier is 4 bytes. If `size < 4`, @return will always be 0. + */ +unsigned ZSTD_isSkippableFrame(const void* buffer, size_t size) +{ + if (size < ZSTD_FRAMEIDSIZE) return 0; + { U32 const magic = MEM_readLE32(buffer); + if ((magic & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) return 1; + } + return 0; +} + +/** ZSTD_frameHeaderSize_internal() : + * srcSize must be large enough to reach header size fields. + * note : only works for formats ZSTD_f_zstd1 and ZSTD_f_zstd1_magicless. + * @return : size of the Frame Header + * or an error code, which can be tested with ZSTD_isError() */ +static size_t ZSTD_frameHeaderSize_internal(const void* src, size_t srcSize, ZSTD_format_e format) +{ + size_t const minInputSize = ZSTD_startingInputLength(format); + RETURN_ERROR_IF(srcSize < minInputSize, srcSize_wrong, ""); + + { BYTE const fhd = ((const BYTE*)src)[minInputSize-1]; + U32 const dictID= fhd & 3; + U32 const singleSegment = (fhd >> 5) & 1; + U32 const fcsId = fhd >> 6; + return minInputSize + !singleSegment + + ZSTD_did_fieldSize[dictID] + ZSTD_fcs_fieldSize[fcsId] + + (singleSegment && !fcsId); + } +} + +/** ZSTD_frameHeaderSize() : + * srcSize must be >= ZSTD_frameHeaderSize_prefix. + * @return : size of the Frame Header, + * or an error code (if srcSize is too small) */ +size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize) +{ + return ZSTD_frameHeaderSize_internal(src, srcSize, ZSTD_f_zstd1); +} + + +/** ZSTD_getFrameHeader_advanced() : + * decode Frame Header, or require larger `srcSize`. + * note : only works for formats ZSTD_f_zstd1 and ZSTD_f_zstd1_magicless + * @return : 0, `zfhPtr` is correctly filled, + * >0, `srcSize` is too small, value is wanted `srcSize` amount, + * or an error code, which can be tested using ZSTD_isError() */ +size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize, ZSTD_format_e format) +{ + const BYTE* ip = (const BYTE*)src; + size_t const minInputSize = ZSTD_startingInputLength(format); + + ZSTD_memset(zfhPtr, 0, sizeof(*zfhPtr)); /* not strictly necessary, but static analyzer do not understand that zfhPtr is only going to be read only if return value is zero, since they are 2 different signals */ + if (srcSize < minInputSize) return minInputSize; + RETURN_ERROR_IF(src==NULL, GENERIC, "invalid parameter"); + + if ( (format != ZSTD_f_zstd1_magicless) + && (MEM_readLE32(src) != ZSTD_MAGICNUMBER) ) { + if ((MEM_readLE32(src) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { + /* skippable frame */ + if (srcSize < ZSTD_SKIPPABLEHEADERSIZE) + return ZSTD_SKIPPABLEHEADERSIZE; /* magic number + frame length */ + ZSTD_memset(zfhPtr, 0, sizeof(*zfhPtr)); + zfhPtr->frameContentSize = MEM_readLE32((const char *)src + ZSTD_FRAMEIDSIZE); + zfhPtr->frameType = ZSTD_skippableFrame; + return 0; + } + RETURN_ERROR(prefix_unknown, ""); + } + + /* ensure there is enough `srcSize` to fully read/decode frame header */ + { size_t const fhsize = ZSTD_frameHeaderSize_internal(src, srcSize, format); + if (srcSize < fhsize) return fhsize; + zfhPtr->headerSize = (U32)fhsize; + } + + { BYTE const fhdByte = ip[minInputSize-1]; + size_t pos = minInputSize; + U32 const dictIDSizeCode = fhdByte&3; + U32 const checksumFlag = (fhdByte>>2)&1; + U32 const singleSegment = (fhdByte>>5)&1; + U32 const fcsID = fhdByte>>6; + U64 windowSize = 0; + U32 dictID = 0; + U64 frameContentSize = ZSTD_CONTENTSIZE_UNKNOWN; + RETURN_ERROR_IF((fhdByte & 0x08) != 0, frameParameter_unsupported, + "reserved bits, must be zero"); + + if (!singleSegment) { + BYTE const wlByte = ip[pos++]; + U32 const windowLog = (wlByte >> 3) + ZSTD_WINDOWLOG_ABSOLUTEMIN; + RETURN_ERROR_IF(windowLog > ZSTD_WINDOWLOG_MAX, frameParameter_windowTooLarge, ""); + windowSize = (1ULL << windowLog); + windowSize += (windowSize >> 3) * (wlByte&7); + } + switch(dictIDSizeCode) + { + default: + assert(0); /* impossible */ + ZSTD_FALLTHROUGH; + case 0 : break; + case 1 : dictID = ip[pos]; pos++; break; + case 2 : dictID = MEM_readLE16(ip+pos); pos+=2; break; + case 3 : dictID = MEM_readLE32(ip+pos); pos+=4; break; + } + switch(fcsID) + { + default: + assert(0); /* impossible */ + ZSTD_FALLTHROUGH; + case 0 : if (singleSegment) frameContentSize = ip[pos]; break; + case 1 : frameContentSize = MEM_readLE16(ip+pos)+256; break; + case 2 : frameContentSize = MEM_readLE32(ip+pos); break; + case 3 : frameContentSize = MEM_readLE64(ip+pos); break; + } + if (singleSegment) windowSize = frameContentSize; + + zfhPtr->frameType = ZSTD_frame; + zfhPtr->frameContentSize = frameContentSize; + zfhPtr->windowSize = windowSize; + zfhPtr->blockSizeMax = (unsigned) MIN(windowSize, ZSTD_BLOCKSIZE_MAX); + zfhPtr->dictID = dictID; + zfhPtr->checksumFlag = checksumFlag; + } + return 0; +} + +/** ZSTD_getFrameHeader() : + * decode Frame Header, or require larger `srcSize`. + * note : this function does not consume input, it only reads it. + * @return : 0, `zfhPtr` is correctly filled, + * >0, `srcSize` is too small, value is wanted `srcSize` amount, + * or an error code, which can be tested using ZSTD_isError() */ +size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize) +{ + return ZSTD_getFrameHeader_advanced(zfhPtr, src, srcSize, ZSTD_f_zstd1); +} + +/** ZSTD_getFrameContentSize() : + * compatible with legacy mode + * @return : decompressed size of the single frame pointed to be `src` if known, otherwise + * - ZSTD_CONTENTSIZE_UNKNOWN if the size cannot be determined + * - ZSTD_CONTENTSIZE_ERROR if an error occurred (e.g. invalid magic number, srcSize too small) */ +unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize) +{ +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1) + if (ZSTD_isLegacy(src, srcSize)) { + unsigned long long const ret = ZSTD_getDecompressedSize_legacy(src, srcSize); + return ret == 0 ? ZSTD_CONTENTSIZE_UNKNOWN : ret; + } +#endif + { ZSTD_frameHeader zfh; + if (ZSTD_getFrameHeader(&zfh, src, srcSize) != 0) + return ZSTD_CONTENTSIZE_ERROR; + if (zfh.frameType == ZSTD_skippableFrame) { + return 0; + } else { + return zfh.frameContentSize; + } } +} + +static size_t readSkippableFrameSize(void const* src, size_t srcSize) +{ + size_t const skippableHeaderSize = ZSTD_SKIPPABLEHEADERSIZE; + U32 sizeU32; + + RETURN_ERROR_IF(srcSize < ZSTD_SKIPPABLEHEADERSIZE, srcSize_wrong, ""); + + sizeU32 = MEM_readLE32((BYTE const*)src + ZSTD_FRAMEIDSIZE); + RETURN_ERROR_IF((U32)(sizeU32 + ZSTD_SKIPPABLEHEADERSIZE) < sizeU32, + frameParameter_unsupported, ""); + { + size_t const skippableSize = skippableHeaderSize + sizeU32; + RETURN_ERROR_IF(skippableSize > srcSize, srcSize_wrong, ""); + return skippableSize; + } +} + +/*! ZSTD_readSkippableFrame() : + * Retrieves a zstd skippable frame containing data given by src, and writes it to dst buffer. + * + * The parameter magicVariant will receive the magicVariant that was supplied when the frame was written, + * i.e. magicNumber - ZSTD_MAGIC_SKIPPABLE_START. This can be NULL if the caller is not interested + * in the magicVariant. + * + * Returns an error if destination buffer is not large enough, or if the frame is not skippable. + * + * @return : number of bytes written or a ZSTD error. + */ +ZSTDLIB_API size_t ZSTD_readSkippableFrame(void* dst, size_t dstCapacity, unsigned* magicVariant, + const void* src, size_t srcSize) +{ + U32 const magicNumber = MEM_readLE32(src); + size_t skippableFrameSize = readSkippableFrameSize(src, srcSize); + size_t skippableContentSize = skippableFrameSize - ZSTD_SKIPPABLEHEADERSIZE; + + /* check input validity */ + RETURN_ERROR_IF(!ZSTD_isSkippableFrame(src, srcSize), frameParameter_unsupported, ""); + RETURN_ERROR_IF(skippableFrameSize < ZSTD_SKIPPABLEHEADERSIZE || skippableFrameSize > srcSize, srcSize_wrong, ""); + RETURN_ERROR_IF(skippableContentSize > dstCapacity, dstSize_tooSmall, ""); + + /* deliver payload */ + if (skippableContentSize > 0 && dst != NULL) + ZSTD_memcpy(dst, (const BYTE *)src + ZSTD_SKIPPABLEHEADERSIZE, skippableContentSize); + if (magicVariant != NULL) + *magicVariant = magicNumber - ZSTD_MAGIC_SKIPPABLE_START; + return skippableContentSize; +} + +/** ZSTD_findDecompressedSize() : + * compatible with legacy mode + * `srcSize` must be the exact length of some number of ZSTD compressed and/or + * skippable frames + * @return : decompressed size of the frames contained */ +unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize) +{ + unsigned long long totalDstSize = 0; + + while (srcSize >= ZSTD_startingInputLength(ZSTD_f_zstd1)) { + U32 const magicNumber = MEM_readLE32(src); + + if ((magicNumber & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { + size_t const skippableSize = readSkippableFrameSize(src, srcSize); + if (ZSTD_isError(skippableSize)) { + return ZSTD_CONTENTSIZE_ERROR; + } + assert(skippableSize <= srcSize); + + src = (const BYTE *)src + skippableSize; + srcSize -= skippableSize; + continue; + } + + { unsigned long long const ret = ZSTD_getFrameContentSize(src, srcSize); + if (ret >= ZSTD_CONTENTSIZE_ERROR) return ret; + + /* check for overflow */ + if (totalDstSize + ret < totalDstSize) return ZSTD_CONTENTSIZE_ERROR; + totalDstSize += ret; + } + { size_t const frameSrcSize = ZSTD_findFrameCompressedSize(src, srcSize); + if (ZSTD_isError(frameSrcSize)) { + return ZSTD_CONTENTSIZE_ERROR; + } + + src = (const BYTE *)src + frameSrcSize; + srcSize -= frameSrcSize; + } + } /* while (srcSize >= ZSTD_frameHeaderSize_prefix) */ + + if (srcSize) return ZSTD_CONTENTSIZE_ERROR; + + return totalDstSize; +} + +/** ZSTD_getDecompressedSize() : + * compatible with legacy mode + * @return : decompressed size if known, 0 otherwise + note : 0 can mean any of the following : + - frame content is empty + - decompressed size field is not present in frame header + - frame header unknown / not supported + - frame header not complete (`srcSize` too small) */ +unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize) +{ + unsigned long long const ret = ZSTD_getFrameContentSize(src, srcSize); + ZSTD_STATIC_ASSERT(ZSTD_CONTENTSIZE_ERROR < ZSTD_CONTENTSIZE_UNKNOWN); + return (ret >= ZSTD_CONTENTSIZE_ERROR) ? 0 : ret; +} + + +/** ZSTD_decodeFrameHeader() : + * `headerSize` must be the size provided by ZSTD_frameHeaderSize(). + * If multiple DDict references are enabled, also will choose the correct DDict to use. + * @return : 0 if success, or an error code, which can be tested using ZSTD_isError() */ +static size_t ZSTD_decodeFrameHeader(ZSTD_DCtx* dctx, const void* src, size_t headerSize) +{ + size_t const result = ZSTD_getFrameHeader_advanced(&(dctx->fParams), src, headerSize, dctx->format); + if (ZSTD_isError(result)) return result; /* invalid header */ + RETURN_ERROR_IF(result>0, srcSize_wrong, "headerSize too small"); + + /* Reference DDict requested by frame if dctx references multiple ddicts */ + if (dctx->refMultipleDDicts == ZSTD_rmd_refMultipleDDicts && dctx->ddictSet) { + ZSTD_DCtx_selectFrameDDict(dctx); + } + +#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION + /* Skip the dictID check in fuzzing mode, because it makes the search + * harder. + */ + RETURN_ERROR_IF(dctx->fParams.dictID && (dctx->dictID != dctx->fParams.dictID), + dictionary_wrong, ""); +#endif + dctx->validateChecksum = (dctx->fParams.checksumFlag && !dctx->forceIgnoreChecksum) ? 1 : 0; + if (dctx->validateChecksum) XXH64_reset(&dctx->xxhState, 0); + dctx->processedCSize += headerSize; + return 0; +} + +static ZSTD_frameSizeInfo ZSTD_errorFrameSizeInfo(size_t ret) +{ + ZSTD_frameSizeInfo frameSizeInfo; + frameSizeInfo.compressedSize = ret; + frameSizeInfo.decompressedBound = ZSTD_CONTENTSIZE_ERROR; + return frameSizeInfo; +} + +static ZSTD_frameSizeInfo ZSTD_findFrameSizeInfo(const void* src, size_t srcSize) +{ + ZSTD_frameSizeInfo frameSizeInfo; + ZSTD_memset(&frameSizeInfo, 0, sizeof(ZSTD_frameSizeInfo)); + +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1) + if (ZSTD_isLegacy(src, srcSize)) + return ZSTD_findFrameSizeInfoLegacy(src, srcSize); +#endif + + if ((srcSize >= ZSTD_SKIPPABLEHEADERSIZE) + && (MEM_readLE32(src) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { + frameSizeInfo.compressedSize = readSkippableFrameSize(src, srcSize); + assert(ZSTD_isError(frameSizeInfo.compressedSize) || + frameSizeInfo.compressedSize <= srcSize); + return frameSizeInfo; + } else { + const BYTE* ip = (const BYTE*)src; + const BYTE* const ipstart = ip; + size_t remainingSize = srcSize; + size_t nbBlocks = 0; + ZSTD_frameHeader zfh; + + /* Extract Frame Header */ + { size_t const ret = ZSTD_getFrameHeader(&zfh, src, srcSize); + if (ZSTD_isError(ret)) + return ZSTD_errorFrameSizeInfo(ret); + if (ret > 0) + return ZSTD_errorFrameSizeInfo(ERROR(srcSize_wrong)); + } + + ip += zfh.headerSize; + remainingSize -= zfh.headerSize; + + /* Iterate over each block */ + while (1) { + blockProperties_t blockProperties; + size_t const cBlockSize = ZSTD_getcBlockSize(ip, remainingSize, &blockProperties); + if (ZSTD_isError(cBlockSize)) + return ZSTD_errorFrameSizeInfo(cBlockSize); + + if (ZSTD_blockHeaderSize + cBlockSize > remainingSize) + return ZSTD_errorFrameSizeInfo(ERROR(srcSize_wrong)); + + ip += ZSTD_blockHeaderSize + cBlockSize; + remainingSize -= ZSTD_blockHeaderSize + cBlockSize; + nbBlocks++; + + if (blockProperties.lastBlock) break; + } + + /* Final frame content checksum */ + if (zfh.checksumFlag) { + if (remainingSize < 4) + return ZSTD_errorFrameSizeInfo(ERROR(srcSize_wrong)); + ip += 4; + } + + frameSizeInfo.compressedSize = (size_t)(ip - ipstart); + frameSizeInfo.decompressedBound = (zfh.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN) + ? zfh.frameContentSize + : nbBlocks * zfh.blockSizeMax; + return frameSizeInfo; + } +} + +/** ZSTD_findFrameCompressedSize() : + * compatible with legacy mode + * `src` must point to the start of a ZSTD frame, ZSTD legacy frame, or skippable frame + * `srcSize` must be at least as large as the frame contained + * @return : the compressed size of the frame starting at `src` */ +size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize) +{ + ZSTD_frameSizeInfo const frameSizeInfo = ZSTD_findFrameSizeInfo(src, srcSize); + return frameSizeInfo.compressedSize; +} + +/** ZSTD_decompressBound() : + * compatible with legacy mode + * `src` must point to the start of a ZSTD frame or a skippeable frame + * `srcSize` must be at least as large as the frame contained + * @return : the maximum decompressed size of the compressed source + */ +unsigned long long ZSTD_decompressBound(const void* src, size_t srcSize) +{ + unsigned long long bound = 0; + /* Iterate over each frame */ + while (srcSize > 0) { + ZSTD_frameSizeInfo const frameSizeInfo = ZSTD_findFrameSizeInfo(src, srcSize); + size_t const compressedSize = frameSizeInfo.compressedSize; + unsigned long long const decompressedBound = frameSizeInfo.decompressedBound; + if (ZSTD_isError(compressedSize) || decompressedBound == ZSTD_CONTENTSIZE_ERROR) + return ZSTD_CONTENTSIZE_ERROR; + assert(srcSize >= compressedSize); + src = (const BYTE*)src + compressedSize; + srcSize -= compressedSize; + bound += decompressedBound; + } + return bound; +} + + +/*-************************************************************* + * Frame decoding + ***************************************************************/ + +/** ZSTD_insertBlock() : + * insert `src` block into `dctx` history. Useful to track uncompressed blocks. */ +size_t ZSTD_insertBlock(ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize) +{ + DEBUGLOG(5, "ZSTD_insertBlock: %u bytes", (unsigned)blockSize); + ZSTD_checkContinuity(dctx, blockStart, blockSize); + dctx->previousDstEnd = (const char*)blockStart + blockSize; + return blockSize; +} + + +static size_t ZSTD_copyRawBlock(void* dst, size_t dstCapacity, + const void* src, size_t srcSize) +{ + DEBUGLOG(5, "ZSTD_copyRawBlock"); + RETURN_ERROR_IF(srcSize > dstCapacity, dstSize_tooSmall, ""); + if (dst == NULL) { + if (srcSize == 0) return 0; + RETURN_ERROR(dstBuffer_null, ""); + } + ZSTD_memcpy(dst, src, srcSize); + return srcSize; +} + +static size_t ZSTD_setRleBlock(void* dst, size_t dstCapacity, + BYTE b, + size_t regenSize) +{ + RETURN_ERROR_IF(regenSize > dstCapacity, dstSize_tooSmall, ""); + if (dst == NULL) { + if (regenSize == 0) return 0; + RETURN_ERROR(dstBuffer_null, ""); + } + ZSTD_memset(dst, b, regenSize); + return regenSize; +} + +static void ZSTD_DCtx_trace_end(ZSTD_DCtx const* dctx, U64 uncompressedSize, U64 compressedSize, unsigned streaming) +{ +#if ZSTD_TRACE + if (dctx->traceCtx && ZSTD_trace_decompress_end != NULL) { + ZSTD_Trace trace; + ZSTD_memset(&trace, 0, sizeof(trace)); + trace.version = ZSTD_VERSION_NUMBER; + trace.streaming = streaming; + if (dctx->ddict) { + trace.dictionaryID = ZSTD_getDictID_fromDDict(dctx->ddict); + trace.dictionarySize = ZSTD_DDict_dictSize(dctx->ddict); + trace.dictionaryIsCold = dctx->ddictIsCold; + } + trace.uncompressedSize = (size_t)uncompressedSize; + trace.compressedSize = (size_t)compressedSize; + trace.dctx = dctx; + ZSTD_trace_decompress_end(dctx->traceCtx, &trace); + } +#else + (void)dctx; + (void)uncompressedSize; + (void)compressedSize; + (void)streaming; +#endif +} + + +/*! ZSTD_decompressFrame() : + * @dctx must be properly initialized + * will update *srcPtr and *srcSizePtr, + * to make *srcPtr progress by one frame. */ +static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void** srcPtr, size_t *srcSizePtr) +{ + const BYTE* const istart = (const BYTE*)(*srcPtr); + const BYTE* ip = istart; + BYTE* const ostart = (BYTE*)dst; + BYTE* const oend = dstCapacity != 0 ? ostart + dstCapacity : ostart; + BYTE* op = ostart; + size_t remainingSrcSize = *srcSizePtr; + + DEBUGLOG(4, "ZSTD_decompressFrame (srcSize:%i)", (int)*srcSizePtr); + + /* check */ + RETURN_ERROR_IF( + remainingSrcSize < ZSTD_FRAMEHEADERSIZE_MIN(dctx->format)+ZSTD_blockHeaderSize, + srcSize_wrong, ""); + + /* Frame Header */ + { size_t const frameHeaderSize = ZSTD_frameHeaderSize_internal( + ip, ZSTD_FRAMEHEADERSIZE_PREFIX(dctx->format), dctx->format); + if (ZSTD_isError(frameHeaderSize)) return frameHeaderSize; + RETURN_ERROR_IF(remainingSrcSize < frameHeaderSize+ZSTD_blockHeaderSize, + srcSize_wrong, ""); + FORWARD_IF_ERROR( ZSTD_decodeFrameHeader(dctx, ip, frameHeaderSize) , ""); + ip += frameHeaderSize; remainingSrcSize -= frameHeaderSize; + } + + /* Loop on each block */ + while (1) { + size_t decodedSize; + blockProperties_t blockProperties; + size_t const cBlockSize = ZSTD_getcBlockSize(ip, remainingSrcSize, &blockProperties); + if (ZSTD_isError(cBlockSize)) return cBlockSize; + + ip += ZSTD_blockHeaderSize; + remainingSrcSize -= ZSTD_blockHeaderSize; + RETURN_ERROR_IF(cBlockSize > remainingSrcSize, srcSize_wrong, ""); + + switch(blockProperties.blockType) + { + case bt_compressed: + decodedSize = ZSTD_decompressBlock_internal(dctx, op, (size_t)(oend-op), ip, cBlockSize, /* frame */ 1, not_streaming); + break; + case bt_raw : + decodedSize = ZSTD_copyRawBlock(op, (size_t)(oend-op), ip, cBlockSize); + break; + case bt_rle : + decodedSize = ZSTD_setRleBlock(op, (size_t)(oend-op), *ip, blockProperties.origSize); + break; + case bt_reserved : + default: + RETURN_ERROR(corruption_detected, "invalid block type"); + } + + if (ZSTD_isError(decodedSize)) return decodedSize; + if (dctx->validateChecksum) + XXH64_update(&dctx->xxhState, op, decodedSize); + if (decodedSize != 0) + op += decodedSize; + assert(ip != NULL); + ip += cBlockSize; + remainingSrcSize -= cBlockSize; + if (blockProperties.lastBlock) break; + } + + if (dctx->fParams.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN) { + RETURN_ERROR_IF((U64)(op-ostart) != dctx->fParams.frameContentSize, + corruption_detected, ""); + } + if (dctx->fParams.checksumFlag) { /* Frame content checksum verification */ + RETURN_ERROR_IF(remainingSrcSize<4, checksum_wrong, ""); + if (!dctx->forceIgnoreChecksum) { + U32 const checkCalc = (U32)XXH64_digest(&dctx->xxhState); + U32 checkRead; + checkRead = MEM_readLE32(ip); + RETURN_ERROR_IF(checkRead != checkCalc, checksum_wrong, ""); + } + ip += 4; + remainingSrcSize -= 4; + } + ZSTD_DCtx_trace_end(dctx, (U64)(op-ostart), (U64)(ip-istart), /* streaming */ 0); + /* Allow caller to get size read */ + *srcPtr = ip; + *srcSizePtr = remainingSrcSize; + return (size_t)(op-ostart); +} + +static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const void* dict, size_t dictSize, + const ZSTD_DDict* ddict) +{ + void* const dststart = dst; + int moreThan1Frame = 0; + + DEBUGLOG(5, "ZSTD_decompressMultiFrame"); + assert(dict==NULL || ddict==NULL); /* either dict or ddict set, not both */ + + if (ddict) { + dict = ZSTD_DDict_dictContent(ddict); + dictSize = ZSTD_DDict_dictSize(ddict); + } + + while (srcSize >= ZSTD_startingInputLength(dctx->format)) { + +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1) + if (ZSTD_isLegacy(src, srcSize)) { + size_t decodedSize; + size_t const frameSize = ZSTD_findFrameCompressedSizeLegacy(src, srcSize); + if (ZSTD_isError(frameSize)) return frameSize; + RETURN_ERROR_IF(dctx->staticSize, memory_allocation, + "legacy support is not compatible with static dctx"); + + decodedSize = ZSTD_decompressLegacy(dst, dstCapacity, src, frameSize, dict, dictSize); + if (ZSTD_isError(decodedSize)) return decodedSize; + + assert(decodedSize <= dstCapacity); + dst = (BYTE*)dst + decodedSize; + dstCapacity -= decodedSize; + + src = (const BYTE*)src + frameSize; + srcSize -= frameSize; + + continue; + } +#endif + + { U32 const magicNumber = MEM_readLE32(src); + DEBUGLOG(4, "reading magic number %08X (expecting %08X)", + (unsigned)magicNumber, ZSTD_MAGICNUMBER); + if ((magicNumber & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { + size_t const skippableSize = readSkippableFrameSize(src, srcSize); + FORWARD_IF_ERROR(skippableSize, "readSkippableFrameSize failed"); + assert(skippableSize <= srcSize); + + src = (const BYTE *)src + skippableSize; + srcSize -= skippableSize; + continue; + } } + + if (ddict) { + /* we were called from ZSTD_decompress_usingDDict */ + FORWARD_IF_ERROR(ZSTD_decompressBegin_usingDDict(dctx, ddict), ""); + } else { + /* this will initialize correctly with no dict if dict == NULL, so + * use this in all cases but ddict */ + FORWARD_IF_ERROR(ZSTD_decompressBegin_usingDict(dctx, dict, dictSize), ""); + } + ZSTD_checkContinuity(dctx, dst, dstCapacity); + + { const size_t res = ZSTD_decompressFrame(dctx, dst, dstCapacity, + &src, &srcSize); + RETURN_ERROR_IF( + (ZSTD_getErrorCode(res) == ZSTD_error_prefix_unknown) + && (moreThan1Frame==1), + srcSize_wrong, + "At least one frame successfully completed, " + "but following bytes are garbage: " + "it's more likely to be a srcSize error, " + "specifying more input bytes than size of frame(s). " + "Note: one could be unlucky, it might be a corruption error instead, " + "happening right at the place where we expect zstd magic bytes. " + "But this is _much_ less likely than a srcSize field error."); + if (ZSTD_isError(res)) return res; + assert(res <= dstCapacity); + if (res != 0) + dst = (BYTE*)dst + res; + dstCapacity -= res; + } + moreThan1Frame = 1; + } /* while (srcSize >= ZSTD_frameHeaderSize_prefix) */ + + RETURN_ERROR_IF(srcSize, srcSize_wrong, "input not entirely consumed"); + + return (size_t)((BYTE*)dst - (BYTE*)dststart); +} + +size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const void* dict, size_t dictSize) +{ + return ZSTD_decompressMultiFrame(dctx, dst, dstCapacity, src, srcSize, dict, dictSize, NULL); +} + + +static ZSTD_DDict const* ZSTD_getDDict(ZSTD_DCtx* dctx) +{ + switch (dctx->dictUses) { + default: + assert(0 /* Impossible */); + ZSTD_FALLTHROUGH; + case ZSTD_dont_use: + ZSTD_clearDict(dctx); + return NULL; + case ZSTD_use_indefinitely: + return dctx->ddict; + case ZSTD_use_once: + dctx->dictUses = ZSTD_dont_use; + return dctx->ddict; + } +} + +size_t ZSTD_decompressDCtx(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ + return ZSTD_decompress_usingDDict(dctx, dst, dstCapacity, src, srcSize, ZSTD_getDDict(dctx)); +} + + +size_t ZSTD_decompress(void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ +#if defined(ZSTD_HEAPMODE) && (ZSTD_HEAPMODE>=1) + size_t regenSize; + ZSTD_DCtx* const dctx = ZSTD_createDCtx_internal(ZSTD_defaultCMem); + RETURN_ERROR_IF(dctx==NULL, memory_allocation, "NULL pointer!"); + regenSize = ZSTD_decompressDCtx(dctx, dst, dstCapacity, src, srcSize); + ZSTD_freeDCtx(dctx); + return regenSize; +#else /* stack mode */ + ZSTD_DCtx dctx; + ZSTD_initDCtx_internal(&dctx); + return ZSTD_decompressDCtx(&dctx, dst, dstCapacity, src, srcSize); +#endif +} + + +/*-************************************** +* Advanced Streaming Decompression API +* Bufferless and synchronous +****************************************/ +size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx) { return dctx->expected; } + +/** + * Similar to ZSTD_nextSrcSizeToDecompress(), but when when a block input can be streamed, + * we allow taking a partial block as the input. Currently only raw uncompressed blocks can + * be streamed. + * + * For blocks that can be streamed, this allows us to reduce the latency until we produce + * output, and avoid copying the input. + * + * @param inputSize - The total amount of input that the caller currently has. + */ +static size_t ZSTD_nextSrcSizeToDecompressWithInputSize(ZSTD_DCtx* dctx, size_t inputSize) { + if (!(dctx->stage == ZSTDds_decompressBlock || dctx->stage == ZSTDds_decompressLastBlock)) + return dctx->expected; + if (dctx->bType != bt_raw) + return dctx->expected; + return BOUNDED(1, inputSize, dctx->expected); +} + +ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx) { + switch(dctx->stage) + { + default: /* should not happen */ + assert(0); + ZSTD_FALLTHROUGH; + case ZSTDds_getFrameHeaderSize: + ZSTD_FALLTHROUGH; + case ZSTDds_decodeFrameHeader: + return ZSTDnit_frameHeader; + case ZSTDds_decodeBlockHeader: + return ZSTDnit_blockHeader; + case ZSTDds_decompressBlock: + return ZSTDnit_block; + case ZSTDds_decompressLastBlock: + return ZSTDnit_lastBlock; + case ZSTDds_checkChecksum: + return ZSTDnit_checksum; + case ZSTDds_decodeSkippableHeader: + ZSTD_FALLTHROUGH; + case ZSTDds_skipFrame: + return ZSTDnit_skippableFrame; + } +} + +static int ZSTD_isSkipFrame(ZSTD_DCtx* dctx) { return dctx->stage == ZSTDds_skipFrame; } + +/** ZSTD_decompressContinue() : + * srcSize : must be the exact nb of bytes expected (see ZSTD_nextSrcSizeToDecompress()) + * @return : nb of bytes generated into `dst` (necessarily <= `dstCapacity) + * or an error code, which can be tested using ZSTD_isError() */ +size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ + DEBUGLOG(5, "ZSTD_decompressContinue (srcSize:%u)", (unsigned)srcSize); + /* Sanity check */ + RETURN_ERROR_IF(srcSize != ZSTD_nextSrcSizeToDecompressWithInputSize(dctx, srcSize), srcSize_wrong, "not allowed"); + ZSTD_checkContinuity(dctx, dst, dstCapacity); + + dctx->processedCSize += srcSize; + + switch (dctx->stage) + { + case ZSTDds_getFrameHeaderSize : + assert(src != NULL); + if (dctx->format == ZSTD_f_zstd1) { /* allows header */ + assert(srcSize >= ZSTD_FRAMEIDSIZE); /* to read skippable magic number */ + if ((MEM_readLE32(src) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { /* skippable frame */ + ZSTD_memcpy(dctx->headerBuffer, src, srcSize); + dctx->expected = ZSTD_SKIPPABLEHEADERSIZE - srcSize; /* remaining to load to get full skippable frame header */ + dctx->stage = ZSTDds_decodeSkippableHeader; + return 0; + } } + dctx->headerSize = ZSTD_frameHeaderSize_internal(src, srcSize, dctx->format); + if (ZSTD_isError(dctx->headerSize)) return dctx->headerSize; + ZSTD_memcpy(dctx->headerBuffer, src, srcSize); + dctx->expected = dctx->headerSize - srcSize; + dctx->stage = ZSTDds_decodeFrameHeader; + return 0; + + case ZSTDds_decodeFrameHeader: + assert(src != NULL); + ZSTD_memcpy(dctx->headerBuffer + (dctx->headerSize - srcSize), src, srcSize); + FORWARD_IF_ERROR(ZSTD_decodeFrameHeader(dctx, dctx->headerBuffer, dctx->headerSize), ""); + dctx->expected = ZSTD_blockHeaderSize; + dctx->stage = ZSTDds_decodeBlockHeader; + return 0; + + case ZSTDds_decodeBlockHeader: + { blockProperties_t bp; + size_t const cBlockSize = ZSTD_getcBlockSize(src, ZSTD_blockHeaderSize, &bp); + if (ZSTD_isError(cBlockSize)) return cBlockSize; + RETURN_ERROR_IF(cBlockSize > dctx->fParams.blockSizeMax, corruption_detected, "Block Size Exceeds Maximum"); + dctx->expected = cBlockSize; + dctx->bType = bp.blockType; + dctx->rleSize = bp.origSize; + if (cBlockSize) { + dctx->stage = bp.lastBlock ? ZSTDds_decompressLastBlock : ZSTDds_decompressBlock; + return 0; + } + /* empty block */ + if (bp.lastBlock) { + if (dctx->fParams.checksumFlag) { + dctx->expected = 4; + dctx->stage = ZSTDds_checkChecksum; + } else { + dctx->expected = 0; /* end of frame */ + dctx->stage = ZSTDds_getFrameHeaderSize; + } + } else { + dctx->expected = ZSTD_blockHeaderSize; /* jump to next header */ + dctx->stage = ZSTDds_decodeBlockHeader; + } + return 0; + } + + case ZSTDds_decompressLastBlock: + case ZSTDds_decompressBlock: + DEBUGLOG(5, "ZSTD_decompressContinue: case ZSTDds_decompressBlock"); + { size_t rSize; + switch(dctx->bType) + { + case bt_compressed: + DEBUGLOG(5, "ZSTD_decompressContinue: case bt_compressed"); + rSize = ZSTD_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize, /* frame */ 1, is_streaming); + dctx->expected = 0; /* Streaming not supported */ + break; + case bt_raw : + assert(srcSize <= dctx->expected); + rSize = ZSTD_copyRawBlock(dst, dstCapacity, src, srcSize); + FORWARD_IF_ERROR(rSize, "ZSTD_copyRawBlock failed"); + assert(rSize == srcSize); + dctx->expected -= rSize; + break; + case bt_rle : + rSize = ZSTD_setRleBlock(dst, dstCapacity, *(const BYTE*)src, dctx->rleSize); + dctx->expected = 0; /* Streaming not supported */ + break; + case bt_reserved : /* should never happen */ + default: + RETURN_ERROR(corruption_detected, "invalid block type"); + } + FORWARD_IF_ERROR(rSize, ""); + RETURN_ERROR_IF(rSize > dctx->fParams.blockSizeMax, corruption_detected, "Decompressed Block Size Exceeds Maximum"); + DEBUGLOG(5, "ZSTD_decompressContinue: decoded size from block : %u", (unsigned)rSize); + dctx->decodedSize += rSize; + if (dctx->validateChecksum) XXH64_update(&dctx->xxhState, dst, rSize); + dctx->previousDstEnd = (char*)dst + rSize; + + /* Stay on the same stage until we are finished streaming the block. */ + if (dctx->expected > 0) { + return rSize; + } + + if (dctx->stage == ZSTDds_decompressLastBlock) { /* end of frame */ + DEBUGLOG(4, "ZSTD_decompressContinue: decoded size from frame : %u", (unsigned)dctx->decodedSize); + RETURN_ERROR_IF( + dctx->fParams.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN + && dctx->decodedSize != dctx->fParams.frameContentSize, + corruption_detected, ""); + if (dctx->fParams.checksumFlag) { /* another round for frame checksum */ + dctx->expected = 4; + dctx->stage = ZSTDds_checkChecksum; + } else { + ZSTD_DCtx_trace_end(dctx, dctx->decodedSize, dctx->processedCSize, /* streaming */ 1); + dctx->expected = 0; /* ends here */ + dctx->stage = ZSTDds_getFrameHeaderSize; + } + } else { + dctx->stage = ZSTDds_decodeBlockHeader; + dctx->expected = ZSTD_blockHeaderSize; + } + return rSize; + } + + case ZSTDds_checkChecksum: + assert(srcSize == 4); /* guaranteed by dctx->expected */ + { + if (dctx->validateChecksum) { + U32 const h32 = (U32)XXH64_digest(&dctx->xxhState); + U32 const check32 = MEM_readLE32(src); + DEBUGLOG(4, "ZSTD_decompressContinue: checksum : calculated %08X :: %08X read", (unsigned)h32, (unsigned)check32); + RETURN_ERROR_IF(check32 != h32, checksum_wrong, ""); + } + ZSTD_DCtx_trace_end(dctx, dctx->decodedSize, dctx->processedCSize, /* streaming */ 1); + dctx->expected = 0; + dctx->stage = ZSTDds_getFrameHeaderSize; + return 0; + } + + case ZSTDds_decodeSkippableHeader: + assert(src != NULL); + assert(srcSize <= ZSTD_SKIPPABLEHEADERSIZE); + ZSTD_memcpy(dctx->headerBuffer + (ZSTD_SKIPPABLEHEADERSIZE - srcSize), src, srcSize); /* complete skippable header */ + dctx->expected = MEM_readLE32(dctx->headerBuffer + ZSTD_FRAMEIDSIZE); /* note : dctx->expected can grow seriously large, beyond local buffer size */ + dctx->stage = ZSTDds_skipFrame; + return 0; + + case ZSTDds_skipFrame: + dctx->expected = 0; + dctx->stage = ZSTDds_getFrameHeaderSize; + return 0; + + default: + assert(0); /* impossible */ + RETURN_ERROR(GENERIC, "impossible to reach"); /* some compiler require default to do something */ + } +} + + +static size_t ZSTD_refDictContent(ZSTD_DCtx* dctx, const void* dict, size_t dictSize) +{ + dctx->dictEnd = dctx->previousDstEnd; + dctx->virtualStart = (const char*)dict - ((const char*)(dctx->previousDstEnd) - (const char*)(dctx->prefixStart)); + dctx->prefixStart = dict; + dctx->previousDstEnd = (const char*)dict + dictSize; +#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION + dctx->dictContentBeginForFuzzing = dctx->prefixStart; + dctx->dictContentEndForFuzzing = dctx->previousDstEnd; +#endif + return 0; +} + +/*! ZSTD_loadDEntropy() : + * dict : must point at beginning of a valid zstd dictionary. + * @return : size of entropy tables read */ +size_t +ZSTD_loadDEntropy(ZSTD_entropyDTables_t* entropy, + const void* const dict, size_t const dictSize) +{ + const BYTE* dictPtr = (const BYTE*)dict; + const BYTE* const dictEnd = dictPtr + dictSize; + + RETURN_ERROR_IF(dictSize <= 8, dictionary_corrupted, "dict is too small"); + assert(MEM_readLE32(dict) == ZSTD_MAGIC_DICTIONARY); /* dict must be valid */ + dictPtr += 8; /* skip header = magic + dictID */ + + ZSTD_STATIC_ASSERT(offsetof(ZSTD_entropyDTables_t, OFTable) == offsetof(ZSTD_entropyDTables_t, LLTable) + sizeof(entropy->LLTable)); + ZSTD_STATIC_ASSERT(offsetof(ZSTD_entropyDTables_t, MLTable) == offsetof(ZSTD_entropyDTables_t, OFTable) + sizeof(entropy->OFTable)); + ZSTD_STATIC_ASSERT(sizeof(entropy->LLTable) + sizeof(entropy->OFTable) + sizeof(entropy->MLTable) >= HUF_DECOMPRESS_WORKSPACE_SIZE); + { void* const workspace = &entropy->LLTable; /* use fse tables as temporary workspace; implies fse tables are grouped together */ + size_t const workspaceSize = sizeof(entropy->LLTable) + sizeof(entropy->OFTable) + sizeof(entropy->MLTable); +#ifdef HUF_FORCE_DECOMPRESS_X1 + /* in minimal huffman, we always use X1 variants */ + size_t const hSize = HUF_readDTableX1_wksp(entropy->hufTable, + dictPtr, dictEnd - dictPtr, + workspace, workspaceSize); +#else + size_t const hSize = HUF_readDTableX2_wksp(entropy->hufTable, + dictPtr, (size_t)(dictEnd - dictPtr), + workspace, workspaceSize); +#endif + RETURN_ERROR_IF(HUF_isError(hSize), dictionary_corrupted, ""); + dictPtr += hSize; + } + + { short offcodeNCount[MaxOff+1]; + unsigned offcodeMaxValue = MaxOff, offcodeLog; + size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, (size_t)(dictEnd-dictPtr)); + RETURN_ERROR_IF(FSE_isError(offcodeHeaderSize), dictionary_corrupted, ""); + RETURN_ERROR_IF(offcodeMaxValue > MaxOff, dictionary_corrupted, ""); + RETURN_ERROR_IF(offcodeLog > OffFSELog, dictionary_corrupted, ""); + ZSTD_buildFSETable( entropy->OFTable, + offcodeNCount, offcodeMaxValue, + OF_base, OF_bits, + offcodeLog, + entropy->workspace, sizeof(entropy->workspace), + /* bmi2 */0); + dictPtr += offcodeHeaderSize; + } + + { short matchlengthNCount[MaxML+1]; + unsigned matchlengthMaxValue = MaxML, matchlengthLog; + size_t const matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dictPtr, (size_t)(dictEnd-dictPtr)); + RETURN_ERROR_IF(FSE_isError(matchlengthHeaderSize), dictionary_corrupted, ""); + RETURN_ERROR_IF(matchlengthMaxValue > MaxML, dictionary_corrupted, ""); + RETURN_ERROR_IF(matchlengthLog > MLFSELog, dictionary_corrupted, ""); + ZSTD_buildFSETable( entropy->MLTable, + matchlengthNCount, matchlengthMaxValue, + ML_base, ML_bits, + matchlengthLog, + entropy->workspace, sizeof(entropy->workspace), + /* bmi2 */ 0); + dictPtr += matchlengthHeaderSize; + } + + { short litlengthNCount[MaxLL+1]; + unsigned litlengthMaxValue = MaxLL, litlengthLog; + size_t const litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dictPtr, (size_t)(dictEnd-dictPtr)); + RETURN_ERROR_IF(FSE_isError(litlengthHeaderSize), dictionary_corrupted, ""); + RETURN_ERROR_IF(litlengthMaxValue > MaxLL, dictionary_corrupted, ""); + RETURN_ERROR_IF(litlengthLog > LLFSELog, dictionary_corrupted, ""); + ZSTD_buildFSETable( entropy->LLTable, + litlengthNCount, litlengthMaxValue, + LL_base, LL_bits, + litlengthLog, + entropy->workspace, sizeof(entropy->workspace), + /* bmi2 */ 0); + dictPtr += litlengthHeaderSize; + } + + RETURN_ERROR_IF(dictPtr+12 > dictEnd, dictionary_corrupted, ""); + { int i; + size_t const dictContentSize = (size_t)(dictEnd - (dictPtr+12)); + for (i=0; i<3; i++) { + U32 const rep = MEM_readLE32(dictPtr); dictPtr += 4; + RETURN_ERROR_IF(rep==0 || rep > dictContentSize, + dictionary_corrupted, ""); + entropy->rep[i] = rep; + } } + + return (size_t)(dictPtr - (const BYTE*)dict); +} + +static size_t ZSTD_decompress_insertDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize) +{ + if (dictSize < 8) return ZSTD_refDictContent(dctx, dict, dictSize); + { U32 const magic = MEM_readLE32(dict); + if (magic != ZSTD_MAGIC_DICTIONARY) { + return ZSTD_refDictContent(dctx, dict, dictSize); /* pure content mode */ + } } + dctx->dictID = MEM_readLE32((const char*)dict + ZSTD_FRAMEIDSIZE); + + /* load entropy tables */ + { size_t const eSize = ZSTD_loadDEntropy(&dctx->entropy, dict, dictSize); + RETURN_ERROR_IF(ZSTD_isError(eSize), dictionary_corrupted, ""); + dict = (const char*)dict + eSize; + dictSize -= eSize; + } + dctx->litEntropy = dctx->fseEntropy = 1; + + /* reference dictionary content */ + return ZSTD_refDictContent(dctx, dict, dictSize); +} + +size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx) +{ + assert(dctx != NULL); +#if ZSTD_TRACE + dctx->traceCtx = (ZSTD_trace_decompress_begin != NULL) ? ZSTD_trace_decompress_begin(dctx) : 0; +#endif + dctx->expected = ZSTD_startingInputLength(dctx->format); /* dctx->format must be properly set */ + dctx->stage = ZSTDds_getFrameHeaderSize; + dctx->processedCSize = 0; + dctx->decodedSize = 0; + dctx->previousDstEnd = NULL; + dctx->prefixStart = NULL; + dctx->virtualStart = NULL; + dctx->dictEnd = NULL; + dctx->entropy.hufTable[0] = (HUF_DTable)((HufLog)*0x1000001); /* cover both little and big endian */ + dctx->litEntropy = dctx->fseEntropy = 0; + dctx->dictID = 0; + dctx->bType = bt_reserved; + ZSTD_STATIC_ASSERT(sizeof(dctx->entropy.rep) == sizeof(repStartValue)); + ZSTD_memcpy(dctx->entropy.rep, repStartValue, sizeof(repStartValue)); /* initial repcodes */ + dctx->LLTptr = dctx->entropy.LLTable; + dctx->MLTptr = dctx->entropy.MLTable; + dctx->OFTptr = dctx->entropy.OFTable; + dctx->HUFptr = dctx->entropy.hufTable; + return 0; +} + +size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize) +{ + FORWARD_IF_ERROR( ZSTD_decompressBegin(dctx) , ""); + if (dict && dictSize) + RETURN_ERROR_IF( + ZSTD_isError(ZSTD_decompress_insertDictionary(dctx, dict, dictSize)), + dictionary_corrupted, ""); + return 0; +} + + +/* ====== ZSTD_DDict ====== */ + +size_t ZSTD_decompressBegin_usingDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict) +{ + DEBUGLOG(4, "ZSTD_decompressBegin_usingDDict"); + assert(dctx != NULL); + if (ddict) { + const char* const dictStart = (const char*)ZSTD_DDict_dictContent(ddict); + size_t const dictSize = ZSTD_DDict_dictSize(ddict); + const void* const dictEnd = dictStart + dictSize; + dctx->ddictIsCold = (dctx->dictEnd != dictEnd); + DEBUGLOG(4, "DDict is %s", + dctx->ddictIsCold ? "~cold~" : "hot!"); + } + FORWARD_IF_ERROR( ZSTD_decompressBegin(dctx) , ""); + if (ddict) { /* NULL ddict is equivalent to no dictionary */ + ZSTD_copyDDictParameters(dctx, ddict); + } + return 0; +} + +/*! ZSTD_getDictID_fromDict() : + * Provides the dictID stored within dictionary. + * if @return == 0, the dictionary is not conformant with Zstandard specification. + * It can still be loaded, but as a content-only dictionary. */ +unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize) +{ + if (dictSize < 8) return 0; + if (MEM_readLE32(dict) != ZSTD_MAGIC_DICTIONARY) return 0; + return MEM_readLE32((const char*)dict + ZSTD_FRAMEIDSIZE); +} + +/*! ZSTD_getDictID_fromFrame() : + * Provides the dictID required to decompress frame stored within `src`. + * If @return == 0, the dictID could not be decoded. + * This could for one of the following reasons : + * - The frame does not require a dictionary (most common case). + * - The frame was built with dictID intentionally removed. + * Needed dictionary is a hidden information. + * Note : this use case also happens when using a non-conformant dictionary. + * - `srcSize` is too small, and as a result, frame header could not be decoded. + * Note : possible if `srcSize < ZSTD_FRAMEHEADERSIZE_MAX`. + * - This is not a Zstandard frame. + * When identifying the exact failure cause, it's possible to use + * ZSTD_getFrameHeader(), which will provide a more precise error code. */ +unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize) +{ + ZSTD_frameHeader zfp = { 0, 0, 0, ZSTD_frame, 0, 0, 0 }; + size_t const hError = ZSTD_getFrameHeader(&zfp, src, srcSize); + if (ZSTD_isError(hError)) return 0; + return zfp.dictID; +} + + +/*! ZSTD_decompress_usingDDict() : +* Decompression using a pre-digested Dictionary +* Use dictionary without significant overhead. */ +size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const ZSTD_DDict* ddict) +{ + /* pass content and size in case legacy frames are encountered */ + return ZSTD_decompressMultiFrame(dctx, dst, dstCapacity, src, srcSize, + NULL, 0, + ddict); +} + + +/*===================================== +* Streaming decompression +*====================================*/ + +ZSTD_DStream* ZSTD_createDStream(void) +{ + DEBUGLOG(3, "ZSTD_createDStream"); + return ZSTD_createDCtx_internal(ZSTD_defaultCMem); +} + +ZSTD_DStream* ZSTD_initStaticDStream(void *workspace, size_t workspaceSize) +{ + return ZSTD_initStaticDCtx(workspace, workspaceSize); +} + +ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem) +{ + return ZSTD_createDCtx_internal(customMem); +} + +size_t ZSTD_freeDStream(ZSTD_DStream* zds) +{ + return ZSTD_freeDCtx(zds); +} + + +/* *** Initialization *** */ + +size_t ZSTD_DStreamInSize(void) { return ZSTD_BLOCKSIZE_MAX + ZSTD_blockHeaderSize; } +size_t ZSTD_DStreamOutSize(void) { return ZSTD_BLOCKSIZE_MAX; } + +size_t ZSTD_DCtx_loadDictionary_advanced(ZSTD_DCtx* dctx, + const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType) +{ + RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong, ""); + ZSTD_clearDict(dctx); + if (dict && dictSize != 0) { + dctx->ddictLocal = ZSTD_createDDict_advanced(dict, dictSize, dictLoadMethod, dictContentType, dctx->customMem); + RETURN_ERROR_IF(dctx->ddictLocal == NULL, memory_allocation, "NULL pointer!"); + dctx->ddict = dctx->ddictLocal; + dctx->dictUses = ZSTD_use_indefinitely; + } + return 0; +} + +size_t ZSTD_DCtx_loadDictionary_byReference(ZSTD_DCtx* dctx, const void* dict, size_t dictSize) +{ + return ZSTD_DCtx_loadDictionary_advanced(dctx, dict, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto); +} + +size_t ZSTD_DCtx_loadDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize) +{ + return ZSTD_DCtx_loadDictionary_advanced(dctx, dict, dictSize, ZSTD_dlm_byCopy, ZSTD_dct_auto); +} + +size_t ZSTD_DCtx_refPrefix_advanced(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType) +{ + FORWARD_IF_ERROR(ZSTD_DCtx_loadDictionary_advanced(dctx, prefix, prefixSize, ZSTD_dlm_byRef, dictContentType), ""); + dctx->dictUses = ZSTD_use_once; + return 0; +} + +size_t ZSTD_DCtx_refPrefix(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize) +{ + return ZSTD_DCtx_refPrefix_advanced(dctx, prefix, prefixSize, ZSTD_dct_rawContent); +} + + +/* ZSTD_initDStream_usingDict() : + * return : expected size, aka ZSTD_startingInputLength(). + * this function cannot fail */ +size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize) +{ + DEBUGLOG(4, "ZSTD_initDStream_usingDict"); + FORWARD_IF_ERROR( ZSTD_DCtx_reset(zds, ZSTD_reset_session_only) , ""); + FORWARD_IF_ERROR( ZSTD_DCtx_loadDictionary(zds, dict, dictSize) , ""); + return ZSTD_startingInputLength(zds->format); +} + +/* note : this variant can't fail */ +size_t ZSTD_initDStream(ZSTD_DStream* zds) +{ + DEBUGLOG(4, "ZSTD_initDStream"); + return ZSTD_initDStream_usingDDict(zds, NULL); +} + +/* ZSTD_initDStream_usingDDict() : + * ddict will just be referenced, and must outlive decompression session + * this function cannot fail */ +size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* dctx, const ZSTD_DDict* ddict) +{ + FORWARD_IF_ERROR( ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only) , ""); + FORWARD_IF_ERROR( ZSTD_DCtx_refDDict(dctx, ddict) , ""); + return ZSTD_startingInputLength(dctx->format); +} + +/* ZSTD_resetDStream() : + * return : expected size, aka ZSTD_startingInputLength(). + * this function cannot fail */ +size_t ZSTD_resetDStream(ZSTD_DStream* dctx) +{ + FORWARD_IF_ERROR(ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only), ""); + return ZSTD_startingInputLength(dctx->format); +} + + +size_t ZSTD_DCtx_refDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict) +{ + RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong, ""); + ZSTD_clearDict(dctx); + if (ddict) { + dctx->ddict = ddict; + dctx->dictUses = ZSTD_use_indefinitely; + if (dctx->refMultipleDDicts == ZSTD_rmd_refMultipleDDicts) { + if (dctx->ddictSet == NULL) { + dctx->ddictSet = ZSTD_createDDictHashSet(dctx->customMem); + if (!dctx->ddictSet) { + RETURN_ERROR(memory_allocation, "Failed to allocate memory for hash set!"); + } + } + assert(!dctx->staticSize); /* Impossible: ddictSet cannot have been allocated if static dctx */ + FORWARD_IF_ERROR(ZSTD_DDictHashSet_addDDict(dctx->ddictSet, ddict, dctx->customMem), ""); + } + } + return 0; +} + +/* ZSTD_DCtx_setMaxWindowSize() : + * note : no direct equivalence in ZSTD_DCtx_setParameter, + * since this version sets windowSize, and the other sets windowLog */ +size_t ZSTD_DCtx_setMaxWindowSize(ZSTD_DCtx* dctx, size_t maxWindowSize) +{ + ZSTD_bounds const bounds = ZSTD_dParam_getBounds(ZSTD_d_windowLogMax); + size_t const min = (size_t)1 << bounds.lowerBound; + size_t const max = (size_t)1 << bounds.upperBound; + RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong, ""); + RETURN_ERROR_IF(maxWindowSize < min, parameter_outOfBound, ""); + RETURN_ERROR_IF(maxWindowSize > max, parameter_outOfBound, ""); + dctx->maxWindowSize = maxWindowSize; + return 0; +} + +size_t ZSTD_DCtx_setFormat(ZSTD_DCtx* dctx, ZSTD_format_e format) +{ + return ZSTD_DCtx_setParameter(dctx, ZSTD_d_format, (int)format); +} + +ZSTD_bounds ZSTD_dParam_getBounds(ZSTD_dParameter dParam) +{ + ZSTD_bounds bounds = { 0, 0, 0 }; + switch(dParam) { + case ZSTD_d_windowLogMax: + bounds.lowerBound = ZSTD_WINDOWLOG_ABSOLUTEMIN; + bounds.upperBound = ZSTD_WINDOWLOG_MAX; + return bounds; + case ZSTD_d_format: + bounds.lowerBound = (int)ZSTD_f_zstd1; + bounds.upperBound = (int)ZSTD_f_zstd1_magicless; + ZSTD_STATIC_ASSERT(ZSTD_f_zstd1 < ZSTD_f_zstd1_magicless); + return bounds; + case ZSTD_d_stableOutBuffer: + bounds.lowerBound = (int)ZSTD_bm_buffered; + bounds.upperBound = (int)ZSTD_bm_stable; + return bounds; + case ZSTD_d_forceIgnoreChecksum: + bounds.lowerBound = (int)ZSTD_d_validateChecksum; + bounds.upperBound = (int)ZSTD_d_ignoreChecksum; + return bounds; + case ZSTD_d_refMultipleDDicts: + bounds.lowerBound = (int)ZSTD_rmd_refSingleDDict; + bounds.upperBound = (int)ZSTD_rmd_refMultipleDDicts; + return bounds; + default:; + } + bounds.error = ERROR(parameter_unsupported); + return bounds; +} + +/* ZSTD_dParam_withinBounds: + * @return 1 if value is within dParam bounds, + * 0 otherwise */ +static int ZSTD_dParam_withinBounds(ZSTD_dParameter dParam, int value) +{ + ZSTD_bounds const bounds = ZSTD_dParam_getBounds(dParam); + if (ZSTD_isError(bounds.error)) return 0; + if (value < bounds.lowerBound) return 0; + if (value > bounds.upperBound) return 0; + return 1; +} + +#define CHECK_DBOUNDS(p,v) { \ + RETURN_ERROR_IF(!ZSTD_dParam_withinBounds(p, v), parameter_outOfBound, ""); \ +} + +size_t ZSTD_DCtx_getParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param, int* value) +{ + switch (param) { + case ZSTD_d_windowLogMax: + *value = (int)ZSTD_highbit32((U32)dctx->maxWindowSize); + return 0; + case ZSTD_d_format: + *value = (int)dctx->format; + return 0; + case ZSTD_d_stableOutBuffer: + *value = (int)dctx->outBufferMode; + return 0; + case ZSTD_d_forceIgnoreChecksum: + *value = (int)dctx->forceIgnoreChecksum; + return 0; + case ZSTD_d_refMultipleDDicts: + *value = (int)dctx->refMultipleDDicts; + return 0; + default:; + } + RETURN_ERROR(parameter_unsupported, ""); +} + +size_t ZSTD_DCtx_setParameter(ZSTD_DCtx* dctx, ZSTD_dParameter dParam, int value) +{ + RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong, ""); + switch(dParam) { + case ZSTD_d_windowLogMax: + if (value == 0) value = ZSTD_WINDOWLOG_LIMIT_DEFAULT; + CHECK_DBOUNDS(ZSTD_d_windowLogMax, value); + dctx->maxWindowSize = ((size_t)1) << value; + return 0; + case ZSTD_d_format: + CHECK_DBOUNDS(ZSTD_d_format, value); + dctx->format = (ZSTD_format_e)value; + return 0; + case ZSTD_d_stableOutBuffer: + CHECK_DBOUNDS(ZSTD_d_stableOutBuffer, value); + dctx->outBufferMode = (ZSTD_bufferMode_e)value; + return 0; + case ZSTD_d_forceIgnoreChecksum: + CHECK_DBOUNDS(ZSTD_d_forceIgnoreChecksum, value); + dctx->forceIgnoreChecksum = (ZSTD_forceIgnoreChecksum_e)value; + return 0; + case ZSTD_d_refMultipleDDicts: + CHECK_DBOUNDS(ZSTD_d_refMultipleDDicts, value); + if (dctx->staticSize != 0) { + RETURN_ERROR(parameter_unsupported, "Static dctx does not support multiple DDicts!"); + } + dctx->refMultipleDDicts = (ZSTD_refMultipleDDicts_e)value; + return 0; + default:; + } + RETURN_ERROR(parameter_unsupported, ""); +} + +size_t ZSTD_DCtx_reset(ZSTD_DCtx* dctx, ZSTD_ResetDirective reset) +{ + if ( (reset == ZSTD_reset_session_only) + || (reset == ZSTD_reset_session_and_parameters) ) { + dctx->streamStage = zdss_init; + dctx->noForwardProgress = 0; + } + if ( (reset == ZSTD_reset_parameters) + || (reset == ZSTD_reset_session_and_parameters) ) { + RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong, ""); + ZSTD_clearDict(dctx); + ZSTD_DCtx_resetParameters(dctx); + } + return 0; +} + + +size_t ZSTD_sizeof_DStream(const ZSTD_DStream* dctx) +{ + return ZSTD_sizeof_DCtx(dctx); +} + +size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long long frameContentSize) +{ + size_t const blockSize = (size_t) MIN(windowSize, ZSTD_BLOCKSIZE_MAX); + /* space is needed to store the litbuffer after the output of a given block without stomping the extDict of a previous run, as well as to cover both windows against wildcopy*/ + unsigned long long const neededRBSize = windowSize + blockSize + ZSTD_BLOCKSIZE_MAX + (WILDCOPY_OVERLENGTH * 2); + unsigned long long const neededSize = MIN(frameContentSize, neededRBSize); + size_t const minRBSize = (size_t) neededSize; + RETURN_ERROR_IF((unsigned long long)minRBSize != neededSize, + frameParameter_windowTooLarge, ""); + return minRBSize; +} + +size_t ZSTD_estimateDStreamSize(size_t windowSize) +{ + size_t const blockSize = MIN(windowSize, ZSTD_BLOCKSIZE_MAX); + size_t const inBuffSize = blockSize; /* no block can be larger */ + size_t const outBuffSize = ZSTD_decodingBufferSize_min(windowSize, ZSTD_CONTENTSIZE_UNKNOWN); + return ZSTD_estimateDCtxSize() + inBuffSize + outBuffSize; +} + +size_t ZSTD_estimateDStreamSize_fromFrame(const void* src, size_t srcSize) +{ + U32 const windowSizeMax = 1U << ZSTD_WINDOWLOG_MAX; /* note : should be user-selectable, but requires an additional parameter (or a dctx) */ + ZSTD_frameHeader zfh; + size_t const err = ZSTD_getFrameHeader(&zfh, src, srcSize); + if (ZSTD_isError(err)) return err; + RETURN_ERROR_IF(err>0, srcSize_wrong, ""); + RETURN_ERROR_IF(zfh.windowSize > windowSizeMax, + frameParameter_windowTooLarge, ""); + return ZSTD_estimateDStreamSize((size_t)zfh.windowSize); +} + + +/* ***** Decompression ***** */ + +static int ZSTD_DCtx_isOverflow(ZSTD_DStream* zds, size_t const neededInBuffSize, size_t const neededOutBuffSize) +{ + return (zds->inBuffSize + zds->outBuffSize) >= (neededInBuffSize + neededOutBuffSize) * ZSTD_WORKSPACETOOLARGE_FACTOR; +} + +static void ZSTD_DCtx_updateOversizedDuration(ZSTD_DStream* zds, size_t const neededInBuffSize, size_t const neededOutBuffSize) +{ + if (ZSTD_DCtx_isOverflow(zds, neededInBuffSize, neededOutBuffSize)) + zds->oversizedDuration++; + else + zds->oversizedDuration = 0; +} + +static int ZSTD_DCtx_isOversizedTooLong(ZSTD_DStream* zds) +{ + return zds->oversizedDuration >= ZSTD_WORKSPACETOOLARGE_MAXDURATION; +} + +/* Checks that the output buffer hasn't changed if ZSTD_obm_stable is used. */ +static size_t ZSTD_checkOutBuffer(ZSTD_DStream const* zds, ZSTD_outBuffer const* output) +{ + ZSTD_outBuffer const expect = zds->expectedOutBuffer; + /* No requirement when ZSTD_obm_stable is not enabled. */ + if (zds->outBufferMode != ZSTD_bm_stable) + return 0; + /* Any buffer is allowed in zdss_init, this must be the same for every other call until + * the context is reset. + */ + if (zds->streamStage == zdss_init) + return 0; + /* The buffer must match our expectation exactly. */ + if (expect.dst == output->dst && expect.pos == output->pos && expect.size == output->size) + return 0; + RETURN_ERROR(dstBuffer_wrong, "ZSTD_d_stableOutBuffer enabled but output differs!"); +} + +/* Calls ZSTD_decompressContinue() with the right parameters for ZSTD_decompressStream() + * and updates the stage and the output buffer state. This call is extracted so it can be + * used both when reading directly from the ZSTD_inBuffer, and in buffered input mode. + * NOTE: You must break after calling this function since the streamStage is modified. + */ +static size_t ZSTD_decompressContinueStream( + ZSTD_DStream* zds, char** op, char* oend, + void const* src, size_t srcSize) { + int const isSkipFrame = ZSTD_isSkipFrame(zds); + if (zds->outBufferMode == ZSTD_bm_buffered) { + size_t const dstSize = isSkipFrame ? 0 : zds->outBuffSize - zds->outStart; + size_t const decodedSize = ZSTD_decompressContinue(zds, + zds->outBuff + zds->outStart, dstSize, src, srcSize); + FORWARD_IF_ERROR(decodedSize, ""); + if (!decodedSize && !isSkipFrame) { + zds->streamStage = zdss_read; + } else { + zds->outEnd = zds->outStart + decodedSize; + zds->streamStage = zdss_flush; + } + } else { + /* Write directly into the output buffer */ + size_t const dstSize = isSkipFrame ? 0 : (size_t)(oend - *op); + size_t const decodedSize = ZSTD_decompressContinue(zds, *op, dstSize, src, srcSize); + FORWARD_IF_ERROR(decodedSize, ""); + *op += decodedSize; + /* Flushing is not needed. */ + zds->streamStage = zdss_read; + assert(*op <= oend); + assert(zds->outBufferMode == ZSTD_bm_stable); + } + return 0; +} + +size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input) +{ + const char* const src = (const char*)input->src; + const char* const istart = input->pos != 0 ? src + input->pos : src; + const char* const iend = input->size != 0 ? src + input->size : src; + const char* ip = istart; + char* const dst = (char*)output->dst; + char* const ostart = output->pos != 0 ? dst + output->pos : dst; + char* const oend = output->size != 0 ? dst + output->size : dst; + char* op = ostart; + U32 someMoreWork = 1; + + DEBUGLOG(5, "ZSTD_decompressStream"); + RETURN_ERROR_IF( + input->pos > input->size, + srcSize_wrong, + "forbidden. in: pos: %u vs size: %u", + (U32)input->pos, (U32)input->size); + RETURN_ERROR_IF( + output->pos > output->size, + dstSize_tooSmall, + "forbidden. out: pos: %u vs size: %u", + (U32)output->pos, (U32)output->size); + DEBUGLOG(5, "input size : %u", (U32)(input->size - input->pos)); + FORWARD_IF_ERROR(ZSTD_checkOutBuffer(zds, output), ""); + + while (someMoreWork) { + switch(zds->streamStage) + { + case zdss_init : + DEBUGLOG(5, "stage zdss_init => transparent reset "); + zds->streamStage = zdss_loadHeader; + zds->lhSize = zds->inPos = zds->outStart = zds->outEnd = 0; +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) + zds->legacyVersion = 0; +#endif + zds->hostageByte = 0; + zds->expectedOutBuffer = *output; + ZSTD_FALLTHROUGH; + + case zdss_loadHeader : + DEBUGLOG(5, "stage zdss_loadHeader (srcSize : %u)", (U32)(iend - ip)); +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) + if (zds->legacyVersion) { + RETURN_ERROR_IF(zds->staticSize, memory_allocation, + "legacy support is incompatible with static dctx"); + { size_t const hint = ZSTD_decompressLegacyStream(zds->legacyContext, zds->legacyVersion, output, input); + if (hint==0) zds->streamStage = zdss_init; + return hint; + } } +#endif + { size_t const hSize = ZSTD_getFrameHeader_advanced(&zds->fParams, zds->headerBuffer, zds->lhSize, zds->format); + if (zds->refMultipleDDicts && zds->ddictSet) { + ZSTD_DCtx_selectFrameDDict(zds); + } + DEBUGLOG(5, "header size : %u", (U32)hSize); + if (ZSTD_isError(hSize)) { +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) + U32 const legacyVersion = ZSTD_isLegacy(istart, iend-istart); + if (legacyVersion) { + ZSTD_DDict const* const ddict = ZSTD_getDDict(zds); + const void* const dict = ddict ? ZSTD_DDict_dictContent(ddict) : NULL; + size_t const dictSize = ddict ? ZSTD_DDict_dictSize(ddict) : 0; + DEBUGLOG(5, "ZSTD_decompressStream: detected legacy version v0.%u", legacyVersion); + RETURN_ERROR_IF(zds->staticSize, memory_allocation, + "legacy support is incompatible with static dctx"); + FORWARD_IF_ERROR(ZSTD_initLegacyStream(&zds->legacyContext, + zds->previousLegacyVersion, legacyVersion, + dict, dictSize), ""); + zds->legacyVersion = zds->previousLegacyVersion = legacyVersion; + { size_t const hint = ZSTD_decompressLegacyStream(zds->legacyContext, legacyVersion, output, input); + if (hint==0) zds->streamStage = zdss_init; /* or stay in stage zdss_loadHeader */ + return hint; + } } +#endif + return hSize; /* error */ + } + if (hSize != 0) { /* need more input */ + size_t const toLoad = hSize - zds->lhSize; /* if hSize!=0, hSize > zds->lhSize */ + size_t const remainingInput = (size_t)(iend-ip); + assert(iend >= ip); + if (toLoad > remainingInput) { /* not enough input to load full header */ + if (remainingInput > 0) { + ZSTD_memcpy(zds->headerBuffer + zds->lhSize, ip, remainingInput); + zds->lhSize += remainingInput; + } + input->pos = input->size; + return (MAX((size_t)ZSTD_FRAMEHEADERSIZE_MIN(zds->format), hSize) - zds->lhSize) + ZSTD_blockHeaderSize; /* remaining header bytes + next block header */ + } + assert(ip != NULL); + ZSTD_memcpy(zds->headerBuffer + zds->lhSize, ip, toLoad); zds->lhSize = hSize; ip += toLoad; + break; + } } + + /* check for single-pass mode opportunity */ + if (zds->fParams.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN + && zds->fParams.frameType != ZSTD_skippableFrame + && (U64)(size_t)(oend-op) >= zds->fParams.frameContentSize) { + size_t const cSize = ZSTD_findFrameCompressedSize(istart, (size_t)(iend-istart)); + if (cSize <= (size_t)(iend-istart)) { + /* shortcut : using single-pass mode */ + size_t const decompressedSize = ZSTD_decompress_usingDDict(zds, op, (size_t)(oend-op), istart, cSize, ZSTD_getDDict(zds)); + if (ZSTD_isError(decompressedSize)) return decompressedSize; + DEBUGLOG(4, "shortcut to single-pass ZSTD_decompress_usingDDict()") + ip = istart + cSize; + op += decompressedSize; + zds->expected = 0; + zds->streamStage = zdss_init; + someMoreWork = 0; + break; + } } + + /* Check output buffer is large enough for ZSTD_odm_stable. */ + if (zds->outBufferMode == ZSTD_bm_stable + && zds->fParams.frameType != ZSTD_skippableFrame + && zds->fParams.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN + && (U64)(size_t)(oend-op) < zds->fParams.frameContentSize) { + RETURN_ERROR(dstSize_tooSmall, "ZSTD_obm_stable passed but ZSTD_outBuffer is too small"); + } + + /* Consume header (see ZSTDds_decodeFrameHeader) */ + DEBUGLOG(4, "Consume header"); + FORWARD_IF_ERROR(ZSTD_decompressBegin_usingDDict(zds, ZSTD_getDDict(zds)), ""); + + if ((MEM_readLE32(zds->headerBuffer) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { /* skippable frame */ + zds->expected = MEM_readLE32(zds->headerBuffer + ZSTD_FRAMEIDSIZE); + zds->stage = ZSTDds_skipFrame; + } else { + FORWARD_IF_ERROR(ZSTD_decodeFrameHeader(zds, zds->headerBuffer, zds->lhSize), ""); + zds->expected = ZSTD_blockHeaderSize; + zds->stage = ZSTDds_decodeBlockHeader; + } + + /* control buffer memory usage */ + DEBUGLOG(4, "Control max memory usage (%u KB <= max %u KB)", + (U32)(zds->fParams.windowSize >>10), + (U32)(zds->maxWindowSize >> 10) ); + zds->fParams.windowSize = MAX(zds->fParams.windowSize, 1U << ZSTD_WINDOWLOG_ABSOLUTEMIN); + RETURN_ERROR_IF(zds->fParams.windowSize > zds->maxWindowSize, + frameParameter_windowTooLarge, ""); + + /* Adapt buffer sizes to frame header instructions */ + { size_t const neededInBuffSize = MAX(zds->fParams.blockSizeMax, 4 /* frame checksum */); + size_t const neededOutBuffSize = zds->outBufferMode == ZSTD_bm_buffered + ? ZSTD_decodingBufferSize_min(zds->fParams.windowSize, zds->fParams.frameContentSize) + : 0; + + ZSTD_DCtx_updateOversizedDuration(zds, neededInBuffSize, neededOutBuffSize); + + { int const tooSmall = (zds->inBuffSize < neededInBuffSize) || (zds->outBuffSize < neededOutBuffSize); + int const tooLarge = ZSTD_DCtx_isOversizedTooLong(zds); + + if (tooSmall || tooLarge) { + size_t const bufferSize = neededInBuffSize + neededOutBuffSize; + DEBUGLOG(4, "inBuff : from %u to %u", + (U32)zds->inBuffSize, (U32)neededInBuffSize); + DEBUGLOG(4, "outBuff : from %u to %u", + (U32)zds->outBuffSize, (U32)neededOutBuffSize); + if (zds->staticSize) { /* static DCtx */ + DEBUGLOG(4, "staticSize : %u", (U32)zds->staticSize); + assert(zds->staticSize >= sizeof(ZSTD_DCtx)); /* controlled at init */ + RETURN_ERROR_IF( + bufferSize > zds->staticSize - sizeof(ZSTD_DCtx), + memory_allocation, ""); + } else { + ZSTD_customFree(zds->inBuff, zds->customMem); + zds->inBuffSize = 0; + zds->outBuffSize = 0; + zds->inBuff = (char*)ZSTD_customMalloc(bufferSize, zds->customMem); + RETURN_ERROR_IF(zds->inBuff == NULL, memory_allocation, ""); + } + zds->inBuffSize = neededInBuffSize; + zds->outBuff = zds->inBuff + zds->inBuffSize; + zds->outBuffSize = neededOutBuffSize; + } } } + zds->streamStage = zdss_read; + ZSTD_FALLTHROUGH; + + case zdss_read: + DEBUGLOG(5, "stage zdss_read"); + { size_t const neededInSize = ZSTD_nextSrcSizeToDecompressWithInputSize(zds, (size_t)(iend - ip)); + DEBUGLOG(5, "neededInSize = %u", (U32)neededInSize); + if (neededInSize==0) { /* end of frame */ + zds->streamStage = zdss_init; + someMoreWork = 0; + break; + } + if ((size_t)(iend-ip) >= neededInSize) { /* decode directly from src */ + FORWARD_IF_ERROR(ZSTD_decompressContinueStream(zds, &op, oend, ip, neededInSize), ""); + ip += neededInSize; + /* Function modifies the stage so we must break */ + break; + } } + if (ip==iend) { someMoreWork = 0; break; } /* no more input */ + zds->streamStage = zdss_load; + ZSTD_FALLTHROUGH; + + case zdss_load: + { size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zds); + size_t const toLoad = neededInSize - zds->inPos; + int const isSkipFrame = ZSTD_isSkipFrame(zds); + size_t loadedSize; + /* At this point we shouldn't be decompressing a block that we can stream. */ + assert(neededInSize == ZSTD_nextSrcSizeToDecompressWithInputSize(zds, iend - ip)); + if (isSkipFrame) { + loadedSize = MIN(toLoad, (size_t)(iend-ip)); + } else { + RETURN_ERROR_IF(toLoad > zds->inBuffSize - zds->inPos, + corruption_detected, + "should never happen"); + loadedSize = ZSTD_limitCopy(zds->inBuff + zds->inPos, toLoad, ip, (size_t)(iend-ip)); + } + ip += loadedSize; + zds->inPos += loadedSize; + if (loadedSize < toLoad) { someMoreWork = 0; break; } /* not enough input, wait for more */ + + /* decode loaded input */ + zds->inPos = 0; /* input is consumed */ + FORWARD_IF_ERROR(ZSTD_decompressContinueStream(zds, &op, oend, zds->inBuff, neededInSize), ""); + /* Function modifies the stage so we must break */ + break; + } + case zdss_flush: + { size_t const toFlushSize = zds->outEnd - zds->outStart; + size_t const flushedSize = ZSTD_limitCopy(op, (size_t)(oend-op), zds->outBuff + zds->outStart, toFlushSize); + op += flushedSize; + zds->outStart += flushedSize; + if (flushedSize == toFlushSize) { /* flush completed */ + zds->streamStage = zdss_read; + if ( (zds->outBuffSize < zds->fParams.frameContentSize) + && (zds->outStart + zds->fParams.blockSizeMax > zds->outBuffSize) ) { + DEBUGLOG(5, "restart filling outBuff from beginning (left:%i, needed:%u)", + (int)(zds->outBuffSize - zds->outStart), + (U32)zds->fParams.blockSizeMax); + zds->outStart = zds->outEnd = 0; + } + break; + } } + /* cannot complete flush */ + someMoreWork = 0; + break; + + default: + assert(0); /* impossible */ + RETURN_ERROR(GENERIC, "impossible to reach"); /* some compiler require default to do something */ + } } + + /* result */ + input->pos = (size_t)(ip - (const char*)(input->src)); + output->pos = (size_t)(op - (char*)(output->dst)); + + /* Update the expected output buffer for ZSTD_obm_stable. */ + zds->expectedOutBuffer = *output; + + if ((ip==istart) && (op==ostart)) { /* no forward progress */ + zds->noForwardProgress ++; + if (zds->noForwardProgress >= ZSTD_NO_FORWARD_PROGRESS_MAX) { + RETURN_ERROR_IF(op==oend, dstSize_tooSmall, ""); + RETURN_ERROR_IF(ip==iend, srcSize_wrong, ""); + assert(0); + } + } else { + zds->noForwardProgress = 0; + } + { size_t nextSrcSizeHint = ZSTD_nextSrcSizeToDecompress(zds); + if (!nextSrcSizeHint) { /* frame fully decoded */ + if (zds->outEnd == zds->outStart) { /* output fully flushed */ + if (zds->hostageByte) { + if (input->pos >= input->size) { + /* can't release hostage (not present) */ + zds->streamStage = zdss_read; + return 1; + } + input->pos++; /* release hostage */ + } /* zds->hostageByte */ + return 0; + } /* zds->outEnd == zds->outStart */ + if (!zds->hostageByte) { /* output not fully flushed; keep last byte as hostage; will be released when all output is flushed */ + input->pos--; /* note : pos > 0, otherwise, impossible to finish reading last block */ + zds->hostageByte=1; + } + return 1; + } /* nextSrcSizeHint==0 */ + nextSrcSizeHint += ZSTD_blockHeaderSize * (ZSTD_nextInputType(zds) == ZSTDnit_block); /* preload header of next block */ + assert(zds->inPos <= nextSrcSizeHint); + nextSrcSizeHint -= zds->inPos; /* part already loaded*/ + return nextSrcSizeHint; + } +} + +size_t ZSTD_decompressStream_simpleArgs ( + ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, size_t* dstPos, + const void* src, size_t srcSize, size_t* srcPos) +{ + ZSTD_outBuffer output = { dst, dstCapacity, *dstPos }; + ZSTD_inBuffer input = { src, srcSize, *srcPos }; + /* ZSTD_compress_generic() will check validity of dstPos and srcPos */ + size_t const cErr = ZSTD_decompressStream(dctx, &output, &input); + *dstPos = output.pos; + *srcPos = input.pos; + return cErr; +} diff --git a/src/blosc2/internal-complibs/zstd-1.5.2/decompress/zstd_decompress_block.c b/src/blosc2/internal-complibs/zstd-1.5.2/decompress/zstd_decompress_block.c new file mode 100644 index 0000000..2e44d30 --- /dev/null +++ b/src/blosc2/internal-complibs/zstd-1.5.2/decompress/zstd_decompress_block.c @@ -0,0 +1,2072 @@ +/* + * Copyright (c) Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/* zstd_decompress_block : + * this module takes care of decompressing _compressed_ block */ + +/*-******************************************************* +* Dependencies +*********************************************************/ +#include "../common/zstd_deps.h" /* ZSTD_memcpy, ZSTD_memmove, ZSTD_memset */ +#include "../common/compiler.h" /* prefetch */ +#include "../common/cpu.h" /* bmi2 */ +#include "../common/mem.h" /* low level memory routines */ +#define FSE_STATIC_LINKING_ONLY +#include "../common/fse.h" +#define HUF_STATIC_LINKING_ONLY +#include "../common/huf.h" +#include "../common/zstd_internal.h" +#include "zstd_decompress_internal.h" /* ZSTD_DCtx */ +#include "zstd_ddict.h" /* ZSTD_DDictDictContent */ +#include "zstd_decompress_block.h" + +/*_******************************************************* +* Macros +**********************************************************/ + +/* These two optional macros force the use one way or another of the two + * ZSTD_decompressSequences implementations. You can't force in both directions + * at the same time. + */ +#if defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT) && \ + defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG) +#error "Cannot force the use of the short and the long ZSTD_decompressSequences variants!" +#endif + + +/*_******************************************************* +* Memory operations +**********************************************************/ +static void ZSTD_copy4(void* dst, const void* src) { ZSTD_memcpy(dst, src, 4); } + + +/*-************************************************************* + * Block decoding + ***************************************************************/ + +/*! ZSTD_getcBlockSize() : + * Provides the size of compressed block from block header `src` */ +size_t ZSTD_getcBlockSize(const void* src, size_t srcSize, + blockProperties_t* bpPtr) +{ + RETURN_ERROR_IF(srcSize < ZSTD_blockHeaderSize, srcSize_wrong, ""); + + { U32 const cBlockHeader = MEM_readLE24(src); + U32 const cSize = cBlockHeader >> 3; + bpPtr->lastBlock = cBlockHeader & 1; + bpPtr->blockType = (blockType_e)((cBlockHeader >> 1) & 3); + bpPtr->origSize = cSize; /* only useful for RLE */ + if (bpPtr->blockType == bt_rle) return 1; + RETURN_ERROR_IF(bpPtr->blockType == bt_reserved, corruption_detected, ""); + return cSize; + } +} + +/* Allocate buffer for literals, either overlapping current dst, or split between dst and litExtraBuffer, or stored entirely within litExtraBuffer */ +static void ZSTD_allocateLiteralsBuffer(ZSTD_DCtx* dctx, void* const dst, const size_t dstCapacity, const size_t litSize, + const streaming_operation streaming, const size_t expectedWriteSize, const unsigned splitImmediately) +{ + if (streaming == not_streaming && dstCapacity > ZSTD_BLOCKSIZE_MAX + WILDCOPY_OVERLENGTH + litSize + WILDCOPY_OVERLENGTH) + { + /* room for litbuffer to fit without read faulting */ + dctx->litBuffer = (BYTE*)dst + ZSTD_BLOCKSIZE_MAX + WILDCOPY_OVERLENGTH; + dctx->litBufferEnd = dctx->litBuffer + litSize; + dctx->litBufferLocation = ZSTD_in_dst; + } + else if (litSize > ZSTD_LITBUFFEREXTRASIZE) + { + /* won't fit in litExtraBuffer, so it will be split between end of dst and extra buffer */ + if (splitImmediately) { + /* won't fit in litExtraBuffer, so it will be split between end of dst and extra buffer */ + dctx->litBuffer = (BYTE*)dst + expectedWriteSize - litSize + ZSTD_LITBUFFEREXTRASIZE - WILDCOPY_OVERLENGTH; + dctx->litBufferEnd = dctx->litBuffer + litSize - ZSTD_LITBUFFEREXTRASIZE; + } + else { + /* initially this will be stored entirely in dst during huffman decoding, it will partially shifted to litExtraBuffer after */ + dctx->litBuffer = (BYTE*)dst + expectedWriteSize - litSize; + dctx->litBufferEnd = (BYTE*)dst + expectedWriteSize; + } + dctx->litBufferLocation = ZSTD_split; + } + else + { + /* fits entirely within litExtraBuffer, so no split is necessary */ + dctx->litBuffer = dctx->litExtraBuffer; + dctx->litBufferEnd = dctx->litBuffer + litSize; + dctx->litBufferLocation = ZSTD_not_in_dst; + } +} + +/* Hidden declaration for fullbench */ +size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, + const void* src, size_t srcSize, + void* dst, size_t dstCapacity, const streaming_operation streaming); +/*! ZSTD_decodeLiteralsBlock() : + * Where it is possible to do so without being stomped by the output during decompression, the literals block will be stored + * in the dstBuffer. If there is room to do so, it will be stored in full in the excess dst space after where the current + * block will be output. Otherwise it will be stored at the end of the current dst blockspace, with a small portion being + * stored in dctx->litExtraBuffer to help keep it "ahead" of the current output write. + * + * @return : nb of bytes read from src (< srcSize ) + * note : symbol not declared but exposed for fullbench */ +size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, + const void* src, size_t srcSize, /* note : srcSize < BLOCKSIZE */ + void* dst, size_t dstCapacity, const streaming_operation streaming) +{ + DEBUGLOG(5, "ZSTD_decodeLiteralsBlock"); + RETURN_ERROR_IF(srcSize < MIN_CBLOCK_SIZE, corruption_detected, ""); + + { const BYTE* const istart = (const BYTE*) src; + symbolEncodingType_e const litEncType = (symbolEncodingType_e)(istart[0] & 3); + + switch(litEncType) + { + case set_repeat: + DEBUGLOG(5, "set_repeat flag : re-using stats from previous compressed literals block"); + RETURN_ERROR_IF(dctx->litEntropy==0, dictionary_corrupted, ""); + ZSTD_FALLTHROUGH; + + case set_compressed: + RETURN_ERROR_IF(srcSize < 5, corruption_detected, "srcSize >= MIN_CBLOCK_SIZE == 3; here we need up to 5 for case 3"); + { size_t lhSize, litSize, litCSize; + U32 singleStream=0; + U32 const lhlCode = (istart[0] >> 2) & 3; + U32 const lhc = MEM_readLE32(istart); + size_t hufSuccess; + size_t expectedWriteSize = MIN(ZSTD_BLOCKSIZE_MAX, dstCapacity); + switch(lhlCode) + { + case 0: case 1: default: /* note : default is impossible, since lhlCode into [0..3] */ + /* 2 - 2 - 10 - 10 */ + singleStream = !lhlCode; + lhSize = 3; + litSize = (lhc >> 4) & 0x3FF; + litCSize = (lhc >> 14) & 0x3FF; + break; + case 2: + /* 2 - 2 - 14 - 14 */ + lhSize = 4; + litSize = (lhc >> 4) & 0x3FFF; + litCSize = lhc >> 18; + break; + case 3: + /* 2 - 2 - 18 - 18 */ + lhSize = 5; + litSize = (lhc >> 4) & 0x3FFFF; + litCSize = (lhc >> 22) + ((size_t)istart[4] << 10); + break; + } + RETURN_ERROR_IF(litSize > 0 && dst == NULL, dstSize_tooSmall, "NULL not handled"); + RETURN_ERROR_IF(litSize > ZSTD_BLOCKSIZE_MAX, corruption_detected, ""); + RETURN_ERROR_IF(litCSize + lhSize > srcSize, corruption_detected, ""); + RETURN_ERROR_IF(expectedWriteSize < litSize , dstSize_tooSmall, ""); + ZSTD_allocateLiteralsBuffer(dctx, dst, dstCapacity, litSize, streaming, expectedWriteSize, 0); + + /* prefetch huffman table if cold */ + if (dctx->ddictIsCold && (litSize > 768 /* heuristic */)) { + PREFETCH_AREA(dctx->HUFptr, sizeof(dctx->entropy.hufTable)); + } + + if (litEncType==set_repeat) { + if (singleStream) { + hufSuccess = HUF_decompress1X_usingDTable_bmi2( + dctx->litBuffer, litSize, istart+lhSize, litCSize, + dctx->HUFptr, ZSTD_DCtx_get_bmi2(dctx)); + } else { + hufSuccess = HUF_decompress4X_usingDTable_bmi2( + dctx->litBuffer, litSize, istart+lhSize, litCSize, + dctx->HUFptr, ZSTD_DCtx_get_bmi2(dctx)); + } + } else { + if (singleStream) { +#if defined(HUF_FORCE_DECOMPRESS_X2) + hufSuccess = HUF_decompress1X_DCtx_wksp( + dctx->entropy.hufTable, dctx->litBuffer, litSize, + istart+lhSize, litCSize, dctx->workspace, + sizeof(dctx->workspace)); +#else + hufSuccess = HUF_decompress1X1_DCtx_wksp_bmi2( + dctx->entropy.hufTable, dctx->litBuffer, litSize, + istart+lhSize, litCSize, dctx->workspace, + sizeof(dctx->workspace), ZSTD_DCtx_get_bmi2(dctx)); +#endif + } else { + hufSuccess = HUF_decompress4X_hufOnly_wksp_bmi2( + dctx->entropy.hufTable, dctx->litBuffer, litSize, + istart+lhSize, litCSize, dctx->workspace, + sizeof(dctx->workspace), ZSTD_DCtx_get_bmi2(dctx)); + } + } + if (dctx->litBufferLocation == ZSTD_split) + { + ZSTD_memcpy(dctx->litExtraBuffer, dctx->litBufferEnd - ZSTD_LITBUFFEREXTRASIZE, ZSTD_LITBUFFEREXTRASIZE); + ZSTD_memmove(dctx->litBuffer + ZSTD_LITBUFFEREXTRASIZE - WILDCOPY_OVERLENGTH, dctx->litBuffer, litSize - ZSTD_LITBUFFEREXTRASIZE); + dctx->litBuffer += ZSTD_LITBUFFEREXTRASIZE - WILDCOPY_OVERLENGTH; + dctx->litBufferEnd -= WILDCOPY_OVERLENGTH; + } + + RETURN_ERROR_IF(HUF_isError(hufSuccess), corruption_detected, ""); + + dctx->litPtr = dctx->litBuffer; + dctx->litSize = litSize; + dctx->litEntropy = 1; + if (litEncType==set_compressed) dctx->HUFptr = dctx->entropy.hufTable; + return litCSize + lhSize; + } + + case set_basic: + { size_t litSize, lhSize; + U32 const lhlCode = ((istart[0]) >> 2) & 3; + size_t expectedWriteSize = MIN(ZSTD_BLOCKSIZE_MAX, dstCapacity); + switch(lhlCode) + { + case 0: case 2: default: /* note : default is impossible, since lhlCode into [0..3] */ + lhSize = 1; + litSize = istart[0] >> 3; + break; + case 1: + lhSize = 2; + litSize = MEM_readLE16(istart) >> 4; + break; + case 3: + lhSize = 3; + litSize = MEM_readLE24(istart) >> 4; + break; + } + + RETURN_ERROR_IF(litSize > 0 && dst == NULL, dstSize_tooSmall, "NULL not handled"); + RETURN_ERROR_IF(expectedWriteSize < litSize, dstSize_tooSmall, ""); + ZSTD_allocateLiteralsBuffer(dctx, dst, dstCapacity, litSize, streaming, expectedWriteSize, 1); + if (lhSize+litSize+WILDCOPY_OVERLENGTH > srcSize) { /* risk reading beyond src buffer with wildcopy */ + RETURN_ERROR_IF(litSize+lhSize > srcSize, corruption_detected, ""); + if (dctx->litBufferLocation == ZSTD_split) + { + ZSTD_memcpy(dctx->litBuffer, istart + lhSize, litSize - ZSTD_LITBUFFEREXTRASIZE); + ZSTD_memcpy(dctx->litExtraBuffer, istart + lhSize + litSize - ZSTD_LITBUFFEREXTRASIZE, ZSTD_LITBUFFEREXTRASIZE); + } + else + { + ZSTD_memcpy(dctx->litBuffer, istart + lhSize, litSize); + } + dctx->litPtr = dctx->litBuffer; + dctx->litSize = litSize; + return lhSize+litSize; + } + /* direct reference into compressed stream */ + dctx->litPtr = istart+lhSize; + dctx->litSize = litSize; + dctx->litBufferEnd = dctx->litPtr + litSize; + dctx->litBufferLocation = ZSTD_not_in_dst; + return lhSize+litSize; + } + + case set_rle: + { U32 const lhlCode = ((istart[0]) >> 2) & 3; + size_t litSize, lhSize; + size_t expectedWriteSize = MIN(ZSTD_BLOCKSIZE_MAX, dstCapacity); + switch(lhlCode) + { + case 0: case 2: default: /* note : default is impossible, since lhlCode into [0..3] */ + lhSize = 1; + litSize = istart[0] >> 3; + break; + case 1: + lhSize = 2; + litSize = MEM_readLE16(istart) >> 4; + break; + case 3: + lhSize = 3; + litSize = MEM_readLE24(istart) >> 4; + RETURN_ERROR_IF(srcSize<4, corruption_detected, "srcSize >= MIN_CBLOCK_SIZE == 3; here we need lhSize+1 = 4"); + break; + } + RETURN_ERROR_IF(litSize > 0 && dst == NULL, dstSize_tooSmall, "NULL not handled"); + RETURN_ERROR_IF(litSize > ZSTD_BLOCKSIZE_MAX, corruption_detected, ""); + RETURN_ERROR_IF(expectedWriteSize < litSize, dstSize_tooSmall, ""); + ZSTD_allocateLiteralsBuffer(dctx, dst, dstCapacity, litSize, streaming, expectedWriteSize, 1); + if (dctx->litBufferLocation == ZSTD_split) + { + ZSTD_memset(dctx->litBuffer, istart[lhSize], litSize - ZSTD_LITBUFFEREXTRASIZE); + ZSTD_memset(dctx->litExtraBuffer, istart[lhSize], ZSTD_LITBUFFEREXTRASIZE); + } + else + { + ZSTD_memset(dctx->litBuffer, istart[lhSize], litSize); + } + dctx->litPtr = dctx->litBuffer; + dctx->litSize = litSize; + return lhSize+1; + } + default: + RETURN_ERROR(corruption_detected, "impossible"); + } + } +} + +/* Default FSE distribution tables. + * These are pre-calculated FSE decoding tables using default distributions as defined in specification : + * https://github.com/facebook/zstd/blob/release/doc/zstd_compression_format.md#default-distributions + * They were generated programmatically with following method : + * - start from default distributions, present in /lib/common/zstd_internal.h + * - generate tables normally, using ZSTD_buildFSETable() + * - printout the content of tables + * - pretify output, report below, test with fuzzer to ensure it's correct */ + +/* Default FSE distribution table for Literal Lengths */ +static const ZSTD_seqSymbol LL_defaultDTable[(1<tableLog = 0; + DTableH->fastMode = 0; + + cell->nbBits = 0; + cell->nextState = 0; + assert(nbAddBits < 255); + cell->nbAdditionalBits = nbAddBits; + cell->baseValue = baseValue; +} + + +/* ZSTD_buildFSETable() : + * generate FSE decoding table for one symbol (ll, ml or off) + * cannot fail if input is valid => + * all inputs are presumed validated at this stage */ +FORCE_INLINE_TEMPLATE +void ZSTD_buildFSETable_body(ZSTD_seqSymbol* dt, + const short* normalizedCounter, unsigned maxSymbolValue, + const U32* baseValue, const U8* nbAdditionalBits, + unsigned tableLog, void* wksp, size_t wkspSize) +{ + ZSTD_seqSymbol* const tableDecode = dt+1; + U32 const maxSV1 = maxSymbolValue + 1; + U32 const tableSize = 1 << tableLog; + + U16* symbolNext = (U16*)wksp; + BYTE* spread = (BYTE*)(symbolNext + MaxSeq + 1); + U32 highThreshold = tableSize - 1; + + + /* Sanity Checks */ + assert(maxSymbolValue <= MaxSeq); + assert(tableLog <= MaxFSELog); + assert(wkspSize >= ZSTD_BUILD_FSE_TABLE_WKSP_SIZE); + (void)wkspSize; + /* Init, lay down lowprob symbols */ + { ZSTD_seqSymbol_header DTableH; + DTableH.tableLog = tableLog; + DTableH.fastMode = 1; + { S16 const largeLimit= (S16)(1 << (tableLog-1)); + U32 s; + for (s=0; s= largeLimit) DTableH.fastMode=0; + assert(normalizedCounter[s]>=0); + symbolNext[s] = (U16)normalizedCounter[s]; + } } } + ZSTD_memcpy(dt, &DTableH, sizeof(DTableH)); + } + + /* Spread symbols */ + assert(tableSize <= 512); + /* Specialized symbol spreading for the case when there are + * no low probability (-1 count) symbols. When compressing + * small blocks we avoid low probability symbols to hit this + * case, since header decoding speed matters more. + */ + if (highThreshold == tableSize - 1) { + size_t const tableMask = tableSize-1; + size_t const step = FSE_TABLESTEP(tableSize); + /* First lay down the symbols in order. + * We use a uint64_t to lay down 8 bytes at a time. This reduces branch + * misses since small blocks generally have small table logs, so nearly + * all symbols have counts <= 8. We ensure we have 8 bytes at the end of + * our buffer to handle the over-write. + */ + { + U64 const add = 0x0101010101010101ull; + size_t pos = 0; + U64 sv = 0; + U32 s; + for (s=0; s highThreshold) position = (position + step) & tableMask; /* lowprob area */ + } } + assert(position == 0); /* position must reach all cells once, otherwise normalizedCounter is incorrect */ + } + + /* Build Decoding table */ + { + U32 u; + for (u=0; u max, corruption_detected, ""); + { U32 const symbol = *(const BYTE*)src; + U32 const baseline = baseValue[symbol]; + U8 const nbBits = nbAdditionalBits[symbol]; + ZSTD_buildSeqTable_rle(DTableSpace, baseline, nbBits); + } + *DTablePtr = DTableSpace; + return 1; + case set_basic : + *DTablePtr = defaultTable; + return 0; + case set_repeat: + RETURN_ERROR_IF(!flagRepeatTable, corruption_detected, ""); + /* prefetch FSE table if used */ + if (ddictIsCold && (nbSeq > 24 /* heuristic */)) { + const void* const pStart = *DTablePtr; + size_t const pSize = sizeof(ZSTD_seqSymbol) * (SEQSYMBOL_TABLE_SIZE(maxLog)); + PREFETCH_AREA(pStart, pSize); + } + return 0; + case set_compressed : + { unsigned tableLog; + S16 norm[MaxSeq+1]; + size_t const headerSize = FSE_readNCount(norm, &max, &tableLog, src, srcSize); + RETURN_ERROR_IF(FSE_isError(headerSize), corruption_detected, ""); + RETURN_ERROR_IF(tableLog > maxLog, corruption_detected, ""); + ZSTD_buildFSETable(DTableSpace, norm, max, baseValue, nbAdditionalBits, tableLog, wksp, wkspSize, bmi2); + *DTablePtr = DTableSpace; + return headerSize; + } + default : + assert(0); + RETURN_ERROR(GENERIC, "impossible"); + } +} + +size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr, + const void* src, size_t srcSize) +{ + const BYTE* const istart = (const BYTE*)src; + const BYTE* const iend = istart + srcSize; + const BYTE* ip = istart; + int nbSeq; + DEBUGLOG(5, "ZSTD_decodeSeqHeaders"); + + /* check */ + RETURN_ERROR_IF(srcSize < MIN_SEQUENCES_SIZE, srcSize_wrong, ""); + + /* SeqHead */ + nbSeq = *ip++; + if (!nbSeq) { + *nbSeqPtr=0; + RETURN_ERROR_IF(srcSize != 1, srcSize_wrong, ""); + return 1; + } + if (nbSeq > 0x7F) { + if (nbSeq == 0xFF) { + RETURN_ERROR_IF(ip+2 > iend, srcSize_wrong, ""); + nbSeq = MEM_readLE16(ip) + LONGNBSEQ; + ip+=2; + } else { + RETURN_ERROR_IF(ip >= iend, srcSize_wrong, ""); + nbSeq = ((nbSeq-0x80)<<8) + *ip++; + } + } + *nbSeqPtr = nbSeq; + + /* FSE table descriptors */ + RETURN_ERROR_IF(ip+1 > iend, srcSize_wrong, ""); /* minimum possible size: 1 byte for symbol encoding types */ + { symbolEncodingType_e const LLtype = (symbolEncodingType_e)(*ip >> 6); + symbolEncodingType_e const OFtype = (symbolEncodingType_e)((*ip >> 4) & 3); + symbolEncodingType_e const MLtype = (symbolEncodingType_e)((*ip >> 2) & 3); + ip++; + + /* Build DTables */ + { size_t const llhSize = ZSTD_buildSeqTable(dctx->entropy.LLTable, &dctx->LLTptr, + LLtype, MaxLL, LLFSELog, + ip, iend-ip, + LL_base, LL_bits, + LL_defaultDTable, dctx->fseEntropy, + dctx->ddictIsCold, nbSeq, + dctx->workspace, sizeof(dctx->workspace), + ZSTD_DCtx_get_bmi2(dctx)); + RETURN_ERROR_IF(ZSTD_isError(llhSize), corruption_detected, "ZSTD_buildSeqTable failed"); + ip += llhSize; + } + + { size_t const ofhSize = ZSTD_buildSeqTable(dctx->entropy.OFTable, &dctx->OFTptr, + OFtype, MaxOff, OffFSELog, + ip, iend-ip, + OF_base, OF_bits, + OF_defaultDTable, dctx->fseEntropy, + dctx->ddictIsCold, nbSeq, + dctx->workspace, sizeof(dctx->workspace), + ZSTD_DCtx_get_bmi2(dctx)); + RETURN_ERROR_IF(ZSTD_isError(ofhSize), corruption_detected, "ZSTD_buildSeqTable failed"); + ip += ofhSize; + } + + { size_t const mlhSize = ZSTD_buildSeqTable(dctx->entropy.MLTable, &dctx->MLTptr, + MLtype, MaxML, MLFSELog, + ip, iend-ip, + ML_base, ML_bits, + ML_defaultDTable, dctx->fseEntropy, + dctx->ddictIsCold, nbSeq, + dctx->workspace, sizeof(dctx->workspace), + ZSTD_DCtx_get_bmi2(dctx)); + RETURN_ERROR_IF(ZSTD_isError(mlhSize), corruption_detected, "ZSTD_buildSeqTable failed"); + ip += mlhSize; + } + } + + return ip-istart; +} + + +typedef struct { + size_t litLength; + size_t matchLength; + size_t offset; +} seq_t; + +typedef struct { + size_t state; + const ZSTD_seqSymbol* table; +} ZSTD_fseState; + +typedef struct { + BIT_DStream_t DStream; + ZSTD_fseState stateLL; + ZSTD_fseState stateOffb; + ZSTD_fseState stateML; + size_t prevOffset[ZSTD_REP_NUM]; +} seqState_t; + +/*! ZSTD_overlapCopy8() : + * Copies 8 bytes from ip to op and updates op and ip where ip <= op. + * If the offset is < 8 then the offset is spread to at least 8 bytes. + * + * Precondition: *ip <= *op + * Postcondition: *op - *op >= 8 + */ +HINT_INLINE void ZSTD_overlapCopy8(BYTE** op, BYTE const** ip, size_t offset) { + assert(*ip <= *op); + if (offset < 8) { + /* close range match, overlap */ + static const U32 dec32table[] = { 0, 1, 2, 1, 4, 4, 4, 4 }; /* added */ + static const int dec64table[] = { 8, 8, 8, 7, 8, 9,10,11 }; /* subtracted */ + int const sub2 = dec64table[offset]; + (*op)[0] = (*ip)[0]; + (*op)[1] = (*ip)[1]; + (*op)[2] = (*ip)[2]; + (*op)[3] = (*ip)[3]; + *ip += dec32table[offset]; + ZSTD_copy4(*op+4, *ip); + *ip -= sub2; + } else { + ZSTD_copy8(*op, *ip); + } + *ip += 8; + *op += 8; + assert(*op - *ip >= 8); +} + +/*! ZSTD_safecopy() : + * Specialized version of memcpy() that is allowed to READ up to WILDCOPY_OVERLENGTH past the input buffer + * and write up to 16 bytes past oend_w (op >= oend_w is allowed). + * This function is only called in the uncommon case where the sequence is near the end of the block. It + * should be fast for a single long sequence, but can be slow for several short sequences. + * + * @param ovtype controls the overlap detection + * - ZSTD_no_overlap: The source and destination are guaranteed to be at least WILDCOPY_VECLEN bytes apart. + * - ZSTD_overlap_src_before_dst: The src and dst may overlap and may be any distance apart. + * The src buffer must be before the dst buffer. + */ +static void ZSTD_safecopy(BYTE* op, const BYTE* const oend_w, BYTE const* ip, ptrdiff_t length, ZSTD_overlap_e ovtype) { + ptrdiff_t const diff = op - ip; + BYTE* const oend = op + length; + + assert((ovtype == ZSTD_no_overlap && (diff <= -8 || diff >= 8 || op >= oend_w)) || + (ovtype == ZSTD_overlap_src_before_dst && diff >= 0)); + + if (length < 8) { + /* Handle short lengths. */ + while (op < oend) *op++ = *ip++; + return; + } + if (ovtype == ZSTD_overlap_src_before_dst) { + /* Copy 8 bytes and ensure the offset >= 8 when there can be overlap. */ + assert(length >= 8); + ZSTD_overlapCopy8(&op, &ip, diff); + length -= 8; + assert(op - ip >= 8); + assert(op <= oend); + } + + if (oend <= oend_w) { + /* No risk of overwrite. */ + ZSTD_wildcopy(op, ip, length, ovtype); + return; + } + if (op <= oend_w) { + /* Wildcopy until we get close to the end. */ + assert(oend > oend_w); + ZSTD_wildcopy(op, ip, oend_w - op, ovtype); + ip += oend_w - op; + op += oend_w - op; + } + /* Handle the leftovers. */ + while (op < oend) *op++ = *ip++; +} + +/* ZSTD_safecopyDstBeforeSrc(): + * This version allows overlap with dst before src, or handles the non-overlap case with dst after src + * Kept separate from more common ZSTD_safecopy case to avoid performance impact to the safecopy common case */ +static void ZSTD_safecopyDstBeforeSrc(BYTE* op, BYTE const* ip, ptrdiff_t length) { + ptrdiff_t const diff = op - ip; + BYTE* const oend = op + length; + + if (length < 8 || diff > -8) { + /* Handle short lengths, close overlaps, and dst not before src. */ + while (op < oend) *op++ = *ip++; + return; + } + + if (op <= oend - WILDCOPY_OVERLENGTH && diff < -WILDCOPY_VECLEN) { + ZSTD_wildcopy(op, ip, oend - WILDCOPY_OVERLENGTH - op, ZSTD_no_overlap); + ip += oend - WILDCOPY_OVERLENGTH - op; + op += oend - WILDCOPY_OVERLENGTH - op; + } + + /* Handle the leftovers. */ + while (op < oend) *op++ = *ip++; +} + +/* ZSTD_execSequenceEnd(): + * This version handles cases that are near the end of the output buffer. It requires + * more careful checks to make sure there is no overflow. By separating out these hard + * and unlikely cases, we can speed up the common cases. + * + * NOTE: This function needs to be fast for a single long sequence, but doesn't need + * to be optimized for many small sequences, since those fall into ZSTD_execSequence(). + */ +FORCE_NOINLINE +size_t ZSTD_execSequenceEnd(BYTE* op, + BYTE* const oend, seq_t sequence, + const BYTE** litPtr, const BYTE* const litLimit, + const BYTE* const prefixStart, const BYTE* const virtualStart, const BYTE* const dictEnd) +{ + BYTE* const oLitEnd = op + sequence.litLength; + size_t const sequenceLength = sequence.litLength + sequence.matchLength; + const BYTE* const iLitEnd = *litPtr + sequence.litLength; + const BYTE* match = oLitEnd - sequence.offset; + BYTE* const oend_w = oend - WILDCOPY_OVERLENGTH; + + /* bounds checks : careful of address space overflow in 32-bit mode */ + RETURN_ERROR_IF(sequenceLength > (size_t)(oend - op), dstSize_tooSmall, "last match must fit within dstBuffer"); + RETURN_ERROR_IF(sequence.litLength > (size_t)(litLimit - *litPtr), corruption_detected, "try to read beyond literal buffer"); + assert(op < op + sequenceLength); + assert(oLitEnd < op + sequenceLength); + + /* copy literals */ + ZSTD_safecopy(op, oend_w, *litPtr, sequence.litLength, ZSTD_no_overlap); + op = oLitEnd; + *litPtr = iLitEnd; + + /* copy Match */ + if (sequence.offset > (size_t)(oLitEnd - prefixStart)) { + /* offset beyond prefix */ + RETURN_ERROR_IF(sequence.offset > (size_t)(oLitEnd - virtualStart), corruption_detected, ""); + match = dictEnd - (prefixStart - match); + if (match + sequence.matchLength <= dictEnd) { + ZSTD_memmove(oLitEnd, match, sequence.matchLength); + return sequenceLength; + } + /* span extDict & currentPrefixSegment */ + { size_t const length1 = dictEnd - match; + ZSTD_memmove(oLitEnd, match, length1); + op = oLitEnd + length1; + sequence.matchLength -= length1; + match = prefixStart; + } + } + ZSTD_safecopy(op, oend_w, match, sequence.matchLength, ZSTD_overlap_src_before_dst); + return sequenceLength; +} + +/* ZSTD_execSequenceEndSplitLitBuffer(): + * This version is intended to be used during instances where the litBuffer is still split. It is kept separate to avoid performance impact for the good case. + */ +FORCE_NOINLINE +size_t ZSTD_execSequenceEndSplitLitBuffer(BYTE* op, + BYTE* const oend, const BYTE* const oend_w, seq_t sequence, + const BYTE** litPtr, const BYTE* const litLimit, + const BYTE* const prefixStart, const BYTE* const virtualStart, const BYTE* const dictEnd) +{ + BYTE* const oLitEnd = op + sequence.litLength; + size_t const sequenceLength = sequence.litLength + sequence.matchLength; + const BYTE* const iLitEnd = *litPtr + sequence.litLength; + const BYTE* match = oLitEnd - sequence.offset; + + + /* bounds checks : careful of address space overflow in 32-bit mode */ + RETURN_ERROR_IF(sequenceLength > (size_t)(oend - op), dstSize_tooSmall, "last match must fit within dstBuffer"); + RETURN_ERROR_IF(sequence.litLength > (size_t)(litLimit - *litPtr), corruption_detected, "try to read beyond literal buffer"); + assert(op < op + sequenceLength); + assert(oLitEnd < op + sequenceLength); + + /* copy literals */ + RETURN_ERROR_IF(op > *litPtr && op < *litPtr + sequence.litLength, dstSize_tooSmall, "output should not catch up to and overwrite literal buffer"); + ZSTD_safecopyDstBeforeSrc(op, *litPtr, sequence.litLength); + op = oLitEnd; + *litPtr = iLitEnd; + + /* copy Match */ + if (sequence.offset > (size_t)(oLitEnd - prefixStart)) { + /* offset beyond prefix */ + RETURN_ERROR_IF(sequence.offset > (size_t)(oLitEnd - virtualStart), corruption_detected, ""); + match = dictEnd - (prefixStart - match); + if (match + sequence.matchLength <= dictEnd) { + ZSTD_memmove(oLitEnd, match, sequence.matchLength); + return sequenceLength; + } + /* span extDict & currentPrefixSegment */ + { size_t const length1 = dictEnd - match; + ZSTD_memmove(oLitEnd, match, length1); + op = oLitEnd + length1; + sequence.matchLength -= length1; + match = prefixStart; + } + } + ZSTD_safecopy(op, oend_w, match, sequence.matchLength, ZSTD_overlap_src_before_dst); + return sequenceLength; +} + +HINT_INLINE +size_t ZSTD_execSequence(BYTE* op, + BYTE* const oend, seq_t sequence, + const BYTE** litPtr, const BYTE* const litLimit, + const BYTE* const prefixStart, const BYTE* const virtualStart, const BYTE* const dictEnd) +{ + BYTE* const oLitEnd = op + sequence.litLength; + size_t const sequenceLength = sequence.litLength + sequence.matchLength; + BYTE* const oMatchEnd = op + sequenceLength; /* risk : address space overflow (32-bits) */ + BYTE* const oend_w = oend - WILDCOPY_OVERLENGTH; /* risk : address space underflow on oend=NULL */ + const BYTE* const iLitEnd = *litPtr + sequence.litLength; + const BYTE* match = oLitEnd - sequence.offset; + + assert(op != NULL /* Precondition */); + assert(oend_w < oend /* No underflow */); + /* Handle edge cases in a slow path: + * - Read beyond end of literals + * - Match end is within WILDCOPY_OVERLIMIT of oend + * - 32-bit mode and the match length overflows + */ + if (UNLIKELY( + iLitEnd > litLimit || + oMatchEnd > oend_w || + (MEM_32bits() && (size_t)(oend - op) < sequenceLength + WILDCOPY_OVERLENGTH))) + return ZSTD_execSequenceEnd(op, oend, sequence, litPtr, litLimit, prefixStart, virtualStart, dictEnd); + + /* Assumptions (everything else goes into ZSTD_execSequenceEnd()) */ + assert(op <= oLitEnd /* No overflow */); + assert(oLitEnd < oMatchEnd /* Non-zero match & no overflow */); + assert(oMatchEnd <= oend /* No underflow */); + assert(iLitEnd <= litLimit /* Literal length is in bounds */); + assert(oLitEnd <= oend_w /* Can wildcopy literals */); + assert(oMatchEnd <= oend_w /* Can wildcopy matches */); + + /* Copy Literals: + * Split out litLength <= 16 since it is nearly always true. +1.6% on gcc-9. + * We likely don't need the full 32-byte wildcopy. + */ + assert(WILDCOPY_OVERLENGTH >= 16); + ZSTD_copy16(op, (*litPtr)); + if (UNLIKELY(sequence.litLength > 16)) { + ZSTD_wildcopy(op + 16, (*litPtr) + 16, sequence.litLength - 16, ZSTD_no_overlap); + } + op = oLitEnd; + *litPtr = iLitEnd; /* update for next sequence */ + + /* Copy Match */ + if (sequence.offset > (size_t)(oLitEnd - prefixStart)) { + /* offset beyond prefix -> go into extDict */ + RETURN_ERROR_IF(UNLIKELY(sequence.offset > (size_t)(oLitEnd - virtualStart)), corruption_detected, ""); + match = dictEnd + (match - prefixStart); + if (match + sequence.matchLength <= dictEnd) { + ZSTD_memmove(oLitEnd, match, sequence.matchLength); + return sequenceLength; + } + /* span extDict & currentPrefixSegment */ + { size_t const length1 = dictEnd - match; + ZSTD_memmove(oLitEnd, match, length1); + op = oLitEnd + length1; + sequence.matchLength -= length1; + match = prefixStart; + } + } + /* Match within prefix of 1 or more bytes */ + assert(op <= oMatchEnd); + assert(oMatchEnd <= oend_w); + assert(match >= prefixStart); + assert(sequence.matchLength >= 1); + + /* Nearly all offsets are >= WILDCOPY_VECLEN bytes, which means we can use wildcopy + * without overlap checking. + */ + if (LIKELY(sequence.offset >= WILDCOPY_VECLEN)) { + /* We bet on a full wildcopy for matches, since we expect matches to be + * longer than literals (in general). In silesia, ~10% of matches are longer + * than 16 bytes. + */ + ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength, ZSTD_no_overlap); + return sequenceLength; + } + assert(sequence.offset < WILDCOPY_VECLEN); + + /* Copy 8 bytes and spread the offset to be >= 8. */ + ZSTD_overlapCopy8(&op, &match, sequence.offset); + + /* If the match length is > 8 bytes, then continue with the wildcopy. */ + if (sequence.matchLength > 8) { + assert(op < oMatchEnd); + ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength - 8, ZSTD_overlap_src_before_dst); + } + return sequenceLength; +} + +HINT_INLINE +size_t ZSTD_execSequenceSplitLitBuffer(BYTE* op, + BYTE* const oend, const BYTE* const oend_w, seq_t sequence, + const BYTE** litPtr, const BYTE* const litLimit, + const BYTE* const prefixStart, const BYTE* const virtualStart, const BYTE* const dictEnd) +{ + BYTE* const oLitEnd = op + sequence.litLength; + size_t const sequenceLength = sequence.litLength + sequence.matchLength; + BYTE* const oMatchEnd = op + sequenceLength; /* risk : address space overflow (32-bits) */ + const BYTE* const iLitEnd = *litPtr + sequence.litLength; + const BYTE* match = oLitEnd - sequence.offset; + + assert(op != NULL /* Precondition */); + assert(oend_w < oend /* No underflow */); + /* Handle edge cases in a slow path: + * - Read beyond end of literals + * - Match end is within WILDCOPY_OVERLIMIT of oend + * - 32-bit mode and the match length overflows + */ + if (UNLIKELY( + iLitEnd > litLimit || + oMatchEnd > oend_w || + (MEM_32bits() && (size_t)(oend - op) < sequenceLength + WILDCOPY_OVERLENGTH))) + return ZSTD_execSequenceEndSplitLitBuffer(op, oend, oend_w, sequence, litPtr, litLimit, prefixStart, virtualStart, dictEnd); + + /* Assumptions (everything else goes into ZSTD_execSequenceEnd()) */ + assert(op <= oLitEnd /* No overflow */); + assert(oLitEnd < oMatchEnd /* Non-zero match & no overflow */); + assert(oMatchEnd <= oend /* No underflow */); + assert(iLitEnd <= litLimit /* Literal length is in bounds */); + assert(oLitEnd <= oend_w /* Can wildcopy literals */); + assert(oMatchEnd <= oend_w /* Can wildcopy matches */); + + /* Copy Literals: + * Split out litLength <= 16 since it is nearly always true. +1.6% on gcc-9. + * We likely don't need the full 32-byte wildcopy. + */ + assert(WILDCOPY_OVERLENGTH >= 16); + ZSTD_copy16(op, (*litPtr)); + if (UNLIKELY(sequence.litLength > 16)) { + ZSTD_wildcopy(op+16, (*litPtr)+16, sequence.litLength-16, ZSTD_no_overlap); + } + op = oLitEnd; + *litPtr = iLitEnd; /* update for next sequence */ + + /* Copy Match */ + if (sequence.offset > (size_t)(oLitEnd - prefixStart)) { + /* offset beyond prefix -> go into extDict */ + RETURN_ERROR_IF(UNLIKELY(sequence.offset > (size_t)(oLitEnd - virtualStart)), corruption_detected, ""); + match = dictEnd + (match - prefixStart); + if (match + sequence.matchLength <= dictEnd) { + ZSTD_memmove(oLitEnd, match, sequence.matchLength); + return sequenceLength; + } + /* span extDict & currentPrefixSegment */ + { size_t const length1 = dictEnd - match; + ZSTD_memmove(oLitEnd, match, length1); + op = oLitEnd + length1; + sequence.matchLength -= length1; + match = prefixStart; + } } + /* Match within prefix of 1 or more bytes */ + assert(op <= oMatchEnd); + assert(oMatchEnd <= oend_w); + assert(match >= prefixStart); + assert(sequence.matchLength >= 1); + + /* Nearly all offsets are >= WILDCOPY_VECLEN bytes, which means we can use wildcopy + * without overlap checking. + */ + if (LIKELY(sequence.offset >= WILDCOPY_VECLEN)) { + /* We bet on a full wildcopy for matches, since we expect matches to be + * longer than literals (in general). In silesia, ~10% of matches are longer + * than 16 bytes. + */ + ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength, ZSTD_no_overlap); + return sequenceLength; + } + assert(sequence.offset < WILDCOPY_VECLEN); + + /* Copy 8 bytes and spread the offset to be >= 8. */ + ZSTD_overlapCopy8(&op, &match, sequence.offset); + + /* If the match length is > 8 bytes, then continue with the wildcopy. */ + if (sequence.matchLength > 8) { + assert(op < oMatchEnd); + ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8, ZSTD_overlap_src_before_dst); + } + return sequenceLength; +} + + +static void +ZSTD_initFseState(ZSTD_fseState* DStatePtr, BIT_DStream_t* bitD, const ZSTD_seqSymbol* dt) +{ + const void* ptr = dt; + const ZSTD_seqSymbol_header* const DTableH = (const ZSTD_seqSymbol_header*)ptr; + DStatePtr->state = BIT_readBits(bitD, DTableH->tableLog); + DEBUGLOG(6, "ZSTD_initFseState : val=%u using %u bits", + (U32)DStatePtr->state, DTableH->tableLog); + BIT_reloadDStream(bitD); + DStatePtr->table = dt + 1; +} + +FORCE_INLINE_TEMPLATE void +ZSTD_updateFseStateWithDInfo(ZSTD_fseState* DStatePtr, BIT_DStream_t* bitD, U16 nextState, U32 nbBits) +{ + size_t const lowBits = BIT_readBits(bitD, nbBits); + DStatePtr->state = nextState + lowBits; +} + +/* We need to add at most (ZSTD_WINDOWLOG_MAX_32 - 1) bits to read the maximum + * offset bits. But we can only read at most (STREAM_ACCUMULATOR_MIN_32 - 1) + * bits before reloading. This value is the maximum number of bytes we read + * after reloading when we are decoding long offsets. + */ +#define LONG_OFFSETS_MAX_EXTRA_BITS_32 \ + (ZSTD_WINDOWLOG_MAX_32 > STREAM_ACCUMULATOR_MIN_32 \ + ? ZSTD_WINDOWLOG_MAX_32 - STREAM_ACCUMULATOR_MIN_32 \ + : 0) + +typedef enum { ZSTD_lo_isRegularOffset, ZSTD_lo_isLongOffset=1 } ZSTD_longOffset_e; + +FORCE_INLINE_TEMPLATE seq_t +ZSTD_decodeSequence(seqState_t* seqState, const ZSTD_longOffset_e longOffsets) +{ + seq_t seq; + const ZSTD_seqSymbol* const llDInfo = seqState->stateLL.table + seqState->stateLL.state; + const ZSTD_seqSymbol* const mlDInfo = seqState->stateML.table + seqState->stateML.state; + const ZSTD_seqSymbol* const ofDInfo = seqState->stateOffb.table + seqState->stateOffb.state; + seq.matchLength = mlDInfo->baseValue; + seq.litLength = llDInfo->baseValue; + { U32 const ofBase = ofDInfo->baseValue; + BYTE const llBits = llDInfo->nbAdditionalBits; + BYTE const mlBits = mlDInfo->nbAdditionalBits; + BYTE const ofBits = ofDInfo->nbAdditionalBits; + BYTE const totalBits = llBits+mlBits+ofBits; + + U16 const llNext = llDInfo->nextState; + U16 const mlNext = mlDInfo->nextState; + U16 const ofNext = ofDInfo->nextState; + U32 const llnbBits = llDInfo->nbBits; + U32 const mlnbBits = mlDInfo->nbBits; + U32 const ofnbBits = ofDInfo->nbBits; + /* + * As gcc has better branch and block analyzers, sometimes it is only + * valuable to mark likelyness for clang, it gives around 3-4% of + * performance. + */ + + /* sequence */ + { size_t offset; + #if defined(__clang__) + if (LIKELY(ofBits > 1)) { + #else + if (ofBits > 1) { + #endif + ZSTD_STATIC_ASSERT(ZSTD_lo_isLongOffset == 1); + ZSTD_STATIC_ASSERT(LONG_OFFSETS_MAX_EXTRA_BITS_32 == 5); + assert(ofBits <= MaxOff); + if (MEM_32bits() && longOffsets && (ofBits >= STREAM_ACCUMULATOR_MIN_32)) { + U32 const extraBits = ofBits - MIN(ofBits, 32 - seqState->DStream.bitsConsumed); + offset = ofBase + (BIT_readBitsFast(&seqState->DStream, ofBits - extraBits) << extraBits); + BIT_reloadDStream(&seqState->DStream); + if (extraBits) offset += BIT_readBitsFast(&seqState->DStream, extraBits); + assert(extraBits <= LONG_OFFSETS_MAX_EXTRA_BITS_32); /* to avoid another reload */ + } else { + offset = ofBase + BIT_readBitsFast(&seqState->DStream, ofBits/*>0*/); /* <= (ZSTD_WINDOWLOG_MAX-1) bits */ + if (MEM_32bits()) BIT_reloadDStream(&seqState->DStream); + } + seqState->prevOffset[2] = seqState->prevOffset[1]; + seqState->prevOffset[1] = seqState->prevOffset[0]; + seqState->prevOffset[0] = offset; + } else { + U32 const ll0 = (llDInfo->baseValue == 0); + if (LIKELY((ofBits == 0))) { + offset = seqState->prevOffset[ll0]; + seqState->prevOffset[1] = seqState->prevOffset[!ll0]; + seqState->prevOffset[0] = offset; + } else { + offset = ofBase + ll0 + BIT_readBitsFast(&seqState->DStream, 1); + { size_t temp = (offset==3) ? seqState->prevOffset[0] - 1 : seqState->prevOffset[offset]; + temp += !temp; /* 0 is not valid; input is corrupted; force offset to 1 */ + if (offset != 1) seqState->prevOffset[2] = seqState->prevOffset[1]; + seqState->prevOffset[1] = seqState->prevOffset[0]; + seqState->prevOffset[0] = offset = temp; + } } } + seq.offset = offset; + } + + #if defined(__clang__) + if (UNLIKELY(mlBits > 0)) + #else + if (mlBits > 0) + #endif + seq.matchLength += BIT_readBitsFast(&seqState->DStream, mlBits/*>0*/); + + if (MEM_32bits() && (mlBits+llBits >= STREAM_ACCUMULATOR_MIN_32-LONG_OFFSETS_MAX_EXTRA_BITS_32)) + BIT_reloadDStream(&seqState->DStream); + if (MEM_64bits() && UNLIKELY(totalBits >= STREAM_ACCUMULATOR_MIN_64-(LLFSELog+MLFSELog+OffFSELog))) + BIT_reloadDStream(&seqState->DStream); + /* Ensure there are enough bits to read the rest of data in 64-bit mode. */ + ZSTD_STATIC_ASSERT(16+LLFSELog+MLFSELog+OffFSELog < STREAM_ACCUMULATOR_MIN_64); + + #if defined(__clang__) + if (UNLIKELY(llBits > 0)) + #else + if (llBits > 0) + #endif + seq.litLength += BIT_readBitsFast(&seqState->DStream, llBits/*>0*/); + + if (MEM_32bits()) + BIT_reloadDStream(&seqState->DStream); + + DEBUGLOG(6, "seq: litL=%u, matchL=%u, offset=%u", + (U32)seq.litLength, (U32)seq.matchLength, (U32)seq.offset); + + ZSTD_updateFseStateWithDInfo(&seqState->stateLL, &seqState->DStream, llNext, llnbBits); /* <= 9 bits */ + ZSTD_updateFseStateWithDInfo(&seqState->stateML, &seqState->DStream, mlNext, mlnbBits); /* <= 9 bits */ + if (MEM_32bits()) BIT_reloadDStream(&seqState->DStream); /* <= 18 bits */ + ZSTD_updateFseStateWithDInfo(&seqState->stateOffb, &seqState->DStream, ofNext, ofnbBits); /* <= 8 bits */ + } + + return seq; +} + +#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION +MEM_STATIC int ZSTD_dictionaryIsActive(ZSTD_DCtx const* dctx, BYTE const* prefixStart, BYTE const* oLitEnd) +{ + size_t const windowSize = dctx->fParams.windowSize; + /* No dictionary used. */ + if (dctx->dictContentEndForFuzzing == NULL) return 0; + /* Dictionary is our prefix. */ + if (prefixStart == dctx->dictContentBeginForFuzzing) return 1; + /* Dictionary is not our ext-dict. */ + if (dctx->dictEnd != dctx->dictContentEndForFuzzing) return 0; + /* Dictionary is not within our window size. */ + if ((size_t)(oLitEnd - prefixStart) >= windowSize) return 0; + /* Dictionary is active. */ + return 1; +} + +MEM_STATIC void ZSTD_assertValidSequence( + ZSTD_DCtx const* dctx, + BYTE const* op, BYTE const* oend, + seq_t const seq, + BYTE const* prefixStart, BYTE const* virtualStart) +{ +#if DEBUGLEVEL >= 1 + size_t const windowSize = dctx->fParams.windowSize; + size_t const sequenceSize = seq.litLength + seq.matchLength; + BYTE const* const oLitEnd = op + seq.litLength; + DEBUGLOG(6, "Checking sequence: litL=%u matchL=%u offset=%u", + (U32)seq.litLength, (U32)seq.matchLength, (U32)seq.offset); + assert(op <= oend); + assert((size_t)(oend - op) >= sequenceSize); + assert(sequenceSize <= ZSTD_BLOCKSIZE_MAX); + if (ZSTD_dictionaryIsActive(dctx, prefixStart, oLitEnd)) { + size_t const dictSize = (size_t)((char const*)dctx->dictContentEndForFuzzing - (char const*)dctx->dictContentBeginForFuzzing); + /* Offset must be within the dictionary. */ + assert(seq.offset <= (size_t)(oLitEnd - virtualStart)); + assert(seq.offset <= windowSize + dictSize); + } else { + /* Offset must be within our window. */ + assert(seq.offset <= windowSize); + } +#else + (void)dctx, (void)op, (void)oend, (void)seq, (void)prefixStart, (void)virtualStart; +#endif +} +#endif + +#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG + + +FORCE_INLINE_TEMPLATE size_t +DONT_VECTORIZE +ZSTD_decompressSequences_bodySplitLitBuffer( ZSTD_DCtx* dctx, + void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset, + const int frame) +{ + const BYTE* ip = (const BYTE*)seqStart; + const BYTE* const iend = ip + seqSize; + BYTE* const ostart = (BYTE*)dst; + BYTE* const oend = ostart + maxDstSize; + BYTE* op = ostart; + const BYTE* litPtr = dctx->litPtr; + const BYTE* litBufferEnd = dctx->litBufferEnd; + const BYTE* const prefixStart = (const BYTE*) (dctx->prefixStart); + const BYTE* const vBase = (const BYTE*) (dctx->virtualStart); + const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd); + DEBUGLOG(5, "ZSTD_decompressSequences_bodySplitLitBuffer"); + (void)frame; + + /* Regen sequences */ + if (nbSeq) { + seqState_t seqState; + dctx->fseEntropy = 1; + { U32 i; for (i=0; ientropy.rep[i]; } + RETURN_ERROR_IF( + ERR_isError(BIT_initDStream(&seqState.DStream, ip, iend-ip)), + corruption_detected, ""); + ZSTD_initFseState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr); + ZSTD_initFseState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr); + ZSTD_initFseState(&seqState.stateML, &seqState.DStream, dctx->MLTptr); + assert(dst != NULL); + + ZSTD_STATIC_ASSERT( + BIT_DStream_unfinished < BIT_DStream_completed && + BIT_DStream_endOfBuffer < BIT_DStream_completed && + BIT_DStream_completed < BIT_DStream_overflow); + + /* decompress without overrunning litPtr begins */ + { + seq_t sequence = ZSTD_decodeSequence(&seqState, isLongOffset); + /* Align the decompression loop to 32 + 16 bytes. + * + * zstd compiled with gcc-9 on an Intel i9-9900k shows 10% decompression + * speed swings based on the alignment of the decompression loop. This + * performance swing is caused by parts of the decompression loop falling + * out of the DSB. The entire decompression loop should fit in the DSB, + * when it can't we get much worse performance. You can measure if you've + * hit the good case or the bad case with this perf command for some + * compressed file test.zst: + * + * perf stat -e cycles -e instructions -e idq.all_dsb_cycles_any_uops \ + * -e idq.all_mite_cycles_any_uops -- ./zstd -tq test.zst + * + * If you see most cycles served out of the MITE you've hit the bad case. + * If you see most cycles served out of the DSB you've hit the good case. + * If it is pretty even then you may be in an okay case. + * + * This issue has been reproduced on the following CPUs: + * - Kabylake: Macbook Pro (15-inch, 2019) 2.4 GHz Intel Core i9 + * Use Instruments->Counters to get DSB/MITE cycles. + * I never got performance swings, but I was able to + * go from the good case of mostly DSB to half of the + * cycles served from MITE. + * - Coffeelake: Intel i9-9900k + * - Coffeelake: Intel i7-9700k + * + * I haven't been able to reproduce the instability or DSB misses on any + * of the following CPUS: + * - Haswell + * - Broadwell: Intel(R) Xeon(R) CPU E5-2680 v4 @ 2.40GH + * - Skylake + * + * Alignment is done for each of the three major decompression loops: + * - ZSTD_decompressSequences_bodySplitLitBuffer - presplit section of the literal buffer + * - ZSTD_decompressSequences_bodySplitLitBuffer - postsplit section of the literal buffer + * - ZSTD_decompressSequences_body + * Alignment choices are made to minimize large swings on bad cases and influence on performance + * from changes external to this code, rather than to overoptimize on the current commit. + * + * If you are seeing performance stability this script can help test. + * It tests on 4 commits in zstd where I saw performance change. + * + * https://gist.github.com/terrelln/9889fc06a423fd5ca6e99351564473f4 + */ +#if defined(__GNUC__) && defined(__x86_64__) + __asm__(".p2align 6"); +# if __GNUC__ >= 7 + /* good for gcc-7, gcc-9, and gcc-11 */ + __asm__("nop"); + __asm__(".p2align 5"); + __asm__("nop"); + __asm__(".p2align 4"); +# if __GNUC__ == 8 || __GNUC__ == 10 + /* good for gcc-8 and gcc-10 */ + __asm__("nop"); + __asm__(".p2align 3"); +# endif +# endif +#endif + + /* Handle the initial state where litBuffer is currently split between dst and litExtraBuffer */ + for (; litPtr + sequence.litLength <= dctx->litBufferEnd; ) { + size_t const oneSeqSize = ZSTD_execSequenceSplitLitBuffer(op, oend, litPtr + sequence.litLength - WILDCOPY_OVERLENGTH, sequence, &litPtr, litBufferEnd, prefixStart, vBase, dictEnd); +#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE) + assert(!ZSTD_isError(oneSeqSize)); + if (frame) ZSTD_assertValidSequence(dctx, op, oend, sequence, prefixStart, vBase); +#endif + if (UNLIKELY(ZSTD_isError(oneSeqSize))) + return oneSeqSize; + DEBUGLOG(6, "regenerated sequence size : %u", (U32)oneSeqSize); + op += oneSeqSize; + if (UNLIKELY(!--nbSeq)) + break; + BIT_reloadDStream(&(seqState.DStream)); + sequence = ZSTD_decodeSequence(&seqState, isLongOffset); + } + + /* If there are more sequences, they will need to read literals from litExtraBuffer; copy over the remainder from dst and update litPtr and litEnd */ + if (nbSeq > 0) { + const size_t leftoverLit = dctx->litBufferEnd - litPtr; + if (leftoverLit) + { + RETURN_ERROR_IF(leftoverLit > (size_t)(oend - op), dstSize_tooSmall, "remaining lit must fit within dstBuffer"); + ZSTD_safecopyDstBeforeSrc(op, litPtr, leftoverLit); + sequence.litLength -= leftoverLit; + op += leftoverLit; + } + litPtr = dctx->litExtraBuffer; + litBufferEnd = dctx->litExtraBuffer + ZSTD_LITBUFFEREXTRASIZE; + dctx->litBufferLocation = ZSTD_not_in_dst; + { + size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litBufferEnd, prefixStart, vBase, dictEnd); +#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE) + assert(!ZSTD_isError(oneSeqSize)); + if (frame) ZSTD_assertValidSequence(dctx, op, oend, sequence, prefixStart, vBase); +#endif + if (UNLIKELY(ZSTD_isError(oneSeqSize))) + return oneSeqSize; + DEBUGLOG(6, "regenerated sequence size : %u", (U32)oneSeqSize); + op += oneSeqSize; + if (--nbSeq) + BIT_reloadDStream(&(seqState.DStream)); + } + } + } + + if (nbSeq > 0) /* there is remaining lit from extra buffer */ + { + +#if defined(__GNUC__) && defined(__x86_64__) + __asm__(".p2align 6"); + __asm__("nop"); +# if __GNUC__ != 7 + /* worse for gcc-7 better for gcc-8, gcc-9, and gcc-10 and clang */ + __asm__(".p2align 4"); + __asm__("nop"); + __asm__(".p2align 3"); +# elif __GNUC__ >= 11 + __asm__(".p2align 3"); +# else + __asm__(".p2align 5"); + __asm__("nop"); + __asm__(".p2align 3"); +# endif +#endif + + for (; ; ) { + seq_t const sequence = ZSTD_decodeSequence(&seqState, isLongOffset); + size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litBufferEnd, prefixStart, vBase, dictEnd); +#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE) + assert(!ZSTD_isError(oneSeqSize)); + if (frame) ZSTD_assertValidSequence(dctx, op, oend, sequence, prefixStart, vBase); +#endif + if (UNLIKELY(ZSTD_isError(oneSeqSize))) + return oneSeqSize; + DEBUGLOG(6, "regenerated sequence size : %u", (U32)oneSeqSize); + op += oneSeqSize; + if (UNLIKELY(!--nbSeq)) + break; + BIT_reloadDStream(&(seqState.DStream)); + } + } + + /* check if reached exact end */ + DEBUGLOG(5, "ZSTD_decompressSequences_bodySplitLitBuffer: after decode loop, remaining nbSeq : %i", nbSeq); + RETURN_ERROR_IF(nbSeq, corruption_detected, ""); + RETURN_ERROR_IF(BIT_reloadDStream(&seqState.DStream) < BIT_DStream_completed, corruption_detected, ""); + /* save reps for next block */ + { U32 i; for (i=0; ientropy.rep[i] = (U32)(seqState.prevOffset[i]); } + } + + /* last literal segment */ + if (dctx->litBufferLocation == ZSTD_split) /* split hasn't been reached yet, first get dst then copy litExtraBuffer */ + { + size_t const lastLLSize = litBufferEnd - litPtr; + RETURN_ERROR_IF(lastLLSize > (size_t)(oend - op), dstSize_tooSmall, ""); + if (op != NULL) { + ZSTD_memmove(op, litPtr, lastLLSize); + op += lastLLSize; + } + litPtr = dctx->litExtraBuffer; + litBufferEnd = dctx->litExtraBuffer + ZSTD_LITBUFFEREXTRASIZE; + dctx->litBufferLocation = ZSTD_not_in_dst; + } + { size_t const lastLLSize = litBufferEnd - litPtr; + RETURN_ERROR_IF(lastLLSize > (size_t)(oend-op), dstSize_tooSmall, ""); + if (op != NULL) { + ZSTD_memcpy(op, litPtr, lastLLSize); + op += lastLLSize; + } + } + + return op-ostart; +} + +FORCE_INLINE_TEMPLATE size_t +DONT_VECTORIZE +ZSTD_decompressSequences_body(ZSTD_DCtx* dctx, + void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset, + const int frame) +{ + const BYTE* ip = (const BYTE*)seqStart; + const BYTE* const iend = ip + seqSize; + BYTE* const ostart = (BYTE*)dst; + BYTE* const oend = dctx->litBufferLocation == ZSTD_not_in_dst ? ostart + maxDstSize : dctx->litBuffer; + BYTE* op = ostart; + const BYTE* litPtr = dctx->litPtr; + const BYTE* const litEnd = litPtr + dctx->litSize; + const BYTE* const prefixStart = (const BYTE*)(dctx->prefixStart); + const BYTE* const vBase = (const BYTE*)(dctx->virtualStart); + const BYTE* const dictEnd = (const BYTE*)(dctx->dictEnd); + DEBUGLOG(5, "ZSTD_decompressSequences_body"); + (void)frame; + + /* Regen sequences */ + if (nbSeq) { + seqState_t seqState; + dctx->fseEntropy = 1; + { U32 i; for (i = 0; i < ZSTD_REP_NUM; i++) seqState.prevOffset[i] = dctx->entropy.rep[i]; } + RETURN_ERROR_IF( + ERR_isError(BIT_initDStream(&seqState.DStream, ip, iend - ip)), + corruption_detected, ""); + ZSTD_initFseState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr); + ZSTD_initFseState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr); + ZSTD_initFseState(&seqState.stateML, &seqState.DStream, dctx->MLTptr); + assert(dst != NULL); + + ZSTD_STATIC_ASSERT( + BIT_DStream_unfinished < BIT_DStream_completed && + BIT_DStream_endOfBuffer < BIT_DStream_completed && + BIT_DStream_completed < BIT_DStream_overflow); + +#if defined(__GNUC__) && defined(__x86_64__) + __asm__(".p2align 6"); + __asm__("nop"); +# if __GNUC__ >= 7 + __asm__(".p2align 5"); + __asm__("nop"); + __asm__(".p2align 3"); +# else + __asm__(".p2align 4"); + __asm__("nop"); + __asm__(".p2align 3"); +# endif +#endif + + for ( ; ; ) { + seq_t const sequence = ZSTD_decodeSequence(&seqState, isLongOffset); + size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litEnd, prefixStart, vBase, dictEnd); +#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE) + assert(!ZSTD_isError(oneSeqSize)); + if (frame) ZSTD_assertValidSequence(dctx, op, oend, sequence, prefixStart, vBase); +#endif + if (UNLIKELY(ZSTD_isError(oneSeqSize))) + return oneSeqSize; + DEBUGLOG(6, "regenerated sequence size : %u", (U32)oneSeqSize); + op += oneSeqSize; + if (UNLIKELY(!--nbSeq)) + break; + BIT_reloadDStream(&(seqState.DStream)); + } + + /* check if reached exact end */ + DEBUGLOG(5, "ZSTD_decompressSequences_body: after decode loop, remaining nbSeq : %i", nbSeq); + RETURN_ERROR_IF(nbSeq, corruption_detected, ""); + RETURN_ERROR_IF(BIT_reloadDStream(&seqState.DStream) < BIT_DStream_completed, corruption_detected, ""); + /* save reps for next block */ + { U32 i; for (i=0; ientropy.rep[i] = (U32)(seqState.prevOffset[i]); } + } + + /* last literal segment */ + { size_t const lastLLSize = litEnd - litPtr; + RETURN_ERROR_IF(lastLLSize > (size_t)(oend-op), dstSize_tooSmall, ""); + if (op != NULL) { + ZSTD_memcpy(op, litPtr, lastLLSize); + op += lastLLSize; + } + } + + return op-ostart; +} + +static size_t +ZSTD_decompressSequences_default(ZSTD_DCtx* dctx, + void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset, + const int frame) +{ + return ZSTD_decompressSequences_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame); +} + +static size_t +ZSTD_decompressSequencesSplitLitBuffer_default(ZSTD_DCtx* dctx, + void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset, + const int frame) +{ + return ZSTD_decompressSequences_bodySplitLitBuffer(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame); +} +#endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG */ + +#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT + +FORCE_INLINE_TEMPLATE size_t +ZSTD_prefetchMatch(size_t prefetchPos, seq_t const sequence, + const BYTE* const prefixStart, const BYTE* const dictEnd) +{ + prefetchPos += sequence.litLength; + { const BYTE* const matchBase = (sequence.offset > prefetchPos) ? dictEnd : prefixStart; + const BYTE* const match = matchBase + prefetchPos - sequence.offset; /* note : this operation can overflow when seq.offset is really too large, which can only happen when input is corrupted. + * No consequence though : memory address is only used for prefetching, not for dereferencing */ + PREFETCH_L1(match); PREFETCH_L1(match+CACHELINE_SIZE); /* note : it's safe to invoke PREFETCH() on any memory address, including invalid ones */ + } + return prefetchPos + sequence.matchLength; +} + +/* This decoding function employs prefetching + * to reduce latency impact of cache misses. + * It's generally employed when block contains a significant portion of long-distance matches + * or when coupled with a "cold" dictionary */ +FORCE_INLINE_TEMPLATE size_t +ZSTD_decompressSequencesLong_body( + ZSTD_DCtx* dctx, + void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset, + const int frame) +{ + const BYTE* ip = (const BYTE*)seqStart; + const BYTE* const iend = ip + seqSize; + BYTE* const ostart = (BYTE*)dst; + BYTE* const oend = dctx->litBufferLocation == ZSTD_in_dst ? dctx->litBuffer : ostart + maxDstSize; + BYTE* op = ostart; + const BYTE* litPtr = dctx->litPtr; + const BYTE* litBufferEnd = dctx->litBufferEnd; + const BYTE* const prefixStart = (const BYTE*) (dctx->prefixStart); + const BYTE* const dictStart = (const BYTE*) (dctx->virtualStart); + const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd); + (void)frame; + + /* Regen sequences */ + if (nbSeq) { +#define STORED_SEQS 8 +#define STORED_SEQS_MASK (STORED_SEQS-1) +#define ADVANCED_SEQS STORED_SEQS + seq_t sequences[STORED_SEQS]; + int const seqAdvance = MIN(nbSeq, ADVANCED_SEQS); + seqState_t seqState; + int seqNb; + size_t prefetchPos = (size_t)(op-prefixStart); /* track position relative to prefixStart */ + + dctx->fseEntropy = 1; + { int i; for (i=0; ientropy.rep[i]; } + assert(dst != NULL); + assert(iend >= ip); + RETURN_ERROR_IF( + ERR_isError(BIT_initDStream(&seqState.DStream, ip, iend-ip)), + corruption_detected, ""); + ZSTD_initFseState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr); + ZSTD_initFseState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr); + ZSTD_initFseState(&seqState.stateML, &seqState.DStream, dctx->MLTptr); + + /* prepare in advance */ + for (seqNb=0; (BIT_reloadDStream(&seqState.DStream) <= BIT_DStream_completed) && (seqNblitBufferLocation == ZSTD_split && litPtr + sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK].litLength > dctx->litBufferEnd) + { + /* lit buffer is reaching split point, empty out the first buffer and transition to litExtraBuffer */ + const size_t leftoverLit = dctx->litBufferEnd - litPtr; + if (leftoverLit) + { + RETURN_ERROR_IF(leftoverLit > (size_t)(oend - op), dstSize_tooSmall, "remaining lit must fit within dstBuffer"); + ZSTD_safecopyDstBeforeSrc(op, litPtr, leftoverLit); + sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK].litLength -= leftoverLit; + op += leftoverLit; + } + litPtr = dctx->litExtraBuffer; + litBufferEnd = dctx->litExtraBuffer + ZSTD_LITBUFFEREXTRASIZE; + dctx->litBufferLocation = ZSTD_not_in_dst; + oneSeqSize = ZSTD_execSequence(op, oend, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK], &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd); +#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE) + assert(!ZSTD_isError(oneSeqSize)); + if (frame) ZSTD_assertValidSequence(dctx, op, oend, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK], prefixStart, dictStart); +#endif + if (ZSTD_isError(oneSeqSize)) return oneSeqSize; + + prefetchPos = ZSTD_prefetchMatch(prefetchPos, sequence, prefixStart, dictEnd); + sequences[seqNb & STORED_SEQS_MASK] = sequence; + op += oneSeqSize; + } + else + { + /* lit buffer is either wholly contained in first or second split, or not split at all*/ + oneSeqSize = dctx->litBufferLocation == ZSTD_split ? + ZSTD_execSequenceSplitLitBuffer(op, oend, litPtr + sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK].litLength - WILDCOPY_OVERLENGTH, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK], &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd) : + ZSTD_execSequence(op, oend, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK], &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd); +#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE) + assert(!ZSTD_isError(oneSeqSize)); + if (frame) ZSTD_assertValidSequence(dctx, op, oend, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK], prefixStart, dictStart); +#endif + if (ZSTD_isError(oneSeqSize)) return oneSeqSize; + + prefetchPos = ZSTD_prefetchMatch(prefetchPos, sequence, prefixStart, dictEnd); + sequences[seqNb & STORED_SEQS_MASK] = sequence; + op += oneSeqSize; + } + } + RETURN_ERROR_IF(seqNblitBufferLocation == ZSTD_split && litPtr + sequence->litLength > dctx->litBufferEnd) + { + const size_t leftoverLit = dctx->litBufferEnd - litPtr; + if (leftoverLit) + { + RETURN_ERROR_IF(leftoverLit > (size_t)(oend - op), dstSize_tooSmall, "remaining lit must fit within dstBuffer"); + ZSTD_safecopyDstBeforeSrc(op, litPtr, leftoverLit); + sequence->litLength -= leftoverLit; + op += leftoverLit; + } + litPtr = dctx->litExtraBuffer; + litBufferEnd = dctx->litExtraBuffer + ZSTD_LITBUFFEREXTRASIZE; + dctx->litBufferLocation = ZSTD_not_in_dst; + { + size_t const oneSeqSize = ZSTD_execSequence(op, oend, *sequence, &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd); +#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE) + assert(!ZSTD_isError(oneSeqSize)); + if (frame) ZSTD_assertValidSequence(dctx, op, oend, sequences[seqNb&STORED_SEQS_MASK], prefixStart, dictStart); +#endif + if (ZSTD_isError(oneSeqSize)) return oneSeqSize; + op += oneSeqSize; + } + } + else + { + size_t const oneSeqSize = dctx->litBufferLocation == ZSTD_split ? + ZSTD_execSequenceSplitLitBuffer(op, oend, litPtr + sequence->litLength - WILDCOPY_OVERLENGTH, *sequence, &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd) : + ZSTD_execSequence(op, oend, *sequence, &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd); +#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE) + assert(!ZSTD_isError(oneSeqSize)); + if (frame) ZSTD_assertValidSequence(dctx, op, oend, sequences[seqNb&STORED_SEQS_MASK], prefixStart, dictStart); +#endif + if (ZSTD_isError(oneSeqSize)) return oneSeqSize; + op += oneSeqSize; + } + } + + /* save reps for next block */ + { U32 i; for (i=0; ientropy.rep[i] = (U32)(seqState.prevOffset[i]); } + } + + /* last literal segment */ + if (dctx->litBufferLocation == ZSTD_split) /* first deplete literal buffer in dst, then copy litExtraBuffer */ + { + size_t const lastLLSize = litBufferEnd - litPtr; + RETURN_ERROR_IF(lastLLSize > (size_t)(oend - op), dstSize_tooSmall, ""); + if (op != NULL) { + ZSTD_memmove(op, litPtr, lastLLSize); + op += lastLLSize; + } + litPtr = dctx->litExtraBuffer; + litBufferEnd = dctx->litExtraBuffer + ZSTD_LITBUFFEREXTRASIZE; + } + { size_t const lastLLSize = litBufferEnd - litPtr; + RETURN_ERROR_IF(lastLLSize > (size_t)(oend-op), dstSize_tooSmall, ""); + if (op != NULL) { + ZSTD_memmove(op, litPtr, lastLLSize); + op += lastLLSize; + } + } + + return op-ostart; +} + +static size_t +ZSTD_decompressSequencesLong_default(ZSTD_DCtx* dctx, + void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset, + const int frame) +{ + return ZSTD_decompressSequencesLong_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame); +} +#endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT */ + + + +#if DYNAMIC_BMI2 + +#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG +static BMI2_TARGET_ATTRIBUTE size_t +DONT_VECTORIZE +ZSTD_decompressSequences_bmi2(ZSTD_DCtx* dctx, + void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset, + const int frame) +{ + return ZSTD_decompressSequences_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame); +} +static BMI2_TARGET_ATTRIBUTE size_t +DONT_VECTORIZE +ZSTD_decompressSequencesSplitLitBuffer_bmi2(ZSTD_DCtx* dctx, + void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset, + const int frame) +{ + return ZSTD_decompressSequences_bodySplitLitBuffer(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame); +} +#endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG */ + +#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT +static BMI2_TARGET_ATTRIBUTE size_t +ZSTD_decompressSequencesLong_bmi2(ZSTD_DCtx* dctx, + void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset, + const int frame) +{ + return ZSTD_decompressSequencesLong_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame); +} +#endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT */ + +#endif /* DYNAMIC_BMI2 */ + +typedef size_t (*ZSTD_decompressSequences_t)( + ZSTD_DCtx* dctx, + void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset, + const int frame); + +#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG +static size_t +ZSTD_decompressSequences(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset, + const int frame) +{ + DEBUGLOG(5, "ZSTD_decompressSequences"); +#if DYNAMIC_BMI2 + if (ZSTD_DCtx_get_bmi2(dctx)) { + return ZSTD_decompressSequences_bmi2(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame); + } +#endif + return ZSTD_decompressSequences_default(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame); +} +static size_t +ZSTD_decompressSequencesSplitLitBuffer(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset, + const int frame) +{ + DEBUGLOG(5, "ZSTD_decompressSequencesSplitLitBuffer"); +#if DYNAMIC_BMI2 + if (ZSTD_DCtx_get_bmi2(dctx)) { + return ZSTD_decompressSequencesSplitLitBuffer_bmi2(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame); + } +#endif + return ZSTD_decompressSequencesSplitLitBuffer_default(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame); +} +#endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG */ + + +#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT +/* ZSTD_decompressSequencesLong() : + * decompression function triggered when a minimum share of offsets is considered "long", + * aka out of cache. + * note : "long" definition seems overloaded here, sometimes meaning "wider than bitstream register", and sometimes meaning "farther than memory cache distance". + * This function will try to mitigate main memory latency through the use of prefetching */ +static size_t +ZSTD_decompressSequencesLong(ZSTD_DCtx* dctx, + void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset, + const int frame) +{ + DEBUGLOG(5, "ZSTD_decompressSequencesLong"); +#if DYNAMIC_BMI2 + if (ZSTD_DCtx_get_bmi2(dctx)) { + return ZSTD_decompressSequencesLong_bmi2(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame); + } +#endif + return ZSTD_decompressSequencesLong_default(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame); +} +#endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT */ + + + +#if !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT) && \ + !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG) +/* ZSTD_getLongOffsetsShare() : + * condition : offTable must be valid + * @return : "share" of long offsets (arbitrarily defined as > (1<<23)) + * compared to maximum possible of (1< 22) total += 1; + } + + assert(tableLog <= OffFSELog); + total <<= (OffFSELog - tableLog); /* scale to OffFSELog */ + + return total; +} +#endif + +size_t +ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, const int frame, const streaming_operation streaming) +{ /* blockType == blockCompressed */ + const BYTE* ip = (const BYTE*)src; + /* isLongOffset must be true if there are long offsets. + * Offsets are long if they are larger than 2^STREAM_ACCUMULATOR_MIN. + * We don't expect that to be the case in 64-bit mode. + * In block mode, window size is not known, so we have to be conservative. + * (note: but it could be evaluated from current-lowLimit) + */ + ZSTD_longOffset_e const isLongOffset = (ZSTD_longOffset_e)(MEM_32bits() && (!frame || (dctx->fParams.windowSize > (1ULL << STREAM_ACCUMULATOR_MIN)))); + DEBUGLOG(5, "ZSTD_decompressBlock_internal (size : %u)", (U32)srcSize); + + RETURN_ERROR_IF(srcSize >= ZSTD_BLOCKSIZE_MAX, srcSize_wrong, ""); + + /* Decode literals section */ + { size_t const litCSize = ZSTD_decodeLiteralsBlock(dctx, src, srcSize, dst, dstCapacity, streaming); + DEBUGLOG(5, "ZSTD_decodeLiteralsBlock : %u", (U32)litCSize); + if (ZSTD_isError(litCSize)) return litCSize; + ip += litCSize; + srcSize -= litCSize; + } + + /* Build Decoding Tables */ + { + /* These macros control at build-time which decompressor implementation + * we use. If neither is defined, we do some inspection and dispatch at + * runtime. + */ +#if !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT) && \ + !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG) + int usePrefetchDecoder = dctx->ddictIsCold; +#endif + int nbSeq; + size_t const seqHSize = ZSTD_decodeSeqHeaders(dctx, &nbSeq, ip, srcSize); + if (ZSTD_isError(seqHSize)) return seqHSize; + ip += seqHSize; + srcSize -= seqHSize; + + RETURN_ERROR_IF(dst == NULL && nbSeq > 0, dstSize_tooSmall, "NULL not handled"); + +#if !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT) && \ + !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG) + if ( !usePrefetchDecoder + && (!frame || (dctx->fParams.windowSize > (1<<24))) + && (nbSeq>ADVANCED_SEQS) ) { /* could probably use a larger nbSeq limit */ + U32 const shareLongOffsets = ZSTD_getLongOffsetsShare(dctx->OFTptr); + U32 const minShare = MEM_64bits() ? 7 : 20; /* heuristic values, correspond to 2.73% and 7.81% */ + usePrefetchDecoder = (shareLongOffsets >= minShare); + } +#endif + + dctx->ddictIsCold = 0; + +#if !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT) && \ + !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG) + if (usePrefetchDecoder) +#endif +#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT + return ZSTD_decompressSequencesLong(dctx, dst, dstCapacity, ip, srcSize, nbSeq, isLongOffset, frame); +#endif + +#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG + /* else */ + if (dctx->litBufferLocation == ZSTD_split) + return ZSTD_decompressSequencesSplitLitBuffer(dctx, dst, dstCapacity, ip, srcSize, nbSeq, isLongOffset, frame); + else + return ZSTD_decompressSequences(dctx, dst, dstCapacity, ip, srcSize, nbSeq, isLongOffset, frame); +#endif + } +} + + +void ZSTD_checkContinuity(ZSTD_DCtx* dctx, const void* dst, size_t dstSize) +{ + if (dst != dctx->previousDstEnd && dstSize > 0) { /* not contiguous */ + dctx->dictEnd = dctx->previousDstEnd; + dctx->virtualStart = (const char*)dst - ((const char*)(dctx->previousDstEnd) - (const char*)(dctx->prefixStart)); + dctx->prefixStart = dst; + dctx->previousDstEnd = dst; + } +} + + +size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize) +{ + size_t dSize; + ZSTD_checkContinuity(dctx, dst, dstCapacity); + dSize = ZSTD_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize, /* frame */ 0, not_streaming); + dctx->previousDstEnd = (char*)dst + dSize; + return dSize; +} diff --git a/src/blosc2/internal-complibs/zstd-1.5.2/decompress/zstd_decompress_block.h b/src/blosc2/internal-complibs/zstd-1.5.2/decompress/zstd_decompress_block.h new file mode 100644 index 0000000..c61a9d0 --- /dev/null +++ b/src/blosc2/internal-complibs/zstd-1.5.2/decompress/zstd_decompress_block.h @@ -0,0 +1,68 @@ +/* + * Copyright (c) Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + + +#ifndef ZSTD_DEC_BLOCK_H +#define ZSTD_DEC_BLOCK_H + +/*-******************************************************* + * Dependencies + *********************************************************/ +#include "../common/zstd_deps.h" /* size_t */ +#include "../zstd.h" /* DCtx, and some public functions */ +#include "../common/zstd_internal.h" /* blockProperties_t, and some public functions */ +#include "zstd_decompress_internal.h" /* ZSTD_seqSymbol */ + + +/* === Prototypes === */ + +/* note: prototypes already published within `zstd.h` : + * ZSTD_decompressBlock() + */ + +/* note: prototypes already published within `zstd_internal.h` : + * ZSTD_getcBlockSize() + * ZSTD_decodeSeqHeaders() + */ + + + /* Streaming state is used to inform allocation of the literal buffer */ +typedef enum { + not_streaming = 0, + is_streaming = 1 +} streaming_operation; + +/* ZSTD_decompressBlock_internal() : + * decompress block, starting at `src`, + * into destination buffer `dst`. + * @return : decompressed block size, + * or an error code (which can be tested using ZSTD_isError()) + */ +size_t ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, const int frame, const streaming_operation streaming); + +/* ZSTD_buildFSETable() : + * generate FSE decoding table for one symbol (ll, ml or off) + * this function must be called with valid parameters only + * (dt is large enough, normalizedCounter distribution total is a power of 2, max is within range, etc.) + * in which case it cannot fail. + * The workspace must be 4-byte aligned and at least ZSTD_BUILD_FSE_TABLE_WKSP_SIZE bytes, which is + * defined in zstd_decompress_internal.h. + * Internal use only. + */ +void ZSTD_buildFSETable(ZSTD_seqSymbol* dt, + const short* normalizedCounter, unsigned maxSymbolValue, + const U32* baseValue, const U8* nbAdditionalBits, + unsigned tableLog, void* wksp, size_t wkspSize, + int bmi2); + + +#endif /* ZSTD_DEC_BLOCK_H */ diff --git a/src/blosc2/internal-complibs/zstd-1.5.2/decompress/zstd_decompress_internal.h b/src/blosc2/internal-complibs/zstd-1.5.2/decompress/zstd_decompress_internal.h new file mode 100644 index 0000000..2b5a538 --- /dev/null +++ b/src/blosc2/internal-complibs/zstd-1.5.2/decompress/zstd_decompress_internal.h @@ -0,0 +1,236 @@ +/* + * Copyright (c) Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + + +/* zstd_decompress_internal: + * objects and definitions shared within lib/decompress modules */ + + #ifndef ZSTD_DECOMPRESS_INTERNAL_H + #define ZSTD_DECOMPRESS_INTERNAL_H + + +/*-******************************************************* + * Dependencies + *********************************************************/ +#include "../common/mem.h" /* BYTE, U16, U32 */ +#include "../common/zstd_internal.h" /* constants : MaxLL, MaxML, MaxOff, LLFSELog, etc. */ + + + +/*-******************************************************* + * Constants + *********************************************************/ +static UNUSED_ATTR const U32 LL_base[MaxLL+1] = { + 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 10, 11, 12, 13, 14, 15, + 16, 18, 20, 22, 24, 28, 32, 40, + 48, 64, 0x80, 0x100, 0x200, 0x400, 0x800, 0x1000, + 0x2000, 0x4000, 0x8000, 0x10000 }; + +static UNUSED_ATTR const U32 OF_base[MaxOff+1] = { + 0, 1, 1, 5, 0xD, 0x1D, 0x3D, 0x7D, + 0xFD, 0x1FD, 0x3FD, 0x7FD, 0xFFD, 0x1FFD, 0x3FFD, 0x7FFD, + 0xFFFD, 0x1FFFD, 0x3FFFD, 0x7FFFD, 0xFFFFD, 0x1FFFFD, 0x3FFFFD, 0x7FFFFD, + 0xFFFFFD, 0x1FFFFFD, 0x3FFFFFD, 0x7FFFFFD, 0xFFFFFFD, 0x1FFFFFFD, 0x3FFFFFFD, 0x7FFFFFFD }; + +static UNUSED_ATTR const U8 OF_bits[MaxOff+1] = { + 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, + 24, 25, 26, 27, 28, 29, 30, 31 }; + +static UNUSED_ATTR const U32 ML_base[MaxML+1] = { + 3, 4, 5, 6, 7, 8, 9, 10, + 11, 12, 13, 14, 15, 16, 17, 18, + 19, 20, 21, 22, 23, 24, 25, 26, + 27, 28, 29, 30, 31, 32, 33, 34, + 35, 37, 39, 41, 43, 47, 51, 59, + 67, 83, 99, 0x83, 0x103, 0x203, 0x403, 0x803, + 0x1003, 0x2003, 0x4003, 0x8003, 0x10003 }; + + +/*-******************************************************* + * Decompression types + *********************************************************/ + typedef struct { + U32 fastMode; + U32 tableLog; + } ZSTD_seqSymbol_header; + + typedef struct { + U16 nextState; + BYTE nbAdditionalBits; + BYTE nbBits; + U32 baseValue; + } ZSTD_seqSymbol; + + #define SEQSYMBOL_TABLE_SIZE(log) (1 + (1 << (log))) + +#define ZSTD_BUILD_FSE_TABLE_WKSP_SIZE (sizeof(S16) * (MaxSeq + 1) + (1u << MaxFSELog) + sizeof(U64)) +#define ZSTD_BUILD_FSE_TABLE_WKSP_SIZE_U32 ((ZSTD_BUILD_FSE_TABLE_WKSP_SIZE + sizeof(U32) - 1) / sizeof(U32)) + +typedef struct { + ZSTD_seqSymbol LLTable[SEQSYMBOL_TABLE_SIZE(LLFSELog)]; /* Note : Space reserved for FSE Tables */ + ZSTD_seqSymbol OFTable[SEQSYMBOL_TABLE_SIZE(OffFSELog)]; /* is also used as temporary workspace while building hufTable during DDict creation */ + ZSTD_seqSymbol MLTable[SEQSYMBOL_TABLE_SIZE(MLFSELog)]; /* and therefore must be at least HUF_DECOMPRESS_WORKSPACE_SIZE large */ + HUF_DTable hufTable[HUF_DTABLE_SIZE(HufLog)]; /* can accommodate HUF_decompress4X */ + U32 rep[ZSTD_REP_NUM]; + U32 workspace[ZSTD_BUILD_FSE_TABLE_WKSP_SIZE_U32]; +} ZSTD_entropyDTables_t; + +typedef enum { ZSTDds_getFrameHeaderSize, ZSTDds_decodeFrameHeader, + ZSTDds_decodeBlockHeader, ZSTDds_decompressBlock, + ZSTDds_decompressLastBlock, ZSTDds_checkChecksum, + ZSTDds_decodeSkippableHeader, ZSTDds_skipFrame } ZSTD_dStage; + +typedef enum { zdss_init=0, zdss_loadHeader, + zdss_read, zdss_load, zdss_flush } ZSTD_dStreamStage; + +typedef enum { + ZSTD_use_indefinitely = -1, /* Use the dictionary indefinitely */ + ZSTD_dont_use = 0, /* Do not use the dictionary (if one exists free it) */ + ZSTD_use_once = 1 /* Use the dictionary once and set to ZSTD_dont_use */ +} ZSTD_dictUses_e; + +/* Hashset for storing references to multiple ZSTD_DDict within ZSTD_DCtx */ +typedef struct { + const ZSTD_DDict** ddictPtrTable; + size_t ddictPtrTableSize; + size_t ddictPtrCount; +} ZSTD_DDictHashSet; + +#ifndef ZSTD_DECODER_INTERNAL_BUFFER +# define ZSTD_DECODER_INTERNAL_BUFFER (1 << 16) +#endif + +#define ZSTD_LBMIN 64 +#define ZSTD_LBMAX (128 << 10) + +/* extra buffer, compensates when dst is not large enough to store litBuffer */ +#define ZSTD_LITBUFFEREXTRASIZE BOUNDED(ZSTD_LBMIN, ZSTD_DECODER_INTERNAL_BUFFER, ZSTD_LBMAX) + +typedef enum { + ZSTD_not_in_dst = 0, /* Stored entirely within litExtraBuffer */ + ZSTD_in_dst = 1, /* Stored entirely within dst (in memory after current output write) */ + ZSTD_split = 2 /* Split between litExtraBuffer and dst */ +} ZSTD_litLocation_e; + +struct ZSTD_DCtx_s +{ + const ZSTD_seqSymbol* LLTptr; + const ZSTD_seqSymbol* MLTptr; + const ZSTD_seqSymbol* OFTptr; + const HUF_DTable* HUFptr; + ZSTD_entropyDTables_t entropy; + U32 workspace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32]; /* space needed when building huffman tables */ + const void* previousDstEnd; /* detect continuity */ + const void* prefixStart; /* start of current segment */ + const void* virtualStart; /* virtual start of previous segment if it was just before current one */ + const void* dictEnd; /* end of previous segment */ + size_t expected; + ZSTD_frameHeader fParams; + U64 processedCSize; + U64 decodedSize; + blockType_e bType; /* used in ZSTD_decompressContinue(), store blockType between block header decoding and block decompression stages */ + ZSTD_dStage stage; + U32 litEntropy; + U32 fseEntropy; + XXH64_state_t xxhState; + size_t headerSize; + ZSTD_format_e format; + ZSTD_forceIgnoreChecksum_e forceIgnoreChecksum; /* User specified: if == 1, will ignore checksums in compressed frame. Default == 0 */ + U32 validateChecksum; /* if == 1, will validate checksum. Is == 1 if (fParams.checksumFlag == 1) and (forceIgnoreChecksum == 0). */ + const BYTE* litPtr; + ZSTD_customMem customMem; + size_t litSize; + size_t rleSize; + size_t staticSize; +#if DYNAMIC_BMI2 != 0 + int bmi2; /* == 1 if the CPU supports BMI2 and 0 otherwise. CPU support is determined dynamically once per context lifetime. */ +#endif + + /* dictionary */ + ZSTD_DDict* ddictLocal; + const ZSTD_DDict* ddict; /* set by ZSTD_initDStream_usingDDict(), or ZSTD_DCtx_refDDict() */ + U32 dictID; + int ddictIsCold; /* if == 1 : dictionary is "new" for working context, and presumed "cold" (not in cpu cache) */ + ZSTD_dictUses_e dictUses; + ZSTD_DDictHashSet* ddictSet; /* Hash set for multiple ddicts */ + ZSTD_refMultipleDDicts_e refMultipleDDicts; /* User specified: if == 1, will allow references to multiple DDicts. Default == 0 (disabled) */ + + /* streaming */ + ZSTD_dStreamStage streamStage; + char* inBuff; + size_t inBuffSize; + size_t inPos; + size_t maxWindowSize; + char* outBuff; + size_t outBuffSize; + size_t outStart; + size_t outEnd; + size_t lhSize; +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) + void* legacyContext; + U32 previousLegacyVersion; + U32 legacyVersion; +#endif + U32 hostageByte; + int noForwardProgress; + ZSTD_bufferMode_e outBufferMode; + ZSTD_outBuffer expectedOutBuffer; + + /* workspace */ + BYTE* litBuffer; + const BYTE* litBufferEnd; + ZSTD_litLocation_e litBufferLocation; + BYTE litExtraBuffer[ZSTD_LITBUFFEREXTRASIZE + WILDCOPY_OVERLENGTH]; /* literal buffer can be split between storage within dst and within this scratch buffer */ + BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX]; + + size_t oversizedDuration; + +#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION + void const* dictContentBeginForFuzzing; + void const* dictContentEndForFuzzing; +#endif + + /* Tracing */ +#if ZSTD_TRACE + ZSTD_TraceCtx traceCtx; +#endif +}; /* typedef'd to ZSTD_DCtx within "zstd.h" */ + +MEM_STATIC int ZSTD_DCtx_get_bmi2(const struct ZSTD_DCtx_s *dctx) { +#if DYNAMIC_BMI2 != 0 + return dctx->bmi2; +#else + (void)dctx; + return 0; +#endif +} + +/*-******************************************************* + * Shared internal functions + *********************************************************/ + +/*! ZSTD_loadDEntropy() : + * dict : must point at beginning of a valid zstd dictionary. + * @return : size of dictionary header (size of magic number + dict ID + entropy tables) */ +size_t ZSTD_loadDEntropy(ZSTD_entropyDTables_t* entropy, + const void* const dict, size_t const dictSize); + +/*! ZSTD_checkContinuity() : + * check if next `dst` follows previous position, where decompression ended. + * If yes, do nothing (continue on current segment). + * If not, classify previous segment as "external dictionary", and start a new segment. + * This function cannot fail. */ +void ZSTD_checkContinuity(ZSTD_DCtx* dctx, const void* dst, size_t dstSize); + + +#endif /* ZSTD_DECOMPRESS_INTERNAL_H */ diff --git a/src/blosc2/internal-complibs/zstd-1.5.2/deprecated/zbuff.h b/src/blosc2/internal-complibs/zstd-1.5.2/deprecated/zbuff.h new file mode 100644 index 0000000..b83ea0f --- /dev/null +++ b/src/blosc2/internal-complibs/zstd-1.5.2/deprecated/zbuff.h @@ -0,0 +1,214 @@ +/* + * Copyright (c) Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/* *************************************************************** +* NOTES/WARNINGS +******************************************************************/ +/* The streaming API defined here is deprecated. + * Consider migrating towards ZSTD_compressStream() API in `zstd.h` + * See 'lib/README.md'. + *****************************************************************/ + + +#if defined (__cplusplus) +extern "C" { +#endif + +#ifndef ZSTD_BUFFERED_H_23987 +#define ZSTD_BUFFERED_H_23987 + +/* ************************************* +* Dependencies +***************************************/ +#include /* size_t */ +#include "../zstd.h" /* ZSTD_CStream, ZSTD_DStream, ZSTDLIB_API */ + + +/* *************************************************************** +* Compiler specifics +*****************************************************************/ +/* Deprecation warnings */ +/* Should these warnings be a problem, + * it is generally possible to disable them, + * typically with -Wno-deprecated-declarations for gcc + * or _CRT_SECURE_NO_WARNINGS in Visual. + * Otherwise, it's also possible to define ZBUFF_DISABLE_DEPRECATE_WARNINGS + */ +#ifdef ZBUFF_DISABLE_DEPRECATE_WARNINGS +# define ZBUFF_DEPRECATED(message) ZSTDLIB_API /* disable deprecation warnings */ +#else +# if defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */ +# define ZBUFF_DEPRECATED(message) [[deprecated(message)]] ZSTDLIB_API +# elif (defined(GNUC) && (GNUC > 4 || (GNUC == 4 && GNUC_MINOR >= 5))) || defined(__clang__) +# define ZBUFF_DEPRECATED(message) ZSTDLIB_API __attribute__((deprecated(message))) +# elif defined(__GNUC__) && (__GNUC__ >= 3) +# define ZBUFF_DEPRECATED(message) ZSTDLIB_API __attribute__((deprecated)) +# elif defined(_MSC_VER) +# define ZBUFF_DEPRECATED(message) ZSTDLIB_API __declspec(deprecated(message)) +# else +# pragma message("WARNING: You need to implement ZBUFF_DEPRECATED for this compiler") +# define ZBUFF_DEPRECATED(message) ZSTDLIB_API +# endif +#endif /* ZBUFF_DISABLE_DEPRECATE_WARNINGS */ + + +/* ************************************* +* Streaming functions +***************************************/ +/* This is the easier "buffered" streaming API, +* using an internal buffer to lift all restrictions on user-provided buffers +* which can be any size, any place, for both input and output. +* ZBUFF and ZSTD are 100% interoperable, +* frames created by one can be decoded by the other one */ + +typedef ZSTD_CStream ZBUFF_CCtx; +ZBUFF_DEPRECATED("use ZSTD_createCStream") ZBUFF_CCtx* ZBUFF_createCCtx(void); +ZBUFF_DEPRECATED("use ZSTD_freeCStream") size_t ZBUFF_freeCCtx(ZBUFF_CCtx* cctx); + +ZBUFF_DEPRECATED("use ZSTD_initCStream") size_t ZBUFF_compressInit(ZBUFF_CCtx* cctx, int compressionLevel); +ZBUFF_DEPRECATED("use ZSTD_initCStream_usingDict") size_t ZBUFF_compressInitDictionary(ZBUFF_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel); + +ZBUFF_DEPRECATED("use ZSTD_compressStream") size_t ZBUFF_compressContinue(ZBUFF_CCtx* cctx, void* dst, size_t* dstCapacityPtr, const void* src, size_t* srcSizePtr); +ZBUFF_DEPRECATED("use ZSTD_flushStream") size_t ZBUFF_compressFlush(ZBUFF_CCtx* cctx, void* dst, size_t* dstCapacityPtr); +ZBUFF_DEPRECATED("use ZSTD_endStream") size_t ZBUFF_compressEnd(ZBUFF_CCtx* cctx, void* dst, size_t* dstCapacityPtr); + +/*-************************************************* +* Streaming compression - howto +* +* A ZBUFF_CCtx object is required to track streaming operation. +* Use ZBUFF_createCCtx() and ZBUFF_freeCCtx() to create/release resources. +* ZBUFF_CCtx objects can be reused multiple times. +* +* Start by initializing ZBUF_CCtx. +* Use ZBUFF_compressInit() to start a new compression operation. +* Use ZBUFF_compressInitDictionary() for a compression which requires a dictionary. +* +* Use ZBUFF_compressContinue() repetitively to consume input stream. +* *srcSizePtr and *dstCapacityPtr can be any size. +* The function will report how many bytes were read or written within *srcSizePtr and *dstCapacityPtr. +* Note that it may not consume the entire input, in which case it's up to the caller to present again remaining data. +* The content of `dst` will be overwritten (up to *dstCapacityPtr) at each call, so save its content if it matters or change @dst . +* @return : a hint to preferred nb of bytes to use as input for next function call (it's just a hint, to improve latency) +* or an error code, which can be tested using ZBUFF_isError(). +* +* At any moment, it's possible to flush whatever data remains within buffer, using ZBUFF_compressFlush(). +* The nb of bytes written into `dst` will be reported into *dstCapacityPtr. +* Note that the function cannot output more than *dstCapacityPtr, +* therefore, some content might still be left into internal buffer if *dstCapacityPtr is too small. +* @return : nb of bytes still present into internal buffer (0 if it's empty) +* or an error code, which can be tested using ZBUFF_isError(). +* +* ZBUFF_compressEnd() instructs to finish a frame. +* It will perform a flush and write frame epilogue. +* The epilogue is required for decoders to consider a frame completed. +* Similar to ZBUFF_compressFlush(), it may not be able to output the entire internal buffer content if *dstCapacityPtr is too small. +* In which case, call again ZBUFF_compressFlush() to complete the flush. +* @return : nb of bytes still present into internal buffer (0 if it's empty) +* or an error code, which can be tested using ZBUFF_isError(). +* +* Hint : _recommended buffer_ sizes (not compulsory) : ZBUFF_recommendedCInSize() / ZBUFF_recommendedCOutSize() +* input : ZBUFF_recommendedCInSize==128 KB block size is the internal unit, use this value to reduce intermediate stages (better latency) +* output : ZBUFF_recommendedCOutSize==ZSTD_compressBound(128 KB) + 3 + 3 : ensures it's always possible to write/flush/end a full block. Skip some buffering. +* By using both, it ensures that input will be entirely consumed, and output will always contain the result, reducing intermediate buffering. +* **************************************************/ + + +typedef ZSTD_DStream ZBUFF_DCtx; +ZBUFF_DEPRECATED("use ZSTD_createDStream") ZBUFF_DCtx* ZBUFF_createDCtx(void); +ZBUFF_DEPRECATED("use ZSTD_freeDStream") size_t ZBUFF_freeDCtx(ZBUFF_DCtx* dctx); + +ZBUFF_DEPRECATED("use ZSTD_initDStream") size_t ZBUFF_decompressInit(ZBUFF_DCtx* dctx); +ZBUFF_DEPRECATED("use ZSTD_initDStream_usingDict") size_t ZBUFF_decompressInitDictionary(ZBUFF_DCtx* dctx, const void* dict, size_t dictSize); + +ZBUFF_DEPRECATED("use ZSTD_decompressStream") size_t ZBUFF_decompressContinue(ZBUFF_DCtx* dctx, + void* dst, size_t* dstCapacityPtr, + const void* src, size_t* srcSizePtr); + +/*-*************************************************************************** +* Streaming decompression howto +* +* A ZBUFF_DCtx object is required to track streaming operations. +* Use ZBUFF_createDCtx() and ZBUFF_freeDCtx() to create/release resources. +* Use ZBUFF_decompressInit() to start a new decompression operation, +* or ZBUFF_decompressInitDictionary() if decompression requires a dictionary. +* Note that ZBUFF_DCtx objects can be re-init multiple times. +* +* Use ZBUFF_decompressContinue() repetitively to consume your input. +* *srcSizePtr and *dstCapacityPtr can be any size. +* The function will report how many bytes were read or written by modifying *srcSizePtr and *dstCapacityPtr. +* Note that it may not consume the entire input, in which case it's up to the caller to present remaining input again. +* The content of `dst` will be overwritten (up to *dstCapacityPtr) at each function call, so save its content if it matters, or change `dst`. +* @return : 0 when a frame is completely decoded and fully flushed, +* 1 when there is still some data left within internal buffer to flush, +* >1 when more data is expected, with value being a suggested next input size (it's just a hint, which helps latency), +* or an error code, which can be tested using ZBUFF_isError(). +* +* Hint : recommended buffer sizes (not compulsory) : ZBUFF_recommendedDInSize() and ZBUFF_recommendedDOutSize() +* output : ZBUFF_recommendedDOutSize== 128 KB block size is the internal unit, it ensures it's always possible to write a full block when decoded. +* input : ZBUFF_recommendedDInSize == 128KB + 3; +* just follow indications from ZBUFF_decompressContinue() to minimize latency. It should always be <= 128 KB + 3 . +* *******************************************************************************/ + + +/* ************************************* +* Tool functions +***************************************/ +ZBUFF_DEPRECATED("use ZSTD_isError") unsigned ZBUFF_isError(size_t errorCode); +ZBUFF_DEPRECATED("use ZSTD_getErrorName") const char* ZBUFF_getErrorName(size_t errorCode); + +/** Functions below provide recommended buffer sizes for Compression or Decompression operations. +* These sizes are just hints, they tend to offer better latency */ +ZBUFF_DEPRECATED("use ZSTD_CStreamInSize") size_t ZBUFF_recommendedCInSize(void); +ZBUFF_DEPRECATED("use ZSTD_CStreamOutSize") size_t ZBUFF_recommendedCOutSize(void); +ZBUFF_DEPRECATED("use ZSTD_DStreamInSize") size_t ZBUFF_recommendedDInSize(void); +ZBUFF_DEPRECATED("use ZSTD_DStreamOutSize") size_t ZBUFF_recommendedDOutSize(void); + +#endif /* ZSTD_BUFFERED_H_23987 */ + + +#ifdef ZBUFF_STATIC_LINKING_ONLY +#ifndef ZBUFF_STATIC_H_30298098432 +#define ZBUFF_STATIC_H_30298098432 + +/* ==================================================================================== + * The definitions in this section are considered experimental. + * They should never be used in association with a dynamic library, as they may change in the future. + * They are provided for advanced usages. + * Use them only in association with static linking. + * ==================================================================================== */ + +/*--- Dependency ---*/ +#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_parameters, ZSTD_customMem */ +#include "../zstd.h" + + +/*--- Custom memory allocator ---*/ +/*! ZBUFF_createCCtx_advanced() : + * Create a ZBUFF compression context using external alloc and free functions */ +ZBUFF_DEPRECATED("use ZSTD_createCStream_advanced") ZBUFF_CCtx* ZBUFF_createCCtx_advanced(ZSTD_customMem customMem); + +/*! ZBUFF_createDCtx_advanced() : + * Create a ZBUFF decompression context using external alloc and free functions */ +ZBUFF_DEPRECATED("use ZSTD_createDStream_advanced") ZBUFF_DCtx* ZBUFF_createDCtx_advanced(ZSTD_customMem customMem); + + +/*--- Advanced Streaming Initialization ---*/ +ZBUFF_DEPRECATED("use ZSTD_initDStream_usingDict") size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* zbc, + const void* dict, size_t dictSize, + ZSTD_parameters params, unsigned long long pledgedSrcSize); + + +#endif /* ZBUFF_STATIC_H_30298098432 */ +#endif /* ZBUFF_STATIC_LINKING_ONLY */ + + +#if defined (__cplusplus) +} +#endif diff --git a/src/blosc2/internal-complibs/zstd-1.5.2/deprecated/zbuff_common.c b/src/blosc2/internal-complibs/zstd-1.5.2/deprecated/zbuff_common.c new file mode 100644 index 0000000..e7d01a0 --- /dev/null +++ b/src/blosc2/internal-complibs/zstd-1.5.2/deprecated/zbuff_common.c @@ -0,0 +1,26 @@ +/* + * Copyright (c) Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/*-************************************* +* Dependencies +***************************************/ +#include "../common/error_private.h" +#include "zbuff.h" + +/*-**************************************** +* ZBUFF Error Management (deprecated) +******************************************/ + +/*! ZBUFF_isError() : +* tells if a return value is an error code */ +unsigned ZBUFF_isError(size_t errorCode) { return ERR_isError(errorCode); } +/*! ZBUFF_getErrorName() : +* provides error code string from function result (useful for debugging) */ +const char* ZBUFF_getErrorName(size_t errorCode) { return ERR_getErrorName(errorCode); } diff --git a/src/blosc2/internal-complibs/zstd-1.5.2/deprecated/zbuff_compress.c b/src/blosc2/internal-complibs/zstd-1.5.2/deprecated/zbuff_compress.c new file mode 100644 index 0000000..51cf158 --- /dev/null +++ b/src/blosc2/internal-complibs/zstd-1.5.2/deprecated/zbuff_compress.c @@ -0,0 +1,167 @@ +/* + * Copyright (c) Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + + + +/* ************************************* +* Dependencies +***************************************/ +#define ZBUFF_STATIC_LINKING_ONLY +#include "zbuff.h" +#include "../common/error_private.h" + + +/*-*********************************************************** +* Streaming compression +* +* A ZBUFF_CCtx object is required to track streaming operation. +* Use ZBUFF_createCCtx() and ZBUFF_freeCCtx() to create/release resources. +* Use ZBUFF_compressInit() to start a new compression operation. +* ZBUFF_CCtx objects can be reused multiple times. +* +* Use ZBUFF_compressContinue() repetitively to consume your input. +* *srcSizePtr and *dstCapacityPtr can be any size. +* The function will report how many bytes were read or written by modifying *srcSizePtr and *dstCapacityPtr. +* Note that it may not consume the entire input, in which case it's up to the caller to call again the function with remaining input. +* The content of dst will be overwritten (up to *dstCapacityPtr) at each function call, so save its content if it matters or change dst . +* @return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to improve latency) +* or an error code, which can be tested using ZBUFF_isError(). +* +* ZBUFF_compressFlush() can be used to instruct ZBUFF to compress and output whatever remains within its buffer. +* Note that it will not output more than *dstCapacityPtr. +* Therefore, some content might still be left into its internal buffer if dst buffer is too small. +* @return : nb of bytes still present into internal buffer (0 if it's empty) +* or an error code, which can be tested using ZBUFF_isError(). +* +* ZBUFF_compressEnd() instructs to finish a frame. +* It will perform a flush and write frame epilogue. +* Similar to ZBUFF_compressFlush(), it may not be able to output the entire internal buffer content if *dstCapacityPtr is too small. +* @return : nb of bytes still present into internal buffer (0 if it's empty) +* or an error code, which can be tested using ZBUFF_isError(). +* +* Hint : recommended buffer sizes (not compulsory) +* input : ZSTD_BLOCKSIZE_MAX (128 KB), internal unit size, it improves latency to use this value. +* output : ZSTD_compressBound(ZSTD_BLOCKSIZE_MAX) + ZSTD_blockHeaderSize + ZBUFF_endFrameSize : ensures it's always possible to write/flush/end a full block at best speed. +* ***********************************************************/ + +ZBUFF_CCtx* ZBUFF_createCCtx(void) +{ + return ZSTD_createCStream(); +} + +ZBUFF_CCtx* ZBUFF_createCCtx_advanced(ZSTD_customMem customMem) +{ + return ZSTD_createCStream_advanced(customMem); +} + +size_t ZBUFF_freeCCtx(ZBUFF_CCtx* zbc) +{ + return ZSTD_freeCStream(zbc); +} + + +/* ====== Initialization ====== */ + +size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* zbc, + const void* dict, size_t dictSize, + ZSTD_parameters params, unsigned long long pledgedSrcSize) +{ + if (pledgedSrcSize==0) pledgedSrcSize = ZSTD_CONTENTSIZE_UNKNOWN; /* preserve "0 == unknown" behavior */ + FORWARD_IF_ERROR(ZSTD_CCtx_reset(zbc, ZSTD_reset_session_only), ""); + FORWARD_IF_ERROR(ZSTD_CCtx_setPledgedSrcSize(zbc, pledgedSrcSize), ""); + + FORWARD_IF_ERROR(ZSTD_checkCParams(params.cParams), ""); + FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(zbc, ZSTD_c_windowLog, params.cParams.windowLog), ""); + FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(zbc, ZSTD_c_hashLog, params.cParams.hashLog), ""); + FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(zbc, ZSTD_c_chainLog, params.cParams.chainLog), ""); + FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(zbc, ZSTD_c_searchLog, params.cParams.searchLog), ""); + FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(zbc, ZSTD_c_minMatch, params.cParams.minMatch), ""); + FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(zbc, ZSTD_c_targetLength, params.cParams.targetLength), ""); + FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(zbc, ZSTD_c_strategy, params.cParams.strategy), ""); + + FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(zbc, ZSTD_c_contentSizeFlag, params.fParams.contentSizeFlag), ""); + FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(zbc, ZSTD_c_checksumFlag, params.fParams.checksumFlag), ""); + FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(zbc, ZSTD_c_dictIDFlag, params.fParams.noDictIDFlag), ""); + + FORWARD_IF_ERROR(ZSTD_CCtx_loadDictionary(zbc, dict, dictSize), ""); + return 0; +} + +size_t ZBUFF_compressInitDictionary(ZBUFF_CCtx* zbc, const void* dict, size_t dictSize, int compressionLevel) +{ + FORWARD_IF_ERROR(ZSTD_CCtx_reset(zbc, ZSTD_reset_session_only), ""); + FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(zbc, ZSTD_c_compressionLevel, compressionLevel), ""); + FORWARD_IF_ERROR(ZSTD_CCtx_loadDictionary(zbc, dict, dictSize), ""); + return 0; +} + +size_t ZBUFF_compressInit(ZBUFF_CCtx* zbc, int compressionLevel) +{ + return ZSTD_initCStream(zbc, compressionLevel); +} + +/* ====== Compression ====== */ + + +size_t ZBUFF_compressContinue(ZBUFF_CCtx* zbc, + void* dst, size_t* dstCapacityPtr, + const void* src, size_t* srcSizePtr) +{ + size_t result; + ZSTD_outBuffer outBuff; + ZSTD_inBuffer inBuff; + outBuff.dst = dst; + outBuff.pos = 0; + outBuff.size = *dstCapacityPtr; + inBuff.src = src; + inBuff.pos = 0; + inBuff.size = *srcSizePtr; + result = ZSTD_compressStream(zbc, &outBuff, &inBuff); + *dstCapacityPtr = outBuff.pos; + *srcSizePtr = inBuff.pos; + return result; +} + + + +/* ====== Finalize ====== */ + +size_t ZBUFF_compressFlush(ZBUFF_CCtx* zbc, void* dst, size_t* dstCapacityPtr) +{ + size_t result; + ZSTD_outBuffer outBuff; + outBuff.dst = dst; + outBuff.pos = 0; + outBuff.size = *dstCapacityPtr; + result = ZSTD_flushStream(zbc, &outBuff); + *dstCapacityPtr = outBuff.pos; + return result; +} + + +size_t ZBUFF_compressEnd(ZBUFF_CCtx* zbc, void* dst, size_t* dstCapacityPtr) +{ + size_t result; + ZSTD_outBuffer outBuff; + outBuff.dst = dst; + outBuff.pos = 0; + outBuff.size = *dstCapacityPtr; + result = ZSTD_endStream(zbc, &outBuff); + *dstCapacityPtr = outBuff.pos; + return result; +} + + + +/* ************************************* +* Tool functions +***************************************/ +size_t ZBUFF_recommendedCInSize(void) { return ZSTD_CStreamInSize(); } +size_t ZBUFF_recommendedCOutSize(void) { return ZSTD_CStreamOutSize(); } diff --git a/src/blosc2/internal-complibs/zstd-1.5.2/deprecated/zbuff_decompress.c b/src/blosc2/internal-complibs/zstd-1.5.2/deprecated/zbuff_decompress.c new file mode 100644 index 0000000..d73c0f3 --- /dev/null +++ b/src/blosc2/internal-complibs/zstd-1.5.2/deprecated/zbuff_decompress.c @@ -0,0 +1,75 @@ +/* + * Copyright (c) Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + + + +/* ************************************* +* Dependencies +***************************************/ +#define ZBUFF_STATIC_LINKING_ONLY +#include "zbuff.h" + + +ZBUFF_DCtx* ZBUFF_createDCtx(void) +{ + return ZSTD_createDStream(); +} + +ZBUFF_DCtx* ZBUFF_createDCtx_advanced(ZSTD_customMem customMem) +{ + return ZSTD_createDStream_advanced(customMem); +} + +size_t ZBUFF_freeDCtx(ZBUFF_DCtx* zbd) +{ + return ZSTD_freeDStream(zbd); +} + + +/* *** Initialization *** */ + +size_t ZBUFF_decompressInitDictionary(ZBUFF_DCtx* zbd, const void* dict, size_t dictSize) +{ + return ZSTD_initDStream_usingDict(zbd, dict, dictSize); +} + +size_t ZBUFF_decompressInit(ZBUFF_DCtx* zbd) +{ + return ZSTD_initDStream(zbd); +} + + +/* *** Decompression *** */ + +size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbd, + void* dst, size_t* dstCapacityPtr, + const void* src, size_t* srcSizePtr) +{ + ZSTD_outBuffer outBuff; + ZSTD_inBuffer inBuff; + size_t result; + outBuff.dst = dst; + outBuff.pos = 0; + outBuff.size = *dstCapacityPtr; + inBuff.src = src; + inBuff.pos = 0; + inBuff.size = *srcSizePtr; + result = ZSTD_decompressStream(zbd, &outBuff, &inBuff); + *dstCapacityPtr = outBuff.pos; + *srcSizePtr = inBuff.pos; + return result; +} + + +/* ************************************* +* Tool functions +***************************************/ +size_t ZBUFF_recommendedDInSize(void) { return ZSTD_DStreamInSize(); } +size_t ZBUFF_recommendedDOutSize(void) { return ZSTD_DStreamOutSize(); } diff --git a/src/blosc2/internal-complibs/zstd-1.5.2/dictBuilder/cover.c b/src/blosc2/internal-complibs/zstd-1.5.2/dictBuilder/cover.c new file mode 100644 index 0000000..028802a --- /dev/null +++ b/src/blosc2/internal-complibs/zstd-1.5.2/dictBuilder/cover.c @@ -0,0 +1,1253 @@ +/* + * Copyright (c) Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/* ***************************************************************************** + * Constructs a dictionary using a heuristic based on the following paper: + * + * Liao, Petri, Moffat, Wirth + * Effective Construction of Relative Lempel-Ziv Dictionaries + * Published in WWW 2016. + * + * Adapted from code originally written by @ot (Giuseppe Ottaviano). + ******************************************************************************/ + +/*-************************************* +* Dependencies +***************************************/ +#include /* fprintf */ +#include /* malloc, free, qsort */ +#include /* memset */ +#include /* clock */ + +#ifndef ZDICT_STATIC_LINKING_ONLY +# define ZDICT_STATIC_LINKING_ONLY +#endif + +#include "../common/mem.h" /* read */ +#include "../common/pool.h" +#include "../common/threading.h" +#include "../common/zstd_internal.h" /* includes zstd.h */ +#include "../zdict.h" +#include "cover.h" + +/*-************************************* +* Constants +***************************************/ +/** +* There are 32bit indexes used to ref samples, so limit samples size to 4GB +* on 64bit builds. +* For 32bit builds we choose 1 GB. +* Most 32bit platforms have 2GB user-mode addressable space and we allocate a large +* contiguous buffer, so 1GB is already a high limit. +*/ +#define COVER_MAX_SAMPLES_SIZE (sizeof(size_t) == 8 ? ((unsigned)-1) : ((unsigned)1 GB)) +#define COVER_DEFAULT_SPLITPOINT 1.0 + +/*-************************************* +* Console display +***************************************/ +#ifndef LOCALDISPLAYLEVEL +static int g_displayLevel = 0; +#endif +#undef DISPLAY +#define DISPLAY(...) \ + { \ + fprintf(stderr, __VA_ARGS__); \ + fflush(stderr); \ + } +#undef LOCALDISPLAYLEVEL +#define LOCALDISPLAYLEVEL(displayLevel, l, ...) \ + if (displayLevel >= l) { \ + DISPLAY(__VA_ARGS__); \ + } /* 0 : no display; 1: errors; 2: default; 3: details; 4: debug */ +#undef DISPLAYLEVEL +#define DISPLAYLEVEL(l, ...) LOCALDISPLAYLEVEL(g_displayLevel, l, __VA_ARGS__) + +#ifndef LOCALDISPLAYUPDATE +static const clock_t g_refreshRate = CLOCKS_PER_SEC * 15 / 100; +static clock_t g_time = 0; +#endif +#undef LOCALDISPLAYUPDATE +#define LOCALDISPLAYUPDATE(displayLevel, l, ...) \ + if (displayLevel >= l) { \ + if ((clock() - g_time > g_refreshRate) || (displayLevel >= 4)) { \ + g_time = clock(); \ + DISPLAY(__VA_ARGS__); \ + } \ + } +#undef DISPLAYUPDATE +#define DISPLAYUPDATE(l, ...) LOCALDISPLAYUPDATE(g_displayLevel, l, __VA_ARGS__) + +/*-************************************* +* Hash table +*************************************** +* A small specialized hash map for storing activeDmers. +* The map does not resize, so if it becomes full it will loop forever. +* Thus, the map must be large enough to store every value. +* The map implements linear probing and keeps its load less than 0.5. +*/ + +#define MAP_EMPTY_VALUE ((U32)-1) +typedef struct COVER_map_pair_t_s { + U32 key; + U32 value; +} COVER_map_pair_t; + +typedef struct COVER_map_s { + COVER_map_pair_t *data; + U32 sizeLog; + U32 size; + U32 sizeMask; +} COVER_map_t; + +/** + * Clear the map. + */ +static void COVER_map_clear(COVER_map_t *map) { + memset(map->data, MAP_EMPTY_VALUE, map->size * sizeof(COVER_map_pair_t)); +} + +/** + * Initializes a map of the given size. + * Returns 1 on success and 0 on failure. + * The map must be destroyed with COVER_map_destroy(). + * The map is only guaranteed to be large enough to hold size elements. + */ +static int COVER_map_init(COVER_map_t *map, U32 size) { + map->sizeLog = ZSTD_highbit32(size) + 2; + map->size = (U32)1 << map->sizeLog; + map->sizeMask = map->size - 1; + map->data = (COVER_map_pair_t *)malloc(map->size * sizeof(COVER_map_pair_t)); + if (!map->data) { + map->sizeLog = 0; + map->size = 0; + return 0; + } + COVER_map_clear(map); + return 1; +} + +/** + * Internal hash function + */ +static const U32 COVER_prime4bytes = 2654435761U; +static U32 COVER_map_hash(COVER_map_t *map, U32 key) { + return (key * COVER_prime4bytes) >> (32 - map->sizeLog); +} + +/** + * Helper function that returns the index that a key should be placed into. + */ +static U32 COVER_map_index(COVER_map_t *map, U32 key) { + const U32 hash = COVER_map_hash(map, key); + U32 i; + for (i = hash;; i = (i + 1) & map->sizeMask) { + COVER_map_pair_t *pos = &map->data[i]; + if (pos->value == MAP_EMPTY_VALUE) { + return i; + } + if (pos->key == key) { + return i; + } + } +} + +/** + * Returns the pointer to the value for key. + * If key is not in the map, it is inserted and the value is set to 0. + * The map must not be full. + */ +static U32 *COVER_map_at(COVER_map_t *map, U32 key) { + COVER_map_pair_t *pos = &map->data[COVER_map_index(map, key)]; + if (pos->value == MAP_EMPTY_VALUE) { + pos->key = key; + pos->value = 0; + } + return &pos->value; +} + +/** + * Deletes key from the map if present. + */ +static void COVER_map_remove(COVER_map_t *map, U32 key) { + U32 i = COVER_map_index(map, key); + COVER_map_pair_t *del = &map->data[i]; + U32 shift = 1; + if (del->value == MAP_EMPTY_VALUE) { + return; + } + for (i = (i + 1) & map->sizeMask;; i = (i + 1) & map->sizeMask) { + COVER_map_pair_t *const pos = &map->data[i]; + /* If the position is empty we are done */ + if (pos->value == MAP_EMPTY_VALUE) { + del->value = MAP_EMPTY_VALUE; + return; + } + /* If pos can be moved to del do so */ + if (((i - COVER_map_hash(map, pos->key)) & map->sizeMask) >= shift) { + del->key = pos->key; + del->value = pos->value; + del = pos; + shift = 1; + } else { + ++shift; + } + } +} + +/** + * Destroys a map that is inited with COVER_map_init(). + */ +static void COVER_map_destroy(COVER_map_t *map) { + if (map->data) { + free(map->data); + } + map->data = NULL; + map->size = 0; +} + +/*-************************************* +* Context +***************************************/ + +typedef struct { + const BYTE *samples; + size_t *offsets; + const size_t *samplesSizes; + size_t nbSamples; + size_t nbTrainSamples; + size_t nbTestSamples; + U32 *suffix; + size_t suffixSize; + U32 *freqs; + U32 *dmerAt; + unsigned d; +} COVER_ctx_t; + +/* We need a global context for qsort... */ +static COVER_ctx_t *g_coverCtx = NULL; + +/*-************************************* +* Helper functions +***************************************/ + +/** + * Returns the sum of the sample sizes. + */ +size_t COVER_sum(const size_t *samplesSizes, unsigned nbSamples) { + size_t sum = 0; + unsigned i; + for (i = 0; i < nbSamples; ++i) { + sum += samplesSizes[i]; + } + return sum; +} + +/** + * Returns -1 if the dmer at lp is less than the dmer at rp. + * Return 0 if the dmers at lp and rp are equal. + * Returns 1 if the dmer at lp is greater than the dmer at rp. + */ +static int COVER_cmp(COVER_ctx_t *ctx, const void *lp, const void *rp) { + U32 const lhs = *(U32 const *)lp; + U32 const rhs = *(U32 const *)rp; + return memcmp(ctx->samples + lhs, ctx->samples + rhs, ctx->d); +} +/** + * Faster version for d <= 8. + */ +static int COVER_cmp8(COVER_ctx_t *ctx, const void *lp, const void *rp) { + U64 const mask = (ctx->d == 8) ? (U64)-1 : (((U64)1 << (8 * ctx->d)) - 1); + U64 const lhs = MEM_readLE64(ctx->samples + *(U32 const *)lp) & mask; + U64 const rhs = MEM_readLE64(ctx->samples + *(U32 const *)rp) & mask; + if (lhs < rhs) { + return -1; + } + return (lhs > rhs); +} + +/** + * Same as COVER_cmp() except ties are broken by pointer value + * NOTE: g_coverCtx must be set to call this function. A global is required because + * qsort doesn't take an opaque pointer. + */ +static int WIN_CDECL COVER_strict_cmp(const void *lp, const void *rp) { + int result = COVER_cmp(g_coverCtx, lp, rp); + if (result == 0) { + result = lp < rp ? -1 : 1; + } + return result; +} +/** + * Faster version for d <= 8. + */ +static int WIN_CDECL COVER_strict_cmp8(const void *lp, const void *rp) { + int result = COVER_cmp8(g_coverCtx, lp, rp); + if (result == 0) { + result = lp < rp ? -1 : 1; + } + return result; +} + +/** + * Returns the first pointer in [first, last) whose element does not compare + * less than value. If no such element exists it returns last. + */ +static const size_t *COVER_lower_bound(const size_t *first, const size_t *last, + size_t value) { + size_t count = last - first; + while (count != 0) { + size_t step = count / 2; + const size_t *ptr = first; + ptr += step; + if (*ptr < value) { + first = ++ptr; + count -= step + 1; + } else { + count = step; + } + } + return first; +} + +/** + * Generic groupBy function. + * Groups an array sorted by cmp into groups with equivalent values. + * Calls grp for each group. + */ +static void +COVER_groupBy(const void *data, size_t count, size_t size, COVER_ctx_t *ctx, + int (*cmp)(COVER_ctx_t *, const void *, const void *), + void (*grp)(COVER_ctx_t *, const void *, const void *)) { + const BYTE *ptr = (const BYTE *)data; + size_t num = 0; + while (num < count) { + const BYTE *grpEnd = ptr + size; + ++num; + while (num < count && cmp(ctx, ptr, grpEnd) == 0) { + grpEnd += size; + ++num; + } + grp(ctx, ptr, grpEnd); + ptr = grpEnd; + } +} + +/*-************************************* +* Cover functions +***************************************/ + +/** + * Called on each group of positions with the same dmer. + * Counts the frequency of each dmer and saves it in the suffix array. + * Fills `ctx->dmerAt`. + */ +static void COVER_group(COVER_ctx_t *ctx, const void *group, + const void *groupEnd) { + /* The group consists of all the positions with the same first d bytes. */ + const U32 *grpPtr = (const U32 *)group; + const U32 *grpEnd = (const U32 *)groupEnd; + /* The dmerId is how we will reference this dmer. + * This allows us to map the whole dmer space to a much smaller space, the + * size of the suffix array. + */ + const U32 dmerId = (U32)(grpPtr - ctx->suffix); + /* Count the number of samples this dmer shows up in */ + U32 freq = 0; + /* Details */ + const size_t *curOffsetPtr = ctx->offsets; + const size_t *offsetsEnd = ctx->offsets + ctx->nbSamples; + /* Once *grpPtr >= curSampleEnd this occurrence of the dmer is in a + * different sample than the last. + */ + size_t curSampleEnd = ctx->offsets[0]; + for (; grpPtr != grpEnd; ++grpPtr) { + /* Save the dmerId for this position so we can get back to it. */ + ctx->dmerAt[*grpPtr] = dmerId; + /* Dictionaries only help for the first reference to the dmer. + * After that zstd can reference the match from the previous reference. + * So only count each dmer once for each sample it is in. + */ + if (*grpPtr < curSampleEnd) { + continue; + } + freq += 1; + /* Binary search to find the end of the sample *grpPtr is in. + * In the common case that grpPtr + 1 == grpEnd we can skip the binary + * search because the loop is over. + */ + if (grpPtr + 1 != grpEnd) { + const size_t *sampleEndPtr = + COVER_lower_bound(curOffsetPtr, offsetsEnd, *grpPtr); + curSampleEnd = *sampleEndPtr; + curOffsetPtr = sampleEndPtr + 1; + } + } + /* At this point we are never going to look at this segment of the suffix + * array again. We take advantage of this fact to save memory. + * We store the frequency of the dmer in the first position of the group, + * which is dmerId. + */ + ctx->suffix[dmerId] = freq; +} + + +/** + * Selects the best segment in an epoch. + * Segments of are scored according to the function: + * + * Let F(d) be the frequency of dmer d. + * Let S_i be the dmer at position i of segment S which has length k. + * + * Score(S) = F(S_1) + F(S_2) + ... + F(S_{k-d+1}) + * + * Once the dmer d is in the dictionary we set F(d) = 0. + */ +static COVER_segment_t COVER_selectSegment(const COVER_ctx_t *ctx, U32 *freqs, + COVER_map_t *activeDmers, U32 begin, + U32 end, + ZDICT_cover_params_t parameters) { + /* Constants */ + const U32 k = parameters.k; + const U32 d = parameters.d; + const U32 dmersInK = k - d + 1; + /* Try each segment (activeSegment) and save the best (bestSegment) */ + COVER_segment_t bestSegment = {0, 0, 0}; + COVER_segment_t activeSegment; + /* Reset the activeDmers in the segment */ + COVER_map_clear(activeDmers); + /* The activeSegment starts at the beginning of the epoch. */ + activeSegment.begin = begin; + activeSegment.end = begin; + activeSegment.score = 0; + /* Slide the activeSegment through the whole epoch. + * Save the best segment in bestSegment. + */ + while (activeSegment.end < end) { + /* The dmerId for the dmer at the next position */ + U32 newDmer = ctx->dmerAt[activeSegment.end]; + /* The entry in activeDmers for this dmerId */ + U32 *newDmerOcc = COVER_map_at(activeDmers, newDmer); + /* If the dmer isn't already present in the segment add its score. */ + if (*newDmerOcc == 0) { + /* The paper suggest using the L-0.5 norm, but experiments show that it + * doesn't help. + */ + activeSegment.score += freqs[newDmer]; + } + /* Add the dmer to the segment */ + activeSegment.end += 1; + *newDmerOcc += 1; + + /* If the window is now too large, drop the first position */ + if (activeSegment.end - activeSegment.begin == dmersInK + 1) { + U32 delDmer = ctx->dmerAt[activeSegment.begin]; + U32 *delDmerOcc = COVER_map_at(activeDmers, delDmer); + activeSegment.begin += 1; + *delDmerOcc -= 1; + /* If this is the last occurrence of the dmer, subtract its score */ + if (*delDmerOcc == 0) { + COVER_map_remove(activeDmers, delDmer); + activeSegment.score -= freqs[delDmer]; + } + } + + /* If this segment is the best so far save it */ + if (activeSegment.score > bestSegment.score) { + bestSegment = activeSegment; + } + } + { + /* Trim off the zero frequency head and tail from the segment. */ + U32 newBegin = bestSegment.end; + U32 newEnd = bestSegment.begin; + U32 pos; + for (pos = bestSegment.begin; pos != bestSegment.end; ++pos) { + U32 freq = freqs[ctx->dmerAt[pos]]; + if (freq != 0) { + newBegin = MIN(newBegin, pos); + newEnd = pos + 1; + } + } + bestSegment.begin = newBegin; + bestSegment.end = newEnd; + } + { + /* Zero out the frequency of each dmer covered by the chosen segment. */ + U32 pos; + for (pos = bestSegment.begin; pos != bestSegment.end; ++pos) { + freqs[ctx->dmerAt[pos]] = 0; + } + } + return bestSegment; +} + +/** + * Check the validity of the parameters. + * Returns non-zero if the parameters are valid and 0 otherwise. + */ +static int COVER_checkParameters(ZDICT_cover_params_t parameters, + size_t maxDictSize) { + /* k and d are required parameters */ + if (parameters.d == 0 || parameters.k == 0) { + return 0; + } + /* k <= maxDictSize */ + if (parameters.k > maxDictSize) { + return 0; + } + /* d <= k */ + if (parameters.d > parameters.k) { + return 0; + } + /* 0 < splitPoint <= 1 */ + if (parameters.splitPoint <= 0 || parameters.splitPoint > 1){ + return 0; + } + return 1; +} + +/** + * Clean up a context initialized with `COVER_ctx_init()`. + */ +static void COVER_ctx_destroy(COVER_ctx_t *ctx) { + if (!ctx) { + return; + } + if (ctx->suffix) { + free(ctx->suffix); + ctx->suffix = NULL; + } + if (ctx->freqs) { + free(ctx->freqs); + ctx->freqs = NULL; + } + if (ctx->dmerAt) { + free(ctx->dmerAt); + ctx->dmerAt = NULL; + } + if (ctx->offsets) { + free(ctx->offsets); + ctx->offsets = NULL; + } +} + +/** + * Prepare a context for dictionary building. + * The context is only dependent on the parameter `d` and can used multiple + * times. + * Returns 0 on success or error code on error. + * The context must be destroyed with `COVER_ctx_destroy()`. + */ +static size_t COVER_ctx_init(COVER_ctx_t *ctx, const void *samplesBuffer, + const size_t *samplesSizes, unsigned nbSamples, + unsigned d, double splitPoint) { + const BYTE *const samples = (const BYTE *)samplesBuffer; + const size_t totalSamplesSize = COVER_sum(samplesSizes, nbSamples); + /* Split samples into testing and training sets */ + const unsigned nbTrainSamples = splitPoint < 1.0 ? (unsigned)((double)nbSamples * splitPoint) : nbSamples; + const unsigned nbTestSamples = splitPoint < 1.0 ? nbSamples - nbTrainSamples : nbSamples; + const size_t trainingSamplesSize = splitPoint < 1.0 ? COVER_sum(samplesSizes, nbTrainSamples) : totalSamplesSize; + const size_t testSamplesSize = splitPoint < 1.0 ? COVER_sum(samplesSizes + nbTrainSamples, nbTestSamples) : totalSamplesSize; + /* Checks */ + if (totalSamplesSize < MAX(d, sizeof(U64)) || + totalSamplesSize >= (size_t)COVER_MAX_SAMPLES_SIZE) { + DISPLAYLEVEL(1, "Total samples size is too large (%u MB), maximum size is %u MB\n", + (unsigned)(totalSamplesSize>>20), (COVER_MAX_SAMPLES_SIZE >> 20)); + return ERROR(srcSize_wrong); + } + /* Check if there are at least 5 training samples */ + if (nbTrainSamples < 5) { + DISPLAYLEVEL(1, "Total number of training samples is %u and is invalid.", nbTrainSamples); + return ERROR(srcSize_wrong); + } + /* Check if there's testing sample */ + if (nbTestSamples < 1) { + DISPLAYLEVEL(1, "Total number of testing samples is %u and is invalid.", nbTestSamples); + return ERROR(srcSize_wrong); + } + /* Zero the context */ + memset(ctx, 0, sizeof(*ctx)); + DISPLAYLEVEL(2, "Training on %u samples of total size %u\n", nbTrainSamples, + (unsigned)trainingSamplesSize); + DISPLAYLEVEL(2, "Testing on %u samples of total size %u\n", nbTestSamples, + (unsigned)testSamplesSize); + ctx->samples = samples; + ctx->samplesSizes = samplesSizes; + ctx->nbSamples = nbSamples; + ctx->nbTrainSamples = nbTrainSamples; + ctx->nbTestSamples = nbTestSamples; + /* Partial suffix array */ + ctx->suffixSize = trainingSamplesSize - MAX(d, sizeof(U64)) + 1; + ctx->suffix = (U32 *)malloc(ctx->suffixSize * sizeof(U32)); + /* Maps index to the dmerID */ + ctx->dmerAt = (U32 *)malloc(ctx->suffixSize * sizeof(U32)); + /* The offsets of each file */ + ctx->offsets = (size_t *)malloc((nbSamples + 1) * sizeof(size_t)); + if (!ctx->suffix || !ctx->dmerAt || !ctx->offsets) { + DISPLAYLEVEL(1, "Failed to allocate scratch buffers\n"); + COVER_ctx_destroy(ctx); + return ERROR(memory_allocation); + } + ctx->freqs = NULL; + ctx->d = d; + + /* Fill offsets from the samplesSizes */ + { + U32 i; + ctx->offsets[0] = 0; + for (i = 1; i <= nbSamples; ++i) { + ctx->offsets[i] = ctx->offsets[i - 1] + samplesSizes[i - 1]; + } + } + DISPLAYLEVEL(2, "Constructing partial suffix array\n"); + { + /* suffix is a partial suffix array. + * It only sorts suffixes by their first parameters.d bytes. + * The sort is stable, so each dmer group is sorted by position in input. + */ + U32 i; + for (i = 0; i < ctx->suffixSize; ++i) { + ctx->suffix[i] = i; + } + /* qsort doesn't take an opaque pointer, so pass as a global. + * On OpenBSD qsort() is not guaranteed to be stable, their mergesort() is. + */ + g_coverCtx = ctx; +#if defined(__OpenBSD__) + mergesort(ctx->suffix, ctx->suffixSize, sizeof(U32), + (ctx->d <= 8 ? &COVER_strict_cmp8 : &COVER_strict_cmp)); +#else + qsort(ctx->suffix, ctx->suffixSize, sizeof(U32), + (ctx->d <= 8 ? &COVER_strict_cmp8 : &COVER_strict_cmp)); +#endif + } + DISPLAYLEVEL(2, "Computing frequencies\n"); + /* For each dmer group (group of positions with the same first d bytes): + * 1. For each position we set dmerAt[position] = dmerID. The dmerID is + * (groupBeginPtr - suffix). This allows us to go from position to + * dmerID so we can look up values in freq. + * 2. We calculate how many samples the dmer occurs in and save it in + * freqs[dmerId]. + */ + COVER_groupBy(ctx->suffix, ctx->suffixSize, sizeof(U32), ctx, + (ctx->d <= 8 ? &COVER_cmp8 : &COVER_cmp), &COVER_group); + ctx->freqs = ctx->suffix; + ctx->suffix = NULL; + return 0; +} + +void COVER_warnOnSmallCorpus(size_t maxDictSize, size_t nbDmers, int displayLevel) +{ + const double ratio = (double)nbDmers / maxDictSize; + if (ratio >= 10) { + return; + } + LOCALDISPLAYLEVEL(displayLevel, 1, + "WARNING: The maximum dictionary size %u is too large " + "compared to the source size %u! " + "size(source)/size(dictionary) = %f, but it should be >= " + "10! This may lead to a subpar dictionary! We recommend " + "training on sources at least 10x, and preferably 100x " + "the size of the dictionary! \n", (U32)maxDictSize, + (U32)nbDmers, ratio); +} + +COVER_epoch_info_t COVER_computeEpochs(U32 maxDictSize, + U32 nbDmers, U32 k, U32 passes) +{ + const U32 minEpochSize = k * 10; + COVER_epoch_info_t epochs; + epochs.num = MAX(1, maxDictSize / k / passes); + epochs.size = nbDmers / epochs.num; + if (epochs.size >= minEpochSize) { + assert(epochs.size * epochs.num <= nbDmers); + return epochs; + } + epochs.size = MIN(minEpochSize, nbDmers); + epochs.num = nbDmers / epochs.size; + assert(epochs.size * epochs.num <= nbDmers); + return epochs; +} + +/** + * Given the prepared context build the dictionary. + */ +static size_t COVER_buildDictionary(const COVER_ctx_t *ctx, U32 *freqs, + COVER_map_t *activeDmers, void *dictBuffer, + size_t dictBufferCapacity, + ZDICT_cover_params_t parameters) { + BYTE *const dict = (BYTE *)dictBuffer; + size_t tail = dictBufferCapacity; + /* Divide the data into epochs. We will select one segment from each epoch. */ + const COVER_epoch_info_t epochs = COVER_computeEpochs( + (U32)dictBufferCapacity, (U32)ctx->suffixSize, parameters.k, 4); + const size_t maxZeroScoreRun = MAX(10, MIN(100, epochs.num >> 3)); + size_t zeroScoreRun = 0; + size_t epoch; + DISPLAYLEVEL(2, "Breaking content into %u epochs of size %u\n", + (U32)epochs.num, (U32)epochs.size); + /* Loop through the epochs until there are no more segments or the dictionary + * is full. + */ + for (epoch = 0; tail > 0; epoch = (epoch + 1) % epochs.num) { + const U32 epochBegin = (U32)(epoch * epochs.size); + const U32 epochEnd = epochBegin + epochs.size; + size_t segmentSize; + /* Select a segment */ + COVER_segment_t segment = COVER_selectSegment( + ctx, freqs, activeDmers, epochBegin, epochEnd, parameters); + /* If the segment covers no dmers, then we are out of content. + * There may be new content in other epochs, for continue for some time. + */ + if (segment.score == 0) { + if (++zeroScoreRun >= maxZeroScoreRun) { + break; + } + continue; + } + zeroScoreRun = 0; + /* Trim the segment if necessary and if it is too small then we are done */ + segmentSize = MIN(segment.end - segment.begin + parameters.d - 1, tail); + if (segmentSize < parameters.d) { + break; + } + /* We fill the dictionary from the back to allow the best segments to be + * referenced with the smallest offsets. + */ + tail -= segmentSize; + memcpy(dict + tail, ctx->samples + segment.begin, segmentSize); + DISPLAYUPDATE( + 2, "\r%u%% ", + (unsigned)(((dictBufferCapacity - tail) * 100) / dictBufferCapacity)); + } + DISPLAYLEVEL(2, "\r%79s\r", ""); + return tail; +} + +ZDICTLIB_API size_t ZDICT_trainFromBuffer_cover( + void *dictBuffer, size_t dictBufferCapacity, + const void *samplesBuffer, const size_t *samplesSizes, unsigned nbSamples, + ZDICT_cover_params_t parameters) +{ + BYTE* const dict = (BYTE*)dictBuffer; + COVER_ctx_t ctx; + COVER_map_t activeDmers; + parameters.splitPoint = 1.0; + /* Initialize global data */ + g_displayLevel = (int)parameters.zParams.notificationLevel; + /* Checks */ + if (!COVER_checkParameters(parameters, dictBufferCapacity)) { + DISPLAYLEVEL(1, "Cover parameters incorrect\n"); + return ERROR(parameter_outOfBound); + } + if (nbSamples == 0) { + DISPLAYLEVEL(1, "Cover must have at least one input file\n"); + return ERROR(srcSize_wrong); + } + if (dictBufferCapacity < ZDICT_DICTSIZE_MIN) { + DISPLAYLEVEL(1, "dictBufferCapacity must be at least %u\n", + ZDICT_DICTSIZE_MIN); + return ERROR(dstSize_tooSmall); + } + /* Initialize context and activeDmers */ + { + size_t const initVal = COVER_ctx_init(&ctx, samplesBuffer, samplesSizes, nbSamples, + parameters.d, parameters.splitPoint); + if (ZSTD_isError(initVal)) { + return initVal; + } + } + COVER_warnOnSmallCorpus(dictBufferCapacity, ctx.suffixSize, g_displayLevel); + if (!COVER_map_init(&activeDmers, parameters.k - parameters.d + 1)) { + DISPLAYLEVEL(1, "Failed to allocate dmer map: out of memory\n"); + COVER_ctx_destroy(&ctx); + return ERROR(memory_allocation); + } + + DISPLAYLEVEL(2, "Building dictionary\n"); + { + const size_t tail = + COVER_buildDictionary(&ctx, ctx.freqs, &activeDmers, dictBuffer, + dictBufferCapacity, parameters); + const size_t dictionarySize = ZDICT_finalizeDictionary( + dict, dictBufferCapacity, dict + tail, dictBufferCapacity - tail, + samplesBuffer, samplesSizes, nbSamples, parameters.zParams); + if (!ZSTD_isError(dictionarySize)) { + DISPLAYLEVEL(2, "Constructed dictionary of size %u\n", + (unsigned)dictionarySize); + } + COVER_ctx_destroy(&ctx); + COVER_map_destroy(&activeDmers); + return dictionarySize; + } +} + + + +size_t COVER_checkTotalCompressedSize(const ZDICT_cover_params_t parameters, + const size_t *samplesSizes, const BYTE *samples, + size_t *offsets, + size_t nbTrainSamples, size_t nbSamples, + BYTE *const dict, size_t dictBufferCapacity) { + size_t totalCompressedSize = ERROR(GENERIC); + /* Pointers */ + ZSTD_CCtx *cctx; + ZSTD_CDict *cdict; + void *dst; + /* Local variables */ + size_t dstCapacity; + size_t i; + /* Allocate dst with enough space to compress the maximum sized sample */ + { + size_t maxSampleSize = 0; + i = parameters.splitPoint < 1.0 ? nbTrainSamples : 0; + for (; i < nbSamples; ++i) { + maxSampleSize = MAX(samplesSizes[i], maxSampleSize); + } + dstCapacity = ZSTD_compressBound(maxSampleSize); + dst = malloc(dstCapacity); + } + /* Create the cctx and cdict */ + cctx = ZSTD_createCCtx(); + cdict = ZSTD_createCDict(dict, dictBufferCapacity, + parameters.zParams.compressionLevel); + if (!dst || !cctx || !cdict) { + goto _compressCleanup; + } + /* Compress each sample and sum their sizes (or error) */ + totalCompressedSize = dictBufferCapacity; + i = parameters.splitPoint < 1.0 ? nbTrainSamples : 0; + for (; i < nbSamples; ++i) { + const size_t size = ZSTD_compress_usingCDict( + cctx, dst, dstCapacity, samples + offsets[i], + samplesSizes[i], cdict); + if (ZSTD_isError(size)) { + totalCompressedSize = size; + goto _compressCleanup; + } + totalCompressedSize += size; + } +_compressCleanup: + ZSTD_freeCCtx(cctx); + ZSTD_freeCDict(cdict); + if (dst) { + free(dst); + } + return totalCompressedSize; +} + + +/** + * Initialize the `COVER_best_t`. + */ +void COVER_best_init(COVER_best_t *best) { + if (best==NULL) return; /* compatible with init on NULL */ + (void)ZSTD_pthread_mutex_init(&best->mutex, NULL); + (void)ZSTD_pthread_cond_init(&best->cond, NULL); + best->liveJobs = 0; + best->dict = NULL; + best->dictSize = 0; + best->compressedSize = (size_t)-1; + memset(&best->parameters, 0, sizeof(best->parameters)); +} + +/** + * Wait until liveJobs == 0. + */ +void COVER_best_wait(COVER_best_t *best) { + if (!best) { + return; + } + ZSTD_pthread_mutex_lock(&best->mutex); + while (best->liveJobs != 0) { + ZSTD_pthread_cond_wait(&best->cond, &best->mutex); + } + ZSTD_pthread_mutex_unlock(&best->mutex); +} + +/** + * Call COVER_best_wait() and then destroy the COVER_best_t. + */ +void COVER_best_destroy(COVER_best_t *best) { + if (!best) { + return; + } + COVER_best_wait(best); + if (best->dict) { + free(best->dict); + } + ZSTD_pthread_mutex_destroy(&best->mutex); + ZSTD_pthread_cond_destroy(&best->cond); +} + +/** + * Called when a thread is about to be launched. + * Increments liveJobs. + */ +void COVER_best_start(COVER_best_t *best) { + if (!best) { + return; + } + ZSTD_pthread_mutex_lock(&best->mutex); + ++best->liveJobs; + ZSTD_pthread_mutex_unlock(&best->mutex); +} + +/** + * Called when a thread finishes executing, both on error or success. + * Decrements liveJobs and signals any waiting threads if liveJobs == 0. + * If this dictionary is the best so far save it and its parameters. + */ +void COVER_best_finish(COVER_best_t *best, ZDICT_cover_params_t parameters, + COVER_dictSelection_t selection) { + void* dict = selection.dictContent; + size_t compressedSize = selection.totalCompressedSize; + size_t dictSize = selection.dictSize; + if (!best) { + return; + } + { + size_t liveJobs; + ZSTD_pthread_mutex_lock(&best->mutex); + --best->liveJobs; + liveJobs = best->liveJobs; + /* If the new dictionary is better */ + if (compressedSize < best->compressedSize) { + /* Allocate space if necessary */ + if (!best->dict || best->dictSize < dictSize) { + if (best->dict) { + free(best->dict); + } + best->dict = malloc(dictSize); + if (!best->dict) { + best->compressedSize = ERROR(GENERIC); + best->dictSize = 0; + ZSTD_pthread_cond_signal(&best->cond); + ZSTD_pthread_mutex_unlock(&best->mutex); + return; + } + } + /* Save the dictionary, parameters, and size */ + if (dict) { + memcpy(best->dict, dict, dictSize); + best->dictSize = dictSize; + best->parameters = parameters; + best->compressedSize = compressedSize; + } + } + if (liveJobs == 0) { + ZSTD_pthread_cond_broadcast(&best->cond); + } + ZSTD_pthread_mutex_unlock(&best->mutex); + } +} + +COVER_dictSelection_t COVER_dictSelectionError(size_t error) { + COVER_dictSelection_t selection = { NULL, 0, error }; + return selection; +} + +unsigned COVER_dictSelectionIsError(COVER_dictSelection_t selection) { + return (ZSTD_isError(selection.totalCompressedSize) || !selection.dictContent); +} + +void COVER_dictSelectionFree(COVER_dictSelection_t selection){ + free(selection.dictContent); +} + +COVER_dictSelection_t COVER_selectDict(BYTE* customDictContent, size_t dictBufferCapacity, + size_t dictContentSize, const BYTE* samplesBuffer, const size_t* samplesSizes, unsigned nbFinalizeSamples, + size_t nbCheckSamples, size_t nbSamples, ZDICT_cover_params_t params, size_t* offsets, size_t totalCompressedSize) { + + size_t largestDict = 0; + size_t largestCompressed = 0; + BYTE* customDictContentEnd = customDictContent + dictContentSize; + + BYTE * largestDictbuffer = (BYTE *)malloc(dictBufferCapacity); + BYTE * candidateDictBuffer = (BYTE *)malloc(dictBufferCapacity); + double regressionTolerance = ((double)params.shrinkDictMaxRegression / 100.0) + 1.00; + + if (!largestDictbuffer || !candidateDictBuffer) { + free(largestDictbuffer); + free(candidateDictBuffer); + return COVER_dictSelectionError(dictContentSize); + } + + /* Initial dictionary size and compressed size */ + memcpy(largestDictbuffer, customDictContent, dictContentSize); + dictContentSize = ZDICT_finalizeDictionary( + largestDictbuffer, dictBufferCapacity, customDictContent, dictContentSize, + samplesBuffer, samplesSizes, nbFinalizeSamples, params.zParams); + + if (ZDICT_isError(dictContentSize)) { + free(largestDictbuffer); + free(candidateDictBuffer); + return COVER_dictSelectionError(dictContentSize); + } + + totalCompressedSize = COVER_checkTotalCompressedSize(params, samplesSizes, + samplesBuffer, offsets, + nbCheckSamples, nbSamples, + largestDictbuffer, dictContentSize); + + if (ZSTD_isError(totalCompressedSize)) { + free(largestDictbuffer); + free(candidateDictBuffer); + return COVER_dictSelectionError(totalCompressedSize); + } + + if (params.shrinkDict == 0) { + COVER_dictSelection_t selection = { largestDictbuffer, dictContentSize, totalCompressedSize }; + free(candidateDictBuffer); + return selection; + } + + largestDict = dictContentSize; + largestCompressed = totalCompressedSize; + dictContentSize = ZDICT_DICTSIZE_MIN; + + /* Largest dict is initially at least ZDICT_DICTSIZE_MIN */ + while (dictContentSize < largestDict) { + memcpy(candidateDictBuffer, largestDictbuffer, largestDict); + dictContentSize = ZDICT_finalizeDictionary( + candidateDictBuffer, dictBufferCapacity, customDictContentEnd - dictContentSize, dictContentSize, + samplesBuffer, samplesSizes, nbFinalizeSamples, params.zParams); + + if (ZDICT_isError(dictContentSize)) { + free(largestDictbuffer); + free(candidateDictBuffer); + return COVER_dictSelectionError(dictContentSize); + + } + + totalCompressedSize = COVER_checkTotalCompressedSize(params, samplesSizes, + samplesBuffer, offsets, + nbCheckSamples, nbSamples, + candidateDictBuffer, dictContentSize); + + if (ZSTD_isError(totalCompressedSize)) { + free(largestDictbuffer); + free(candidateDictBuffer); + return COVER_dictSelectionError(totalCompressedSize); + } + + if (totalCompressedSize <= largestCompressed * regressionTolerance) { + COVER_dictSelection_t selection = { candidateDictBuffer, dictContentSize, totalCompressedSize }; + free(largestDictbuffer); + return selection; + } + dictContentSize *= 2; + } + dictContentSize = largestDict; + totalCompressedSize = largestCompressed; + { + COVER_dictSelection_t selection = { largestDictbuffer, dictContentSize, totalCompressedSize }; + free(candidateDictBuffer); + return selection; + } +} + +/** + * Parameters for COVER_tryParameters(). + */ +typedef struct COVER_tryParameters_data_s { + const COVER_ctx_t *ctx; + COVER_best_t *best; + size_t dictBufferCapacity; + ZDICT_cover_params_t parameters; +} COVER_tryParameters_data_t; + +/** + * Tries a set of parameters and updates the COVER_best_t with the results. + * This function is thread safe if zstd is compiled with multithreaded support. + * It takes its parameters as an *OWNING* opaque pointer to support threading. + */ +static void COVER_tryParameters(void *opaque) +{ + /* Save parameters as local variables */ + COVER_tryParameters_data_t *const data = (COVER_tryParameters_data_t*)opaque; + const COVER_ctx_t *const ctx = data->ctx; + const ZDICT_cover_params_t parameters = data->parameters; + size_t dictBufferCapacity = data->dictBufferCapacity; + size_t totalCompressedSize = ERROR(GENERIC); + /* Allocate space for hash table, dict, and freqs */ + COVER_map_t activeDmers; + BYTE* const dict = (BYTE*)malloc(dictBufferCapacity); + COVER_dictSelection_t selection = COVER_dictSelectionError(ERROR(GENERIC)); + U32* const freqs = (U32*)malloc(ctx->suffixSize * sizeof(U32)); + if (!COVER_map_init(&activeDmers, parameters.k - parameters.d + 1)) { + DISPLAYLEVEL(1, "Failed to allocate dmer map: out of memory\n"); + goto _cleanup; + } + if (!dict || !freqs) { + DISPLAYLEVEL(1, "Failed to allocate buffers: out of memory\n"); + goto _cleanup; + } + /* Copy the frequencies because we need to modify them */ + memcpy(freqs, ctx->freqs, ctx->suffixSize * sizeof(U32)); + /* Build the dictionary */ + { + const size_t tail = COVER_buildDictionary(ctx, freqs, &activeDmers, dict, + dictBufferCapacity, parameters); + selection = COVER_selectDict(dict + tail, dictBufferCapacity, dictBufferCapacity - tail, + ctx->samples, ctx->samplesSizes, (unsigned)ctx->nbTrainSamples, ctx->nbTrainSamples, ctx->nbSamples, parameters, ctx->offsets, + totalCompressedSize); + + if (COVER_dictSelectionIsError(selection)) { + DISPLAYLEVEL(1, "Failed to select dictionary\n"); + goto _cleanup; + } + } +_cleanup: + free(dict); + COVER_best_finish(data->best, parameters, selection); + free(data); + COVER_map_destroy(&activeDmers); + COVER_dictSelectionFree(selection); + free(freqs); +} + +ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_cover( + void* dictBuffer, size_t dictBufferCapacity, const void* samplesBuffer, + const size_t* samplesSizes, unsigned nbSamples, + ZDICT_cover_params_t* parameters) +{ + /* constants */ + const unsigned nbThreads = parameters->nbThreads; + const double splitPoint = + parameters->splitPoint <= 0.0 ? COVER_DEFAULT_SPLITPOINT : parameters->splitPoint; + const unsigned kMinD = parameters->d == 0 ? 6 : parameters->d; + const unsigned kMaxD = parameters->d == 0 ? 8 : parameters->d; + const unsigned kMinK = parameters->k == 0 ? 50 : parameters->k; + const unsigned kMaxK = parameters->k == 0 ? 2000 : parameters->k; + const unsigned kSteps = parameters->steps == 0 ? 40 : parameters->steps; + const unsigned kStepSize = MAX((kMaxK - kMinK) / kSteps, 1); + const unsigned kIterations = + (1 + (kMaxD - kMinD) / 2) * (1 + (kMaxK - kMinK) / kStepSize); + const unsigned shrinkDict = 0; + /* Local variables */ + const int displayLevel = parameters->zParams.notificationLevel; + unsigned iteration = 1; + unsigned d; + unsigned k; + COVER_best_t best; + POOL_ctx *pool = NULL; + int warned = 0; + + /* Checks */ + if (splitPoint <= 0 || splitPoint > 1) { + LOCALDISPLAYLEVEL(displayLevel, 1, "Incorrect parameters\n"); + return ERROR(parameter_outOfBound); + } + if (kMinK < kMaxD || kMaxK < kMinK) { + LOCALDISPLAYLEVEL(displayLevel, 1, "Incorrect parameters\n"); + return ERROR(parameter_outOfBound); + } + if (nbSamples == 0) { + DISPLAYLEVEL(1, "Cover must have at least one input file\n"); + return ERROR(srcSize_wrong); + } + if (dictBufferCapacity < ZDICT_DICTSIZE_MIN) { + DISPLAYLEVEL(1, "dictBufferCapacity must be at least %u\n", + ZDICT_DICTSIZE_MIN); + return ERROR(dstSize_tooSmall); + } + if (nbThreads > 1) { + pool = POOL_create(nbThreads, 1); + if (!pool) { + return ERROR(memory_allocation); + } + } + /* Initialization */ + COVER_best_init(&best); + /* Turn down global display level to clean up display at level 2 and below */ + g_displayLevel = displayLevel == 0 ? 0 : displayLevel - 1; + /* Loop through d first because each new value needs a new context */ + LOCALDISPLAYLEVEL(displayLevel, 2, "Trying %u different sets of parameters\n", + kIterations); + for (d = kMinD; d <= kMaxD; d += 2) { + /* Initialize the context for this value of d */ + COVER_ctx_t ctx; + LOCALDISPLAYLEVEL(displayLevel, 3, "d=%u\n", d); + { + const size_t initVal = COVER_ctx_init(&ctx, samplesBuffer, samplesSizes, nbSamples, d, splitPoint); + if (ZSTD_isError(initVal)) { + LOCALDISPLAYLEVEL(displayLevel, 1, "Failed to initialize context\n"); + COVER_best_destroy(&best); + POOL_free(pool); + return initVal; + } + } + if (!warned) { + COVER_warnOnSmallCorpus(dictBufferCapacity, ctx.suffixSize, displayLevel); + warned = 1; + } + /* Loop through k reusing the same context */ + for (k = kMinK; k <= kMaxK; k += kStepSize) { + /* Prepare the arguments */ + COVER_tryParameters_data_t *data = (COVER_tryParameters_data_t *)malloc( + sizeof(COVER_tryParameters_data_t)); + LOCALDISPLAYLEVEL(displayLevel, 3, "k=%u\n", k); + if (!data) { + LOCALDISPLAYLEVEL(displayLevel, 1, "Failed to allocate parameters\n"); + COVER_best_destroy(&best); + COVER_ctx_destroy(&ctx); + POOL_free(pool); + return ERROR(memory_allocation); + } + data->ctx = &ctx; + data->best = &best; + data->dictBufferCapacity = dictBufferCapacity; + data->parameters = *parameters; + data->parameters.k = k; + data->parameters.d = d; + data->parameters.splitPoint = splitPoint; + data->parameters.steps = kSteps; + data->parameters.shrinkDict = shrinkDict; + data->parameters.zParams.notificationLevel = g_displayLevel; + /* Check the parameters */ + if (!COVER_checkParameters(data->parameters, dictBufferCapacity)) { + DISPLAYLEVEL(1, "Cover parameters incorrect\n"); + free(data); + continue; + } + /* Call the function and pass ownership of data to it */ + COVER_best_start(&best); + if (pool) { + POOL_add(pool, &COVER_tryParameters, data); + } else { + COVER_tryParameters(data); + } + /* Print status */ + LOCALDISPLAYUPDATE(displayLevel, 2, "\r%u%% ", + (unsigned)((iteration * 100) / kIterations)); + ++iteration; + } + COVER_best_wait(&best); + COVER_ctx_destroy(&ctx); + } + LOCALDISPLAYLEVEL(displayLevel, 2, "\r%79s\r", ""); + /* Fill the output buffer and parameters with output of the best parameters */ + { + const size_t dictSize = best.dictSize; + if (ZSTD_isError(best.compressedSize)) { + const size_t compressedSize = best.compressedSize; + COVER_best_destroy(&best); + POOL_free(pool); + return compressedSize; + } + *parameters = best.parameters; + memcpy(dictBuffer, best.dict, dictSize); + COVER_best_destroy(&best); + POOL_free(pool); + return dictSize; + } +} diff --git a/src/blosc2/internal-complibs/zstd-1.5.2/dictBuilder/cover.h b/src/blosc2/internal-complibs/zstd-1.5.2/dictBuilder/cover.h new file mode 100644 index 0000000..1aacddd --- /dev/null +++ b/src/blosc2/internal-complibs/zstd-1.5.2/dictBuilder/cover.h @@ -0,0 +1,158 @@ +/* + * Copyright (c) Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZDICT_STATIC_LINKING_ONLY +# define ZDICT_STATIC_LINKING_ONLY +#endif + +#include /* fprintf */ +#include /* malloc, free, qsort */ +#include /* memset */ +#include /* clock */ +#include "../common/mem.h" /* read */ +#include "../common/pool.h" +#include "../common/threading.h" +#include "../common/zstd_internal.h" /* includes zstd.h */ +#include "../zdict.h" + +/** + * COVER_best_t is used for two purposes: + * 1. Synchronizing threads. + * 2. Saving the best parameters and dictionary. + * + * All of the methods except COVER_best_init() are thread safe if zstd is + * compiled with multithreaded support. + */ +typedef struct COVER_best_s { + ZSTD_pthread_mutex_t mutex; + ZSTD_pthread_cond_t cond; + size_t liveJobs; + void *dict; + size_t dictSize; + ZDICT_cover_params_t parameters; + size_t compressedSize; +} COVER_best_t; + +/** + * A segment is a range in the source as well as the score of the segment. + */ +typedef struct { + U32 begin; + U32 end; + U32 score; +} COVER_segment_t; + +/** + *Number of epochs and size of each epoch. + */ +typedef struct { + U32 num; + U32 size; +} COVER_epoch_info_t; + +/** + * Struct used for the dictionary selection function. + */ +typedef struct COVER_dictSelection { + BYTE* dictContent; + size_t dictSize; + size_t totalCompressedSize; +} COVER_dictSelection_t; + +/** + * Computes the number of epochs and the size of each epoch. + * We will make sure that each epoch gets at least 10 * k bytes. + * + * The COVER algorithms divide the data up into epochs of equal size and + * select one segment from each epoch. + * + * @param maxDictSize The maximum allowed dictionary size. + * @param nbDmers The number of dmers we are training on. + * @param k The parameter k (segment size). + * @param passes The target number of passes over the dmer corpus. + * More passes means a better dictionary. + */ +COVER_epoch_info_t COVER_computeEpochs(U32 maxDictSize, U32 nbDmers, + U32 k, U32 passes); + +/** + * Warns the user when their corpus is too small. + */ +void COVER_warnOnSmallCorpus(size_t maxDictSize, size_t nbDmers, int displayLevel); + +/** + * Checks total compressed size of a dictionary + */ +size_t COVER_checkTotalCompressedSize(const ZDICT_cover_params_t parameters, + const size_t *samplesSizes, const BYTE *samples, + size_t *offsets, + size_t nbTrainSamples, size_t nbSamples, + BYTE *const dict, size_t dictBufferCapacity); + +/** + * Returns the sum of the sample sizes. + */ +size_t COVER_sum(const size_t *samplesSizes, unsigned nbSamples) ; + +/** + * Initialize the `COVER_best_t`. + */ +void COVER_best_init(COVER_best_t *best); + +/** + * Wait until liveJobs == 0. + */ +void COVER_best_wait(COVER_best_t *best); + +/** + * Call COVER_best_wait() and then destroy the COVER_best_t. + */ +void COVER_best_destroy(COVER_best_t *best); + +/** + * Called when a thread is about to be launched. + * Increments liveJobs. + */ +void COVER_best_start(COVER_best_t *best); + +/** + * Called when a thread finishes executing, both on error or success. + * Decrements liveJobs and signals any waiting threads if liveJobs == 0. + * If this dictionary is the best so far save it and its parameters. + */ +void COVER_best_finish(COVER_best_t *best, ZDICT_cover_params_t parameters, + COVER_dictSelection_t selection); +/** + * Error function for COVER_selectDict function. Checks if the return + * value is an error. + */ +unsigned COVER_dictSelectionIsError(COVER_dictSelection_t selection); + + /** + * Error function for COVER_selectDict function. Returns a struct where + * return.totalCompressedSize is a ZSTD error. + */ +COVER_dictSelection_t COVER_dictSelectionError(size_t error); + +/** + * Always call after selectDict is called to free up used memory from + * newly created dictionary. + */ +void COVER_dictSelectionFree(COVER_dictSelection_t selection); + +/** + * Called to finalize the dictionary and select one based on whether or not + * the shrink-dict flag was enabled. If enabled the dictionary used is the + * smallest dictionary within a specified regression of the compressed size + * from the largest dictionary. + */ + COVER_dictSelection_t COVER_selectDict(BYTE* customDictContent, size_t dictBufferCapacity, + size_t dictContentSize, const BYTE* samplesBuffer, const size_t* samplesSizes, unsigned nbFinalizeSamples, + size_t nbCheckSamples, size_t nbSamples, ZDICT_cover_params_t params, size_t* offsets, size_t totalCompressedSize); diff --git a/src/blosc2/internal-complibs/zstd-1.5.2/dictBuilder/divsufsort.c b/src/blosc2/internal-complibs/zstd-1.5.2/dictBuilder/divsufsort.c new file mode 100644 index 0000000..a2870fb --- /dev/null +++ b/src/blosc2/internal-complibs/zstd-1.5.2/dictBuilder/divsufsort.c @@ -0,0 +1,1913 @@ +/* + * divsufsort.c for libdivsufsort-lite + * Copyright (c) 2003-2008 Yuta Mori All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +/*- Compiler specifics -*/ +#ifdef __clang__ +#pragma clang diagnostic ignored "-Wshorten-64-to-32" +#endif + +#if defined(_MSC_VER) +# pragma warning(disable : 4244) +# pragma warning(disable : 4127) /* C4127 : Condition expression is constant */ +#endif + + +/*- Dependencies -*/ +#include +#include +#include + +#include "divsufsort.h" + +/*- Constants -*/ +#if defined(INLINE) +# undef INLINE +#endif +#if !defined(INLINE) +# define INLINE __inline +#endif +#if defined(ALPHABET_SIZE) && (ALPHABET_SIZE < 1) +# undef ALPHABET_SIZE +#endif +#if !defined(ALPHABET_SIZE) +# define ALPHABET_SIZE (256) +#endif +#define BUCKET_A_SIZE (ALPHABET_SIZE) +#define BUCKET_B_SIZE (ALPHABET_SIZE * ALPHABET_SIZE) +#if defined(SS_INSERTIONSORT_THRESHOLD) +# if SS_INSERTIONSORT_THRESHOLD < 1 +# undef SS_INSERTIONSORT_THRESHOLD +# define SS_INSERTIONSORT_THRESHOLD (1) +# endif +#else +# define SS_INSERTIONSORT_THRESHOLD (8) +#endif +#if defined(SS_BLOCKSIZE) +# if SS_BLOCKSIZE < 0 +# undef SS_BLOCKSIZE +# define SS_BLOCKSIZE (0) +# elif 32768 <= SS_BLOCKSIZE +# undef SS_BLOCKSIZE +# define SS_BLOCKSIZE (32767) +# endif +#else +# define SS_BLOCKSIZE (1024) +#endif +/* minstacksize = log(SS_BLOCKSIZE) / log(3) * 2 */ +#if SS_BLOCKSIZE == 0 +# define SS_MISORT_STACKSIZE (96) +#elif SS_BLOCKSIZE <= 4096 +# define SS_MISORT_STACKSIZE (16) +#else +# define SS_MISORT_STACKSIZE (24) +#endif +#define SS_SMERGE_STACKSIZE (32) +#define TR_INSERTIONSORT_THRESHOLD (8) +#define TR_STACKSIZE (64) + + +/*- Macros -*/ +#ifndef SWAP +# define SWAP(_a, _b) do { t = (_a); (_a) = (_b); (_b) = t; } while(0) +#endif /* SWAP */ +#ifndef MIN +# define MIN(_a, _b) (((_a) < (_b)) ? (_a) : (_b)) +#endif /* MIN */ +#ifndef MAX +# define MAX(_a, _b) (((_a) > (_b)) ? (_a) : (_b)) +#endif /* MAX */ +#define STACK_PUSH(_a, _b, _c, _d)\ + do {\ + assert(ssize < STACK_SIZE);\ + stack[ssize].a = (_a), stack[ssize].b = (_b),\ + stack[ssize].c = (_c), stack[ssize++].d = (_d);\ + } while(0) +#define STACK_PUSH5(_a, _b, _c, _d, _e)\ + do {\ + assert(ssize < STACK_SIZE);\ + stack[ssize].a = (_a), stack[ssize].b = (_b),\ + stack[ssize].c = (_c), stack[ssize].d = (_d), stack[ssize++].e = (_e);\ + } while(0) +#define STACK_POP(_a, _b, _c, _d)\ + do {\ + assert(0 <= ssize);\ + if(ssize == 0) { return; }\ + (_a) = stack[--ssize].a, (_b) = stack[ssize].b,\ + (_c) = stack[ssize].c, (_d) = stack[ssize].d;\ + } while(0) +#define STACK_POP5(_a, _b, _c, _d, _e)\ + do {\ + assert(0 <= ssize);\ + if(ssize == 0) { return; }\ + (_a) = stack[--ssize].a, (_b) = stack[ssize].b,\ + (_c) = stack[ssize].c, (_d) = stack[ssize].d, (_e) = stack[ssize].e;\ + } while(0) +#define BUCKET_A(_c0) bucket_A[(_c0)] +#if ALPHABET_SIZE == 256 +#define BUCKET_B(_c0, _c1) (bucket_B[((_c1) << 8) | (_c0)]) +#define BUCKET_BSTAR(_c0, _c1) (bucket_B[((_c0) << 8) | (_c1)]) +#else +#define BUCKET_B(_c0, _c1) (bucket_B[(_c1) * ALPHABET_SIZE + (_c0)]) +#define BUCKET_BSTAR(_c0, _c1) (bucket_B[(_c0) * ALPHABET_SIZE + (_c1)]) +#endif + + +/*- Private Functions -*/ + +static const int lg_table[256]= { + -1,0,1,1,2,2,2,2,3,3,3,3,3,3,3,3,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4, + 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5, + 6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6, + 6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6, + 7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7, + 7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7, + 7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7, + 7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7 +}; + +#if (SS_BLOCKSIZE == 0) || (SS_INSERTIONSORT_THRESHOLD < SS_BLOCKSIZE) + +static INLINE +int +ss_ilg(int n) { +#if SS_BLOCKSIZE == 0 + return (n & 0xffff0000) ? + ((n & 0xff000000) ? + 24 + lg_table[(n >> 24) & 0xff] : + 16 + lg_table[(n >> 16) & 0xff]) : + ((n & 0x0000ff00) ? + 8 + lg_table[(n >> 8) & 0xff] : + 0 + lg_table[(n >> 0) & 0xff]); +#elif SS_BLOCKSIZE < 256 + return lg_table[n]; +#else + return (n & 0xff00) ? + 8 + lg_table[(n >> 8) & 0xff] : + 0 + lg_table[(n >> 0) & 0xff]; +#endif +} + +#endif /* (SS_BLOCKSIZE == 0) || (SS_INSERTIONSORT_THRESHOLD < SS_BLOCKSIZE) */ + +#if SS_BLOCKSIZE != 0 + +static const int sqq_table[256] = { + 0, 16, 22, 27, 32, 35, 39, 42, 45, 48, 50, 53, 55, 57, 59, 61, + 64, 65, 67, 69, 71, 73, 75, 76, 78, 80, 81, 83, 84, 86, 87, 89, + 90, 91, 93, 94, 96, 97, 98, 99, 101, 102, 103, 104, 106, 107, 108, 109, +110, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, +128, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, +143, 144, 144, 145, 146, 147, 148, 149, 150, 150, 151, 152, 153, 154, 155, 155, +156, 157, 158, 159, 160, 160, 161, 162, 163, 163, 164, 165, 166, 167, 167, 168, +169, 170, 170, 171, 172, 173, 173, 174, 175, 176, 176, 177, 178, 178, 179, 180, +181, 181, 182, 183, 183, 184, 185, 185, 186, 187, 187, 188, 189, 189, 190, 191, +192, 192, 193, 193, 194, 195, 195, 196, 197, 197, 198, 199, 199, 200, 201, 201, +202, 203, 203, 204, 204, 205, 206, 206, 207, 208, 208, 209, 209, 210, 211, 211, +212, 212, 213, 214, 214, 215, 215, 216, 217, 217, 218, 218, 219, 219, 220, 221, +221, 222, 222, 223, 224, 224, 225, 225, 226, 226, 227, 227, 228, 229, 229, 230, +230, 231, 231, 232, 232, 233, 234, 234, 235, 235, 236, 236, 237, 237, 238, 238, +239, 240, 240, 241, 241, 242, 242, 243, 243, 244, 244, 245, 245, 246, 246, 247, +247, 248, 248, 249, 249, 250, 250, 251, 251, 252, 252, 253, 253, 254, 254, 255 +}; + +static INLINE +int +ss_isqrt(int x) { + int y, e; + + if(x >= (SS_BLOCKSIZE * SS_BLOCKSIZE)) { return SS_BLOCKSIZE; } + e = (x & 0xffff0000) ? + ((x & 0xff000000) ? + 24 + lg_table[(x >> 24) & 0xff] : + 16 + lg_table[(x >> 16) & 0xff]) : + ((x & 0x0000ff00) ? + 8 + lg_table[(x >> 8) & 0xff] : + 0 + lg_table[(x >> 0) & 0xff]); + + if(e >= 16) { + y = sqq_table[x >> ((e - 6) - (e & 1))] << ((e >> 1) - 7); + if(e >= 24) { y = (y + 1 + x / y) >> 1; } + y = (y + 1 + x / y) >> 1; + } else if(e >= 8) { + y = (sqq_table[x >> ((e - 6) - (e & 1))] >> (7 - (e >> 1))) + 1; + } else { + return sqq_table[x] >> 4; + } + + return (x < (y * y)) ? y - 1 : y; +} + +#endif /* SS_BLOCKSIZE != 0 */ + + +/*---------------------------------------------------------------------------*/ + +/* Compares two suffixes. */ +static INLINE +int +ss_compare(const unsigned char *T, + const int *p1, const int *p2, + int depth) { + const unsigned char *U1, *U2, *U1n, *U2n; + + for(U1 = T + depth + *p1, + U2 = T + depth + *p2, + U1n = T + *(p1 + 1) + 2, + U2n = T + *(p2 + 1) + 2; + (U1 < U1n) && (U2 < U2n) && (*U1 == *U2); + ++U1, ++U2) { + } + + return U1 < U1n ? + (U2 < U2n ? *U1 - *U2 : 1) : + (U2 < U2n ? -1 : 0); +} + + +/*---------------------------------------------------------------------------*/ + +#if (SS_BLOCKSIZE != 1) && (SS_INSERTIONSORT_THRESHOLD != 1) + +/* Insertionsort for small size groups */ +static +void +ss_insertionsort(const unsigned char *T, const int *PA, + int *first, int *last, int depth) { + int *i, *j; + int t; + int r; + + for(i = last - 2; first <= i; --i) { + for(t = *i, j = i + 1; 0 < (r = ss_compare(T, PA + t, PA + *j, depth));) { + do { *(j - 1) = *j; } while((++j < last) && (*j < 0)); + if(last <= j) { break; } + } + if(r == 0) { *j = ~*j; } + *(j - 1) = t; + } +} + +#endif /* (SS_BLOCKSIZE != 1) && (SS_INSERTIONSORT_THRESHOLD != 1) */ + + +/*---------------------------------------------------------------------------*/ + +#if (SS_BLOCKSIZE == 0) || (SS_INSERTIONSORT_THRESHOLD < SS_BLOCKSIZE) + +static INLINE +void +ss_fixdown(const unsigned char *Td, const int *PA, + int *SA, int i, int size) { + int j, k; + int v; + int c, d, e; + + for(v = SA[i], c = Td[PA[v]]; (j = 2 * i + 1) < size; SA[i] = SA[k], i = k) { + d = Td[PA[SA[k = j++]]]; + if(d < (e = Td[PA[SA[j]]])) { k = j; d = e; } + if(d <= c) { break; } + } + SA[i] = v; +} + +/* Simple top-down heapsort. */ +static +void +ss_heapsort(const unsigned char *Td, const int *PA, int *SA, int size) { + int i, m; + int t; + + m = size; + if((size % 2) == 0) { + m--; + if(Td[PA[SA[m / 2]]] < Td[PA[SA[m]]]) { SWAP(SA[m], SA[m / 2]); } + } + + for(i = m / 2 - 1; 0 <= i; --i) { ss_fixdown(Td, PA, SA, i, m); } + if((size % 2) == 0) { SWAP(SA[0], SA[m]); ss_fixdown(Td, PA, SA, 0, m); } + for(i = m - 1; 0 < i; --i) { + t = SA[0], SA[0] = SA[i]; + ss_fixdown(Td, PA, SA, 0, i); + SA[i] = t; + } +} + + +/*---------------------------------------------------------------------------*/ + +/* Returns the median of three elements. */ +static INLINE +int * +ss_median3(const unsigned char *Td, const int *PA, + int *v1, int *v2, int *v3) { + int *t; + if(Td[PA[*v1]] > Td[PA[*v2]]) { SWAP(v1, v2); } + if(Td[PA[*v2]] > Td[PA[*v3]]) { + if(Td[PA[*v1]] > Td[PA[*v3]]) { return v1; } + else { return v3; } + } + return v2; +} + +/* Returns the median of five elements. */ +static INLINE +int * +ss_median5(const unsigned char *Td, const int *PA, + int *v1, int *v2, int *v3, int *v4, int *v5) { + int *t; + if(Td[PA[*v2]] > Td[PA[*v3]]) { SWAP(v2, v3); } + if(Td[PA[*v4]] > Td[PA[*v5]]) { SWAP(v4, v5); } + if(Td[PA[*v2]] > Td[PA[*v4]]) { SWAP(v2, v4); SWAP(v3, v5); } + if(Td[PA[*v1]] > Td[PA[*v3]]) { SWAP(v1, v3); } + if(Td[PA[*v1]] > Td[PA[*v4]]) { SWAP(v1, v4); SWAP(v3, v5); } + if(Td[PA[*v3]] > Td[PA[*v4]]) { return v4; } + return v3; +} + +/* Returns the pivot element. */ +static INLINE +int * +ss_pivot(const unsigned char *Td, const int *PA, int *first, int *last) { + int *middle; + int t; + + t = last - first; + middle = first + t / 2; + + if(t <= 512) { + if(t <= 32) { + return ss_median3(Td, PA, first, middle, last - 1); + } else { + t >>= 2; + return ss_median5(Td, PA, first, first + t, middle, last - 1 - t, last - 1); + } + } + t >>= 3; + first = ss_median3(Td, PA, first, first + t, first + (t << 1)); + middle = ss_median3(Td, PA, middle - t, middle, middle + t); + last = ss_median3(Td, PA, last - 1 - (t << 1), last - 1 - t, last - 1); + return ss_median3(Td, PA, first, middle, last); +} + + +/*---------------------------------------------------------------------------*/ + +/* Binary partition for substrings. */ +static INLINE +int * +ss_partition(const int *PA, + int *first, int *last, int depth) { + int *a, *b; + int t; + for(a = first - 1, b = last;;) { + for(; (++a < b) && ((PA[*a] + depth) >= (PA[*a + 1] + 1));) { *a = ~*a; } + for(; (a < --b) && ((PA[*b] + depth) < (PA[*b + 1] + 1));) { } + if(b <= a) { break; } + t = ~*b; + *b = *a; + *a = t; + } + if(first < a) { *first = ~*first; } + return a; +} + +/* Multikey introsort for medium size groups. */ +static +void +ss_mintrosort(const unsigned char *T, const int *PA, + int *first, int *last, + int depth) { +#define STACK_SIZE SS_MISORT_STACKSIZE + struct { int *a, *b, c; int d; } stack[STACK_SIZE]; + const unsigned char *Td; + int *a, *b, *c, *d, *e, *f; + int s, t; + int ssize; + int limit; + int v, x = 0; + + for(ssize = 0, limit = ss_ilg(last - first);;) { + + if((last - first) <= SS_INSERTIONSORT_THRESHOLD) { +#if 1 < SS_INSERTIONSORT_THRESHOLD + if(1 < (last - first)) { ss_insertionsort(T, PA, first, last, depth); } +#endif + STACK_POP(first, last, depth, limit); + continue; + } + + Td = T + depth; + if(limit-- == 0) { ss_heapsort(Td, PA, first, last - first); } + if(limit < 0) { + for(a = first + 1, v = Td[PA[*first]]; a < last; ++a) { + if((x = Td[PA[*a]]) != v) { + if(1 < (a - first)) { break; } + v = x; + first = a; + } + } + if(Td[PA[*first] - 1] < v) { + first = ss_partition(PA, first, a, depth); + } + if((a - first) <= (last - a)) { + if(1 < (a - first)) { + STACK_PUSH(a, last, depth, -1); + last = a, depth += 1, limit = ss_ilg(a - first); + } else { + first = a, limit = -1; + } + } else { + if(1 < (last - a)) { + STACK_PUSH(first, a, depth + 1, ss_ilg(a - first)); + first = a, limit = -1; + } else { + last = a, depth += 1, limit = ss_ilg(a - first); + } + } + continue; + } + + /* choose pivot */ + a = ss_pivot(Td, PA, first, last); + v = Td[PA[*a]]; + SWAP(*first, *a); + + /* partition */ + for(b = first; (++b < last) && ((x = Td[PA[*b]]) == v);) { } + if(((a = b) < last) && (x < v)) { + for(; (++b < last) && ((x = Td[PA[*b]]) <= v);) { + if(x == v) { SWAP(*b, *a); ++a; } + } + } + for(c = last; (b < --c) && ((x = Td[PA[*c]]) == v);) { } + if((b < (d = c)) && (x > v)) { + for(; (b < --c) && ((x = Td[PA[*c]]) >= v);) { + if(x == v) { SWAP(*c, *d); --d; } + } + } + for(; b < c;) { + SWAP(*b, *c); + for(; (++b < c) && ((x = Td[PA[*b]]) <= v);) { + if(x == v) { SWAP(*b, *a); ++a; } + } + for(; (b < --c) && ((x = Td[PA[*c]]) >= v);) { + if(x == v) { SWAP(*c, *d); --d; } + } + } + + if(a <= d) { + c = b - 1; + + if((s = a - first) > (t = b - a)) { s = t; } + for(e = first, f = b - s; 0 < s; --s, ++e, ++f) { SWAP(*e, *f); } + if((s = d - c) > (t = last - d - 1)) { s = t; } + for(e = b, f = last - s; 0 < s; --s, ++e, ++f) { SWAP(*e, *f); } + + a = first + (b - a), c = last - (d - c); + b = (v <= Td[PA[*a] - 1]) ? a : ss_partition(PA, a, c, depth); + + if((a - first) <= (last - c)) { + if((last - c) <= (c - b)) { + STACK_PUSH(b, c, depth + 1, ss_ilg(c - b)); + STACK_PUSH(c, last, depth, limit); + last = a; + } else if((a - first) <= (c - b)) { + STACK_PUSH(c, last, depth, limit); + STACK_PUSH(b, c, depth + 1, ss_ilg(c - b)); + last = a; + } else { + STACK_PUSH(c, last, depth, limit); + STACK_PUSH(first, a, depth, limit); + first = b, last = c, depth += 1, limit = ss_ilg(c - b); + } + } else { + if((a - first) <= (c - b)) { + STACK_PUSH(b, c, depth + 1, ss_ilg(c - b)); + STACK_PUSH(first, a, depth, limit); + first = c; + } else if((last - c) <= (c - b)) { + STACK_PUSH(first, a, depth, limit); + STACK_PUSH(b, c, depth + 1, ss_ilg(c - b)); + first = c; + } else { + STACK_PUSH(first, a, depth, limit); + STACK_PUSH(c, last, depth, limit); + first = b, last = c, depth += 1, limit = ss_ilg(c - b); + } + } + } else { + limit += 1; + if(Td[PA[*first] - 1] < v) { + first = ss_partition(PA, first, last, depth); + limit = ss_ilg(last - first); + } + depth += 1; + } + } +#undef STACK_SIZE +} + +#endif /* (SS_BLOCKSIZE == 0) || (SS_INSERTIONSORT_THRESHOLD < SS_BLOCKSIZE) */ + + +/*---------------------------------------------------------------------------*/ + +#if SS_BLOCKSIZE != 0 + +static INLINE +void +ss_blockswap(int *a, int *b, int n) { + int t; + for(; 0 < n; --n, ++a, ++b) { + t = *a, *a = *b, *b = t; + } +} + +static INLINE +void +ss_rotate(int *first, int *middle, int *last) { + int *a, *b, t; + int l, r; + l = middle - first, r = last - middle; + for(; (0 < l) && (0 < r);) { + if(l == r) { ss_blockswap(first, middle, l); break; } + if(l < r) { + a = last - 1, b = middle - 1; + t = *a; + do { + *a-- = *b, *b-- = *a; + if(b < first) { + *a = t; + last = a; + if((r -= l + 1) <= l) { break; } + a -= 1, b = middle - 1; + t = *a; + } + } while(1); + } else { + a = first, b = middle; + t = *a; + do { + *a++ = *b, *b++ = *a; + if(last <= b) { + *a = t; + first = a + 1; + if((l -= r + 1) <= r) { break; } + a += 1, b = middle; + t = *a; + } + } while(1); + } + } +} + + +/*---------------------------------------------------------------------------*/ + +static +void +ss_inplacemerge(const unsigned char *T, const int *PA, + int *first, int *middle, int *last, + int depth) { + const int *p; + int *a, *b; + int len, half; + int q, r; + int x; + + for(;;) { + if(*(last - 1) < 0) { x = 1; p = PA + ~*(last - 1); } + else { x = 0; p = PA + *(last - 1); } + for(a = first, len = middle - first, half = len >> 1, r = -1; + 0 < len; + len = half, half >>= 1) { + b = a + half; + q = ss_compare(T, PA + ((0 <= *b) ? *b : ~*b), p, depth); + if(q < 0) { + a = b + 1; + half -= (len & 1) ^ 1; + } else { + r = q; + } + } + if(a < middle) { + if(r == 0) { *a = ~*a; } + ss_rotate(a, middle, last); + last -= middle - a; + middle = a; + if(first == middle) { break; } + } + --last; + if(x != 0) { while(*--last < 0) { } } + if(middle == last) { break; } + } +} + + +/*---------------------------------------------------------------------------*/ + +/* Merge-forward with internal buffer. */ +static +void +ss_mergeforward(const unsigned char *T, const int *PA, + int *first, int *middle, int *last, + int *buf, int depth) { + int *a, *b, *c, *bufend; + int t; + int r; + + bufend = buf + (middle - first) - 1; + ss_blockswap(buf, first, middle - first); + + for(t = *(a = first), b = buf, c = middle;;) { + r = ss_compare(T, PA + *b, PA + *c, depth); + if(r < 0) { + do { + *a++ = *b; + if(bufend <= b) { *bufend = t; return; } + *b++ = *a; + } while(*b < 0); + } else if(r > 0) { + do { + *a++ = *c, *c++ = *a; + if(last <= c) { + while(b < bufend) { *a++ = *b, *b++ = *a; } + *a = *b, *b = t; + return; + } + } while(*c < 0); + } else { + *c = ~*c; + do { + *a++ = *b; + if(bufend <= b) { *bufend = t; return; } + *b++ = *a; + } while(*b < 0); + + do { + *a++ = *c, *c++ = *a; + if(last <= c) { + while(b < bufend) { *a++ = *b, *b++ = *a; } + *a = *b, *b = t; + return; + } + } while(*c < 0); + } + } +} + +/* Merge-backward with internal buffer. */ +static +void +ss_mergebackward(const unsigned char *T, const int *PA, + int *first, int *middle, int *last, + int *buf, int depth) { + const int *p1, *p2; + int *a, *b, *c, *bufend; + int t; + int r; + int x; + + bufend = buf + (last - middle) - 1; + ss_blockswap(buf, middle, last - middle); + + x = 0; + if(*bufend < 0) { p1 = PA + ~*bufend; x |= 1; } + else { p1 = PA + *bufend; } + if(*(middle - 1) < 0) { p2 = PA + ~*(middle - 1); x |= 2; } + else { p2 = PA + *(middle - 1); } + for(t = *(a = last - 1), b = bufend, c = middle - 1;;) { + r = ss_compare(T, p1, p2, depth); + if(0 < r) { + if(x & 1) { do { *a-- = *b, *b-- = *a; } while(*b < 0); x ^= 1; } + *a-- = *b; + if(b <= buf) { *buf = t; break; } + *b-- = *a; + if(*b < 0) { p1 = PA + ~*b; x |= 1; } + else { p1 = PA + *b; } + } else if(r < 0) { + if(x & 2) { do { *a-- = *c, *c-- = *a; } while(*c < 0); x ^= 2; } + *a-- = *c, *c-- = *a; + if(c < first) { + while(buf < b) { *a-- = *b, *b-- = *a; } + *a = *b, *b = t; + break; + } + if(*c < 0) { p2 = PA + ~*c; x |= 2; } + else { p2 = PA + *c; } + } else { + if(x & 1) { do { *a-- = *b, *b-- = *a; } while(*b < 0); x ^= 1; } + *a-- = ~*b; + if(b <= buf) { *buf = t; break; } + *b-- = *a; + if(x & 2) { do { *a-- = *c, *c-- = *a; } while(*c < 0); x ^= 2; } + *a-- = *c, *c-- = *a; + if(c < first) { + while(buf < b) { *a-- = *b, *b-- = *a; } + *a = *b, *b = t; + break; + } + if(*b < 0) { p1 = PA + ~*b; x |= 1; } + else { p1 = PA + *b; } + if(*c < 0) { p2 = PA + ~*c; x |= 2; } + else { p2 = PA + *c; } + } + } +} + +/* D&C based merge. */ +static +void +ss_swapmerge(const unsigned char *T, const int *PA, + int *first, int *middle, int *last, + int *buf, int bufsize, int depth) { +#define STACK_SIZE SS_SMERGE_STACKSIZE +#define GETIDX(a) ((0 <= (a)) ? (a) : (~(a))) +#define MERGE_CHECK(a, b, c)\ + do {\ + if(((c) & 1) ||\ + (((c) & 2) && (ss_compare(T, PA + GETIDX(*((a) - 1)), PA + *(a), depth) == 0))) {\ + *(a) = ~*(a);\ + }\ + if(((c) & 4) && ((ss_compare(T, PA + GETIDX(*((b) - 1)), PA + *(b), depth) == 0))) {\ + *(b) = ~*(b);\ + }\ + } while(0) + struct { int *a, *b, *c; int d; } stack[STACK_SIZE]; + int *l, *r, *lm, *rm; + int m, len, half; + int ssize; + int check, next; + + for(check = 0, ssize = 0;;) { + if((last - middle) <= bufsize) { + if((first < middle) && (middle < last)) { + ss_mergebackward(T, PA, first, middle, last, buf, depth); + } + MERGE_CHECK(first, last, check); + STACK_POP(first, middle, last, check); + continue; + } + + if((middle - first) <= bufsize) { + if(first < middle) { + ss_mergeforward(T, PA, first, middle, last, buf, depth); + } + MERGE_CHECK(first, last, check); + STACK_POP(first, middle, last, check); + continue; + } + + for(m = 0, len = MIN(middle - first, last - middle), half = len >> 1; + 0 < len; + len = half, half >>= 1) { + if(ss_compare(T, PA + GETIDX(*(middle + m + half)), + PA + GETIDX(*(middle - m - half - 1)), depth) < 0) { + m += half + 1; + half -= (len & 1) ^ 1; + } + } + + if(0 < m) { + lm = middle - m, rm = middle + m; + ss_blockswap(lm, middle, m); + l = r = middle, next = 0; + if(rm < last) { + if(*rm < 0) { + *rm = ~*rm; + if(first < lm) { for(; *--l < 0;) { } next |= 4; } + next |= 1; + } else if(first < lm) { + for(; *r < 0; ++r) { } + next |= 2; + } + } + + if((l - first) <= (last - r)) { + STACK_PUSH(r, rm, last, (next & 3) | (check & 4)); + middle = lm, last = l, check = (check & 3) | (next & 4); + } else { + if((next & 2) && (r == middle)) { next ^= 6; } + STACK_PUSH(first, lm, l, (check & 3) | (next & 4)); + first = r, middle = rm, check = (next & 3) | (check & 4); + } + } else { + if(ss_compare(T, PA + GETIDX(*(middle - 1)), PA + *middle, depth) == 0) { + *middle = ~*middle; + } + MERGE_CHECK(first, last, check); + STACK_POP(first, middle, last, check); + } + } +#undef STACK_SIZE +} + +#endif /* SS_BLOCKSIZE != 0 */ + + +/*---------------------------------------------------------------------------*/ + +/* Substring sort */ +static +void +sssort(const unsigned char *T, const int *PA, + int *first, int *last, + int *buf, int bufsize, + int depth, int n, int lastsuffix) { + int *a; +#if SS_BLOCKSIZE != 0 + int *b, *middle, *curbuf; + int j, k, curbufsize, limit; +#endif + int i; + + if(lastsuffix != 0) { ++first; } + +#if SS_BLOCKSIZE == 0 + ss_mintrosort(T, PA, first, last, depth); +#else + if((bufsize < SS_BLOCKSIZE) && + (bufsize < (last - first)) && + (bufsize < (limit = ss_isqrt(last - first)))) { + if(SS_BLOCKSIZE < limit) { limit = SS_BLOCKSIZE; } + buf = middle = last - limit, bufsize = limit; + } else { + middle = last, limit = 0; + } + for(a = first, i = 0; SS_BLOCKSIZE < (middle - a); a += SS_BLOCKSIZE, ++i) { +#if SS_INSERTIONSORT_THRESHOLD < SS_BLOCKSIZE + ss_mintrosort(T, PA, a, a + SS_BLOCKSIZE, depth); +#elif 1 < SS_BLOCKSIZE + ss_insertionsort(T, PA, a, a + SS_BLOCKSIZE, depth); +#endif + curbufsize = last - (a + SS_BLOCKSIZE); + curbuf = a + SS_BLOCKSIZE; + if(curbufsize <= bufsize) { curbufsize = bufsize, curbuf = buf; } + for(b = a, k = SS_BLOCKSIZE, j = i; j & 1; b -= k, k <<= 1, j >>= 1) { + ss_swapmerge(T, PA, b - k, b, b + k, curbuf, curbufsize, depth); + } + } +#if SS_INSERTIONSORT_THRESHOLD < SS_BLOCKSIZE + ss_mintrosort(T, PA, a, middle, depth); +#elif 1 < SS_BLOCKSIZE + ss_insertionsort(T, PA, a, middle, depth); +#endif + for(k = SS_BLOCKSIZE; i != 0; k <<= 1, i >>= 1) { + if(i & 1) { + ss_swapmerge(T, PA, a - k, a, middle, buf, bufsize, depth); + a -= k; + } + } + if(limit != 0) { +#if SS_INSERTIONSORT_THRESHOLD < SS_BLOCKSIZE + ss_mintrosort(T, PA, middle, last, depth); +#elif 1 < SS_BLOCKSIZE + ss_insertionsort(T, PA, middle, last, depth); +#endif + ss_inplacemerge(T, PA, first, middle, last, depth); + } +#endif + + if(lastsuffix != 0) { + /* Insert last type B* suffix. */ + int PAi[2]; PAi[0] = PA[*(first - 1)], PAi[1] = n - 2; + for(a = first, i = *(first - 1); + (a < last) && ((*a < 0) || (0 < ss_compare(T, &(PAi[0]), PA + *a, depth))); + ++a) { + *(a - 1) = *a; + } + *(a - 1) = i; + } +} + + +/*---------------------------------------------------------------------------*/ + +static INLINE +int +tr_ilg(int n) { + return (n & 0xffff0000) ? + ((n & 0xff000000) ? + 24 + lg_table[(n >> 24) & 0xff] : + 16 + lg_table[(n >> 16) & 0xff]) : + ((n & 0x0000ff00) ? + 8 + lg_table[(n >> 8) & 0xff] : + 0 + lg_table[(n >> 0) & 0xff]); +} + + +/*---------------------------------------------------------------------------*/ + +/* Simple insertionsort for small size groups. */ +static +void +tr_insertionsort(const int *ISAd, int *first, int *last) { + int *a, *b; + int t, r; + + for(a = first + 1; a < last; ++a) { + for(t = *a, b = a - 1; 0 > (r = ISAd[t] - ISAd[*b]);) { + do { *(b + 1) = *b; } while((first <= --b) && (*b < 0)); + if(b < first) { break; } + } + if(r == 0) { *b = ~*b; } + *(b + 1) = t; + } +} + + +/*---------------------------------------------------------------------------*/ + +static INLINE +void +tr_fixdown(const int *ISAd, int *SA, int i, int size) { + int j, k; + int v; + int c, d, e; + + for(v = SA[i], c = ISAd[v]; (j = 2 * i + 1) < size; SA[i] = SA[k], i = k) { + d = ISAd[SA[k = j++]]; + if(d < (e = ISAd[SA[j]])) { k = j; d = e; } + if(d <= c) { break; } + } + SA[i] = v; +} + +/* Simple top-down heapsort. */ +static +void +tr_heapsort(const int *ISAd, int *SA, int size) { + int i, m; + int t; + + m = size; + if((size % 2) == 0) { + m--; + if(ISAd[SA[m / 2]] < ISAd[SA[m]]) { SWAP(SA[m], SA[m / 2]); } + } + + for(i = m / 2 - 1; 0 <= i; --i) { tr_fixdown(ISAd, SA, i, m); } + if((size % 2) == 0) { SWAP(SA[0], SA[m]); tr_fixdown(ISAd, SA, 0, m); } + for(i = m - 1; 0 < i; --i) { + t = SA[0], SA[0] = SA[i]; + tr_fixdown(ISAd, SA, 0, i); + SA[i] = t; + } +} + + +/*---------------------------------------------------------------------------*/ + +/* Returns the median of three elements. */ +static INLINE +int * +tr_median3(const int *ISAd, int *v1, int *v2, int *v3) { + int *t; + if(ISAd[*v1] > ISAd[*v2]) { SWAP(v1, v2); } + if(ISAd[*v2] > ISAd[*v3]) { + if(ISAd[*v1] > ISAd[*v3]) { return v1; } + else { return v3; } + } + return v2; +} + +/* Returns the median of five elements. */ +static INLINE +int * +tr_median5(const int *ISAd, + int *v1, int *v2, int *v3, int *v4, int *v5) { + int *t; + if(ISAd[*v2] > ISAd[*v3]) { SWAP(v2, v3); } + if(ISAd[*v4] > ISAd[*v5]) { SWAP(v4, v5); } + if(ISAd[*v2] > ISAd[*v4]) { SWAP(v2, v4); SWAP(v3, v5); } + if(ISAd[*v1] > ISAd[*v3]) { SWAP(v1, v3); } + if(ISAd[*v1] > ISAd[*v4]) { SWAP(v1, v4); SWAP(v3, v5); } + if(ISAd[*v3] > ISAd[*v4]) { return v4; } + return v3; +} + +/* Returns the pivot element. */ +static INLINE +int * +tr_pivot(const int *ISAd, int *first, int *last) { + int *middle; + int t; + + t = last - first; + middle = first + t / 2; + + if(t <= 512) { + if(t <= 32) { + return tr_median3(ISAd, first, middle, last - 1); + } else { + t >>= 2; + return tr_median5(ISAd, first, first + t, middle, last - 1 - t, last - 1); + } + } + t >>= 3; + first = tr_median3(ISAd, first, first + t, first + (t << 1)); + middle = tr_median3(ISAd, middle - t, middle, middle + t); + last = tr_median3(ISAd, last - 1 - (t << 1), last - 1 - t, last - 1); + return tr_median3(ISAd, first, middle, last); +} + + +/*---------------------------------------------------------------------------*/ + +typedef struct _trbudget_t trbudget_t; +struct _trbudget_t { + int chance; + int remain; + int incval; + int count; +}; + +static INLINE +void +trbudget_init(trbudget_t *budget, int chance, int incval) { + budget->chance = chance; + budget->remain = budget->incval = incval; +} + +static INLINE +int +trbudget_check(trbudget_t *budget, int size) { + if(size <= budget->remain) { budget->remain -= size; return 1; } + if(budget->chance == 0) { budget->count += size; return 0; } + budget->remain += budget->incval - size; + budget->chance -= 1; + return 1; +} + + +/*---------------------------------------------------------------------------*/ + +static INLINE +void +tr_partition(const int *ISAd, + int *first, int *middle, int *last, + int **pa, int **pb, int v) { + int *a, *b, *c, *d, *e, *f; + int t, s; + int x = 0; + + for(b = middle - 1; (++b < last) && ((x = ISAd[*b]) == v);) { } + if(((a = b) < last) && (x < v)) { + for(; (++b < last) && ((x = ISAd[*b]) <= v);) { + if(x == v) { SWAP(*b, *a); ++a; } + } + } + for(c = last; (b < --c) && ((x = ISAd[*c]) == v);) { } + if((b < (d = c)) && (x > v)) { + for(; (b < --c) && ((x = ISAd[*c]) >= v);) { + if(x == v) { SWAP(*c, *d); --d; } + } + } + for(; b < c;) { + SWAP(*b, *c); + for(; (++b < c) && ((x = ISAd[*b]) <= v);) { + if(x == v) { SWAP(*b, *a); ++a; } + } + for(; (b < --c) && ((x = ISAd[*c]) >= v);) { + if(x == v) { SWAP(*c, *d); --d; } + } + } + + if(a <= d) { + c = b - 1; + if((s = a - first) > (t = b - a)) { s = t; } + for(e = first, f = b - s; 0 < s; --s, ++e, ++f) { SWAP(*e, *f); } + if((s = d - c) > (t = last - d - 1)) { s = t; } + for(e = b, f = last - s; 0 < s; --s, ++e, ++f) { SWAP(*e, *f); } + first += (b - a), last -= (d - c); + } + *pa = first, *pb = last; +} + +static +void +tr_copy(int *ISA, const int *SA, + int *first, int *a, int *b, int *last, + int depth) { + /* sort suffixes of middle partition + by using sorted order of suffixes of left and right partition. */ + int *c, *d, *e; + int s, v; + + v = b - SA - 1; + for(c = first, d = a - 1; c <= d; ++c) { + if((0 <= (s = *c - depth)) && (ISA[s] == v)) { + *++d = s; + ISA[s] = d - SA; + } + } + for(c = last - 1, e = d + 1, d = b; e < d; --c) { + if((0 <= (s = *c - depth)) && (ISA[s] == v)) { + *--d = s; + ISA[s] = d - SA; + } + } +} + +static +void +tr_partialcopy(int *ISA, const int *SA, + int *first, int *a, int *b, int *last, + int depth) { + int *c, *d, *e; + int s, v; + int rank, lastrank, newrank = -1; + + v = b - SA - 1; + lastrank = -1; + for(c = first, d = a - 1; c <= d; ++c) { + if((0 <= (s = *c - depth)) && (ISA[s] == v)) { + *++d = s; + rank = ISA[s + depth]; + if(lastrank != rank) { lastrank = rank; newrank = d - SA; } + ISA[s] = newrank; + } + } + + lastrank = -1; + for(e = d; first <= e; --e) { + rank = ISA[*e]; + if(lastrank != rank) { lastrank = rank; newrank = e - SA; } + if(newrank != rank) { ISA[*e] = newrank; } + } + + lastrank = -1; + for(c = last - 1, e = d + 1, d = b; e < d; --c) { + if((0 <= (s = *c - depth)) && (ISA[s] == v)) { + *--d = s; + rank = ISA[s + depth]; + if(lastrank != rank) { lastrank = rank; newrank = d - SA; } + ISA[s] = newrank; + } + } +} + +static +void +tr_introsort(int *ISA, const int *ISAd, + int *SA, int *first, int *last, + trbudget_t *budget) { +#define STACK_SIZE TR_STACKSIZE + struct { const int *a; int *b, *c; int d, e; }stack[STACK_SIZE]; + int *a, *b, *c; + int t; + int v, x = 0; + int incr = ISAd - ISA; + int limit, next; + int ssize, trlink = -1; + + for(ssize = 0, limit = tr_ilg(last - first);;) { + + if(limit < 0) { + if(limit == -1) { + /* tandem repeat partition */ + tr_partition(ISAd - incr, first, first, last, &a, &b, last - SA - 1); + + /* update ranks */ + if(a < last) { + for(c = first, v = a - SA - 1; c < a; ++c) { ISA[*c] = v; } + } + if(b < last) { + for(c = a, v = b - SA - 1; c < b; ++c) { ISA[*c] = v; } + } + + /* push */ + if(1 < (b - a)) { + STACK_PUSH5(NULL, a, b, 0, 0); + STACK_PUSH5(ISAd - incr, first, last, -2, trlink); + trlink = ssize - 2; + } + if((a - first) <= (last - b)) { + if(1 < (a - first)) { + STACK_PUSH5(ISAd, b, last, tr_ilg(last - b), trlink); + last = a, limit = tr_ilg(a - first); + } else if(1 < (last - b)) { + first = b, limit = tr_ilg(last - b); + } else { + STACK_POP5(ISAd, first, last, limit, trlink); + } + } else { + if(1 < (last - b)) { + STACK_PUSH5(ISAd, first, a, tr_ilg(a - first), trlink); + first = b, limit = tr_ilg(last - b); + } else if(1 < (a - first)) { + last = a, limit = tr_ilg(a - first); + } else { + STACK_POP5(ISAd, first, last, limit, trlink); + } + } + } else if(limit == -2) { + /* tandem repeat copy */ + a = stack[--ssize].b, b = stack[ssize].c; + if(stack[ssize].d == 0) { + tr_copy(ISA, SA, first, a, b, last, ISAd - ISA); + } else { + if(0 <= trlink) { stack[trlink].d = -1; } + tr_partialcopy(ISA, SA, first, a, b, last, ISAd - ISA); + } + STACK_POP5(ISAd, first, last, limit, trlink); + } else { + /* sorted partition */ + if(0 <= *first) { + a = first; + do { ISA[*a] = a - SA; } while((++a < last) && (0 <= *a)); + first = a; + } + if(first < last) { + a = first; do { *a = ~*a; } while(*++a < 0); + next = (ISA[*a] != ISAd[*a]) ? tr_ilg(a - first + 1) : -1; + if(++a < last) { for(b = first, v = a - SA - 1; b < a; ++b) { ISA[*b] = v; } } + + /* push */ + if(trbudget_check(budget, a - first)) { + if((a - first) <= (last - a)) { + STACK_PUSH5(ISAd, a, last, -3, trlink); + ISAd += incr, last = a, limit = next; + } else { + if(1 < (last - a)) { + STACK_PUSH5(ISAd + incr, first, a, next, trlink); + first = a, limit = -3; + } else { + ISAd += incr, last = a, limit = next; + } + } + } else { + if(0 <= trlink) { stack[trlink].d = -1; } + if(1 < (last - a)) { + first = a, limit = -3; + } else { + STACK_POP5(ISAd, first, last, limit, trlink); + } + } + } else { + STACK_POP5(ISAd, first, last, limit, trlink); + } + } + continue; + } + + if((last - first) <= TR_INSERTIONSORT_THRESHOLD) { + tr_insertionsort(ISAd, first, last); + limit = -3; + continue; + } + + if(limit-- == 0) { + tr_heapsort(ISAd, first, last - first); + for(a = last - 1; first < a; a = b) { + for(x = ISAd[*a], b = a - 1; (first <= b) && (ISAd[*b] == x); --b) { *b = ~*b; } + } + limit = -3; + continue; + } + + /* choose pivot */ + a = tr_pivot(ISAd, first, last); + SWAP(*first, *a); + v = ISAd[*first]; + + /* partition */ + tr_partition(ISAd, first, first + 1, last, &a, &b, v); + if((last - first) != (b - a)) { + next = (ISA[*a] != v) ? tr_ilg(b - a) : -1; + + /* update ranks */ + for(c = first, v = a - SA - 1; c < a; ++c) { ISA[*c] = v; } + if(b < last) { for(c = a, v = b - SA - 1; c < b; ++c) { ISA[*c] = v; } } + + /* push */ + if((1 < (b - a)) && (trbudget_check(budget, b - a))) { + if((a - first) <= (last - b)) { + if((last - b) <= (b - a)) { + if(1 < (a - first)) { + STACK_PUSH5(ISAd + incr, a, b, next, trlink); + STACK_PUSH5(ISAd, b, last, limit, trlink); + last = a; + } else if(1 < (last - b)) { + STACK_PUSH5(ISAd + incr, a, b, next, trlink); + first = b; + } else { + ISAd += incr, first = a, last = b, limit = next; + } + } else if((a - first) <= (b - a)) { + if(1 < (a - first)) { + STACK_PUSH5(ISAd, b, last, limit, trlink); + STACK_PUSH5(ISAd + incr, a, b, next, trlink); + last = a; + } else { + STACK_PUSH5(ISAd, b, last, limit, trlink); + ISAd += incr, first = a, last = b, limit = next; + } + } else { + STACK_PUSH5(ISAd, b, last, limit, trlink); + STACK_PUSH5(ISAd, first, a, limit, trlink); + ISAd += incr, first = a, last = b, limit = next; + } + } else { + if((a - first) <= (b - a)) { + if(1 < (last - b)) { + STACK_PUSH5(ISAd + incr, a, b, next, trlink); + STACK_PUSH5(ISAd, first, a, limit, trlink); + first = b; + } else if(1 < (a - first)) { + STACK_PUSH5(ISAd + incr, a, b, next, trlink); + last = a; + } else { + ISAd += incr, first = a, last = b, limit = next; + } + } else if((last - b) <= (b - a)) { + if(1 < (last - b)) { + STACK_PUSH5(ISAd, first, a, limit, trlink); + STACK_PUSH5(ISAd + incr, a, b, next, trlink); + first = b; + } else { + STACK_PUSH5(ISAd, first, a, limit, trlink); + ISAd += incr, first = a, last = b, limit = next; + } + } else { + STACK_PUSH5(ISAd, first, a, limit, trlink); + STACK_PUSH5(ISAd, b, last, limit, trlink); + ISAd += incr, first = a, last = b, limit = next; + } + } + } else { + if((1 < (b - a)) && (0 <= trlink)) { stack[trlink].d = -1; } + if((a - first) <= (last - b)) { + if(1 < (a - first)) { + STACK_PUSH5(ISAd, b, last, limit, trlink); + last = a; + } else if(1 < (last - b)) { + first = b; + } else { + STACK_POP5(ISAd, first, last, limit, trlink); + } + } else { + if(1 < (last - b)) { + STACK_PUSH5(ISAd, first, a, limit, trlink); + first = b; + } else if(1 < (a - first)) { + last = a; + } else { + STACK_POP5(ISAd, first, last, limit, trlink); + } + } + } + } else { + if(trbudget_check(budget, last - first)) { + limit = tr_ilg(last - first), ISAd += incr; + } else { + if(0 <= trlink) { stack[trlink].d = -1; } + STACK_POP5(ISAd, first, last, limit, trlink); + } + } + } +#undef STACK_SIZE +} + + + +/*---------------------------------------------------------------------------*/ + +/* Tandem repeat sort */ +static +void +trsort(int *ISA, int *SA, int n, int depth) { + int *ISAd; + int *first, *last; + trbudget_t budget; + int t, skip, unsorted; + + trbudget_init(&budget, tr_ilg(n) * 2 / 3, n); +/* trbudget_init(&budget, tr_ilg(n) * 3 / 4, n); */ + for(ISAd = ISA + depth; -n < *SA; ISAd += ISAd - ISA) { + first = SA; + skip = 0; + unsorted = 0; + do { + if((t = *first) < 0) { first -= t; skip += t; } + else { + if(skip != 0) { *(first + skip) = skip; skip = 0; } + last = SA + ISA[t] + 1; + if(1 < (last - first)) { + budget.count = 0; + tr_introsort(ISA, ISAd, SA, first, last, &budget); + if(budget.count != 0) { unsorted += budget.count; } + else { skip = first - last; } + } else if((last - first) == 1) { + skip = -1; + } + first = last; + } + } while(first < (SA + n)); + if(skip != 0) { *(first + skip) = skip; } + if(unsorted == 0) { break; } + } +} + + +/*---------------------------------------------------------------------------*/ + +/* Sorts suffixes of type B*. */ +static +int +sort_typeBstar(const unsigned char *T, int *SA, + int *bucket_A, int *bucket_B, + int n, int openMP) { + int *PAb, *ISAb, *buf; +#ifdef LIBBSC_OPENMP + int *curbuf; + int l; +#endif + int i, j, k, t, m, bufsize; + int c0, c1; +#ifdef LIBBSC_OPENMP + int d0, d1; +#endif + (void)openMP; + + /* Initialize bucket arrays. */ + for(i = 0; i < BUCKET_A_SIZE; ++i) { bucket_A[i] = 0; } + for(i = 0; i < BUCKET_B_SIZE; ++i) { bucket_B[i] = 0; } + + /* Count the number of occurrences of the first one or two characters of each + type A, B and B* suffix. Moreover, store the beginning position of all + type B* suffixes into the array SA. */ + for(i = n - 1, m = n, c0 = T[n - 1]; 0 <= i;) { + /* type A suffix. */ + do { ++BUCKET_A(c1 = c0); } while((0 <= --i) && ((c0 = T[i]) >= c1)); + if(0 <= i) { + /* type B* suffix. */ + ++BUCKET_BSTAR(c0, c1); + SA[--m] = i; + /* type B suffix. */ + for(--i, c1 = c0; (0 <= i) && ((c0 = T[i]) <= c1); --i, c1 = c0) { + ++BUCKET_B(c0, c1); + } + } + } + m = n - m; +/* +note: + A type B* suffix is lexicographically smaller than a type B suffix that + begins with the same first two characters. +*/ + + /* Calculate the index of start/end point of each bucket. */ + for(c0 = 0, i = 0, j = 0; c0 < ALPHABET_SIZE; ++c0) { + t = i + BUCKET_A(c0); + BUCKET_A(c0) = i + j; /* start point */ + i = t + BUCKET_B(c0, c0); + for(c1 = c0 + 1; c1 < ALPHABET_SIZE; ++c1) { + j += BUCKET_BSTAR(c0, c1); + BUCKET_BSTAR(c0, c1) = j; /* end point */ + i += BUCKET_B(c0, c1); + } + } + + if(0 < m) { + /* Sort the type B* suffixes by their first two characters. */ + PAb = SA + n - m; ISAb = SA + m; + for(i = m - 2; 0 <= i; --i) { + t = PAb[i], c0 = T[t], c1 = T[t + 1]; + SA[--BUCKET_BSTAR(c0, c1)] = i; + } + t = PAb[m - 1], c0 = T[t], c1 = T[t + 1]; + SA[--BUCKET_BSTAR(c0, c1)] = m - 1; + + /* Sort the type B* substrings using sssort. */ +#ifdef LIBBSC_OPENMP + if (openMP) + { + buf = SA + m; + c0 = ALPHABET_SIZE - 2, c1 = ALPHABET_SIZE - 1, j = m; +#pragma omp parallel default(shared) private(bufsize, curbuf, k, l, d0, d1) + { + bufsize = (n - (2 * m)) / omp_get_num_threads(); + curbuf = buf + omp_get_thread_num() * bufsize; + k = 0; + for(;;) { + #pragma omp critical(sssort_lock) + { + if(0 < (l = j)) { + d0 = c0, d1 = c1; + do { + k = BUCKET_BSTAR(d0, d1); + if(--d1 <= d0) { + d1 = ALPHABET_SIZE - 1; + if(--d0 < 0) { break; } + } + } while(((l - k) <= 1) && (0 < (l = k))); + c0 = d0, c1 = d1, j = k; + } + } + if(l == 0) { break; } + sssort(T, PAb, SA + k, SA + l, + curbuf, bufsize, 2, n, *(SA + k) == (m - 1)); + } + } + } + else + { + buf = SA + m, bufsize = n - (2 * m); + for(c0 = ALPHABET_SIZE - 2, j = m; 0 < j; --c0) { + for(c1 = ALPHABET_SIZE - 1; c0 < c1; j = i, --c1) { + i = BUCKET_BSTAR(c0, c1); + if(1 < (j - i)) { + sssort(T, PAb, SA + i, SA + j, + buf, bufsize, 2, n, *(SA + i) == (m - 1)); + } + } + } + } +#else + buf = SA + m, bufsize = n - (2 * m); + for(c0 = ALPHABET_SIZE - 2, j = m; 0 < j; --c0) { + for(c1 = ALPHABET_SIZE - 1; c0 < c1; j = i, --c1) { + i = BUCKET_BSTAR(c0, c1); + if(1 < (j - i)) { + sssort(T, PAb, SA + i, SA + j, + buf, bufsize, 2, n, *(SA + i) == (m - 1)); + } + } + } +#endif + + /* Compute ranks of type B* substrings. */ + for(i = m - 1; 0 <= i; --i) { + if(0 <= SA[i]) { + j = i; + do { ISAb[SA[i]] = i; } while((0 <= --i) && (0 <= SA[i])); + SA[i + 1] = i - j; + if(i <= 0) { break; } + } + j = i; + do { ISAb[SA[i] = ~SA[i]] = j; } while(SA[--i] < 0); + ISAb[SA[i]] = j; + } + + /* Construct the inverse suffix array of type B* suffixes using trsort. */ + trsort(ISAb, SA, m, 1); + + /* Set the sorted order of type B* suffixes. */ + for(i = n - 1, j = m, c0 = T[n - 1]; 0 <= i;) { + for(--i, c1 = c0; (0 <= i) && ((c0 = T[i]) >= c1); --i, c1 = c0) { } + if(0 <= i) { + t = i; + for(--i, c1 = c0; (0 <= i) && ((c0 = T[i]) <= c1); --i, c1 = c0) { } + SA[ISAb[--j]] = ((t == 0) || (1 < (t - i))) ? t : ~t; + } + } + + /* Calculate the index of start/end point of each bucket. */ + BUCKET_B(ALPHABET_SIZE - 1, ALPHABET_SIZE - 1) = n; /* end point */ + for(c0 = ALPHABET_SIZE - 2, k = m - 1; 0 <= c0; --c0) { + i = BUCKET_A(c0 + 1) - 1; + for(c1 = ALPHABET_SIZE - 1; c0 < c1; --c1) { + t = i - BUCKET_B(c0, c1); + BUCKET_B(c0, c1) = i; /* end point */ + + /* Move all type B* suffixes to the correct position. */ + for(i = t, j = BUCKET_BSTAR(c0, c1); + j <= k; + --i, --k) { SA[i] = SA[k]; } + } + BUCKET_BSTAR(c0, c0 + 1) = i - BUCKET_B(c0, c0) + 1; /* start point */ + BUCKET_B(c0, c0) = i; /* end point */ + } + } + + return m; +} + +/* Constructs the suffix array by using the sorted order of type B* suffixes. */ +static +void +construct_SA(const unsigned char *T, int *SA, + int *bucket_A, int *bucket_B, + int n, int m) { + int *i, *j, *k; + int s; + int c0, c1, c2; + + if(0 < m) { + /* Construct the sorted order of type B suffixes by using + the sorted order of type B* suffixes. */ + for(c1 = ALPHABET_SIZE - 2; 0 <= c1; --c1) { + /* Scan the suffix array from right to left. */ + for(i = SA + BUCKET_BSTAR(c1, c1 + 1), + j = SA + BUCKET_A(c1 + 1) - 1, k = NULL, c2 = -1; + i <= j; + --j) { + if(0 < (s = *j)) { + assert(T[s] == c1); + assert(((s + 1) < n) && (T[s] <= T[s + 1])); + assert(T[s - 1] <= T[s]); + *j = ~s; + c0 = T[--s]; + if((0 < s) && (T[s - 1] > c0)) { s = ~s; } + if(c0 != c2) { + if(0 <= c2) { BUCKET_B(c2, c1) = k - SA; } + k = SA + BUCKET_B(c2 = c0, c1); + } + assert(k < j); assert(k != NULL); + *k-- = s; + } else { + assert(((s == 0) && (T[s] == c1)) || (s < 0)); + *j = ~s; + } + } + } + } + + /* Construct the suffix array by using + the sorted order of type B suffixes. */ + k = SA + BUCKET_A(c2 = T[n - 1]); + *k++ = (T[n - 2] < c2) ? ~(n - 1) : (n - 1); + /* Scan the suffix array from left to right. */ + for(i = SA, j = SA + n; i < j; ++i) { + if(0 < (s = *i)) { + assert(T[s - 1] >= T[s]); + c0 = T[--s]; + if((s == 0) || (T[s - 1] < c0)) { s = ~s; } + if(c0 != c2) { + BUCKET_A(c2) = k - SA; + k = SA + BUCKET_A(c2 = c0); + } + assert(i < k); + *k++ = s; + } else { + assert(s < 0); + *i = ~s; + } + } +} + +/* Constructs the burrows-wheeler transformed string directly + by using the sorted order of type B* suffixes. */ +static +int +construct_BWT(const unsigned char *T, int *SA, + int *bucket_A, int *bucket_B, + int n, int m) { + int *i, *j, *k, *orig; + int s; + int c0, c1, c2; + + if(0 < m) { + /* Construct the sorted order of type B suffixes by using + the sorted order of type B* suffixes. */ + for(c1 = ALPHABET_SIZE - 2; 0 <= c1; --c1) { + /* Scan the suffix array from right to left. */ + for(i = SA + BUCKET_BSTAR(c1, c1 + 1), + j = SA + BUCKET_A(c1 + 1) - 1, k = NULL, c2 = -1; + i <= j; + --j) { + if(0 < (s = *j)) { + assert(T[s] == c1); + assert(((s + 1) < n) && (T[s] <= T[s + 1])); + assert(T[s - 1] <= T[s]); + c0 = T[--s]; + *j = ~((int)c0); + if((0 < s) && (T[s - 1] > c0)) { s = ~s; } + if(c0 != c2) { + if(0 <= c2) { BUCKET_B(c2, c1) = k - SA; } + k = SA + BUCKET_B(c2 = c0, c1); + } + assert(k < j); assert(k != NULL); + *k-- = s; + } else if(s != 0) { + *j = ~s; +#ifndef NDEBUG + } else { + assert(T[s] == c1); +#endif + } + } + } + } + + /* Construct the BWTed string by using + the sorted order of type B suffixes. */ + k = SA + BUCKET_A(c2 = T[n - 1]); + *k++ = (T[n - 2] < c2) ? ~((int)T[n - 2]) : (n - 1); + /* Scan the suffix array from left to right. */ + for(i = SA, j = SA + n, orig = SA; i < j; ++i) { + if(0 < (s = *i)) { + assert(T[s - 1] >= T[s]); + c0 = T[--s]; + *i = c0; + if((0 < s) && (T[s - 1] < c0)) { s = ~((int)T[s - 1]); } + if(c0 != c2) { + BUCKET_A(c2) = k - SA; + k = SA + BUCKET_A(c2 = c0); + } + assert(i < k); + *k++ = s; + } else if(s != 0) { + *i = ~s; + } else { + orig = i; + } + } + + return orig - SA; +} + +/* Constructs the burrows-wheeler transformed string directly + by using the sorted order of type B* suffixes. */ +static +int +construct_BWT_indexes(const unsigned char *T, int *SA, + int *bucket_A, int *bucket_B, + int n, int m, + unsigned char * num_indexes, int * indexes) { + int *i, *j, *k, *orig; + int s; + int c0, c1, c2; + + int mod = n / 8; + { + mod |= mod >> 1; mod |= mod >> 2; + mod |= mod >> 4; mod |= mod >> 8; + mod |= mod >> 16; mod >>= 1; + + *num_indexes = (unsigned char)((n - 1) / (mod + 1)); + } + + if(0 < m) { + /* Construct the sorted order of type B suffixes by using + the sorted order of type B* suffixes. */ + for(c1 = ALPHABET_SIZE - 2; 0 <= c1; --c1) { + /* Scan the suffix array from right to left. */ + for(i = SA + BUCKET_BSTAR(c1, c1 + 1), + j = SA + BUCKET_A(c1 + 1) - 1, k = NULL, c2 = -1; + i <= j; + --j) { + if(0 < (s = *j)) { + assert(T[s] == c1); + assert(((s + 1) < n) && (T[s] <= T[s + 1])); + assert(T[s - 1] <= T[s]); + + if ((s & mod) == 0) indexes[s / (mod + 1) - 1] = j - SA; + + c0 = T[--s]; + *j = ~((int)c0); + if((0 < s) && (T[s - 1] > c0)) { s = ~s; } + if(c0 != c2) { + if(0 <= c2) { BUCKET_B(c2, c1) = k - SA; } + k = SA + BUCKET_B(c2 = c0, c1); + } + assert(k < j); assert(k != NULL); + *k-- = s; + } else if(s != 0) { + *j = ~s; +#ifndef NDEBUG + } else { + assert(T[s] == c1); +#endif + } + } + } + } + + /* Construct the BWTed string by using + the sorted order of type B suffixes. */ + k = SA + BUCKET_A(c2 = T[n - 1]); + if (T[n - 2] < c2) { + if (((n - 1) & mod) == 0) indexes[(n - 1) / (mod + 1) - 1] = k - SA; + *k++ = ~((int)T[n - 2]); + } + else { + *k++ = n - 1; + } + + /* Scan the suffix array from left to right. */ + for(i = SA, j = SA + n, orig = SA; i < j; ++i) { + if(0 < (s = *i)) { + assert(T[s - 1] >= T[s]); + + if ((s & mod) == 0) indexes[s / (mod + 1) - 1] = i - SA; + + c0 = T[--s]; + *i = c0; + if(c0 != c2) { + BUCKET_A(c2) = k - SA; + k = SA + BUCKET_A(c2 = c0); + } + assert(i < k); + if((0 < s) && (T[s - 1] < c0)) { + if ((s & mod) == 0) indexes[s / (mod + 1) - 1] = k - SA; + *k++ = ~((int)T[s - 1]); + } else + *k++ = s; + } else if(s != 0) { + *i = ~s; + } else { + orig = i; + } + } + + return orig - SA; +} + + +/*---------------------------------------------------------------------------*/ + +/*- Function -*/ + +int +divsufsort(const unsigned char *T, int *SA, int n, int openMP) { + int *bucket_A, *bucket_B; + int m; + int err = 0; + + /* Check arguments. */ + if((T == NULL) || (SA == NULL) || (n < 0)) { return -1; } + else if(n == 0) { return 0; } + else if(n == 1) { SA[0] = 0; return 0; } + else if(n == 2) { m = (T[0] < T[1]); SA[m ^ 1] = 0, SA[m] = 1; return 0; } + + bucket_A = (int *)malloc(BUCKET_A_SIZE * sizeof(int)); + bucket_B = (int *)malloc(BUCKET_B_SIZE * sizeof(int)); + + /* Suffixsort. */ + if((bucket_A != NULL) && (bucket_B != NULL)) { + m = sort_typeBstar(T, SA, bucket_A, bucket_B, n, openMP); + construct_SA(T, SA, bucket_A, bucket_B, n, m); + } else { + err = -2; + } + + free(bucket_B); + free(bucket_A); + + return err; +} + +int +divbwt(const unsigned char *T, unsigned char *U, int *A, int n, unsigned char * num_indexes, int * indexes, int openMP) { + int *B; + int *bucket_A, *bucket_B; + int m, pidx, i; + + /* Check arguments. */ + if((T == NULL) || (U == NULL) || (n < 0)) { return -1; } + else if(n <= 1) { if(n == 1) { U[0] = T[0]; } return n; } + + if((B = A) == NULL) { B = (int *)malloc((size_t)(n + 1) * sizeof(int)); } + bucket_A = (int *)malloc(BUCKET_A_SIZE * sizeof(int)); + bucket_B = (int *)malloc(BUCKET_B_SIZE * sizeof(int)); + + /* Burrows-Wheeler Transform. */ + if((B != NULL) && (bucket_A != NULL) && (bucket_B != NULL)) { + m = sort_typeBstar(T, B, bucket_A, bucket_B, n, openMP); + + if (num_indexes == NULL || indexes == NULL) { + pidx = construct_BWT(T, B, bucket_A, bucket_B, n, m); + } else { + pidx = construct_BWT_indexes(T, B, bucket_A, bucket_B, n, m, num_indexes, indexes); + } + + /* Copy to output string. */ + U[0] = T[n - 1]; + for(i = 0; i < pidx; ++i) { U[i + 1] = (unsigned char)B[i]; } + for(i += 1; i < n; ++i) { U[i] = (unsigned char)B[i]; } + pidx += 1; + } else { + pidx = -2; + } + + free(bucket_B); + free(bucket_A); + if(A == NULL) { free(B); } + + return pidx; +} diff --git a/src/blosc2/internal-complibs/zstd-1.5.2/dictBuilder/divsufsort.h b/src/blosc2/internal-complibs/zstd-1.5.2/dictBuilder/divsufsort.h new file mode 100644 index 0000000..5440994 --- /dev/null +++ b/src/blosc2/internal-complibs/zstd-1.5.2/dictBuilder/divsufsort.h @@ -0,0 +1,67 @@ +/* + * divsufsort.h for libdivsufsort-lite + * Copyright (c) 2003-2008 Yuta Mori All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef _DIVSUFSORT_H +#define _DIVSUFSORT_H 1 + +#ifdef __cplusplus +extern "C" { +#endif /* __cplusplus */ + + +/*- Prototypes -*/ + +/** + * Constructs the suffix array of a given string. + * @param T [0..n-1] The input string. + * @param SA [0..n-1] The output array of suffixes. + * @param n The length of the given string. + * @param openMP enables OpenMP optimization. + * @return 0 if no error occurred, -1 or -2 otherwise. + */ +int +divsufsort(const unsigned char *T, int *SA, int n, int openMP); + +/** + * Constructs the burrows-wheeler transformed string of a given string. + * @param T [0..n-1] The input string. + * @param U [0..n-1] The output string. (can be T) + * @param A [0..n-1] The temporary array. (can be NULL) + * @param n The length of the given string. + * @param num_indexes The length of secondary indexes array. (can be NULL) + * @param indexes The secondary indexes array. (can be NULL) + * @param openMP enables OpenMP optimization. + * @return The primary index if no error occurred, -1 or -2 otherwise. + */ +int +divbwt(const unsigned char *T, unsigned char *U, int *A, int n, unsigned char * num_indexes, int * indexes, int openMP); + + +#ifdef __cplusplus +} /* extern "C" */ +#endif /* __cplusplus */ + +#endif /* _DIVSUFSORT_H */ diff --git a/src/blosc2/internal-complibs/zstd-1.5.2/dictBuilder/fastcover.c b/src/blosc2/internal-complibs/zstd-1.5.2/dictBuilder/fastcover.c new file mode 100644 index 0000000..3352859 --- /dev/null +++ b/src/blosc2/internal-complibs/zstd-1.5.2/dictBuilder/fastcover.c @@ -0,0 +1,766 @@ +/* + * Copyright (c) Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/*-************************************* +* Dependencies +***************************************/ +#include /* fprintf */ +#include /* malloc, free, qsort */ +#include /* memset */ +#include /* clock */ + +#ifndef ZDICT_STATIC_LINKING_ONLY +# define ZDICT_STATIC_LINKING_ONLY +#endif + +#include "../common/mem.h" /* read */ +#include "../common/pool.h" +#include "../common/threading.h" +#include "../common/zstd_internal.h" /* includes zstd.h */ +#include "../compress/zstd_compress_internal.h" /* ZSTD_hash*() */ +#include "../zdict.h" +#include "cover.h" + + +/*-************************************* +* Constants +***************************************/ +/** +* There are 32bit indexes used to ref samples, so limit samples size to 4GB +* on 64bit builds. +* For 32bit builds we choose 1 GB. +* Most 32bit platforms have 2GB user-mode addressable space and we allocate a large +* contiguous buffer, so 1GB is already a high limit. +*/ +#define FASTCOVER_MAX_SAMPLES_SIZE (sizeof(size_t) == 8 ? ((unsigned)-1) : ((unsigned)1 GB)) +#define FASTCOVER_MAX_F 31 +#define FASTCOVER_MAX_ACCEL 10 +#define FASTCOVER_DEFAULT_SPLITPOINT 0.75 +#define DEFAULT_F 20 +#define DEFAULT_ACCEL 1 + + +/*-************************************* +* Console display +***************************************/ +#ifndef LOCALDISPLAYLEVEL +static int g_displayLevel = 0; +#endif +#undef DISPLAY +#define DISPLAY(...) \ + { \ + fprintf(stderr, __VA_ARGS__); \ + fflush(stderr); \ + } +#undef LOCALDISPLAYLEVEL +#define LOCALDISPLAYLEVEL(displayLevel, l, ...) \ + if (displayLevel >= l) { \ + DISPLAY(__VA_ARGS__); \ + } /* 0 : no display; 1: errors; 2: default; 3: details; 4: debug */ +#undef DISPLAYLEVEL +#define DISPLAYLEVEL(l, ...) LOCALDISPLAYLEVEL(g_displayLevel, l, __VA_ARGS__) + +#ifndef LOCALDISPLAYUPDATE +static const clock_t g_refreshRate = CLOCKS_PER_SEC * 15 / 100; +static clock_t g_time = 0; +#endif +#undef LOCALDISPLAYUPDATE +#define LOCALDISPLAYUPDATE(displayLevel, l, ...) \ + if (displayLevel >= l) { \ + if ((clock() - g_time > g_refreshRate) || (displayLevel >= 4)) { \ + g_time = clock(); \ + DISPLAY(__VA_ARGS__); \ + } \ + } +#undef DISPLAYUPDATE +#define DISPLAYUPDATE(l, ...) LOCALDISPLAYUPDATE(g_displayLevel, l, __VA_ARGS__) + + +/*-************************************* +* Hash Functions +***************************************/ +/** + * Hash the d-byte value pointed to by p and mod 2^f into the frequency vector + */ +static size_t FASTCOVER_hashPtrToIndex(const void* p, U32 f, unsigned d) { + if (d == 6) { + return ZSTD_hash6Ptr(p, f); + } + return ZSTD_hash8Ptr(p, f); +} + + +/*-************************************* +* Acceleration +***************************************/ +typedef struct { + unsigned finalize; /* Percentage of training samples used for ZDICT_finalizeDictionary */ + unsigned skip; /* Number of dmer skipped between each dmer counted in computeFrequency */ +} FASTCOVER_accel_t; + + +static const FASTCOVER_accel_t FASTCOVER_defaultAccelParameters[FASTCOVER_MAX_ACCEL+1] = { + { 100, 0 }, /* accel = 0, should not happen because accel = 0 defaults to accel = 1 */ + { 100, 0 }, /* accel = 1 */ + { 50, 1 }, /* accel = 2 */ + { 34, 2 }, /* accel = 3 */ + { 25, 3 }, /* accel = 4 */ + { 20, 4 }, /* accel = 5 */ + { 17, 5 }, /* accel = 6 */ + { 14, 6 }, /* accel = 7 */ + { 13, 7 }, /* accel = 8 */ + { 11, 8 }, /* accel = 9 */ + { 10, 9 }, /* accel = 10 */ +}; + + +/*-************************************* +* Context +***************************************/ +typedef struct { + const BYTE *samples; + size_t *offsets; + const size_t *samplesSizes; + size_t nbSamples; + size_t nbTrainSamples; + size_t nbTestSamples; + size_t nbDmers; + U32 *freqs; + unsigned d; + unsigned f; + FASTCOVER_accel_t accelParams; +} FASTCOVER_ctx_t; + + +/*-************************************* +* Helper functions +***************************************/ +/** + * Selects the best segment in an epoch. + * Segments of are scored according to the function: + * + * Let F(d) be the frequency of all dmers with hash value d. + * Let S_i be hash value of the dmer at position i of segment S which has length k. + * + * Score(S) = F(S_1) + F(S_2) + ... + F(S_{k-d+1}) + * + * Once the dmer with hash value d is in the dictionary we set F(d) = 0. + */ +static COVER_segment_t FASTCOVER_selectSegment(const FASTCOVER_ctx_t *ctx, + U32 *freqs, U32 begin, U32 end, + ZDICT_cover_params_t parameters, + U16* segmentFreqs) { + /* Constants */ + const U32 k = parameters.k; + const U32 d = parameters.d; + const U32 f = ctx->f; + const U32 dmersInK = k - d + 1; + + /* Try each segment (activeSegment) and save the best (bestSegment) */ + COVER_segment_t bestSegment = {0, 0, 0}; + COVER_segment_t activeSegment; + + /* Reset the activeDmers in the segment */ + /* The activeSegment starts at the beginning of the epoch. */ + activeSegment.begin = begin; + activeSegment.end = begin; + activeSegment.score = 0; + + /* Slide the activeSegment through the whole epoch. + * Save the best segment in bestSegment. + */ + while (activeSegment.end < end) { + /* Get hash value of current dmer */ + const size_t idx = FASTCOVER_hashPtrToIndex(ctx->samples + activeSegment.end, f, d); + + /* Add frequency of this index to score if this is the first occurrence of index in active segment */ + if (segmentFreqs[idx] == 0) { + activeSegment.score += freqs[idx]; + } + /* Increment end of segment and segmentFreqs*/ + activeSegment.end += 1; + segmentFreqs[idx] += 1; + /* If the window is now too large, drop the first position */ + if (activeSegment.end - activeSegment.begin == dmersInK + 1) { + /* Get hash value of the dmer to be eliminated from active segment */ + const size_t delIndex = FASTCOVER_hashPtrToIndex(ctx->samples + activeSegment.begin, f, d); + segmentFreqs[delIndex] -= 1; + /* Subtract frequency of this index from score if this is the last occurrence of this index in active segment */ + if (segmentFreqs[delIndex] == 0) { + activeSegment.score -= freqs[delIndex]; + } + /* Increment start of segment */ + activeSegment.begin += 1; + } + + /* If this segment is the best so far save it */ + if (activeSegment.score > bestSegment.score) { + bestSegment = activeSegment; + } + } + + /* Zero out rest of segmentFreqs array */ + while (activeSegment.begin < end) { + const size_t delIndex = FASTCOVER_hashPtrToIndex(ctx->samples + activeSegment.begin, f, d); + segmentFreqs[delIndex] -= 1; + activeSegment.begin += 1; + } + + { + /* Zero the frequency of hash value of each dmer covered by the chosen segment. */ + U32 pos; + for (pos = bestSegment.begin; pos != bestSegment.end; ++pos) { + const size_t i = FASTCOVER_hashPtrToIndex(ctx->samples + pos, f, d); + freqs[i] = 0; + } + } + + return bestSegment; +} + + +static int FASTCOVER_checkParameters(ZDICT_cover_params_t parameters, + size_t maxDictSize, unsigned f, + unsigned accel) { + /* k, d, and f are required parameters */ + if (parameters.d == 0 || parameters.k == 0) { + return 0; + } + /* d has to be 6 or 8 */ + if (parameters.d != 6 && parameters.d != 8) { + return 0; + } + /* k <= maxDictSize */ + if (parameters.k > maxDictSize) { + return 0; + } + /* d <= k */ + if (parameters.d > parameters.k) { + return 0; + } + /* 0 < f <= FASTCOVER_MAX_F*/ + if (f > FASTCOVER_MAX_F || f == 0) { + return 0; + } + /* 0 < splitPoint <= 1 */ + if (parameters.splitPoint <= 0 || parameters.splitPoint > 1) { + return 0; + } + /* 0 < accel <= 10 */ + if (accel > 10 || accel == 0) { + return 0; + } + return 1; +} + + +/** + * Clean up a context initialized with `FASTCOVER_ctx_init()`. + */ +static void +FASTCOVER_ctx_destroy(FASTCOVER_ctx_t* ctx) +{ + if (!ctx) return; + + free(ctx->freqs); + ctx->freqs = NULL; + + free(ctx->offsets); + ctx->offsets = NULL; +} + + +/** + * Calculate for frequency of hash value of each dmer in ctx->samples + */ +static void +FASTCOVER_computeFrequency(U32* freqs, const FASTCOVER_ctx_t* ctx) +{ + const unsigned f = ctx->f; + const unsigned d = ctx->d; + const unsigned skip = ctx->accelParams.skip; + const unsigned readLength = MAX(d, 8); + size_t i; + assert(ctx->nbTrainSamples >= 5); + assert(ctx->nbTrainSamples <= ctx->nbSamples); + for (i = 0; i < ctx->nbTrainSamples; i++) { + size_t start = ctx->offsets[i]; /* start of current dmer */ + size_t const currSampleEnd = ctx->offsets[i+1]; + while (start + readLength <= currSampleEnd) { + const size_t dmerIndex = FASTCOVER_hashPtrToIndex(ctx->samples + start, f, d); + freqs[dmerIndex]++; + start = start + skip + 1; + } + } +} + + +/** + * Prepare a context for dictionary building. + * The context is only dependent on the parameter `d` and can used multiple + * times. + * Returns 0 on success or error code on error. + * The context must be destroyed with `FASTCOVER_ctx_destroy()`. + */ +static size_t +FASTCOVER_ctx_init(FASTCOVER_ctx_t* ctx, + const void* samplesBuffer, + const size_t* samplesSizes, unsigned nbSamples, + unsigned d, double splitPoint, unsigned f, + FASTCOVER_accel_t accelParams) +{ + const BYTE* const samples = (const BYTE*)samplesBuffer; + const size_t totalSamplesSize = COVER_sum(samplesSizes, nbSamples); + /* Split samples into testing and training sets */ + const unsigned nbTrainSamples = splitPoint < 1.0 ? (unsigned)((double)nbSamples * splitPoint) : nbSamples; + const unsigned nbTestSamples = splitPoint < 1.0 ? nbSamples - nbTrainSamples : nbSamples; + const size_t trainingSamplesSize = splitPoint < 1.0 ? COVER_sum(samplesSizes, nbTrainSamples) : totalSamplesSize; + const size_t testSamplesSize = splitPoint < 1.0 ? COVER_sum(samplesSizes + nbTrainSamples, nbTestSamples) : totalSamplesSize; + + /* Checks */ + if (totalSamplesSize < MAX(d, sizeof(U64)) || + totalSamplesSize >= (size_t)FASTCOVER_MAX_SAMPLES_SIZE) { + DISPLAYLEVEL(1, "Total samples size is too large (%u MB), maximum size is %u MB\n", + (unsigned)(totalSamplesSize >> 20), (FASTCOVER_MAX_SAMPLES_SIZE >> 20)); + return ERROR(srcSize_wrong); + } + + /* Check if there are at least 5 training samples */ + if (nbTrainSamples < 5) { + DISPLAYLEVEL(1, "Total number of training samples is %u and is invalid\n", nbTrainSamples); + return ERROR(srcSize_wrong); + } + + /* Check if there's testing sample */ + if (nbTestSamples < 1) { + DISPLAYLEVEL(1, "Total number of testing samples is %u and is invalid.\n", nbTestSamples); + return ERROR(srcSize_wrong); + } + + /* Zero the context */ + memset(ctx, 0, sizeof(*ctx)); + DISPLAYLEVEL(2, "Training on %u samples of total size %u\n", nbTrainSamples, + (unsigned)trainingSamplesSize); + DISPLAYLEVEL(2, "Testing on %u samples of total size %u\n", nbTestSamples, + (unsigned)testSamplesSize); + + ctx->samples = samples; + ctx->samplesSizes = samplesSizes; + ctx->nbSamples = nbSamples; + ctx->nbTrainSamples = nbTrainSamples; + ctx->nbTestSamples = nbTestSamples; + ctx->nbDmers = trainingSamplesSize - MAX(d, sizeof(U64)) + 1; + ctx->d = d; + ctx->f = f; + ctx->accelParams = accelParams; + + /* The offsets of each file */ + ctx->offsets = (size_t*)calloc((nbSamples + 1), sizeof(size_t)); + if (ctx->offsets == NULL) { + DISPLAYLEVEL(1, "Failed to allocate scratch buffers \n"); + FASTCOVER_ctx_destroy(ctx); + return ERROR(memory_allocation); + } + + /* Fill offsets from the samplesSizes */ + { U32 i; + ctx->offsets[0] = 0; + assert(nbSamples >= 5); + for (i = 1; i <= nbSamples; ++i) { + ctx->offsets[i] = ctx->offsets[i - 1] + samplesSizes[i - 1]; + } + } + + /* Initialize frequency array of size 2^f */ + ctx->freqs = (U32*)calloc(((U64)1 << f), sizeof(U32)); + if (ctx->freqs == NULL) { + DISPLAYLEVEL(1, "Failed to allocate frequency table \n"); + FASTCOVER_ctx_destroy(ctx); + return ERROR(memory_allocation); + } + + DISPLAYLEVEL(2, "Computing frequencies\n"); + FASTCOVER_computeFrequency(ctx->freqs, ctx); + + return 0; +} + + +/** + * Given the prepared context build the dictionary. + */ +static size_t +FASTCOVER_buildDictionary(const FASTCOVER_ctx_t* ctx, + U32* freqs, + void* dictBuffer, size_t dictBufferCapacity, + ZDICT_cover_params_t parameters, + U16* segmentFreqs) +{ + BYTE *const dict = (BYTE *)dictBuffer; + size_t tail = dictBufferCapacity; + /* Divide the data into epochs. We will select one segment from each epoch. */ + const COVER_epoch_info_t epochs = COVER_computeEpochs( + (U32)dictBufferCapacity, (U32)ctx->nbDmers, parameters.k, 1); + const size_t maxZeroScoreRun = 10; + size_t zeroScoreRun = 0; + size_t epoch; + DISPLAYLEVEL(2, "Breaking content into %u epochs of size %u\n", + (U32)epochs.num, (U32)epochs.size); + /* Loop through the epochs until there are no more segments or the dictionary + * is full. + */ + for (epoch = 0; tail > 0; epoch = (epoch + 1) % epochs.num) { + const U32 epochBegin = (U32)(epoch * epochs.size); + const U32 epochEnd = epochBegin + epochs.size; + size_t segmentSize; + /* Select a segment */ + COVER_segment_t segment = FASTCOVER_selectSegment( + ctx, freqs, epochBegin, epochEnd, parameters, segmentFreqs); + + /* If the segment covers no dmers, then we are out of content. + * There may be new content in other epochs, for continue for some time. + */ + if (segment.score == 0) { + if (++zeroScoreRun >= maxZeroScoreRun) { + break; + } + continue; + } + zeroScoreRun = 0; + + /* Trim the segment if necessary and if it is too small then we are done */ + segmentSize = MIN(segment.end - segment.begin + parameters.d - 1, tail); + if (segmentSize < parameters.d) { + break; + } + + /* We fill the dictionary from the back to allow the best segments to be + * referenced with the smallest offsets. + */ + tail -= segmentSize; + memcpy(dict + tail, ctx->samples + segment.begin, segmentSize); + DISPLAYUPDATE( + 2, "\r%u%% ", + (unsigned)(((dictBufferCapacity - tail) * 100) / dictBufferCapacity)); + } + DISPLAYLEVEL(2, "\r%79s\r", ""); + return tail; +} + +/** + * Parameters for FASTCOVER_tryParameters(). + */ +typedef struct FASTCOVER_tryParameters_data_s { + const FASTCOVER_ctx_t* ctx; + COVER_best_t* best; + size_t dictBufferCapacity; + ZDICT_cover_params_t parameters; +} FASTCOVER_tryParameters_data_t; + + +/** + * Tries a set of parameters and updates the COVER_best_t with the results. + * This function is thread safe if zstd is compiled with multithreaded support. + * It takes its parameters as an *OWNING* opaque pointer to support threading. + */ +static void FASTCOVER_tryParameters(void* opaque) +{ + /* Save parameters as local variables */ + FASTCOVER_tryParameters_data_t *const data = (FASTCOVER_tryParameters_data_t*)opaque; + const FASTCOVER_ctx_t *const ctx = data->ctx; + const ZDICT_cover_params_t parameters = data->parameters; + size_t dictBufferCapacity = data->dictBufferCapacity; + size_t totalCompressedSize = ERROR(GENERIC); + /* Initialize array to keep track of frequency of dmer within activeSegment */ + U16* segmentFreqs = (U16*)calloc(((U64)1 << ctx->f), sizeof(U16)); + /* Allocate space for hash table, dict, and freqs */ + BYTE *const dict = (BYTE*)malloc(dictBufferCapacity); + COVER_dictSelection_t selection = COVER_dictSelectionError(ERROR(GENERIC)); + U32* freqs = (U32*) malloc(((U64)1 << ctx->f) * sizeof(U32)); + if (!segmentFreqs || !dict || !freqs) { + DISPLAYLEVEL(1, "Failed to allocate buffers: out of memory\n"); + goto _cleanup; + } + /* Copy the frequencies because we need to modify them */ + memcpy(freqs, ctx->freqs, ((U64)1 << ctx->f) * sizeof(U32)); + /* Build the dictionary */ + { const size_t tail = FASTCOVER_buildDictionary(ctx, freqs, dict, dictBufferCapacity, + parameters, segmentFreqs); + + const unsigned nbFinalizeSamples = (unsigned)(ctx->nbTrainSamples * ctx->accelParams.finalize / 100); + selection = COVER_selectDict(dict + tail, dictBufferCapacity, dictBufferCapacity - tail, + ctx->samples, ctx->samplesSizes, nbFinalizeSamples, ctx->nbTrainSamples, ctx->nbSamples, parameters, ctx->offsets, + totalCompressedSize); + + if (COVER_dictSelectionIsError(selection)) { + DISPLAYLEVEL(1, "Failed to select dictionary\n"); + goto _cleanup; + } + } +_cleanup: + free(dict); + COVER_best_finish(data->best, parameters, selection); + free(data); + free(segmentFreqs); + COVER_dictSelectionFree(selection); + free(freqs); +} + + +static void +FASTCOVER_convertToCoverParams(ZDICT_fastCover_params_t fastCoverParams, + ZDICT_cover_params_t* coverParams) +{ + coverParams->k = fastCoverParams.k; + coverParams->d = fastCoverParams.d; + coverParams->steps = fastCoverParams.steps; + coverParams->nbThreads = fastCoverParams.nbThreads; + coverParams->splitPoint = fastCoverParams.splitPoint; + coverParams->zParams = fastCoverParams.zParams; + coverParams->shrinkDict = fastCoverParams.shrinkDict; +} + + +static void +FASTCOVER_convertToFastCoverParams(ZDICT_cover_params_t coverParams, + ZDICT_fastCover_params_t* fastCoverParams, + unsigned f, unsigned accel) +{ + fastCoverParams->k = coverParams.k; + fastCoverParams->d = coverParams.d; + fastCoverParams->steps = coverParams.steps; + fastCoverParams->nbThreads = coverParams.nbThreads; + fastCoverParams->splitPoint = coverParams.splitPoint; + fastCoverParams->f = f; + fastCoverParams->accel = accel; + fastCoverParams->zParams = coverParams.zParams; + fastCoverParams->shrinkDict = coverParams.shrinkDict; +} + + +ZDICTLIB_API size_t +ZDICT_trainFromBuffer_fastCover(void* dictBuffer, size_t dictBufferCapacity, + const void* samplesBuffer, + const size_t* samplesSizes, unsigned nbSamples, + ZDICT_fastCover_params_t parameters) +{ + BYTE* const dict = (BYTE*)dictBuffer; + FASTCOVER_ctx_t ctx; + ZDICT_cover_params_t coverParams; + FASTCOVER_accel_t accelParams; + /* Initialize global data */ + g_displayLevel = (int)parameters.zParams.notificationLevel; + /* Assign splitPoint and f if not provided */ + parameters.splitPoint = 1.0; + parameters.f = parameters.f == 0 ? DEFAULT_F : parameters.f; + parameters.accel = parameters.accel == 0 ? DEFAULT_ACCEL : parameters.accel; + /* Convert to cover parameter */ + memset(&coverParams, 0 , sizeof(coverParams)); + FASTCOVER_convertToCoverParams(parameters, &coverParams); + /* Checks */ + if (!FASTCOVER_checkParameters(coverParams, dictBufferCapacity, parameters.f, + parameters.accel)) { + DISPLAYLEVEL(1, "FASTCOVER parameters incorrect\n"); + return ERROR(parameter_outOfBound); + } + if (nbSamples == 0) { + DISPLAYLEVEL(1, "FASTCOVER must have at least one input file\n"); + return ERROR(srcSize_wrong); + } + if (dictBufferCapacity < ZDICT_DICTSIZE_MIN) { + DISPLAYLEVEL(1, "dictBufferCapacity must be at least %u\n", + ZDICT_DICTSIZE_MIN); + return ERROR(dstSize_tooSmall); + } + /* Assign corresponding FASTCOVER_accel_t to accelParams*/ + accelParams = FASTCOVER_defaultAccelParameters[parameters.accel]; + /* Initialize context */ + { + size_t const initVal = FASTCOVER_ctx_init(&ctx, samplesBuffer, samplesSizes, nbSamples, + coverParams.d, parameters.splitPoint, parameters.f, + accelParams); + if (ZSTD_isError(initVal)) { + DISPLAYLEVEL(1, "Failed to initialize context\n"); + return initVal; + } + } + COVER_warnOnSmallCorpus(dictBufferCapacity, ctx.nbDmers, g_displayLevel); + /* Build the dictionary */ + DISPLAYLEVEL(2, "Building dictionary\n"); + { + /* Initialize array to keep track of frequency of dmer within activeSegment */ + U16* segmentFreqs = (U16 *)calloc(((U64)1 << parameters.f), sizeof(U16)); + const size_t tail = FASTCOVER_buildDictionary(&ctx, ctx.freqs, dictBuffer, + dictBufferCapacity, coverParams, segmentFreqs); + const unsigned nbFinalizeSamples = (unsigned)(ctx.nbTrainSamples * ctx.accelParams.finalize / 100); + const size_t dictionarySize = ZDICT_finalizeDictionary( + dict, dictBufferCapacity, dict + tail, dictBufferCapacity - tail, + samplesBuffer, samplesSizes, nbFinalizeSamples, coverParams.zParams); + if (!ZSTD_isError(dictionarySize)) { + DISPLAYLEVEL(2, "Constructed dictionary of size %u\n", + (unsigned)dictionarySize); + } + FASTCOVER_ctx_destroy(&ctx); + free(segmentFreqs); + return dictionarySize; + } +} + + +ZDICTLIB_API size_t +ZDICT_optimizeTrainFromBuffer_fastCover( + void* dictBuffer, size_t dictBufferCapacity, + const void* samplesBuffer, + const size_t* samplesSizes, unsigned nbSamples, + ZDICT_fastCover_params_t* parameters) +{ + ZDICT_cover_params_t coverParams; + FASTCOVER_accel_t accelParams; + /* constants */ + const unsigned nbThreads = parameters->nbThreads; + const double splitPoint = + parameters->splitPoint <= 0.0 ? FASTCOVER_DEFAULT_SPLITPOINT : parameters->splitPoint; + const unsigned kMinD = parameters->d == 0 ? 6 : parameters->d; + const unsigned kMaxD = parameters->d == 0 ? 8 : parameters->d; + const unsigned kMinK = parameters->k == 0 ? 50 : parameters->k; + const unsigned kMaxK = parameters->k == 0 ? 2000 : parameters->k; + const unsigned kSteps = parameters->steps == 0 ? 40 : parameters->steps; + const unsigned kStepSize = MAX((kMaxK - kMinK) / kSteps, 1); + const unsigned kIterations = + (1 + (kMaxD - kMinD) / 2) * (1 + (kMaxK - kMinK) / kStepSize); + const unsigned f = parameters->f == 0 ? DEFAULT_F : parameters->f; + const unsigned accel = parameters->accel == 0 ? DEFAULT_ACCEL : parameters->accel; + const unsigned shrinkDict = 0; + /* Local variables */ + const int displayLevel = (int)parameters->zParams.notificationLevel; + unsigned iteration = 1; + unsigned d; + unsigned k; + COVER_best_t best; + POOL_ctx *pool = NULL; + int warned = 0; + /* Checks */ + if (splitPoint <= 0 || splitPoint > 1) { + LOCALDISPLAYLEVEL(displayLevel, 1, "Incorrect splitPoint\n"); + return ERROR(parameter_outOfBound); + } + if (accel == 0 || accel > FASTCOVER_MAX_ACCEL) { + LOCALDISPLAYLEVEL(displayLevel, 1, "Incorrect accel\n"); + return ERROR(parameter_outOfBound); + } + if (kMinK < kMaxD || kMaxK < kMinK) { + LOCALDISPLAYLEVEL(displayLevel, 1, "Incorrect k\n"); + return ERROR(parameter_outOfBound); + } + if (nbSamples == 0) { + LOCALDISPLAYLEVEL(displayLevel, 1, "FASTCOVER must have at least one input file\n"); + return ERROR(srcSize_wrong); + } + if (dictBufferCapacity < ZDICT_DICTSIZE_MIN) { + LOCALDISPLAYLEVEL(displayLevel, 1, "dictBufferCapacity must be at least %u\n", + ZDICT_DICTSIZE_MIN); + return ERROR(dstSize_tooSmall); + } + if (nbThreads > 1) { + pool = POOL_create(nbThreads, 1); + if (!pool) { + return ERROR(memory_allocation); + } + } + /* Initialization */ + COVER_best_init(&best); + memset(&coverParams, 0 , sizeof(coverParams)); + FASTCOVER_convertToCoverParams(*parameters, &coverParams); + accelParams = FASTCOVER_defaultAccelParameters[accel]; + /* Turn down global display level to clean up display at level 2 and below */ + g_displayLevel = displayLevel == 0 ? 0 : displayLevel - 1; + /* Loop through d first because each new value needs a new context */ + LOCALDISPLAYLEVEL(displayLevel, 2, "Trying %u different sets of parameters\n", + kIterations); + for (d = kMinD; d <= kMaxD; d += 2) { + /* Initialize the context for this value of d */ + FASTCOVER_ctx_t ctx; + LOCALDISPLAYLEVEL(displayLevel, 3, "d=%u\n", d); + { + size_t const initVal = FASTCOVER_ctx_init(&ctx, samplesBuffer, samplesSizes, nbSamples, d, splitPoint, f, accelParams); + if (ZSTD_isError(initVal)) { + LOCALDISPLAYLEVEL(displayLevel, 1, "Failed to initialize context\n"); + COVER_best_destroy(&best); + POOL_free(pool); + return initVal; + } + } + if (!warned) { + COVER_warnOnSmallCorpus(dictBufferCapacity, ctx.nbDmers, displayLevel); + warned = 1; + } + /* Loop through k reusing the same context */ + for (k = kMinK; k <= kMaxK; k += kStepSize) { + /* Prepare the arguments */ + FASTCOVER_tryParameters_data_t *data = (FASTCOVER_tryParameters_data_t *)malloc( + sizeof(FASTCOVER_tryParameters_data_t)); + LOCALDISPLAYLEVEL(displayLevel, 3, "k=%u\n", k); + if (!data) { + LOCALDISPLAYLEVEL(displayLevel, 1, "Failed to allocate parameters\n"); + COVER_best_destroy(&best); + FASTCOVER_ctx_destroy(&ctx); + POOL_free(pool); + return ERROR(memory_allocation); + } + data->ctx = &ctx; + data->best = &best; + data->dictBufferCapacity = dictBufferCapacity; + data->parameters = coverParams; + data->parameters.k = k; + data->parameters.d = d; + data->parameters.splitPoint = splitPoint; + data->parameters.steps = kSteps; + data->parameters.shrinkDict = shrinkDict; + data->parameters.zParams.notificationLevel = (unsigned)g_displayLevel; + /* Check the parameters */ + if (!FASTCOVER_checkParameters(data->parameters, dictBufferCapacity, + data->ctx->f, accel)) { + DISPLAYLEVEL(1, "FASTCOVER parameters incorrect\n"); + free(data); + continue; + } + /* Call the function and pass ownership of data to it */ + COVER_best_start(&best); + if (pool) { + POOL_add(pool, &FASTCOVER_tryParameters, data); + } else { + FASTCOVER_tryParameters(data); + } + /* Print status */ + LOCALDISPLAYUPDATE(displayLevel, 2, "\r%u%% ", + (unsigned)((iteration * 100) / kIterations)); + ++iteration; + } + COVER_best_wait(&best); + FASTCOVER_ctx_destroy(&ctx); + } + LOCALDISPLAYLEVEL(displayLevel, 2, "\r%79s\r", ""); + /* Fill the output buffer and parameters with output of the best parameters */ + { + const size_t dictSize = best.dictSize; + if (ZSTD_isError(best.compressedSize)) { + const size_t compressedSize = best.compressedSize; + COVER_best_destroy(&best); + POOL_free(pool); + return compressedSize; + } + FASTCOVER_convertToFastCoverParams(best.parameters, parameters, f, accel); + memcpy(dictBuffer, best.dict, dictSize); + COVER_best_destroy(&best); + POOL_free(pool); + return dictSize; + } + +} diff --git a/src/blosc2/internal-complibs/zstd-1.5.2/dictBuilder/zdict.c b/src/blosc2/internal-complibs/zstd-1.5.2/dictBuilder/zdict.c new file mode 100644 index 0000000..006aba7 --- /dev/null +++ b/src/blosc2/internal-complibs/zstd-1.5.2/dictBuilder/zdict.c @@ -0,0 +1,1205 @@ +/* + * Copyright (c) Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + + +/*-************************************** +* Tuning parameters +****************************************/ +#define MINRATIO 4 /* minimum nb of apparition to be selected in dictionary */ +#define ZDICT_MAX_SAMPLES_SIZE (2000U << 20) +#define ZDICT_MIN_SAMPLES_SIZE (ZDICT_CONTENTSIZE_MIN * MINRATIO) + + +/*-************************************** +* Compiler Options +****************************************/ +/* Unix Large Files support (>4GB) */ +#define _FILE_OFFSET_BITS 64 +#if (defined(__sun__) && (!defined(__LP64__))) /* Sun Solaris 32-bits requires specific definitions */ +# ifndef _LARGEFILE_SOURCE +# define _LARGEFILE_SOURCE +# endif +#elif ! defined(__LP64__) /* No point defining Large file for 64 bit */ +# ifndef _LARGEFILE64_SOURCE +# define _LARGEFILE64_SOURCE +# endif +#endif + + +/*-************************************* +* Dependencies +***************************************/ +#include /* malloc, free */ +#include /* memset */ +#include /* fprintf, fopen, ftello64 */ +#include /* clock */ + +#ifndef ZDICT_STATIC_LINKING_ONLY +# define ZDICT_STATIC_LINKING_ONLY +#endif +#define HUF_STATIC_LINKING_ONLY + +#include "../common/mem.h" /* read */ +#include "../common/fse.h" /* FSE_normalizeCount, FSE_writeNCount */ +#include "../common/huf.h" /* HUF_buildCTable, HUF_writeCTable */ +#include "../common/zstd_internal.h" /* includes zstd.h */ +#include "../common/xxhash.h" /* XXH64 */ +#include "../compress/zstd_compress_internal.h" /* ZSTD_loadCEntropy() */ +#include "../zdict.h" +#include "divsufsort.h" + + +/*-************************************* +* Constants +***************************************/ +#define KB *(1 <<10) +#define MB *(1 <<20) +#define GB *(1U<<30) + +#define DICTLISTSIZE_DEFAULT 10000 + +#define NOISELENGTH 32 + +static const U32 g_selectivity_default = 9; + + +/*-************************************* +* Console display +***************************************/ +#undef DISPLAY +#define DISPLAY(...) { fprintf(stderr, __VA_ARGS__); fflush( stderr ); } +#undef DISPLAYLEVEL +#define DISPLAYLEVEL(l, ...) if (notificationLevel>=l) { DISPLAY(__VA_ARGS__); } /* 0 : no display; 1: errors; 2: default; 3: details; 4: debug */ + +static clock_t ZDICT_clockSpan(clock_t nPrevious) { return clock() - nPrevious; } + +static void ZDICT_printHex(const void* ptr, size_t length) +{ + const BYTE* const b = (const BYTE*)ptr; + size_t u; + for (u=0; u126) c = '.'; /* non-printable char */ + DISPLAY("%c", c); + } +} + + +/*-******************************************************** +* Helper functions +**********************************************************/ +unsigned ZDICT_isError(size_t errorCode) { return ERR_isError(errorCode); } + +const char* ZDICT_getErrorName(size_t errorCode) { return ERR_getErrorName(errorCode); } + +unsigned ZDICT_getDictID(const void* dictBuffer, size_t dictSize) +{ + if (dictSize < 8) return 0; + if (MEM_readLE32(dictBuffer) != ZSTD_MAGIC_DICTIONARY) return 0; + return MEM_readLE32((const char*)dictBuffer + 4); +} + +size_t ZDICT_getDictHeaderSize(const void* dictBuffer, size_t dictSize) +{ + size_t headerSize; + if (dictSize <= 8 || MEM_readLE32(dictBuffer) != ZSTD_MAGIC_DICTIONARY) return ERROR(dictionary_corrupted); + + { ZSTD_compressedBlockState_t* bs = (ZSTD_compressedBlockState_t*)malloc(sizeof(ZSTD_compressedBlockState_t)); + U32* wksp = (U32*)malloc(HUF_WORKSPACE_SIZE); + if (!bs || !wksp) { + headerSize = ERROR(memory_allocation); + } else { + ZSTD_reset_compressedBlockState(bs); + headerSize = ZSTD_loadCEntropy(bs, wksp, dictBuffer, dictSize); + } + + free(bs); + free(wksp); + } + + return headerSize; +} + +/*-******************************************************** +* Dictionary training functions +**********************************************************/ +static unsigned ZDICT_NbCommonBytes (size_t val) +{ + if (MEM_isLittleEndian()) { + if (MEM_64bits()) { +# if defined(_MSC_VER) && defined(_WIN64) + if (val != 0) { + unsigned long r; + _BitScanForward64(&r, (U64)val); + return (unsigned)(r >> 3); + } else { + /* Should not reach this code path */ + __assume(0); + } +# elif defined(__GNUC__) && (__GNUC__ >= 3) + return (unsigned)(__builtin_ctzll((U64)val) >> 3); +# else + static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, 0, 3, 1, 3, 1, 4, 2, 7, 0, 2, 3, 6, 1, 5, 3, 5, 1, 3, 4, 4, 2, 5, 6, 7, 7, 0, 1, 2, 3, 3, 4, 6, 2, 6, 5, 5, 3, 4, 5, 6, 7, 1, 2, 4, 6, 4, 4, 5, 7, 2, 6, 5, 7, 6, 7, 7 }; + return DeBruijnBytePos[((U64)((val & -(long long)val) * 0x0218A392CDABBD3FULL)) >> 58]; +# endif + } else { /* 32 bits */ +# if defined(_MSC_VER) + if (val != 0) { + unsigned long r; + _BitScanForward(&r, (U32)val); + return (unsigned)(r >> 3); + } else { + /* Should not reach this code path */ + __assume(0); + } +# elif defined(__GNUC__) && (__GNUC__ >= 3) + return (unsigned)(__builtin_ctz((U32)val) >> 3); +# else + static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, 3, 2, 2, 1, 3, 2, 0, 1, 3, 3, 1, 2, 2, 2, 2, 0, 3, 1, 2, 0, 1, 0, 1, 1 }; + return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27]; +# endif + } + } else { /* Big Endian CPU */ + if (MEM_64bits()) { +# if defined(_MSC_VER) && defined(_WIN64) + if (val != 0) { + unsigned long r; + _BitScanReverse64(&r, val); + return (unsigned)(r >> 3); + } else { + /* Should not reach this code path */ + __assume(0); + } +# elif defined(__GNUC__) && (__GNUC__ >= 3) + return (unsigned)(__builtin_clzll(val) >> 3); +# else + unsigned r; + const unsigned n32 = sizeof(size_t)*4; /* calculate this way due to compiler complaining in 32-bits mode */ + if (!(val>>n32)) { r=4; } else { r=0; val>>=n32; } + if (!(val>>16)) { r+=2; val>>=8; } else { val>>=24; } + r += (!val); + return r; +# endif + } else { /* 32 bits */ +# if defined(_MSC_VER) + if (val != 0) { + unsigned long r; + _BitScanReverse(&r, (unsigned long)val); + return (unsigned)(r >> 3); + } else { + /* Should not reach this code path */ + __assume(0); + } +# elif defined(__GNUC__) && (__GNUC__ >= 3) + return (unsigned)(__builtin_clz((U32)val) >> 3); +# else + unsigned r; + if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; } + r += (!val); + return r; +# endif + } } +} + + +/*! ZDICT_count() : + Count the nb of common bytes between 2 pointers. + Note : this function presumes end of buffer followed by noisy guard band. +*/ +static size_t ZDICT_count(const void* pIn, const void* pMatch) +{ + const char* const pStart = (const char*)pIn; + for (;;) { + size_t const diff = MEM_readST(pMatch) ^ MEM_readST(pIn); + if (!diff) { + pIn = (const char*)pIn+sizeof(size_t); + pMatch = (const char*)pMatch+sizeof(size_t); + continue; + } + pIn = (const char*)pIn+ZDICT_NbCommonBytes(diff); + return (size_t)((const char*)pIn - pStart); + } +} + + +typedef struct { + U32 pos; + U32 length; + U32 savings; +} dictItem; + +static void ZDICT_initDictItem(dictItem* d) +{ + d->pos = 1; + d->length = 0; + d->savings = (U32)(-1); +} + + +#define LLIMIT 64 /* heuristic determined experimentally */ +#define MINMATCHLENGTH 7 /* heuristic determined experimentally */ +static dictItem ZDICT_analyzePos( + BYTE* doneMarks, + const int* suffix, U32 start, + const void* buffer, U32 minRatio, U32 notificationLevel) +{ + U32 lengthList[LLIMIT] = {0}; + U32 cumulLength[LLIMIT] = {0}; + U32 savings[LLIMIT] = {0}; + const BYTE* b = (const BYTE*)buffer; + size_t maxLength = LLIMIT; + size_t pos = (size_t)suffix[start]; + U32 end = start; + dictItem solution; + + /* init */ + memset(&solution, 0, sizeof(solution)); + doneMarks[pos] = 1; + + /* trivial repetition cases */ + if ( (MEM_read16(b+pos+0) == MEM_read16(b+pos+2)) + ||(MEM_read16(b+pos+1) == MEM_read16(b+pos+3)) + ||(MEM_read16(b+pos+2) == MEM_read16(b+pos+4)) ) { + /* skip and mark segment */ + U16 const pattern16 = MEM_read16(b+pos+4); + U32 u, patternEnd = 6; + while (MEM_read16(b+pos+patternEnd) == pattern16) patternEnd+=2 ; + if (b[pos+patternEnd] == b[pos+patternEnd-1]) patternEnd++; + for (u=1; u= MINMATCHLENGTH); + } + + /* look backward */ + { size_t length; + do { + length = ZDICT_count(b + pos, b + *(suffix+start-1)); + if (length >=MINMATCHLENGTH) start--; + } while(length >= MINMATCHLENGTH); + } + + /* exit if not found a minimum nb of repetitions */ + if (end-start < minRatio) { + U32 idx; + for(idx=start; idx= %i at pos %7u ", (unsigned)(end-start), MINMATCHLENGTH, (unsigned)pos); + DISPLAYLEVEL(4, "\n"); + + for (mml = MINMATCHLENGTH ; ; mml++) { + BYTE currentChar = 0; + U32 currentCount = 0; + U32 currentID = refinedStart; + U32 id; + U32 selectedCount = 0; + U32 selectedID = currentID; + for (id =refinedStart; id < refinedEnd; id++) { + if (b[suffix[id] + mml] != currentChar) { + if (currentCount > selectedCount) { + selectedCount = currentCount; + selectedID = currentID; + } + currentID = id; + currentChar = b[ suffix[id] + mml]; + currentCount = 0; + } + currentCount ++; + } + if (currentCount > selectedCount) { /* for last */ + selectedCount = currentCount; + selectedID = currentID; + } + + if (selectedCount < minRatio) + break; + refinedStart = selectedID; + refinedEnd = refinedStart + selectedCount; + } + + /* evaluate gain based on new dict */ + start = refinedStart; + pos = suffix[refinedStart]; + end = start; + memset(lengthList, 0, sizeof(lengthList)); + + /* look forward */ + { size_t length; + do { + end++; + length = ZDICT_count(b + pos, b + suffix[end]); + if (length >= LLIMIT) length = LLIMIT-1; + lengthList[length]++; + } while (length >=MINMATCHLENGTH); + } + + /* look backward */ + { size_t length = MINMATCHLENGTH; + while ((length >= MINMATCHLENGTH) & (start > 0)) { + length = ZDICT_count(b + pos, b + suffix[start - 1]); + if (length >= LLIMIT) length = LLIMIT - 1; + lengthList[length]++; + if (length >= MINMATCHLENGTH) start--; + } + } + + /* largest useful length */ + memset(cumulLength, 0, sizeof(cumulLength)); + cumulLength[maxLength-1] = lengthList[maxLength-1]; + for (i=(int)(maxLength-2); i>=0; i--) + cumulLength[i] = cumulLength[i+1] + lengthList[i]; + + for (i=LLIMIT-1; i>=MINMATCHLENGTH; i--) if (cumulLength[i]>=minRatio) break; + maxLength = i; + + /* reduce maxLength in case of final into repetitive data */ + { U32 l = (U32)maxLength; + BYTE const c = b[pos + maxLength-1]; + while (b[pos+l-2]==c) l--; + maxLength = l; + } + if (maxLength < MINMATCHLENGTH) return solution; /* skip : no long-enough solution */ + + /* calculate savings */ + savings[5] = 0; + for (i=MINMATCHLENGTH; i<=(int)maxLength; i++) + savings[i] = savings[i-1] + (lengthList[i] * (i-3)); + + DISPLAYLEVEL(4, "Selected dict at position %u, of length %u : saves %u (ratio: %.2f) \n", + (unsigned)pos, (unsigned)maxLength, (unsigned)savings[maxLength], (double)savings[maxLength] / (double)maxLength); + + solution.pos = (U32)pos; + solution.length = (U32)maxLength; + solution.savings = savings[maxLength]; + + /* mark positions done */ + { U32 id; + for (id=start; id solution.length) length = solution.length; + } + pEnd = (U32)(testedPos + length); + for (p=testedPos; ppos; + const U32 eltEnd = elt.pos + elt.length; + const char* const buf = (const char*) buffer; + + /* tail overlap */ + U32 u; for (u=1; u elt.pos) && (table[u].pos <= eltEnd)) { /* overlap, existing > new */ + /* append */ + U32 const addedLength = table[u].pos - elt.pos; + table[u].length += addedLength; + table[u].pos = elt.pos; + table[u].savings += elt.savings * addedLength / elt.length; /* rough approx */ + table[u].savings += elt.length / 8; /* rough approx bonus */ + elt = table[u]; + /* sort : improve rank */ + while ((u>1) && (table[u-1].savings < elt.savings)) + table[u] = table[u-1], u--; + table[u] = elt; + return u; + } } + + /* front overlap */ + for (u=1; u= elt.pos) && (table[u].pos < elt.pos)) { /* overlap, existing < new */ + /* append */ + int const addedLength = (int)eltEnd - (int)(table[u].pos + table[u].length); + table[u].savings += elt.length / 8; /* rough approx bonus */ + if (addedLength > 0) { /* otherwise, elt fully included into existing */ + table[u].length += addedLength; + table[u].savings += elt.savings * addedLength / elt.length; /* rough approx */ + } + /* sort : improve rank */ + elt = table[u]; + while ((u>1) && (table[u-1].savings < elt.savings)) + table[u] = table[u-1], u--; + table[u] = elt; + return u; + } + + if (MEM_read64(buf + table[u].pos) == MEM_read64(buf + elt.pos + 1)) { + if (isIncluded(buf + table[u].pos, buf + elt.pos + 1, table[u].length)) { + size_t const addedLength = MAX( (int)elt.length - (int)table[u].length , 1 ); + table[u].pos = elt.pos; + table[u].savings += (U32)(elt.savings * addedLength / elt.length); + table[u].length = MIN(elt.length, table[u].length + 1); + return u; + } + } + } + + return 0; +} + + +static void ZDICT_removeDictItem(dictItem* table, U32 id) +{ + /* convention : table[0].pos stores nb of elts */ + U32 const max = table[0].pos; + U32 u; + if (!id) return; /* protection, should never happen */ + for (u=id; upos--; +} + + +static void ZDICT_insertDictItem(dictItem* table, U32 maxSize, dictItem elt, const void* buffer) +{ + /* merge if possible */ + U32 mergeId = ZDICT_tryMerge(table, elt, 0, buffer); + if (mergeId) { + U32 newMerge = 1; + while (newMerge) { + newMerge = ZDICT_tryMerge(table, table[mergeId], mergeId, buffer); + if (newMerge) ZDICT_removeDictItem(table, mergeId); + mergeId = newMerge; + } + return; + } + + /* insert */ + { U32 current; + U32 nextElt = table->pos; + if (nextElt >= maxSize) nextElt = maxSize-1; + current = nextElt-1; + while (table[current].savings < elt.savings) { + table[current+1] = table[current]; + current--; + } + table[current+1] = elt; + table->pos = nextElt+1; + } +} + + +static U32 ZDICT_dictSize(const dictItem* dictList) +{ + U32 u, dictSize = 0; + for (u=1; u=l) { \ + if (ZDICT_clockSpan(displayClock) > refreshRate) \ + { displayClock = clock(); DISPLAY(__VA_ARGS__); \ + if (notificationLevel>=4) fflush(stderr); } } + + /* init */ + DISPLAYLEVEL(2, "\r%70s\r", ""); /* clean display line */ + if (!suffix0 || !reverseSuffix || !doneMarks || !filePos) { + result = ERROR(memory_allocation); + goto _cleanup; + } + if (minRatio < MINRATIO) minRatio = MINRATIO; + memset(doneMarks, 0, bufferSize+16); + + /* limit sample set size (divsufsort limitation)*/ + if (bufferSize > ZDICT_MAX_SAMPLES_SIZE) DISPLAYLEVEL(3, "sample set too large : reduced to %u MB ...\n", (unsigned)(ZDICT_MAX_SAMPLES_SIZE>>20)); + while (bufferSize > ZDICT_MAX_SAMPLES_SIZE) bufferSize -= fileSizes[--nbFiles]; + + /* sort */ + DISPLAYLEVEL(2, "sorting %u files of total size %u MB ...\n", nbFiles, (unsigned)(bufferSize>>20)); + { int const divSuftSortResult = divsufsort((const unsigned char*)buffer, suffix, (int)bufferSize, 0); + if (divSuftSortResult != 0) { result = ERROR(GENERIC); goto _cleanup; } + } + suffix[bufferSize] = (int)bufferSize; /* leads into noise */ + suffix0[0] = (int)bufferSize; /* leads into noise */ + /* build reverse suffix sort */ + { size_t pos; + for (pos=0; pos < bufferSize; pos++) + reverseSuffix[suffix[pos]] = (U32)pos; + /* note filePos tracks borders between samples. + It's not used at this stage, but planned to become useful in a later update */ + filePos[0] = 0; + for (pos=1; pos> 21); + } +} + + +typedef struct +{ + ZSTD_CDict* dict; /* dictionary */ + ZSTD_CCtx* zc; /* working context */ + void* workPlace; /* must be ZSTD_BLOCKSIZE_MAX allocated */ +} EStats_ress_t; + +#define MAXREPOFFSET 1024 + +static void ZDICT_countEStats(EStats_ress_t esr, const ZSTD_parameters* params, + unsigned* countLit, unsigned* offsetcodeCount, unsigned* matchlengthCount, unsigned* litlengthCount, U32* repOffsets, + const void* src, size_t srcSize, + U32 notificationLevel) +{ + size_t const blockSizeMax = MIN (ZSTD_BLOCKSIZE_MAX, 1 << params->cParams.windowLog); + size_t cSize; + + if (srcSize > blockSizeMax) srcSize = blockSizeMax; /* protection vs large samples */ + { size_t const errorCode = ZSTD_compressBegin_usingCDict(esr.zc, esr.dict); + if (ZSTD_isError(errorCode)) { DISPLAYLEVEL(1, "warning : ZSTD_compressBegin_usingCDict failed \n"); return; } + + } + cSize = ZSTD_compressBlock(esr.zc, esr.workPlace, ZSTD_BLOCKSIZE_MAX, src, srcSize); + if (ZSTD_isError(cSize)) { DISPLAYLEVEL(3, "warning : could not compress sample size %u \n", (unsigned)srcSize); return; } + + if (cSize) { /* if == 0; block is not compressible */ + const seqStore_t* const seqStorePtr = ZSTD_getSeqStore(esr.zc); + + /* literals stats */ + { const BYTE* bytePtr; + for(bytePtr = seqStorePtr->litStart; bytePtr < seqStorePtr->lit; bytePtr++) + countLit[*bytePtr]++; + } + + /* seqStats */ + { U32 const nbSeq = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart); + ZSTD_seqToCodes(seqStorePtr); + + { const BYTE* codePtr = seqStorePtr->ofCode; + U32 u; + for (u=0; umlCode; + U32 u; + for (u=0; ullCode; + U32 u; + for (u=0; u= 2) { /* rep offsets */ + const seqDef* const seq = seqStorePtr->sequencesStart; + U32 offset1 = seq[0].offBase - ZSTD_REP_NUM; + U32 offset2 = seq[1].offBase - ZSTD_REP_NUM; + if (offset1 >= MAXREPOFFSET) offset1 = 0; + if (offset2 >= MAXREPOFFSET) offset2 = 0; + repOffsets[offset1] += 3; + repOffsets[offset2] += 1; + } } } +} + +static size_t ZDICT_totalSampleSize(const size_t* fileSizes, unsigned nbFiles) +{ + size_t total=0; + unsigned u; + for (u=0; u0; u--) { + offsetCount_t tmp; + if (table[u-1].count >= table[u].count) break; + tmp = table[u-1]; + table[u-1] = table[u]; + table[u] = tmp; + } +} + +/* ZDICT_flatLit() : + * rewrite `countLit` to contain a mostly flat but still compressible distribution of literals. + * necessary to avoid generating a non-compressible distribution that HUF_writeCTable() cannot encode. + */ +static void ZDICT_flatLit(unsigned* countLit) +{ + int u; + for (u=1; u<256; u++) countLit[u] = 2; + countLit[0] = 4; + countLit[253] = 1; + countLit[254] = 1; +} + +#define OFFCODE_MAX 30 /* only applicable to first block */ +static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize, + int compressionLevel, + const void* srcBuffer, const size_t* fileSizes, unsigned nbFiles, + const void* dictBuffer, size_t dictBufferSize, + unsigned notificationLevel) +{ + unsigned countLit[256]; + HUF_CREATE_STATIC_CTABLE(hufTable, 255); + unsigned offcodeCount[OFFCODE_MAX+1]; + short offcodeNCount[OFFCODE_MAX+1]; + U32 offcodeMax = ZSTD_highbit32((U32)(dictBufferSize + 128 KB)); + unsigned matchLengthCount[MaxML+1]; + short matchLengthNCount[MaxML+1]; + unsigned litLengthCount[MaxLL+1]; + short litLengthNCount[MaxLL+1]; + U32 repOffset[MAXREPOFFSET]; + offsetCount_t bestRepOffset[ZSTD_REP_NUM+1]; + EStats_ress_t esr = { NULL, NULL, NULL }; + ZSTD_parameters params; + U32 u, huffLog = 11, Offlog = OffFSELog, mlLog = MLFSELog, llLog = LLFSELog, total; + size_t pos = 0, errorCode; + size_t eSize = 0; + size_t const totalSrcSize = ZDICT_totalSampleSize(fileSizes, nbFiles); + size_t const averageSampleSize = totalSrcSize / (nbFiles + !nbFiles); + BYTE* dstPtr = (BYTE*)dstBuffer; + + /* init */ + DEBUGLOG(4, "ZDICT_analyzeEntropy"); + if (offcodeMax>OFFCODE_MAX) { eSize = ERROR(dictionaryCreation_failed); goto _cleanup; } /* too large dictionary */ + for (u=0; u<256; u++) countLit[u] = 1; /* any character must be described */ + for (u=0; u<=offcodeMax; u++) offcodeCount[u] = 1; + for (u=0; u<=MaxML; u++) matchLengthCount[u] = 1; + for (u=0; u<=MaxLL; u++) litLengthCount[u] = 1; + memset(repOffset, 0, sizeof(repOffset)); + repOffset[1] = repOffset[4] = repOffset[8] = 1; + memset(bestRepOffset, 0, sizeof(bestRepOffset)); + if (compressionLevel==0) compressionLevel = ZSTD_CLEVEL_DEFAULT; + params = ZSTD_getParams(compressionLevel, averageSampleSize, dictBufferSize); + + esr.dict = ZSTD_createCDict_advanced(dictBuffer, dictBufferSize, ZSTD_dlm_byRef, ZSTD_dct_rawContent, params.cParams, ZSTD_defaultCMem); + esr.zc = ZSTD_createCCtx(); + esr.workPlace = malloc(ZSTD_BLOCKSIZE_MAX); + if (!esr.dict || !esr.zc || !esr.workPlace) { + eSize = ERROR(memory_allocation); + DISPLAYLEVEL(1, "Not enough memory \n"); + goto _cleanup; + } + + /* collect stats on all samples */ + for (u=0; u= 4) { + /* writeStats */ + DISPLAYLEVEL(4, "Offset Code Frequencies : \n"); + for (u=0; u<=offcodeMax; u++) { + DISPLAYLEVEL(4, "%2u :%7u \n", u, offcodeCount[u]); + } } + + /* analyze, build stats, starting with literals */ + { size_t maxNbBits = HUF_buildCTable (hufTable, countLit, 255, huffLog); + if (HUF_isError(maxNbBits)) { + eSize = maxNbBits; + DISPLAYLEVEL(1, " HUF_buildCTable error \n"); + goto _cleanup; + } + if (maxNbBits==8) { /* not compressible : will fail on HUF_writeCTable() */ + DISPLAYLEVEL(2, "warning : pathological dataset : literals are not compressible : samples are noisy or too regular \n"); + ZDICT_flatLit(countLit); /* replace distribution by a fake "mostly flat but still compressible" distribution, that HUF_writeCTable() can encode */ + maxNbBits = HUF_buildCTable (hufTable, countLit, 255, huffLog); + assert(maxNbBits==9); + } + huffLog = (U32)maxNbBits; + } + + /* looking for most common first offsets */ + { U32 offset; + for (offset=1; offset dictBufferCapacity) { + dictContentSize = dictBufferCapacity - hSize; + } + + /* Pad the dictionary content with zeros if it is too small */ + if (dictContentSize < minContentSize) { + RETURN_ERROR_IF(hSize + minContentSize > dictBufferCapacity, dstSize_tooSmall, + "dictBufferCapacity too small to fit max repcode"); + paddingSize = minContentSize - dictContentSize; + } else { + paddingSize = 0; + } + + { + size_t const dictSize = hSize + paddingSize + dictContentSize; + + /* The dictionary consists of the header, optional padding, and the content. + * The padding comes before the content because the "best" position in the + * dictionary is the last byte. + */ + BYTE* const outDictHeader = (BYTE*)dictBuffer; + BYTE* const outDictPadding = outDictHeader + hSize; + BYTE* const outDictContent = outDictPadding + paddingSize; + + assert(dictSize <= dictBufferCapacity); + assert(outDictContent + dictContentSize == (BYTE*)dictBuffer + dictSize); + + /* First copy the customDictContent into its final location. + * `customDictContent` and `dictBuffer` may overlap, so we must + * do this before any other writes into the output buffer. + * Then copy the header & padding into the output buffer. + */ + memmove(outDictContent, customDictContent, dictContentSize); + memcpy(outDictHeader, header, hSize); + memset(outDictPadding, 0, paddingSize); + + return dictSize; + } +} + + +static size_t ZDICT_addEntropyTablesFromBuffer_advanced( + void* dictBuffer, size_t dictContentSize, size_t dictBufferCapacity, + const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples, + ZDICT_params_t params) +{ + int const compressionLevel = (params.compressionLevel == 0) ? ZSTD_CLEVEL_DEFAULT : params.compressionLevel; + U32 const notificationLevel = params.notificationLevel; + size_t hSize = 8; + + /* calculate entropy tables */ + DISPLAYLEVEL(2, "\r%70s\r", ""); /* clean display line */ + DISPLAYLEVEL(2, "statistics ... \n"); + { size_t const eSize = ZDICT_analyzeEntropy((char*)dictBuffer+hSize, dictBufferCapacity-hSize, + compressionLevel, + samplesBuffer, samplesSizes, nbSamples, + (char*)dictBuffer + dictBufferCapacity - dictContentSize, dictContentSize, + notificationLevel); + if (ZDICT_isError(eSize)) return eSize; + hSize += eSize; + } + + /* add dictionary header (after entropy tables) */ + MEM_writeLE32(dictBuffer, ZSTD_MAGIC_DICTIONARY); + { U64 const randomID = XXH64((char*)dictBuffer + dictBufferCapacity - dictContentSize, dictContentSize, 0); + U32 const compliantID = (randomID % ((1U<<31)-32768)) + 32768; + U32 const dictID = params.dictID ? params.dictID : compliantID; + MEM_writeLE32((char*)dictBuffer+4, dictID); + } + + if (hSize + dictContentSize < dictBufferCapacity) + memmove((char*)dictBuffer + hSize, (char*)dictBuffer + dictBufferCapacity - dictContentSize, dictContentSize); + return MIN(dictBufferCapacity, hSize+dictContentSize); +} + +/*! ZDICT_trainFromBuffer_unsafe_legacy() : +* Warning : `samplesBuffer` must be followed by noisy guard band !!! +* @return : size of dictionary, or an error code which can be tested with ZDICT_isError() +*/ +static size_t ZDICT_trainFromBuffer_unsafe_legacy( + void* dictBuffer, size_t maxDictSize, + const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples, + ZDICT_legacy_params_t params) +{ + U32 const dictListSize = MAX(MAX(DICTLISTSIZE_DEFAULT, nbSamples), (U32)(maxDictSize/16)); + dictItem* const dictList = (dictItem*)malloc(dictListSize * sizeof(*dictList)); + unsigned const selectivity = params.selectivityLevel == 0 ? g_selectivity_default : params.selectivityLevel; + unsigned const minRep = (selectivity > 30) ? MINRATIO : nbSamples >> selectivity; + size_t const targetDictSize = maxDictSize; + size_t const samplesBuffSize = ZDICT_totalSampleSize(samplesSizes, nbSamples); + size_t dictSize = 0; + U32 const notificationLevel = params.zParams.notificationLevel; + + /* checks */ + if (!dictList) return ERROR(memory_allocation); + if (maxDictSize < ZDICT_DICTSIZE_MIN) { free(dictList); return ERROR(dstSize_tooSmall); } /* requested dictionary size is too small */ + if (samplesBuffSize < ZDICT_MIN_SAMPLES_SIZE) { free(dictList); return ERROR(dictionaryCreation_failed); } /* not enough source to create dictionary */ + + /* init */ + ZDICT_initDictItem(dictList); + + /* build dictionary */ + ZDICT_trainBuffer_legacy(dictList, dictListSize, + samplesBuffer, samplesBuffSize, + samplesSizes, nbSamples, + minRep, notificationLevel); + + /* display best matches */ + if (params.zParams.notificationLevel>= 3) { + unsigned const nb = MIN(25, dictList[0].pos); + unsigned const dictContentSize = ZDICT_dictSize(dictList); + unsigned u; + DISPLAYLEVEL(3, "\n %u segments found, of total size %u \n", (unsigned)dictList[0].pos-1, dictContentSize); + DISPLAYLEVEL(3, "list %u best segments \n", nb-1); + for (u=1; u samplesBuffSize) || ((pos + length) > samplesBuffSize)) { + free(dictList); + return ERROR(GENERIC); /* should never happen */ + } + DISPLAYLEVEL(3, "%3u:%3u bytes at pos %8u, savings %7u bytes |", + u, length, pos, (unsigned)dictList[u].savings); + ZDICT_printHex((const char*)samplesBuffer+pos, printedLength); + DISPLAYLEVEL(3, "| \n"); + } } + + + /* create dictionary */ + { unsigned dictContentSize = ZDICT_dictSize(dictList); + if (dictContentSize < ZDICT_CONTENTSIZE_MIN) { free(dictList); return ERROR(dictionaryCreation_failed); } /* dictionary content too small */ + if (dictContentSize < targetDictSize/4) { + DISPLAYLEVEL(2, "! warning : selected content significantly smaller than requested (%u < %u) \n", dictContentSize, (unsigned)maxDictSize); + if (samplesBuffSize < 10 * targetDictSize) + DISPLAYLEVEL(2, "! consider increasing the number of samples (total size : %u MB)\n", (unsigned)(samplesBuffSize>>20)); + if (minRep > MINRATIO) { + DISPLAYLEVEL(2, "! consider increasing selectivity to produce larger dictionary (-s%u) \n", selectivity+1); + DISPLAYLEVEL(2, "! note : larger dictionaries are not necessarily better, test its efficiency on samples \n"); + } + } + + if ((dictContentSize > targetDictSize*3) && (nbSamples > 2*MINRATIO) && (selectivity>1)) { + unsigned proposedSelectivity = selectivity-1; + while ((nbSamples >> proposedSelectivity) <= MINRATIO) { proposedSelectivity--; } + DISPLAYLEVEL(2, "! note : calculated dictionary significantly larger than requested (%u > %u) \n", dictContentSize, (unsigned)maxDictSize); + DISPLAYLEVEL(2, "! consider increasing dictionary size, or produce denser dictionary (-s%u) \n", proposedSelectivity); + DISPLAYLEVEL(2, "! always test dictionary efficiency on real samples \n"); + } + + /* limit dictionary size */ + { U32 const max = dictList->pos; /* convention : nb of useful elts within dictList */ + U32 currentSize = 0; + U32 n; for (n=1; n targetDictSize) { currentSize -= dictList[n].length; break; } + } + dictList->pos = n; + dictContentSize = currentSize; + } + + /* build dict content */ + { U32 u; + BYTE* ptr = (BYTE*)dictBuffer + maxDictSize; + for (u=1; upos; u++) { + U32 l = dictList[u].length; + ptr -= l; + if (ptr<(BYTE*)dictBuffer) { free(dictList); return ERROR(GENERIC); } /* should not happen */ + memcpy(ptr, (const char*)samplesBuffer+dictList[u].pos, l); + } } + + dictSize = ZDICT_addEntropyTablesFromBuffer_advanced(dictBuffer, dictContentSize, maxDictSize, + samplesBuffer, samplesSizes, nbSamples, + params.zParams); + } + + /* clean up */ + free(dictList); + return dictSize; +} + + +/* ZDICT_trainFromBuffer_legacy() : + * issue : samplesBuffer need to be followed by a noisy guard band. + * work around : duplicate the buffer, and add the noise */ +size_t ZDICT_trainFromBuffer_legacy(void* dictBuffer, size_t dictBufferCapacity, + const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples, + ZDICT_legacy_params_t params) +{ + size_t result; + void* newBuff; + size_t const sBuffSize = ZDICT_totalSampleSize(samplesSizes, nbSamples); + if (sBuffSize < ZDICT_MIN_SAMPLES_SIZE) return 0; /* not enough content => no dictionary */ + + newBuff = malloc(sBuffSize + NOISELENGTH); + if (!newBuff) return ERROR(memory_allocation); + + memcpy(newBuff, samplesBuffer, sBuffSize); + ZDICT_fillNoise((char*)newBuff + sBuffSize, NOISELENGTH); /* guard band, for end of buffer condition */ + + result = + ZDICT_trainFromBuffer_unsafe_legacy(dictBuffer, dictBufferCapacity, newBuff, + samplesSizes, nbSamples, params); + free(newBuff); + return result; +} + + +size_t ZDICT_trainFromBuffer(void* dictBuffer, size_t dictBufferCapacity, + const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples) +{ + ZDICT_fastCover_params_t params; + DEBUGLOG(3, "ZDICT_trainFromBuffer"); + memset(¶ms, 0, sizeof(params)); + params.d = 8; + params.steps = 4; + /* Use default level since no compression level information is available */ + params.zParams.compressionLevel = ZSTD_CLEVEL_DEFAULT; +#if defined(DEBUGLEVEL) && (DEBUGLEVEL>=1) + params.zParams.notificationLevel = DEBUGLEVEL; +#endif + return ZDICT_optimizeTrainFromBuffer_fastCover(dictBuffer, dictBufferCapacity, + samplesBuffer, samplesSizes, nbSamples, + ¶ms); +} + +size_t ZDICT_addEntropyTablesFromBuffer(void* dictBuffer, size_t dictContentSize, size_t dictBufferCapacity, + const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples) +{ + ZDICT_params_t params; + memset(¶ms, 0, sizeof(params)); + return ZDICT_addEntropyTablesFromBuffer_advanced(dictBuffer, dictContentSize, dictBufferCapacity, + samplesBuffer, samplesSizes, nbSamples, + params); +} diff --git a/src/blosc2/internal-complibs/zstd-1.5.2/dll/example/Makefile b/src/blosc2/internal-complibs/zstd-1.5.2/dll/example/Makefile new file mode 100644 index 0000000..03b034d --- /dev/null +++ b/src/blosc2/internal-complibs/zstd-1.5.2/dll/example/Makefile @@ -0,0 +1,48 @@ +# ################################################################ +# Copyright (c) Yann Collet, Facebook, Inc. +# All rights reserved. +# +# This source code is licensed under both the BSD-style license (found in the +# LICENSE file in the root directory of this source tree) and the GPLv2 (found +# in the COPYING file in the root directory of this source tree). +# You may select, at your option, one of the above-listed licenses. +# ################################################################ + +VOID := /dev/null +ZSTDDIR := ../include +LIBDIR := ../static +DLLDIR := ../dll + +CFLAGS ?= -O3 # can select custom flags. For example : CFLAGS="-O2 -g" make +CFLAGS += -Wall -Wextra -Wundef -Wcast-qual -Wcast-align -Wshadow -Wswitch-enum \ + -Wdeclaration-after-statement -Wstrict-prototypes \ + -Wpointer-arith -Wstrict-aliasing=1 +CFLAGS += $(MOREFLAGS) +CPPFLAGS:= -I$(ZSTDDIR) -DXXH_NAMESPACE=ZSTD_ +FLAGS := $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) + + +# Define *.exe as extension for Windows systems +ifneq (,$(filter Windows%,$(OS))) +EXT =.exe +else +EXT = +endif + +.PHONY: default fullbench-dll fullbench-lib + + +default: all + +all: fullbench-dll fullbench-lib + + +fullbench-lib: fullbench.c datagen.c + $(CC) $(FLAGS) $^ -o $@$(EXT) $(LIBDIR)/libzstd_static.lib + +fullbench-dll: fullbench.c datagen.c + $(CC) $(FLAGS) $^ -o $@$(EXT) -DZSTD_DLL_IMPORT=1 $(DLLDIR)/libzstd.dll + +clean: + @$(RM) fullbench-dll$(EXT) fullbench-lib$(EXT) \ + @echo Cleaning completed diff --git a/src/blosc2/internal-complibs/zstd-1.5.2/dll/example/README.md b/src/blosc2/internal-complibs/zstd-1.5.2/dll/example/README.md new file mode 100644 index 0000000..9e30fd5 --- /dev/null +++ b/src/blosc2/internal-complibs/zstd-1.5.2/dll/example/README.md @@ -0,0 +1,63 @@ +# ZSTD Windows binary package + +## The package contents + +- `zstd.exe` : Command Line Utility, supporting gzip-like arguments +- `dll\libzstd.dll` : The ZSTD dynamic library (DLL) +- `dll\libzstd.lib` : The import library of the ZSTD dynamic library (DLL) for Visual C++ +- `example\` : The example of usage of the ZSTD library +- `include\` : Header files required by the ZSTD library +- `static\libzstd_static.lib` : The static ZSTD library (LIB) + +## Usage of Command Line Interface + +Command Line Interface (CLI) supports gzip-like arguments. +By default CLI takes an input file and compresses it to an output file: + + Usage: zstd [arg] [input] [output] + +The full list of commands for CLI can be obtained with `-h` or `-H`. The ratio can +be improved with commands from `-3` to `-16` but higher levels also have slower +compression. CLI includes in-memory compression benchmark module with compression +levels starting from `-b` and ending with `-e` with iteration time of `-i` seconds. +CLI supports aggregation of parameters i.e. `-b1`, `-e18`, and `-i1` can be joined +into `-b1e18i1`. + +## The example of usage of static and dynamic ZSTD libraries with gcc/MinGW + +Use `cd example` and `make` to build `fullbench-dll` and `fullbench-lib`. +`fullbench-dll` uses a dynamic ZSTD library from the `dll` directory. +`fullbench-lib` uses a static ZSTD library from the `lib` directory. + +## Using ZSTD DLL with gcc/MinGW + +The header files from `include\` and the dynamic library `dll\libzstd.dll` +are required to compile a project using gcc/MinGW. +The dynamic library has to be added to linking options. +It means that if a project that uses ZSTD consists of a single `test-dll.c` +file it should be linked with `dll\libzstd.dll`. For example: + + gcc $(CFLAGS) -Iinclude\ test-dll.c -o test-dll dll\libzstd.dll + +The compiled executable will require ZSTD DLL which is available at `dll\libzstd.dll`. + +## The example of usage of static and dynamic ZSTD libraries with Visual C++ + +Open `example\fullbench-dll.sln` to compile `fullbench-dll` that uses a +dynamic ZSTD library from the `dll` directory. The solution works with Visual C++ +2010 or newer. When one will open the solution with Visual C++ newer than 2010 +then the solution will upgraded to the current version. + +## Using ZSTD DLL with Visual C++ + +The header files from `include\` and the import library `dll\libzstd.lib` +are required to compile a project using Visual C++. + +1. The path to header files should be added to `Additional Include Directories` that can + be found in project properties `C/C++` then `General`. +2. The import library has to be added to `Additional Dependencies` that can + be found in project properties `Linker` then `Input`. + If one will provide only the name `libzstd.lib` without a full path to the library + the directory has to be added to `Linker\General\Additional Library Directories`. + +The compiled executable will require ZSTD DLL which is available at `dll\libzstd.dll`. diff --git a/src/blosc2/internal-complibs/zstd-1.5.2/dll/example/fullbench-dll.sln b/src/blosc2/internal-complibs/zstd-1.5.2/dll/example/fullbench-dll.sln new file mode 100644 index 0000000..ef8d4c0 --- /dev/null +++ b/src/blosc2/internal-complibs/zstd-1.5.2/dll/example/fullbench-dll.sln @@ -0,0 +1,25 @@ +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Express 2012 for Windows Desktop +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "fullbench-dll", "fullbench-dll.vcxproj", "{13992FD2-077E-4954-B065-A428198201A9}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Win32 = Debug|Win32 + Debug|x64 = Debug|x64 + Release|Win32 = Release|Win32 + Release|x64 = Release|x64 + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {13992FD2-077E-4954-B065-A428198201A9}.Debug|Win32.ActiveCfg = Debug|Win32 + {13992FD2-077E-4954-B065-A428198201A9}.Debug|Win32.Build.0 = Debug|Win32 + {13992FD2-077E-4954-B065-A428198201A9}.Debug|x64.ActiveCfg = Debug|x64 + {13992FD2-077E-4954-B065-A428198201A9}.Debug|x64.Build.0 = Debug|x64 + {13992FD2-077E-4954-B065-A428198201A9}.Release|Win32.ActiveCfg = Release|Win32 + {13992FD2-077E-4954-B065-A428198201A9}.Release|Win32.Build.0 = Release|Win32 + {13992FD2-077E-4954-B065-A428198201A9}.Release|x64.ActiveCfg = Release|x64 + {13992FD2-077E-4954-B065-A428198201A9}.Release|x64.Build.0 = Release|x64 + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection +EndGlobal diff --git a/src/blosc2/internal-complibs/zstd-1.5.2/dll/example/fullbench-dll.vcxproj b/src/blosc2/internal-complibs/zstd-1.5.2/dll/example/fullbench-dll.vcxproj new file mode 100644 index 0000000..fbea783 --- /dev/null +++ b/src/blosc2/internal-complibs/zstd-1.5.2/dll/example/fullbench-dll.vcxproj @@ -0,0 +1,181 @@ + + + + + Debug + Win32 + + + Debug + x64 + + + Release + Win32 + + + Release + x64 + + + + {00000000-1CC8-4FD7-9281-6B8DBB9D3DF8} + Win32Proj + fullbench-dll + $(SolutionDir)bin\$(Platform)_$(Configuration)\ + $(SolutionDir)bin\obj\$(RootNamespace)_$(Platform)_$(Configuration)\ + + + + Application + true + MultiByte + + + Application + true + MultiByte + + + Application + false + true + MultiByte + + + Application + false + true + MultiByte + + + + + + + + + + + + + + + + + + + true + $(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath); + false + + + true + $(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath); + false + + + false + $(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath); + false + + + false + $(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath); + false + + + + + + Level4 + Disabled + WIN32;_DEBUG;_CONSOLE;ZSTD_DLL_IMPORT=1;%(PreprocessorDefinitions) + true + false + ..\include + + + Console + true + $(SolutionDir)..\dll;%(AdditionalLibraryDirectories) + libzstd.lib;%(AdditionalDependencies) + false + + + + + + + Level4 + Disabled + WIN32;_DEBUG;_CONSOLE;ZSTD_DLL_IMPORT=1;%(PreprocessorDefinitions) + true + false + ..\include + + + Console + true + $(SolutionDir)..\dll;%(AdditionalLibraryDirectories) + libzstd.lib;%(AdditionalDependencies) + + + + + Level4 + + + MaxSpeed + true + true + WIN32;_DEBUG;_CONSOLE;ZSTD_DLL_IMPORT=1;%(PreprocessorDefinitions) + false + ..\include + false + MultiThreaded + + + Console + true + true + true + $(SolutionDir)..\dll;%(AdditionalLibraryDirectories) + libzstd.lib;%(AdditionalDependencies) + false + + + + + Level4 + + + MaxSpeed + true + true + WIN32;_DEBUG;_CONSOLE;ZSTD_DLL_IMPORT=1;%(PreprocessorDefinitions) + false + false + ..\include + MultiThreaded + + + Console + true + true + true + $(SolutionDir)..\dll;%(AdditionalLibraryDirectories) + libzstd.lib;%(AdditionalDependencies) + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/blosc2/internal-complibs/zstd-1.5.2/legacy/zstd_legacy.h b/src/blosc2/internal-complibs/zstd-1.5.2/legacy/zstd_legacy.h new file mode 100644 index 0000000..a6f1174 --- /dev/null +++ b/src/blosc2/internal-complibs/zstd-1.5.2/legacy/zstd_legacy.h @@ -0,0 +1,415 @@ +/* + * Copyright (c) Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_LEGACY_H +#define ZSTD_LEGACY_H + +#if defined (__cplusplus) +extern "C" { +#endif + +/* ************************************* +* Includes +***************************************/ +#include "../common/mem.h" /* MEM_STATIC */ +#include "../common/error_private.h" /* ERROR */ +#include "../common/zstd_internal.h" /* ZSTD_inBuffer, ZSTD_outBuffer, ZSTD_frameSizeInfo */ + +#if !defined (ZSTD_LEGACY_SUPPORT) || (ZSTD_LEGACY_SUPPORT == 0) +# undef ZSTD_LEGACY_SUPPORT +# define ZSTD_LEGACY_SUPPORT 8 +#endif + +#if (ZSTD_LEGACY_SUPPORT <= 1) +# include "zstd_v01.h" +#endif +#if (ZSTD_LEGACY_SUPPORT <= 2) +# include "zstd_v02.h" +#endif +#if (ZSTD_LEGACY_SUPPORT <= 3) +# include "zstd_v03.h" +#endif +#if (ZSTD_LEGACY_SUPPORT <= 4) +# include "zstd_v04.h" +#endif +#if (ZSTD_LEGACY_SUPPORT <= 5) +# include "zstd_v05.h" +#endif +#if (ZSTD_LEGACY_SUPPORT <= 6) +# include "zstd_v06.h" +#endif +#if (ZSTD_LEGACY_SUPPORT <= 7) +# include "zstd_v07.h" +#endif + +/** ZSTD_isLegacy() : + @return : > 0 if supported by legacy decoder. 0 otherwise. + return value is the version. +*/ +MEM_STATIC unsigned ZSTD_isLegacy(const void* src, size_t srcSize) +{ + U32 magicNumberLE; + if (srcSize<4) return 0; + magicNumberLE = MEM_readLE32(src); + switch(magicNumberLE) + { +#if (ZSTD_LEGACY_SUPPORT <= 1) + case ZSTDv01_magicNumberLE:return 1; +#endif +#if (ZSTD_LEGACY_SUPPORT <= 2) + case ZSTDv02_magicNumber : return 2; +#endif +#if (ZSTD_LEGACY_SUPPORT <= 3) + case ZSTDv03_magicNumber : return 3; +#endif +#if (ZSTD_LEGACY_SUPPORT <= 4) + case ZSTDv04_magicNumber : return 4; +#endif +#if (ZSTD_LEGACY_SUPPORT <= 5) + case ZSTDv05_MAGICNUMBER : return 5; +#endif +#if (ZSTD_LEGACY_SUPPORT <= 6) + case ZSTDv06_MAGICNUMBER : return 6; +#endif +#if (ZSTD_LEGACY_SUPPORT <= 7) + case ZSTDv07_MAGICNUMBER : return 7; +#endif + default : return 0; + } +} + + +MEM_STATIC unsigned long long ZSTD_getDecompressedSize_legacy(const void* src, size_t srcSize) +{ + U32 const version = ZSTD_isLegacy(src, srcSize); + if (version < 5) return 0; /* no decompressed size in frame header, or not a legacy format */ +#if (ZSTD_LEGACY_SUPPORT <= 5) + if (version==5) { + ZSTDv05_parameters fParams; + size_t const frResult = ZSTDv05_getFrameParams(&fParams, src, srcSize); + if (frResult != 0) return 0; + return fParams.srcSize; + } +#endif +#if (ZSTD_LEGACY_SUPPORT <= 6) + if (version==6) { + ZSTDv06_frameParams fParams; + size_t const frResult = ZSTDv06_getFrameParams(&fParams, src, srcSize); + if (frResult != 0) return 0; + return fParams.frameContentSize; + } +#endif +#if (ZSTD_LEGACY_SUPPORT <= 7) + if (version==7) { + ZSTDv07_frameParams fParams; + size_t const frResult = ZSTDv07_getFrameParams(&fParams, src, srcSize); + if (frResult != 0) return 0; + return fParams.frameContentSize; + } +#endif + return 0; /* should not be possible */ +} + + +MEM_STATIC size_t ZSTD_decompressLegacy( + void* dst, size_t dstCapacity, + const void* src, size_t compressedSize, + const void* dict,size_t dictSize) +{ + U32 const version = ZSTD_isLegacy(src, compressedSize); + (void)dst; (void)dstCapacity; (void)dict; (void)dictSize; /* unused when ZSTD_LEGACY_SUPPORT >= 8 */ + switch(version) + { +#if (ZSTD_LEGACY_SUPPORT <= 1) + case 1 : + return ZSTDv01_decompress(dst, dstCapacity, src, compressedSize); +#endif +#if (ZSTD_LEGACY_SUPPORT <= 2) + case 2 : + return ZSTDv02_decompress(dst, dstCapacity, src, compressedSize); +#endif +#if (ZSTD_LEGACY_SUPPORT <= 3) + case 3 : + return ZSTDv03_decompress(dst, dstCapacity, src, compressedSize); +#endif +#if (ZSTD_LEGACY_SUPPORT <= 4) + case 4 : + return ZSTDv04_decompress(dst, dstCapacity, src, compressedSize); +#endif +#if (ZSTD_LEGACY_SUPPORT <= 5) + case 5 : + { size_t result; + ZSTDv05_DCtx* const zd = ZSTDv05_createDCtx(); + if (zd==NULL) return ERROR(memory_allocation); + result = ZSTDv05_decompress_usingDict(zd, dst, dstCapacity, src, compressedSize, dict, dictSize); + ZSTDv05_freeDCtx(zd); + return result; + } +#endif +#if (ZSTD_LEGACY_SUPPORT <= 6) + case 6 : + { size_t result; + ZSTDv06_DCtx* const zd = ZSTDv06_createDCtx(); + if (zd==NULL) return ERROR(memory_allocation); + result = ZSTDv06_decompress_usingDict(zd, dst, dstCapacity, src, compressedSize, dict, dictSize); + ZSTDv06_freeDCtx(zd); + return result; + } +#endif +#if (ZSTD_LEGACY_SUPPORT <= 7) + case 7 : + { size_t result; + ZSTDv07_DCtx* const zd = ZSTDv07_createDCtx(); + if (zd==NULL) return ERROR(memory_allocation); + result = ZSTDv07_decompress_usingDict(zd, dst, dstCapacity, src, compressedSize, dict, dictSize); + ZSTDv07_freeDCtx(zd); + return result; + } +#endif + default : + return ERROR(prefix_unknown); + } +} + +MEM_STATIC ZSTD_frameSizeInfo ZSTD_findFrameSizeInfoLegacy(const void *src, size_t srcSize) +{ + ZSTD_frameSizeInfo frameSizeInfo; + U32 const version = ZSTD_isLegacy(src, srcSize); + switch(version) + { +#if (ZSTD_LEGACY_SUPPORT <= 1) + case 1 : + ZSTDv01_findFrameSizeInfoLegacy(src, srcSize, + &frameSizeInfo.compressedSize, + &frameSizeInfo.decompressedBound); + break; +#endif +#if (ZSTD_LEGACY_SUPPORT <= 2) + case 2 : + ZSTDv02_findFrameSizeInfoLegacy(src, srcSize, + &frameSizeInfo.compressedSize, + &frameSizeInfo.decompressedBound); + break; +#endif +#if (ZSTD_LEGACY_SUPPORT <= 3) + case 3 : + ZSTDv03_findFrameSizeInfoLegacy(src, srcSize, + &frameSizeInfo.compressedSize, + &frameSizeInfo.decompressedBound); + break; +#endif +#if (ZSTD_LEGACY_SUPPORT <= 4) + case 4 : + ZSTDv04_findFrameSizeInfoLegacy(src, srcSize, + &frameSizeInfo.compressedSize, + &frameSizeInfo.decompressedBound); + break; +#endif +#if (ZSTD_LEGACY_SUPPORT <= 5) + case 5 : + ZSTDv05_findFrameSizeInfoLegacy(src, srcSize, + &frameSizeInfo.compressedSize, + &frameSizeInfo.decompressedBound); + break; +#endif +#if (ZSTD_LEGACY_SUPPORT <= 6) + case 6 : + ZSTDv06_findFrameSizeInfoLegacy(src, srcSize, + &frameSizeInfo.compressedSize, + &frameSizeInfo.decompressedBound); + break; +#endif +#if (ZSTD_LEGACY_SUPPORT <= 7) + case 7 : + ZSTDv07_findFrameSizeInfoLegacy(src, srcSize, + &frameSizeInfo.compressedSize, + &frameSizeInfo.decompressedBound); + break; +#endif + default : + frameSizeInfo.compressedSize = ERROR(prefix_unknown); + frameSizeInfo.decompressedBound = ZSTD_CONTENTSIZE_ERROR; + break; + } + if (!ZSTD_isError(frameSizeInfo.compressedSize) && frameSizeInfo.compressedSize > srcSize) { + frameSizeInfo.compressedSize = ERROR(srcSize_wrong); + frameSizeInfo.decompressedBound = ZSTD_CONTENTSIZE_ERROR; + } + return frameSizeInfo; +} + +MEM_STATIC size_t ZSTD_findFrameCompressedSizeLegacy(const void *src, size_t srcSize) +{ + ZSTD_frameSizeInfo frameSizeInfo = ZSTD_findFrameSizeInfoLegacy(src, srcSize); + return frameSizeInfo.compressedSize; +} + +MEM_STATIC size_t ZSTD_freeLegacyStreamContext(void* legacyContext, U32 version) +{ + switch(version) + { + default : + case 1 : + case 2 : + case 3 : + (void)legacyContext; + return ERROR(version_unsupported); +#if (ZSTD_LEGACY_SUPPORT <= 4) + case 4 : return ZBUFFv04_freeDCtx((ZBUFFv04_DCtx*)legacyContext); +#endif +#if (ZSTD_LEGACY_SUPPORT <= 5) + case 5 : return ZBUFFv05_freeDCtx((ZBUFFv05_DCtx*)legacyContext); +#endif +#if (ZSTD_LEGACY_SUPPORT <= 6) + case 6 : return ZBUFFv06_freeDCtx((ZBUFFv06_DCtx*)legacyContext); +#endif +#if (ZSTD_LEGACY_SUPPORT <= 7) + case 7 : return ZBUFFv07_freeDCtx((ZBUFFv07_DCtx*)legacyContext); +#endif + } +} + + +MEM_STATIC size_t ZSTD_initLegacyStream(void** legacyContext, U32 prevVersion, U32 newVersion, + const void* dict, size_t dictSize) +{ + DEBUGLOG(5, "ZSTD_initLegacyStream for v0.%u", newVersion); + if (prevVersion != newVersion) ZSTD_freeLegacyStreamContext(*legacyContext, prevVersion); + switch(newVersion) + { + default : + case 1 : + case 2 : + case 3 : + (void)dict; (void)dictSize; + return 0; +#if (ZSTD_LEGACY_SUPPORT <= 4) + case 4 : + { + ZBUFFv04_DCtx* dctx = (prevVersion != newVersion) ? ZBUFFv04_createDCtx() : (ZBUFFv04_DCtx*)*legacyContext; + if (dctx==NULL) return ERROR(memory_allocation); + ZBUFFv04_decompressInit(dctx); + ZBUFFv04_decompressWithDictionary(dctx, dict, dictSize); + *legacyContext = dctx; + return 0; + } +#endif +#if (ZSTD_LEGACY_SUPPORT <= 5) + case 5 : + { + ZBUFFv05_DCtx* dctx = (prevVersion != newVersion) ? ZBUFFv05_createDCtx() : (ZBUFFv05_DCtx*)*legacyContext; + if (dctx==NULL) return ERROR(memory_allocation); + ZBUFFv05_decompressInitDictionary(dctx, dict, dictSize); + *legacyContext = dctx; + return 0; + } +#endif +#if (ZSTD_LEGACY_SUPPORT <= 6) + case 6 : + { + ZBUFFv06_DCtx* dctx = (prevVersion != newVersion) ? ZBUFFv06_createDCtx() : (ZBUFFv06_DCtx*)*legacyContext; + if (dctx==NULL) return ERROR(memory_allocation); + ZBUFFv06_decompressInitDictionary(dctx, dict, dictSize); + *legacyContext = dctx; + return 0; + } +#endif +#if (ZSTD_LEGACY_SUPPORT <= 7) + case 7 : + { + ZBUFFv07_DCtx* dctx = (prevVersion != newVersion) ? ZBUFFv07_createDCtx() : (ZBUFFv07_DCtx*)*legacyContext; + if (dctx==NULL) return ERROR(memory_allocation); + ZBUFFv07_decompressInitDictionary(dctx, dict, dictSize); + *legacyContext = dctx; + return 0; + } +#endif + } +} + + + +MEM_STATIC size_t ZSTD_decompressLegacyStream(void* legacyContext, U32 version, + ZSTD_outBuffer* output, ZSTD_inBuffer* input) +{ + DEBUGLOG(5, "ZSTD_decompressLegacyStream for v0.%u", version); + switch(version) + { + default : + case 1 : + case 2 : + case 3 : + (void)legacyContext; (void)output; (void)input; + return ERROR(version_unsupported); +#if (ZSTD_LEGACY_SUPPORT <= 4) + case 4 : + { + ZBUFFv04_DCtx* dctx = (ZBUFFv04_DCtx*) legacyContext; + const void* src = (const char*)input->src + input->pos; + size_t readSize = input->size - input->pos; + void* dst = (char*)output->dst + output->pos; + size_t decodedSize = output->size - output->pos; + size_t const hintSize = ZBUFFv04_decompressContinue(dctx, dst, &decodedSize, src, &readSize); + output->pos += decodedSize; + input->pos += readSize; + return hintSize; + } +#endif +#if (ZSTD_LEGACY_SUPPORT <= 5) + case 5 : + { + ZBUFFv05_DCtx* dctx = (ZBUFFv05_DCtx*) legacyContext; + const void* src = (const char*)input->src + input->pos; + size_t readSize = input->size - input->pos; + void* dst = (char*)output->dst + output->pos; + size_t decodedSize = output->size - output->pos; + size_t const hintSize = ZBUFFv05_decompressContinue(dctx, dst, &decodedSize, src, &readSize); + output->pos += decodedSize; + input->pos += readSize; + return hintSize; + } +#endif +#if (ZSTD_LEGACY_SUPPORT <= 6) + case 6 : + { + ZBUFFv06_DCtx* dctx = (ZBUFFv06_DCtx*) legacyContext; + const void* src = (const char*)input->src + input->pos; + size_t readSize = input->size - input->pos; + void* dst = (char*)output->dst + output->pos; + size_t decodedSize = output->size - output->pos; + size_t const hintSize = ZBUFFv06_decompressContinue(dctx, dst, &decodedSize, src, &readSize); + output->pos += decodedSize; + input->pos += readSize; + return hintSize; + } +#endif +#if (ZSTD_LEGACY_SUPPORT <= 7) + case 7 : + { + ZBUFFv07_DCtx* dctx = (ZBUFFv07_DCtx*) legacyContext; + const void* src = (const char*)input->src + input->pos; + size_t readSize = input->size - input->pos; + void* dst = (char*)output->dst + output->pos; + size_t decodedSize = output->size - output->pos; + size_t const hintSize = ZBUFFv07_decompressContinue(dctx, dst, &decodedSize, src, &readSize); + output->pos += decodedSize; + input->pos += readSize; + return hintSize; + } +#endif + } +} + + +#if defined (__cplusplus) +} +#endif + +#endif /* ZSTD_LEGACY_H */ diff --git a/src/blosc2/internal-complibs/zstd-1.5.2/libzstd.mk b/src/blosc2/internal-complibs/zstd-1.5.2/libzstd.mk new file mode 100644 index 0000000..6e9a643 --- /dev/null +++ b/src/blosc2/internal-complibs/zstd-1.5.2/libzstd.mk @@ -0,0 +1,203 @@ +# ################################################################ +# Copyright (c) Yann Collet, Facebook, Inc. +# All rights reserved. +# +# This source code is licensed under both the BSD-style license (found in the +# LICENSE file in the root directory of this source tree) and the GPLv2 (found +# in the COPYING file in the root directory of this source tree). +# You may select, at your option, one of the above-listed licenses. +# ################################################################ + +################################################################## +# Input Variables +################################################################## + +# Zstd lib directory +LIBZSTD ?= ./ + +# Legacy support +ZSTD_LEGACY_SUPPORT ?= 5 +ZSTD_LEGACY_MULTITHREADED_API ?= 0 + +# Build size optimizations +HUF_FORCE_DECOMPRESS_X1 ?= 0 +HUF_FORCE_DECOMPRESS_X2 ?= 0 +ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT ?= 0 +ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG ?= 0 +ZSTD_NO_INLINE ?= 0 +ZSTD_STRIP_ERROR_STRINGS ?= 0 + +# Assembly support +ZSTD_NO_ASM ?= 0 + +################################################################## +# libzstd helpers +################################################################## + +VOID ?= /dev/null + +# Make 4.3 doesn't support '\#' anymore (https://lwn.net/Articles/810071/) +NUM_SYMBOL := \# + +# define silent mode as default (verbose mode with V=1 or VERBOSE=1) +$(V)$(VERBOSE).SILENT: + +# When cross-compiling from linux to windows, +# one might need to specify TARGET_SYSTEM as "Windows." +# Building from Fedora fails without it. +# (but Ubuntu and Debian don't need to set anything) +TARGET_SYSTEM ?= $(OS) + +# Version numbers +LIBVER_SRC := $(LIBZSTD)/zstd.h +LIBVER_MAJOR_SCRIPT:=`sed -n '/define ZSTD_VERSION_MAJOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < $(LIBVER_SRC)` +LIBVER_MINOR_SCRIPT:=`sed -n '/define ZSTD_VERSION_MINOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < $(LIBVER_SRC)` +LIBVER_PATCH_SCRIPT:=`sed -n '/define ZSTD_VERSION_RELEASE/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < $(LIBVER_SRC)` +LIBVER_SCRIPT:= $(LIBVER_MAJOR_SCRIPT).$(LIBVER_MINOR_SCRIPT).$(LIBVER_PATCH_SCRIPT) +LIBVER_MAJOR := $(shell echo $(LIBVER_MAJOR_SCRIPT)) +LIBVER_MINOR := $(shell echo $(LIBVER_MINOR_SCRIPT)) +LIBVER_PATCH := $(shell echo $(LIBVER_PATCH_SCRIPT)) +LIBVER := $(shell echo $(LIBVER_SCRIPT)) +CCVER := $(shell $(CC) --version) +ZSTD_VERSION?= $(LIBVER) + +# ZSTD_LIB_MINIFY is a helper variable that +# configures a bunch of other variables to space-optimized defaults. +ZSTD_LIB_MINIFY ?= 0 +ifneq ($(ZSTD_LIB_MINIFY), 0) + HAVE_CC_OZ ?= $(shell echo "" | $(CC) -Oz -x c -c - -o /dev/null 2> /dev/null && echo 1 || echo 0) + ZSTD_LEGACY_SUPPORT ?= 0 + ZSTD_LIB_DEPRECATED ?= 0 + HUF_FORCE_DECOMPRESS_X1 ?= 1 + ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT ?= 1 + ZSTD_NO_INLINE ?= 1 + ZSTD_STRIP_ERROR_STRINGS ?= 1 +ifneq ($(HAVE_CC_OZ), 0) + # Some compilers (clang) support an even more space-optimized setting. + CFLAGS += -Oz +else + CFLAGS += -Os +endif + CFLAGS += -fno-stack-protector -fomit-frame-pointer -fno-ident \ + -DDYNAMIC_BMI2=0 -DNDEBUG +else + CFLAGS ?= -O3 +endif + +DEBUGLEVEL ?= 0 +CPPFLAGS += -DXXH_NAMESPACE=ZSTD_ -DDEBUGLEVEL=$(DEBUGLEVEL) +ifeq ($(TARGET_SYSTEM),Windows_NT) # MinGW assumed + CPPFLAGS += -D__USE_MINGW_ANSI_STDIO # compatibility with %zu formatting +endif +DEBUGFLAGS= -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \ + -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \ + -Wstrict-prototypes -Wundef -Wpointer-arith \ + -Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \ + -Wredundant-decls -Wmissing-prototypes -Wc++-compat +CFLAGS += $(DEBUGFLAGS) $(MOREFLAGS) +ASFLAGS += $(DEBUGFLAGS) $(MOREFLAGS) $(CFLAGS) +LDFLAGS += $(MOREFLAGS) +FLAGS = $(CPPFLAGS) $(CFLAGS) $(ASFLAGS) $(LDFLAGS) + +ifndef ALREADY_APPENDED_NOEXECSTACK +export ALREADY_APPENDED_NOEXECSTACK := 1 +ifeq ($(shell echo "int main(int argc, char* argv[]) { (void)argc; (void)argv; return 0; }" | $(CC) $(FLAGS) -z noexecstack -x c -Werror - -o $(VOID) 2>$(VOID) && echo 1 || echo 0),1) +LDFLAGS += -z noexecstack +endif +ifeq ($(shell echo | $(CC) $(FLAGS) -Wa,--noexecstack -x assembler -Werror -c - -o $(VOID) 2>$(VOID) && echo 1 || echo 0),1) +CFLAGS += -Wa,--noexecstack +# CFLAGS are also added to ASFLAGS +else ifeq ($(shell echo | $(CC) $(FLAGS) -Qunused-arguments -Wa,--noexecstack -x assembler -Werror -c - -o $(VOID) 2>$(VOID) && echo 1 || echo 0),1) +# See e.g.: https://github.com/android/ndk/issues/171 +CFLAGS += -Qunused-arguments -Wa,--noexecstack +# CFLAGS are also added to ASFLAGS +endif +endif + +HAVE_COLORNEVER = $(shell echo a | grep --color=never a > /dev/null 2> /dev/null && echo 1 || echo 0) +GREP_OPTIONS ?= +ifeq ($HAVE_COLORNEVER, 1) + GREP_OPTIONS += --color=never +endif +GREP = grep $(GREP_OPTIONS) +SED_ERE_OPT ?= -E + +ZSTD_COMMON_FILES := $(sort $(wildcard $(LIBZSTD)/common/*.c)) +ZSTD_COMPRESS_FILES := $(sort $(wildcard $(LIBZSTD)/compress/*.c)) +ZSTD_DECOMPRESS_FILES := $(sort $(wildcard $(LIBZSTD)/decompress/*.c)) +ZSTD_DICTBUILDER_FILES := $(sort $(wildcard $(LIBZSTD)/dictBuilder/*.c)) +ZSTD_DEPRECATED_FILES := $(sort $(wildcard $(LIBZSTD)/deprecated/*.c)) +ZSTD_LEGACY_FILES := + +ZSTD_DECOMPRESS_AMD64_ASM_FILES := $(sort $(wildcard $(LIBZSTD)/decompress/*_amd64.S)) + +ifneq ($(ZSTD_NO_ASM), 0) + CPPFLAGS += -DZSTD_DISABLE_ASM +else + # Unconditionally add the ASM files they are disabled by + # macros in the .S file. + ZSTD_DECOMPRESS_FILES += $(ZSTD_DECOMPRESS_AMD64_ASM_FILES) +endif + +ifneq ($(HUF_FORCE_DECOMPRESS_X1), 0) + CFLAGS += -DHUF_FORCE_DECOMPRESS_X1 +endif + +ifneq ($(HUF_FORCE_DECOMPRESS_X2), 0) + CFLAGS += -DHUF_FORCE_DECOMPRESS_X2 +endif + +ifneq ($(ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT), 0) + CFLAGS += -DZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT +endif + +ifneq ($(ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG), 0) + CFLAGS += -DZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG +endif + +ifneq ($(ZSTD_NO_INLINE), 0) + CFLAGS += -DZSTD_NO_INLINE +endif + +ifneq ($(ZSTD_STRIP_ERROR_STRINGS), 0) + CFLAGS += -DZSTD_STRIP_ERROR_STRINGS +endif + +ifneq ($(ZSTD_LEGACY_MULTITHREADED_API), 0) + CFLAGS += -DZSTD_LEGACY_MULTITHREADED_API +endif + +ifneq ($(ZSTD_LEGACY_SUPPORT), 0) +ifeq ($(shell test $(ZSTD_LEGACY_SUPPORT) -lt 8; echo $$?), 0) + ZSTD_LEGACY_FILES += $(shell ls $(LIBZSTD)/legacy/*.c | $(GREP) 'v0[$(ZSTD_LEGACY_SUPPORT)-7]') +endif +endif +CPPFLAGS += -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) + +UNAME := $(shell uname) + +ifndef BUILD_DIR +ifeq ($(UNAME), Darwin) + ifeq ($(shell md5 < /dev/null > /dev/null; echo $$?), 0) + HASH ?= md5 + endif +else ifeq ($(UNAME), FreeBSD) + HASH ?= gmd5sum +else ifeq ($(UNAME), NetBSD) + HASH ?= md5 -n +else ifeq ($(UNAME), OpenBSD) + HASH ?= md5 +endif +HASH ?= md5sum + +HASH_DIR = conf_$(shell echo $(CC) $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(ZSTD_FILES) | $(HASH) | cut -f 1 -d " " ) +HAVE_HASH :=$(shell echo 1 | $(HASH) > /dev/null && echo 1 || echo 0) +ifeq ($(HAVE_HASH),0) + $(info warning : could not find HASH ($(HASH)), needed to differentiate builds using different flags) + BUILD_DIR := obj/generic_noconf +endif +endif # BUILD_DIR + +ZSTD_SUBDIR := $(LIBZSTD)/common $(LIBZSTD)/compress $(LIBZSTD)/decompress $(LIBZSTD)/dictBuilder $(LIBZSTD)/legacy $(LIBZSTD)/deprecated +vpath %.c $(ZSTD_SUBDIR) +vpath %.S $(ZSTD_SUBDIR) diff --git a/src/blosc2/internal-complibs/zstd-1.5.2/libzstd.pc.in b/src/blosc2/internal-complibs/zstd-1.5.2/libzstd.pc.in new file mode 100644 index 0000000..43ebaec --- /dev/null +++ b/src/blosc2/internal-complibs/zstd-1.5.2/libzstd.pc.in @@ -0,0 +1,16 @@ +# ZSTD - standard compression algorithm +# Copyright (C) 2014-2016, Yann Collet, Facebook +# BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + +prefix=@PREFIX@ +exec_prefix=@EXEC_PREFIX@ +includedir=@INCLUDEDIR@ +libdir=@LIBDIR@ + +Name: zstd +Description: fast lossless compression algorithm library +URL: http://www.zstd.net/ +Version: @VERSION@ +Libs: -L${libdir} -lzstd +Libs.private: @LIBS_PRIVATE@ +Cflags: -I${includedir} diff --git a/src/blosc2/internal-complibs/zstd-1.5.2/module.modulemap b/src/blosc2/internal-complibs/zstd-1.5.2/module.modulemap new file mode 100644 index 0000000..bbb9397 --- /dev/null +++ b/src/blosc2/internal-complibs/zstd-1.5.2/module.modulemap @@ -0,0 +1,25 @@ +module libzstd [extern_c] { + header "zstd.h" + export * + config_macros [exhaustive] /* zstd.h */ \ + ZSTD_STATIC_LINKING_ONLY, \ + ZSTDLIB_VISIBLE, \ + ZSTD_DLL_EXPORT, \ + ZSTDLIB_STATIC_API, \ + ZSTD_DISABLE_DEPRECATE_WARNINGS, \ + ZSTD_CLEVEL_DEFAULT, \ + /* zdict.h */ ZDICT_STATIC_LINKING_ONLY, \ + ZDICTLIB_VISIBILITY, \ + ZDICT_DISABLE_DEPRECATE_WARNINGS, \ + /* zstd_errors.h */ ZSTDERRORLIB_VISIBILITY + + module dictbuilder [extern_c] { + header "zdict.h" + export * + } + + module errors [extern_c] { + header "zstd_errors.h" + export * + } +} diff --git a/src/blosc2/internal-complibs/zstd-1.5.2/zdict.h b/src/blosc2/internal-complibs/zstd-1.5.2/zdict.h new file mode 100644 index 0000000..f1e139a --- /dev/null +++ b/src/blosc2/internal-complibs/zstd-1.5.2/zdict.h @@ -0,0 +1,452 @@ +/* + * Copyright (c) Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef DICTBUILDER_H_001 +#define DICTBUILDER_H_001 + +#if defined (__cplusplus) +extern "C" { +#endif + + +/*====== Dependencies ======*/ +#include /* size_t */ + + +/* ===== ZDICTLIB_API : control library symbols visibility ===== */ +#ifndef ZDICTLIB_VISIBILITY +# if defined(__GNUC__) && (__GNUC__ >= 4) +# define ZDICTLIB_VISIBILITY __attribute__ ((visibility ("default"))) +# else +# define ZDICTLIB_VISIBILITY +# endif +#endif +#if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1) +# define ZDICTLIB_API __declspec(dllexport) ZDICTLIB_VISIBILITY +#elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1) +# define ZDICTLIB_API __declspec(dllimport) ZDICTLIB_VISIBILITY /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/ +#else +# define ZDICTLIB_API ZDICTLIB_VISIBILITY +#endif + +/******************************************************************************* + * Zstd dictionary builder + * + * FAQ + * === + * Why should I use a dictionary? + * ------------------------------ + * + * Zstd can use dictionaries to improve compression ratio of small data. + * Traditionally small files don't compress well because there is very little + * repetition in a single sample, since it is small. But, if you are compressing + * many similar files, like a bunch of JSON records that share the same + * structure, you can train a dictionary on ahead of time on some samples of + * these files. Then, zstd can use the dictionary to find repetitions that are + * present across samples. This can vastly improve compression ratio. + * + * When is a dictionary useful? + * ---------------------------- + * + * Dictionaries are useful when compressing many small files that are similar. + * The larger a file is, the less benefit a dictionary will have. Generally, + * we don't expect dictionary compression to be effective past 100KB. And the + * smaller a file is, the more we would expect the dictionary to help. + * + * How do I use a dictionary? + * -------------------------- + * + * Simply pass the dictionary to the zstd compressor with + * `ZSTD_CCtx_loadDictionary()`. The same dictionary must then be passed to + * the decompressor, using `ZSTD_DCtx_loadDictionary()`. There are other + * more advanced functions that allow selecting some options, see zstd.h for + * complete documentation. + * + * What is a zstd dictionary? + * -------------------------- + * + * A zstd dictionary has two pieces: Its header, and its content. The header + * contains a magic number, the dictionary ID, and entropy tables. These + * entropy tables allow zstd to save on header costs in the compressed file, + * which really matters for small data. The content is just bytes, which are + * repeated content that is common across many samples. + * + * What is a raw content dictionary? + * --------------------------------- + * + * A raw content dictionary is just bytes. It doesn't have a zstd dictionary + * header, a dictionary ID, or entropy tables. Any buffer is a valid raw + * content dictionary. + * + * How do I train a dictionary? + * ---------------------------- + * + * Gather samples from your use case. These samples should be similar to each + * other. If you have several use cases, you could try to train one dictionary + * per use case. + * + * Pass those samples to `ZDICT_trainFromBuffer()` and that will train your + * dictionary. There are a few advanced versions of this function, but this + * is a great starting point. If you want to further tune your dictionary + * you could try `ZDICT_optimizeTrainFromBuffer_cover()`. If that is too slow + * you can try `ZDICT_optimizeTrainFromBuffer_fastCover()`. + * + * If the dictionary training function fails, that is likely because you + * either passed too few samples, or a dictionary would not be effective + * for your data. Look at the messages that the dictionary trainer printed, + * if it doesn't say too few samples, then a dictionary would not be effective. + * + * How large should my dictionary be? + * ---------------------------------- + * + * A reasonable dictionary size, the `dictBufferCapacity`, is about 100KB. + * The zstd CLI defaults to a 110KB dictionary. You likely don't need a + * dictionary larger than that. But, most use cases can get away with a + * smaller dictionary. The advanced dictionary builders can automatically + * shrink the dictionary for you, and select a the smallest size that + * doesn't hurt compression ratio too much. See the `shrinkDict` parameter. + * A smaller dictionary can save memory, and potentially speed up + * compression. + * + * How many samples should I provide to the dictionary builder? + * ------------------------------------------------------------ + * + * We generally recommend passing ~100x the size of the dictionary + * in samples. A few thousand should suffice. Having too few samples + * can hurt the dictionaries effectiveness. Having more samples will + * only improve the dictionaries effectiveness. But having too many + * samples can slow down the dictionary builder. + * + * How do I determine if a dictionary will be effective? + * ----------------------------------------------------- + * + * Simply train a dictionary and try it out. You can use zstd's built in + * benchmarking tool to test the dictionary effectiveness. + * + * # Benchmark levels 1-3 without a dictionary + * zstd -b1e3 -r /path/to/my/files + * # Benchmark levels 1-3 with a dictionary + * zstd -b1e3 -r /path/to/my/files -D /path/to/my/dictionary + * + * When should I retrain a dictionary? + * ----------------------------------- + * + * You should retrain a dictionary when its effectiveness drops. Dictionary + * effectiveness drops as the data you are compressing changes. Generally, we do + * expect dictionaries to "decay" over time, as your data changes, but the rate + * at which they decay depends on your use case. Internally, we regularly + * retrain dictionaries, and if the new dictionary performs significantly + * better than the old dictionary, we will ship the new dictionary. + * + * I have a raw content dictionary, how do I turn it into a zstd dictionary? + * ------------------------------------------------------------------------- + * + * If you have a raw content dictionary, e.g. by manually constructing it, or + * using a third-party dictionary builder, you can turn it into a zstd + * dictionary by using `ZDICT_finalizeDictionary()`. You'll also have to + * provide some samples of the data. It will add the zstd header to the + * raw content, which contains a dictionary ID and entropy tables, which + * will improve compression ratio, and allow zstd to write the dictionary ID + * into the frame, if you so choose. + * + * Do I have to use zstd's dictionary builder? + * ------------------------------------------- + * + * No! You can construct dictionary content however you please, it is just + * bytes. It will always be valid as a raw content dictionary. If you want + * a zstd dictionary, which can improve compression ratio, use + * `ZDICT_finalizeDictionary()`. + * + * What is the attack surface of a zstd dictionary? + * ------------------------------------------------ + * + * Zstd is heavily fuzz tested, including loading fuzzed dictionaries, so + * zstd should never crash, or access out-of-bounds memory no matter what + * the dictionary is. However, if an attacker can control the dictionary + * during decompression, they can cause zstd to generate arbitrary bytes, + * just like if they controlled the compressed data. + * + ******************************************************************************/ + + +/*! ZDICT_trainFromBuffer(): + * Train a dictionary from an array of samples. + * Redirect towards ZDICT_optimizeTrainFromBuffer_fastCover() single-threaded, with d=8, steps=4, + * f=20, and accel=1. + * Samples must be stored concatenated in a single flat buffer `samplesBuffer`, + * supplied with an array of sizes `samplesSizes`, providing the size of each sample, in order. + * The resulting dictionary will be saved into `dictBuffer`. + * @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`) + * or an error code, which can be tested with ZDICT_isError(). + * Note: Dictionary training will fail if there are not enough samples to construct a + * dictionary, or if most of the samples are too small (< 8 bytes being the lower limit). + * If dictionary training fails, you should use zstd without a dictionary, as the dictionary + * would've been ineffective anyways. If you believe your samples would benefit from a dictionary + * please open an issue with details, and we can look into it. + * Note: ZDICT_trainFromBuffer()'s memory usage is about 6 MB. + * Tips: In general, a reasonable dictionary has a size of ~ 100 KB. + * It's possible to select smaller or larger size, just by specifying `dictBufferCapacity`. + * In general, it's recommended to provide a few thousands samples, though this can vary a lot. + * It's recommended that total size of all samples be about ~x100 times the target size of dictionary. + */ +ZDICTLIB_API size_t ZDICT_trainFromBuffer(void* dictBuffer, size_t dictBufferCapacity, + const void* samplesBuffer, + const size_t* samplesSizes, unsigned nbSamples); + +typedef struct { + int compressionLevel; /*< optimize for a specific zstd compression level; 0 means default */ + unsigned notificationLevel; /*< Write log to stderr; 0 = none (default); 1 = errors; 2 = progression; 3 = details; 4 = debug; */ + unsigned dictID; /*< force dictID value; 0 means auto mode (32-bits random value) + * NOTE: The zstd format reserves some dictionary IDs for future use. + * You may use them in private settings, but be warned that they + * may be used by zstd in a public dictionary registry in the future. + * These dictionary IDs are: + * - low range : <= 32767 + * - high range : >= (2^31) + */ +} ZDICT_params_t; + +/*! ZDICT_finalizeDictionary(): + * Given a custom content as a basis for dictionary, and a set of samples, + * finalize dictionary by adding headers and statistics according to the zstd + * dictionary format. + * + * Samples must be stored concatenated in a flat buffer `samplesBuffer`, + * supplied with an array of sizes `samplesSizes`, providing the size of each + * sample in order. The samples are used to construct the statistics, so they + * should be representative of what you will compress with this dictionary. + * + * The compression level can be set in `parameters`. You should pass the + * compression level you expect to use in production. The statistics for each + * compression level differ, so tuning the dictionary for the compression level + * can help quite a bit. + * + * You can set an explicit dictionary ID in `parameters`, or allow us to pick + * a random dictionary ID for you, but we can't guarantee no collisions. + * + * The dstDictBuffer and the dictContent may overlap, and the content will be + * appended to the end of the header. If the header + the content doesn't fit in + * maxDictSize the beginning of the content is truncated to make room, since it + * is presumed that the most profitable content is at the end of the dictionary, + * since that is the cheapest to reference. + * + * `maxDictSize` must be >= max(dictContentSize, ZSTD_DICTSIZE_MIN). + * + * @return: size of dictionary stored into `dstDictBuffer` (<= `maxDictSize`), + * or an error code, which can be tested by ZDICT_isError(). + * Note: ZDICT_finalizeDictionary() will push notifications into stderr if + * instructed to, using notificationLevel>0. + * NOTE: This function currently may fail in several edge cases including: + * * Not enough samples + * * Samples are uncompressible + * * Samples are all exactly the same + */ +ZDICTLIB_API size_t ZDICT_finalizeDictionary(void* dstDictBuffer, size_t maxDictSize, + const void* dictContent, size_t dictContentSize, + const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples, + ZDICT_params_t parameters); + + +/*====== Helper functions ======*/ +ZDICTLIB_API unsigned ZDICT_getDictID(const void* dictBuffer, size_t dictSize); /**< extracts dictID; @return zero if error (not a valid dictionary) */ +ZDICTLIB_API size_t ZDICT_getDictHeaderSize(const void* dictBuffer, size_t dictSize); /* returns dict header size; returns a ZSTD error code on failure */ +ZDICTLIB_API unsigned ZDICT_isError(size_t errorCode); +ZDICTLIB_API const char* ZDICT_getErrorName(size_t errorCode); + + + +#ifdef ZDICT_STATIC_LINKING_ONLY + +/* ==================================================================================== + * The definitions in this section are considered experimental. + * They should never be used with a dynamic library, as they may change in the future. + * They are provided for advanced usages. + * Use them only in association with static linking. + * ==================================================================================== */ + +#define ZDICT_DICTSIZE_MIN 256 +/* Deprecated: Remove in v1.6.0 */ +#define ZDICT_CONTENTSIZE_MIN 128 + +/*! ZDICT_cover_params_t: + * k and d are the only required parameters. + * For others, value 0 means default. + */ +typedef struct { + unsigned k; /* Segment size : constraint: 0 < k : Reasonable range [16, 2048+] */ + unsigned d; /* dmer size : constraint: 0 < d <= k : Reasonable range [6, 16] */ + unsigned steps; /* Number of steps : Only used for optimization : 0 means default (40) : Higher means more parameters checked */ + unsigned nbThreads; /* Number of threads : constraint: 0 < nbThreads : 1 means single-threaded : Only used for optimization : Ignored if ZSTD_MULTITHREAD is not defined */ + double splitPoint; /* Percentage of samples used for training: Only used for optimization : the first nbSamples * splitPoint samples will be used to training, the last nbSamples * (1 - splitPoint) samples will be used for testing, 0 means default (1.0), 1.0 when all samples are used for both training and testing */ + unsigned shrinkDict; /* Train dictionaries to shrink in size starting from the minimum size and selects the smallest dictionary that is shrinkDictMaxRegression% worse than the largest dictionary. 0 means no shrinking and 1 means shrinking */ + unsigned shrinkDictMaxRegression; /* Sets shrinkDictMaxRegression so that a smaller dictionary can be at worse shrinkDictMaxRegression% worse than the max dict size dictionary. */ + ZDICT_params_t zParams; +} ZDICT_cover_params_t; + +typedef struct { + unsigned k; /* Segment size : constraint: 0 < k : Reasonable range [16, 2048+] */ + unsigned d; /* dmer size : constraint: 0 < d <= k : Reasonable range [6, 16] */ + unsigned f; /* log of size of frequency array : constraint: 0 < f <= 31 : 1 means default(20)*/ + unsigned steps; /* Number of steps : Only used for optimization : 0 means default (40) : Higher means more parameters checked */ + unsigned nbThreads; /* Number of threads : constraint: 0 < nbThreads : 1 means single-threaded : Only used for optimization : Ignored if ZSTD_MULTITHREAD is not defined */ + double splitPoint; /* Percentage of samples used for training: Only used for optimization : the first nbSamples * splitPoint samples will be used to training, the last nbSamples * (1 - splitPoint) samples will be used for testing, 0 means default (0.75), 1.0 when all samples are used for both training and testing */ + unsigned accel; /* Acceleration level: constraint: 0 < accel <= 10, higher means faster and less accurate, 0 means default(1) */ + unsigned shrinkDict; /* Train dictionaries to shrink in size starting from the minimum size and selects the smallest dictionary that is shrinkDictMaxRegression% worse than the largest dictionary. 0 means no shrinking and 1 means shrinking */ + unsigned shrinkDictMaxRegression; /* Sets shrinkDictMaxRegression so that a smaller dictionary can be at worse shrinkDictMaxRegression% worse than the max dict size dictionary. */ + + ZDICT_params_t zParams; +} ZDICT_fastCover_params_t; + +/*! ZDICT_trainFromBuffer_cover(): + * Train a dictionary from an array of samples using the COVER algorithm. + * Samples must be stored concatenated in a single flat buffer `samplesBuffer`, + * supplied with an array of sizes `samplesSizes`, providing the size of each sample, in order. + * The resulting dictionary will be saved into `dictBuffer`. + * @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`) + * or an error code, which can be tested with ZDICT_isError(). + * See ZDICT_trainFromBuffer() for details on failure modes. + * Note: ZDICT_trainFromBuffer_cover() requires about 9 bytes of memory for each input byte. + * Tips: In general, a reasonable dictionary has a size of ~ 100 KB. + * It's possible to select smaller or larger size, just by specifying `dictBufferCapacity`. + * In general, it's recommended to provide a few thousands samples, though this can vary a lot. + * It's recommended that total size of all samples be about ~x100 times the target size of dictionary. + */ +ZDICTLIB_API size_t ZDICT_trainFromBuffer_cover( + void *dictBuffer, size_t dictBufferCapacity, + const void *samplesBuffer, const size_t *samplesSizes, unsigned nbSamples, + ZDICT_cover_params_t parameters); + +/*! ZDICT_optimizeTrainFromBuffer_cover(): + * The same requirements as above hold for all the parameters except `parameters`. + * This function tries many parameter combinations and picks the best parameters. + * `*parameters` is filled with the best parameters found, + * dictionary constructed with those parameters is stored in `dictBuffer`. + * + * All of the parameters d, k, steps are optional. + * If d is non-zero then we don't check multiple values of d, otherwise we check d = {6, 8}. + * if steps is zero it defaults to its default value. + * If k is non-zero then we don't check multiple values of k, otherwise we check steps values in [50, 2000]. + * + * @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`) + * or an error code, which can be tested with ZDICT_isError(). + * On success `*parameters` contains the parameters selected. + * See ZDICT_trainFromBuffer() for details on failure modes. + * Note: ZDICT_optimizeTrainFromBuffer_cover() requires about 8 bytes of memory for each input byte and additionally another 5 bytes of memory for each byte of memory for each thread. + */ +ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_cover( + void* dictBuffer, size_t dictBufferCapacity, + const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples, + ZDICT_cover_params_t* parameters); + +/*! ZDICT_trainFromBuffer_fastCover(): + * Train a dictionary from an array of samples using a modified version of COVER algorithm. + * Samples must be stored concatenated in a single flat buffer `samplesBuffer`, + * supplied with an array of sizes `samplesSizes`, providing the size of each sample, in order. + * d and k are required. + * All other parameters are optional, will use default values if not provided + * The resulting dictionary will be saved into `dictBuffer`. + * @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`) + * or an error code, which can be tested with ZDICT_isError(). + * See ZDICT_trainFromBuffer() for details on failure modes. + * Note: ZDICT_trainFromBuffer_fastCover() requires 6 * 2^f bytes of memory. + * Tips: In general, a reasonable dictionary has a size of ~ 100 KB. + * It's possible to select smaller or larger size, just by specifying `dictBufferCapacity`. + * In general, it's recommended to provide a few thousands samples, though this can vary a lot. + * It's recommended that total size of all samples be about ~x100 times the target size of dictionary. + */ +ZDICTLIB_API size_t ZDICT_trainFromBuffer_fastCover(void *dictBuffer, + size_t dictBufferCapacity, const void *samplesBuffer, + const size_t *samplesSizes, unsigned nbSamples, + ZDICT_fastCover_params_t parameters); + +/*! ZDICT_optimizeTrainFromBuffer_fastCover(): + * The same requirements as above hold for all the parameters except `parameters`. + * This function tries many parameter combinations (specifically, k and d combinations) + * and picks the best parameters. `*parameters` is filled with the best parameters found, + * dictionary constructed with those parameters is stored in `dictBuffer`. + * All of the parameters d, k, steps, f, and accel are optional. + * If d is non-zero then we don't check multiple values of d, otherwise we check d = {6, 8}. + * if steps is zero it defaults to its default value. + * If k is non-zero then we don't check multiple values of k, otherwise we check steps values in [50, 2000]. + * If f is zero, default value of 20 is used. + * If accel is zero, default value of 1 is used. + * + * @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`) + * or an error code, which can be tested with ZDICT_isError(). + * On success `*parameters` contains the parameters selected. + * See ZDICT_trainFromBuffer() for details on failure modes. + * Note: ZDICT_optimizeTrainFromBuffer_fastCover() requires about 6 * 2^f bytes of memory for each thread. + */ +ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_fastCover(void* dictBuffer, + size_t dictBufferCapacity, const void* samplesBuffer, + const size_t* samplesSizes, unsigned nbSamples, + ZDICT_fastCover_params_t* parameters); + +typedef struct { + unsigned selectivityLevel; /* 0 means default; larger => select more => larger dictionary */ + ZDICT_params_t zParams; +} ZDICT_legacy_params_t; + +/*! ZDICT_trainFromBuffer_legacy(): + * Train a dictionary from an array of samples. + * Samples must be stored concatenated in a single flat buffer `samplesBuffer`, + * supplied with an array of sizes `samplesSizes`, providing the size of each sample, in order. + * The resulting dictionary will be saved into `dictBuffer`. + * `parameters` is optional and can be provided with values set to 0 to mean "default". + * @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`) + * or an error code, which can be tested with ZDICT_isError(). + * See ZDICT_trainFromBuffer() for details on failure modes. + * Tips: In general, a reasonable dictionary has a size of ~ 100 KB. + * It's possible to select smaller or larger size, just by specifying `dictBufferCapacity`. + * In general, it's recommended to provide a few thousands samples, though this can vary a lot. + * It's recommended that total size of all samples be about ~x100 times the target size of dictionary. + * Note: ZDICT_trainFromBuffer_legacy() will send notifications into stderr if instructed to, using notificationLevel>0. + */ +ZDICTLIB_API size_t ZDICT_trainFromBuffer_legacy( + void* dictBuffer, size_t dictBufferCapacity, + const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples, + ZDICT_legacy_params_t parameters); + + +/* Deprecation warnings */ +/* It is generally possible to disable deprecation warnings from compiler, + for example with -Wno-deprecated-declarations for gcc + or _CRT_SECURE_NO_WARNINGS in Visual. + Otherwise, it's also possible to manually define ZDICT_DISABLE_DEPRECATE_WARNINGS */ +#ifdef ZDICT_DISABLE_DEPRECATE_WARNINGS +# define ZDICT_DEPRECATED(message) ZDICTLIB_API /* disable deprecation warnings */ +#else +# define ZDICT_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) +# if defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */ +# define ZDICT_DEPRECATED(message) [[deprecated(message)]] ZDICTLIB_API +# elif defined(__clang__) || (ZDICT_GCC_VERSION >= 405) +# define ZDICT_DEPRECATED(message) ZDICTLIB_API __attribute__((deprecated(message))) +# elif (ZDICT_GCC_VERSION >= 301) +# define ZDICT_DEPRECATED(message) ZDICTLIB_API __attribute__((deprecated)) +# elif defined(_MSC_VER) +# define ZDICT_DEPRECATED(message) ZDICTLIB_API __declspec(deprecated(message)) +# else +# pragma message("WARNING: You need to implement ZDICT_DEPRECATED for this compiler") +# define ZDICT_DEPRECATED(message) ZDICTLIB_API +# endif +#endif /* ZDICT_DISABLE_DEPRECATE_WARNINGS */ + +ZDICT_DEPRECATED("use ZDICT_finalizeDictionary() instead") +size_t ZDICT_addEntropyTablesFromBuffer(void* dictBuffer, size_t dictContentSize, size_t dictBufferCapacity, + const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples); + + +#endif /* ZDICT_STATIC_LINKING_ONLY */ + +#if defined (__cplusplus) +} +#endif + +#endif /* DICTBUILDER_H_001 */ diff --git a/src/blosc2/internal-complibs/zstd-1.5.2/zstd.h b/src/blosc2/internal-complibs/zstd-1.5.2/zstd.h new file mode 100644 index 0000000..a88ae7b --- /dev/null +++ b/src/blosc2/internal-complibs/zstd-1.5.2/zstd.h @@ -0,0 +1,2575 @@ +/* + * Copyright (c) Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ +#if defined (__cplusplus) +extern "C" { +#endif + +#ifndef ZSTD_H_235446 +#define ZSTD_H_235446 + +/* ====== Dependency ======*/ +#include /* INT_MAX */ +#include /* size_t */ + + +/* ===== ZSTDLIB_API : control library symbols visibility ===== */ +#ifndef ZSTDLIB_VISIBLE +# if defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__) +# define ZSTDLIB_VISIBLE __attribute__ ((visibility ("default"))) +# define ZSTDLIB_HIDDEN __attribute__ ((visibility ("hidden"))) +# else +# define ZSTDLIB_VISIBLE +# define ZSTDLIB_HIDDEN +# endif +#endif +#if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1) +# define ZSTDLIB_API __declspec(dllexport) ZSTDLIB_VISIBLE +#elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1) +# define ZSTDLIB_API __declspec(dllimport) ZSTDLIB_VISIBLE /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/ +#else +# define ZSTDLIB_API ZSTDLIB_VISIBLE +#endif + + +/******************************************************************************* + Introduction + + zstd, short for Zstandard, is a fast lossless compression algorithm, targeting + real-time compression scenarios at zlib-level and better compression ratios. + The zstd compression library provides in-memory compression and decompression + functions. + + The library supports regular compression levels from 1 up to ZSTD_maxCLevel(), + which is currently 22. Levels >= 20, labeled `--ultra`, should be used with + caution, as they require more memory. The library also offers negative + compression levels, which extend the range of speed vs. ratio preferences. + The lower the level, the faster the speed (at the cost of compression). + + Compression can be done in: + - a single step (described as Simple API) + - a single step, reusing a context (described as Explicit context) + - unbounded multiple steps (described as Streaming compression) + + The compression ratio achievable on small data can be highly improved using + a dictionary. Dictionary compression can be performed in: + - a single step (described as Simple dictionary API) + - a single step, reusing a dictionary (described as Bulk-processing + dictionary API) + + Advanced experimental functions can be accessed using + `#define ZSTD_STATIC_LINKING_ONLY` before including zstd.h. + + Advanced experimental APIs should never be used with a dynamically-linked + library. They are not "stable"; their definitions or signatures may change in + the future. Only static linking is allowed. +*******************************************************************************/ + +/*------ Version ------*/ +#define ZSTD_VERSION_MAJOR 1 +#define ZSTD_VERSION_MINOR 5 +#define ZSTD_VERSION_RELEASE 2 +#define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE) + +/*! ZSTD_versionNumber() : + * Return runtime library version, the value is (MAJOR*100*100 + MINOR*100 + RELEASE). */ +ZSTDLIB_API unsigned ZSTD_versionNumber(void); + +#define ZSTD_LIB_VERSION ZSTD_VERSION_MAJOR.ZSTD_VERSION_MINOR.ZSTD_VERSION_RELEASE +#define ZSTD_QUOTE(str) #str +#define ZSTD_EXPAND_AND_QUOTE(str) ZSTD_QUOTE(str) +#define ZSTD_VERSION_STRING ZSTD_EXPAND_AND_QUOTE(ZSTD_LIB_VERSION) + +/*! ZSTD_versionString() : + * Return runtime library version, like "1.4.5". Requires v1.3.0+. */ +ZSTDLIB_API const char* ZSTD_versionString(void); + +/* ************************************* + * Default constant + ***************************************/ +#ifndef ZSTD_CLEVEL_DEFAULT +# define ZSTD_CLEVEL_DEFAULT 3 +#endif + +/* ************************************* + * Constants + ***************************************/ + +/* All magic numbers are supposed read/written to/from files/memory using little-endian convention */ +#define ZSTD_MAGICNUMBER 0xFD2FB528 /* valid since v0.8.0 */ +#define ZSTD_MAGIC_DICTIONARY 0xEC30A437 /* valid since v0.7.0 */ +#define ZSTD_MAGIC_SKIPPABLE_START 0x184D2A50 /* all 16 values, from 0x184D2A50 to 0x184D2A5F, signal the beginning of a skippable frame */ +#define ZSTD_MAGIC_SKIPPABLE_MASK 0xFFFFFFF0 + +#define ZSTD_BLOCKSIZELOG_MAX 17 +#define ZSTD_BLOCKSIZE_MAX (1<= `ZSTD_compressBound(srcSize)`. + * @return : compressed size written into `dst` (<= `dstCapacity), + * or an error code if it fails (which can be tested using ZSTD_isError()). */ +ZSTDLIB_API size_t ZSTD_compress( void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + int compressionLevel); + +/*! ZSTD_decompress() : + * `compressedSize` : must be the _exact_ size of some number of compressed and/or skippable frames. + * `dstCapacity` is an upper bound of originalSize to regenerate. + * If user cannot imply a maximum upper bound, it's better to use streaming mode to decompress data. + * @return : the number of bytes decompressed into `dst` (<= `dstCapacity`), + * or an errorCode if it fails (which can be tested using ZSTD_isError()). */ +ZSTDLIB_API size_t ZSTD_decompress( void* dst, size_t dstCapacity, + const void* src, size_t compressedSize); + +/*! ZSTD_getFrameContentSize() : requires v1.3.0+ + * `src` should point to the start of a ZSTD encoded frame. + * `srcSize` must be at least as large as the frame header. + * hint : any size >= `ZSTD_frameHeaderSize_max` is large enough. + * @return : - decompressed size of `src` frame content, if known + * - ZSTD_CONTENTSIZE_UNKNOWN if the size cannot be determined + * - ZSTD_CONTENTSIZE_ERROR if an error occurred (e.g. invalid magic number, srcSize too small) + * note 1 : a 0 return value means the frame is valid but "empty". + * note 2 : decompressed size is an optional field, it may not be present, typically in streaming mode. + * When `return==ZSTD_CONTENTSIZE_UNKNOWN`, data to decompress could be any size. + * In which case, it's necessary to use streaming mode to decompress data. + * Optionally, application can rely on some implicit limit, + * as ZSTD_decompress() only needs an upper bound of decompressed size. + * (For example, data could be necessarily cut into blocks <= 16 KB). + * note 3 : decompressed size is always present when compression is completed using single-pass functions, + * such as ZSTD_compress(), ZSTD_compressCCtx() ZSTD_compress_usingDict() or ZSTD_compress_usingCDict(). + * note 4 : decompressed size can be very large (64-bits value), + * potentially larger than what local system can handle as a single memory segment. + * In which case, it's necessary to use streaming mode to decompress data. + * note 5 : If source is untrusted, decompressed size could be wrong or intentionally modified. + * Always ensure return value fits within application's authorized limits. + * Each application can set its own limits. + * note 6 : This function replaces ZSTD_getDecompressedSize() */ +#define ZSTD_CONTENTSIZE_UNKNOWN (0ULL - 1) +#define ZSTD_CONTENTSIZE_ERROR (0ULL - 2) +ZSTDLIB_API unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize); + +/*! ZSTD_getDecompressedSize() : + * NOTE: This function is now obsolete, in favor of ZSTD_getFrameContentSize(). + * Both functions work the same way, but ZSTD_getDecompressedSize() blends + * "empty", "unknown" and "error" results to the same return value (0), + * while ZSTD_getFrameContentSize() gives them separate return values. + * @return : decompressed size of `src` frame content _if known and not empty_, 0 otherwise. */ +ZSTDLIB_API unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize); + +/*! ZSTD_findFrameCompressedSize() : Requires v1.4.0+ + * `src` should point to the start of a ZSTD frame or skippable frame. + * `srcSize` must be >= first frame size + * @return : the compressed size of the first frame starting at `src`, + * suitable to pass as `srcSize` to `ZSTD_decompress` or similar, + * or an error code if input is invalid */ +ZSTDLIB_API size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize); + + +/*====== Helper functions ======*/ +#define ZSTD_COMPRESSBOUND(srcSize) ((srcSize) + ((srcSize)>>8) + (((srcSize) < (128<<10)) ? (((128<<10) - (srcSize)) >> 11) /* margin, from 64 to 0 */ : 0)) /* this formula ensures that bound(A) + bound(B) <= bound(A+B) as long as A and B >= 128 KB */ +ZSTDLIB_API size_t ZSTD_compressBound(size_t srcSize); /*!< maximum compressed size in worst case single-pass scenario */ +ZSTDLIB_API unsigned ZSTD_isError(size_t code); /*!< tells if a `size_t` function result is an error code */ +ZSTDLIB_API const char* ZSTD_getErrorName(size_t code); /*!< provides readable string from an error code */ +ZSTDLIB_API int ZSTD_minCLevel(void); /*!< minimum negative compression level allowed, requires v1.4.0+ */ +ZSTDLIB_API int ZSTD_maxCLevel(void); /*!< maximum compression level available */ +ZSTDLIB_API int ZSTD_defaultCLevel(void); /*!< default compression level, specified by ZSTD_CLEVEL_DEFAULT, requires v1.5.0+ */ + + +/*************************************** +* Explicit context +***************************************/ +/*= Compression context + * When compressing many times, + * it is recommended to allocate a context just once, + * and re-use it for each successive compression operation. + * This will make workload friendlier for system's memory. + * Note : re-using context is just a speed / resource optimization. + * It doesn't change the compression ratio, which remains identical. + * Note 2 : In multi-threaded environments, + * use one different context per thread for parallel execution. + */ +typedef struct ZSTD_CCtx_s ZSTD_CCtx; +ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx(void); +ZSTDLIB_API size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx); /* accept NULL pointer */ + +/*! ZSTD_compressCCtx() : + * Same as ZSTD_compress(), using an explicit ZSTD_CCtx. + * Important : in order to behave similarly to `ZSTD_compress()`, + * this function compresses at requested compression level, + * __ignoring any other parameter__ . + * If any advanced parameter was set using the advanced API, + * they will all be reset. Only `compressionLevel` remains. + */ +ZSTDLIB_API size_t ZSTD_compressCCtx(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + int compressionLevel); + +/*= Decompression context + * When decompressing many times, + * it is recommended to allocate a context only once, + * and re-use it for each successive compression operation. + * This will make workload friendlier for system's memory. + * Use one context per thread for parallel execution. */ +typedef struct ZSTD_DCtx_s ZSTD_DCtx; +ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx(void); +ZSTDLIB_API size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx); /* accept NULL pointer */ + +/*! ZSTD_decompressDCtx() : + * Same as ZSTD_decompress(), + * requires an allocated ZSTD_DCtx. + * Compatible with sticky parameters. + */ +ZSTDLIB_API size_t ZSTD_decompressDCtx(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize); + + +/********************************************* +* Advanced compression API (Requires v1.4.0+) +**********************************************/ + +/* API design : + * Parameters are pushed one by one into an existing context, + * using ZSTD_CCtx_set*() functions. + * Pushed parameters are sticky : they are valid for next compressed frame, and any subsequent frame. + * "sticky" parameters are applicable to `ZSTD_compress2()` and `ZSTD_compressStream*()` ! + * __They do not apply to "simple" one-shot variants such as ZSTD_compressCCtx()__ . + * + * It's possible to reset all parameters to "default" using ZSTD_CCtx_reset(). + * + * This API supersedes all other "advanced" API entry points in the experimental section. + * In the future, we expect to remove from experimental API entry points which are redundant with this API. + */ + + +/* Compression strategies, listed from fastest to strongest */ +typedef enum { ZSTD_fast=1, + ZSTD_dfast=2, + ZSTD_greedy=3, + ZSTD_lazy=4, + ZSTD_lazy2=5, + ZSTD_btlazy2=6, + ZSTD_btopt=7, + ZSTD_btultra=8, + ZSTD_btultra2=9 + /* note : new strategies _might_ be added in the future. + Only the order (from fast to strong) is guaranteed */ +} ZSTD_strategy; + +typedef enum { + + /* compression parameters + * Note: When compressing with a ZSTD_CDict these parameters are superseded + * by the parameters used to construct the ZSTD_CDict. + * See ZSTD_CCtx_refCDict() for more info (superseded-by-cdict). */ + ZSTD_c_compressionLevel=100, /* Set compression parameters according to pre-defined cLevel table. + * Note that exact compression parameters are dynamically determined, + * depending on both compression level and srcSize (when known). + * Default level is ZSTD_CLEVEL_DEFAULT==3. + * Special: value 0 means default, which is controlled by ZSTD_CLEVEL_DEFAULT. + * Note 1 : it's possible to pass a negative compression level. + * Note 2 : setting a level does not automatically set all other compression parameters + * to default. Setting this will however eventually dynamically impact the compression + * parameters which have not been manually set. The manually set + * ones will 'stick'. */ + /* Advanced compression parameters : + * It's possible to pin down compression parameters to some specific values. + * In which case, these values are no longer dynamically selected by the compressor */ + ZSTD_c_windowLog=101, /* Maximum allowed back-reference distance, expressed as power of 2. + * This will set a memory budget for streaming decompression, + * with larger values requiring more memory + * and typically compressing more. + * Must be clamped between ZSTD_WINDOWLOG_MIN and ZSTD_WINDOWLOG_MAX. + * Special: value 0 means "use default windowLog". + * Note: Using a windowLog greater than ZSTD_WINDOWLOG_LIMIT_DEFAULT + * requires explicitly allowing such size at streaming decompression stage. */ + ZSTD_c_hashLog=102, /* Size of the initial probe table, as a power of 2. + * Resulting memory usage is (1 << (hashLog+2)). + * Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX. + * Larger tables improve compression ratio of strategies <= dFast, + * and improve speed of strategies > dFast. + * Special: value 0 means "use default hashLog". */ + ZSTD_c_chainLog=103, /* Size of the multi-probe search table, as a power of 2. + * Resulting memory usage is (1 << (chainLog+2)). + * Must be clamped between ZSTD_CHAINLOG_MIN and ZSTD_CHAINLOG_MAX. + * Larger tables result in better and slower compression. + * This parameter is useless for "fast" strategy. + * It's still useful when using "dfast" strategy, + * in which case it defines a secondary probe table. + * Special: value 0 means "use default chainLog". */ + ZSTD_c_searchLog=104, /* Number of search attempts, as a power of 2. + * More attempts result in better and slower compression. + * This parameter is useless for "fast" and "dFast" strategies. + * Special: value 0 means "use default searchLog". */ + ZSTD_c_minMatch=105, /* Minimum size of searched matches. + * Note that Zstandard can still find matches of smaller size, + * it just tweaks its search algorithm to look for this size and larger. + * Larger values increase compression and decompression speed, but decrease ratio. + * Must be clamped between ZSTD_MINMATCH_MIN and ZSTD_MINMATCH_MAX. + * Note that currently, for all strategies < btopt, effective minimum is 4. + * , for all strategies > fast, effective maximum is 6. + * Special: value 0 means "use default minMatchLength". */ + ZSTD_c_targetLength=106, /* Impact of this field depends on strategy. + * For strategies btopt, btultra & btultra2: + * Length of Match considered "good enough" to stop search. + * Larger values make compression stronger, and slower. + * For strategy fast: + * Distance between match sampling. + * Larger values make compression faster, and weaker. + * Special: value 0 means "use default targetLength". */ + ZSTD_c_strategy=107, /* See ZSTD_strategy enum definition. + * The higher the value of selected strategy, the more complex it is, + * resulting in stronger and slower compression. + * Special: value 0 means "use default strategy". */ + /* LDM mode parameters */ + ZSTD_c_enableLongDistanceMatching=160, /* Enable long distance matching. + * This parameter is designed to improve compression ratio + * for large inputs, by finding large matches at long distance. + * It increases memory usage and window size. + * Note: enabling this parameter increases default ZSTD_c_windowLog to 128 MB + * except when expressly set to a different value. + * Note: will be enabled by default if ZSTD_c_windowLog >= 128 MB and + * compression strategy >= ZSTD_btopt (== compression level 16+) */ + ZSTD_c_ldmHashLog=161, /* Size of the table for long distance matching, as a power of 2. + * Larger values increase memory usage and compression ratio, + * but decrease compression speed. + * Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX + * default: windowlog - 7. + * Special: value 0 means "automatically determine hashlog". */ + ZSTD_c_ldmMinMatch=162, /* Minimum match size for long distance matcher. + * Larger/too small values usually decrease compression ratio. + * Must be clamped between ZSTD_LDM_MINMATCH_MIN and ZSTD_LDM_MINMATCH_MAX. + * Special: value 0 means "use default value" (default: 64). */ + ZSTD_c_ldmBucketSizeLog=163, /* Log size of each bucket in the LDM hash table for collision resolution. + * Larger values improve collision resolution but decrease compression speed. + * The maximum value is ZSTD_LDM_BUCKETSIZELOG_MAX. + * Special: value 0 means "use default value" (default: 3). */ + ZSTD_c_ldmHashRateLog=164, /* Frequency of inserting/looking up entries into the LDM hash table. + * Must be clamped between 0 and (ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN). + * Default is MAX(0, (windowLog - ldmHashLog)), optimizing hash table usage. + * Larger values improve compression speed. + * Deviating far from default value will likely result in a compression ratio decrease. + * Special: value 0 means "automatically determine hashRateLog". */ + + /* frame parameters */ + ZSTD_c_contentSizeFlag=200, /* Content size will be written into frame header _whenever known_ (default:1) + * Content size must be known at the beginning of compression. + * This is automatically the case when using ZSTD_compress2(), + * For streaming scenarios, content size must be provided with ZSTD_CCtx_setPledgedSrcSize() */ + ZSTD_c_checksumFlag=201, /* A 32-bits checksum of content is written at end of frame (default:0) */ + ZSTD_c_dictIDFlag=202, /* When applicable, dictionary's ID is written into frame header (default:1) */ + + /* multi-threading parameters */ + /* These parameters are only active if multi-threading is enabled (compiled with build macro ZSTD_MULTITHREAD). + * Otherwise, trying to set any other value than default (0) will be a no-op and return an error. + * In a situation where it's unknown if the linked library supports multi-threading or not, + * setting ZSTD_c_nbWorkers to any value >= 1 and consulting the return value provides a quick way to check this property. + */ + ZSTD_c_nbWorkers=400, /* Select how many threads will be spawned to compress in parallel. + * When nbWorkers >= 1, triggers asynchronous mode when invoking ZSTD_compressStream*() : + * ZSTD_compressStream*() consumes input and flush output if possible, but immediately gives back control to caller, + * while compression is performed in parallel, within worker thread(s). + * (note : a strong exception to this rule is when first invocation of ZSTD_compressStream2() sets ZSTD_e_end : + * in which case, ZSTD_compressStream2() delegates to ZSTD_compress2(), which is always a blocking call). + * More workers improve speed, but also increase memory usage. + * Default value is `0`, aka "single-threaded mode" : no worker is spawned, + * compression is performed inside Caller's thread, and all invocations are blocking */ + ZSTD_c_jobSize=401, /* Size of a compression job. This value is enforced only when nbWorkers >= 1. + * Each compression job is completed in parallel, so this value can indirectly impact the nb of active threads. + * 0 means default, which is dynamically determined based on compression parameters. + * Job size must be a minimum of overlap size, or ZSTDMT_JOBSIZE_MIN (= 512 KB), whichever is largest. + * The minimum size is automatically and transparently enforced. */ + ZSTD_c_overlapLog=402, /* Control the overlap size, as a fraction of window size. + * The overlap size is an amount of data reloaded from previous job at the beginning of a new job. + * It helps preserve compression ratio, while each job is compressed in parallel. + * This value is enforced only when nbWorkers >= 1. + * Larger values increase compression ratio, but decrease speed. + * Possible values range from 0 to 9 : + * - 0 means "default" : value will be determined by the library, depending on strategy + * - 1 means "no overlap" + * - 9 means "full overlap", using a full window size. + * Each intermediate rank increases/decreases load size by a factor 2 : + * 9: full window; 8: w/2; 7: w/4; 6: w/8; 5:w/16; 4: w/32; 3:w/64; 2:w/128; 1:no overlap; 0:default + * default value varies between 6 and 9, depending on strategy */ + + /* note : additional experimental parameters are also available + * within the experimental section of the API. + * At the time of this writing, they include : + * ZSTD_c_rsyncable + * ZSTD_c_format + * ZSTD_c_forceMaxWindow + * ZSTD_c_forceAttachDict + * ZSTD_c_literalCompressionMode + * ZSTD_c_targetCBlockSize + * ZSTD_c_srcSizeHint + * ZSTD_c_enableDedicatedDictSearch + * ZSTD_c_stableInBuffer + * ZSTD_c_stableOutBuffer + * ZSTD_c_blockDelimiters + * ZSTD_c_validateSequences + * ZSTD_c_useBlockSplitter + * ZSTD_c_useRowMatchFinder + * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them. + * note : never ever use experimentalParam? names directly; + * also, the enums values themselves are unstable and can still change. + */ + ZSTD_c_experimentalParam1=500, + ZSTD_c_experimentalParam2=10, + ZSTD_c_experimentalParam3=1000, + ZSTD_c_experimentalParam4=1001, + ZSTD_c_experimentalParam5=1002, + ZSTD_c_experimentalParam6=1003, + ZSTD_c_experimentalParam7=1004, + ZSTD_c_experimentalParam8=1005, + ZSTD_c_experimentalParam9=1006, + ZSTD_c_experimentalParam10=1007, + ZSTD_c_experimentalParam11=1008, + ZSTD_c_experimentalParam12=1009, + ZSTD_c_experimentalParam13=1010, + ZSTD_c_experimentalParam14=1011, + ZSTD_c_experimentalParam15=1012 +} ZSTD_cParameter; + +typedef struct { + size_t error; + int lowerBound; + int upperBound; +} ZSTD_bounds; + +/*! ZSTD_cParam_getBounds() : + * All parameters must belong to an interval with lower and upper bounds, + * otherwise they will either trigger an error or be automatically clamped. + * @return : a structure, ZSTD_bounds, which contains + * - an error status field, which must be tested using ZSTD_isError() + * - lower and upper bounds, both inclusive + */ +ZSTDLIB_API ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter cParam); + +/*! ZSTD_CCtx_setParameter() : + * Set one compression parameter, selected by enum ZSTD_cParameter. + * All parameters have valid bounds. Bounds can be queried using ZSTD_cParam_getBounds(). + * Providing a value beyond bound will either clamp it, or trigger an error (depending on parameter). + * Setting a parameter is generally only possible during frame initialization (before starting compression). + * Exception : when using multi-threading mode (nbWorkers >= 1), + * the following parameters can be updated _during_ compression (within same frame): + * => compressionLevel, hashLog, chainLog, searchLog, minMatch, targetLength and strategy. + * new parameters will be active for next job only (after a flush()). + * @return : an error code (which can be tested using ZSTD_isError()). + */ +ZSTDLIB_API size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int value); + +/*! ZSTD_CCtx_setPledgedSrcSize() : + * Total input data size to be compressed as a single frame. + * Value will be written in frame header, unless if explicitly forbidden using ZSTD_c_contentSizeFlag. + * This value will also be controlled at end of frame, and trigger an error if not respected. + * @result : 0, or an error code (which can be tested with ZSTD_isError()). + * Note 1 : pledgedSrcSize==0 actually means zero, aka an empty frame. + * In order to mean "unknown content size", pass constant ZSTD_CONTENTSIZE_UNKNOWN. + * ZSTD_CONTENTSIZE_UNKNOWN is default value for any new frame. + * Note 2 : pledgedSrcSize is only valid once, for the next frame. + * It's discarded at the end of the frame, and replaced by ZSTD_CONTENTSIZE_UNKNOWN. + * Note 3 : Whenever all input data is provided and consumed in a single round, + * for example with ZSTD_compress2(), + * or invoking immediately ZSTD_compressStream2(,,,ZSTD_e_end), + * this value is automatically overridden by srcSize instead. + */ +ZSTDLIB_API size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long long pledgedSrcSize); + +typedef enum { + ZSTD_reset_session_only = 1, + ZSTD_reset_parameters = 2, + ZSTD_reset_session_and_parameters = 3 +} ZSTD_ResetDirective; + +/*! ZSTD_CCtx_reset() : + * There are 2 different things that can be reset, independently or jointly : + * - The session : will stop compressing current frame, and make CCtx ready to start a new one. + * Useful after an error, or to interrupt any ongoing compression. + * Any internal data not yet flushed is cancelled. + * Compression parameters and dictionary remain unchanged. + * They will be used to compress next frame. + * Resetting session never fails. + * - The parameters : changes all parameters back to "default". + * This removes any reference to any dictionary too. + * Parameters can only be changed between 2 sessions (i.e. no compression is currently ongoing) + * otherwise the reset fails, and function returns an error value (which can be tested using ZSTD_isError()) + * - Both : similar to resetting the session, followed by resetting parameters. + */ +ZSTDLIB_API size_t ZSTD_CCtx_reset(ZSTD_CCtx* cctx, ZSTD_ResetDirective reset); + +/*! ZSTD_compress2() : + * Behave the same as ZSTD_compressCCtx(), but compression parameters are set using the advanced API. + * ZSTD_compress2() always starts a new frame. + * Should cctx hold data from a previously unfinished frame, everything about it is forgotten. + * - Compression parameters are pushed into CCtx before starting compression, using ZSTD_CCtx_set*() + * - The function is always blocking, returns when compression is completed. + * Hint : compression runs faster if `dstCapacity` >= `ZSTD_compressBound(srcSize)`. + * @return : compressed size written into `dst` (<= `dstCapacity), + * or an error code if it fails (which can be tested using ZSTD_isError()). + */ +ZSTDLIB_API size_t ZSTD_compress2( ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize); + + +/*********************************************** +* Advanced decompression API (Requires v1.4.0+) +************************************************/ + +/* The advanced API pushes parameters one by one into an existing DCtx context. + * Parameters are sticky, and remain valid for all following frames + * using the same DCtx context. + * It's possible to reset parameters to default values using ZSTD_DCtx_reset(). + * Note : This API is compatible with existing ZSTD_decompressDCtx() and ZSTD_decompressStream(). + * Therefore, no new decompression function is necessary. + */ + +typedef enum { + + ZSTD_d_windowLogMax=100, /* Select a size limit (in power of 2) beyond which + * the streaming API will refuse to allocate memory buffer + * in order to protect the host from unreasonable memory requirements. + * This parameter is only useful in streaming mode, since no internal buffer is allocated in single-pass mode. + * By default, a decompression context accepts window sizes <= (1 << ZSTD_WINDOWLOG_LIMIT_DEFAULT). + * Special: value 0 means "use default maximum windowLog". */ + + /* note : additional experimental parameters are also available + * within the experimental section of the API. + * At the time of this writing, they include : + * ZSTD_d_format + * ZSTD_d_stableOutBuffer + * ZSTD_d_forceIgnoreChecksum + * ZSTD_d_refMultipleDDicts + * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them. + * note : never ever use experimentalParam? names directly + */ + ZSTD_d_experimentalParam1=1000, + ZSTD_d_experimentalParam2=1001, + ZSTD_d_experimentalParam3=1002, + ZSTD_d_experimentalParam4=1003 + +} ZSTD_dParameter; + +/*! ZSTD_dParam_getBounds() : + * All parameters must belong to an interval with lower and upper bounds, + * otherwise they will either trigger an error or be automatically clamped. + * @return : a structure, ZSTD_bounds, which contains + * - an error status field, which must be tested using ZSTD_isError() + * - both lower and upper bounds, inclusive + */ +ZSTDLIB_API ZSTD_bounds ZSTD_dParam_getBounds(ZSTD_dParameter dParam); + +/*! ZSTD_DCtx_setParameter() : + * Set one compression parameter, selected by enum ZSTD_dParameter. + * All parameters have valid bounds. Bounds can be queried using ZSTD_dParam_getBounds(). + * Providing a value beyond bound will either clamp it, or trigger an error (depending on parameter). + * Setting a parameter is only possible during frame initialization (before starting decompression). + * @return : 0, or an error code (which can be tested using ZSTD_isError()). + */ +ZSTDLIB_API size_t ZSTD_DCtx_setParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param, int value); + +/*! ZSTD_DCtx_reset() : + * Return a DCtx to clean state. + * Session and parameters can be reset jointly or separately. + * Parameters can only be reset when no active frame is being decompressed. + * @return : 0, or an error code, which can be tested with ZSTD_isError() + */ +ZSTDLIB_API size_t ZSTD_DCtx_reset(ZSTD_DCtx* dctx, ZSTD_ResetDirective reset); + + +/**************************** +* Streaming +****************************/ + +typedef struct ZSTD_inBuffer_s { + const void* src; /**< start of input buffer */ + size_t size; /**< size of input buffer */ + size_t pos; /**< position where reading stopped. Will be updated. Necessarily 0 <= pos <= size */ +} ZSTD_inBuffer; + +typedef struct ZSTD_outBuffer_s { + void* dst; /**< start of output buffer */ + size_t size; /**< size of output buffer */ + size_t pos; /**< position where writing stopped. Will be updated. Necessarily 0 <= pos <= size */ +} ZSTD_outBuffer; + + + +/*-*********************************************************************** +* Streaming compression - HowTo +* +* A ZSTD_CStream object is required to track streaming operation. +* Use ZSTD_createCStream() and ZSTD_freeCStream() to create/release resources. +* ZSTD_CStream objects can be reused multiple times on consecutive compression operations. +* It is recommended to re-use ZSTD_CStream since it will play nicer with system's memory, by re-using already allocated memory. +* +* For parallel execution, use one separate ZSTD_CStream per thread. +* +* note : since v1.3.0, ZSTD_CStream and ZSTD_CCtx are the same thing. +* +* Parameters are sticky : when starting a new compression on the same context, +* it will re-use the same sticky parameters as previous compression session. +* When in doubt, it's recommended to fully initialize the context before usage. +* Use ZSTD_CCtx_reset() to reset the context and ZSTD_CCtx_setParameter(), +* ZSTD_CCtx_setPledgedSrcSize(), or ZSTD_CCtx_loadDictionary() and friends to +* set more specific parameters, the pledged source size, or load a dictionary. +* +* Use ZSTD_compressStream2() with ZSTD_e_continue as many times as necessary to +* consume input stream. The function will automatically update both `pos` +* fields within `input` and `output`. +* Note that the function may not consume the entire input, for example, because +* the output buffer is already full, in which case `input.pos < input.size`. +* The caller must check if input has been entirely consumed. +* If not, the caller must make some room to receive more compressed data, +* and then present again remaining input data. +* note: ZSTD_e_continue is guaranteed to make some forward progress when called, +* but doesn't guarantee maximal forward progress. This is especially relevant +* when compressing with multiple threads. The call won't block if it can +* consume some input, but if it can't it will wait for some, but not all, +* output to be flushed. +* @return : provides a minimum amount of data remaining to be flushed from internal buffers +* or an error code, which can be tested using ZSTD_isError(). +* +* At any moment, it's possible to flush whatever data might remain stuck within internal buffer, +* using ZSTD_compressStream2() with ZSTD_e_flush. `output->pos` will be updated. +* Note that, if `output->size` is too small, a single invocation with ZSTD_e_flush might not be enough (return code > 0). +* In which case, make some room to receive more compressed data, and call again ZSTD_compressStream2() with ZSTD_e_flush. +* You must continue calling ZSTD_compressStream2() with ZSTD_e_flush until it returns 0, at which point you can change the +* operation. +* note: ZSTD_e_flush will flush as much output as possible, meaning when compressing with multiple threads, it will +* block until the flush is complete or the output buffer is full. +* @return : 0 if internal buffers are entirely flushed, +* >0 if some data still present within internal buffer (the value is minimal estimation of remaining size), +* or an error code, which can be tested using ZSTD_isError(). +* +* Calling ZSTD_compressStream2() with ZSTD_e_end instructs to finish a frame. +* It will perform a flush and write frame epilogue. +* The epilogue is required for decoders to consider a frame completed. +* flush operation is the same, and follows same rules as calling ZSTD_compressStream2() with ZSTD_e_flush. +* You must continue calling ZSTD_compressStream2() with ZSTD_e_end until it returns 0, at which point you are free to +* start a new frame. +* note: ZSTD_e_end will flush as much output as possible, meaning when compressing with multiple threads, it will +* block until the flush is complete or the output buffer is full. +* @return : 0 if frame fully completed and fully flushed, +* >0 if some data still present within internal buffer (the value is minimal estimation of remaining size), +* or an error code, which can be tested using ZSTD_isError(). +* +* *******************************************************************/ + +typedef ZSTD_CCtx ZSTD_CStream; /**< CCtx and CStream are now effectively same object (>= v1.3.0) */ + /* Continue to distinguish them for compatibility with older versions <= v1.2.0 */ +/*===== ZSTD_CStream management functions =====*/ +ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream(void); +ZSTDLIB_API size_t ZSTD_freeCStream(ZSTD_CStream* zcs); /* accept NULL pointer */ + +/*===== Streaming compression functions =====*/ +typedef enum { + ZSTD_e_continue=0, /* collect more data, encoder decides when to output compressed result, for optimal compression ratio */ + ZSTD_e_flush=1, /* flush any data provided so far, + * it creates (at least) one new block, that can be decoded immediately on reception; + * frame will continue: any future data can still reference previously compressed data, improving compression. + * note : multithreaded compression will block to flush as much output as possible. */ + ZSTD_e_end=2 /* flush any remaining data _and_ close current frame. + * note that frame is only closed after compressed data is fully flushed (return value == 0). + * After that point, any additional data starts a new frame. + * note : each frame is independent (does not reference any content from previous frame). + : note : multithreaded compression will block to flush as much output as possible. */ +} ZSTD_EndDirective; + +/*! ZSTD_compressStream2() : Requires v1.4.0+ + * Behaves about the same as ZSTD_compressStream, with additional control on end directive. + * - Compression parameters are pushed into CCtx before starting compression, using ZSTD_CCtx_set*() + * - Compression parameters cannot be changed once compression is started (save a list of exceptions in multi-threading mode) + * - output->pos must be <= dstCapacity, input->pos must be <= srcSize + * - output->pos and input->pos will be updated. They are guaranteed to remain below their respective limit. + * - endOp must be a valid directive + * - When nbWorkers==0 (default), function is blocking : it completes its job before returning to caller. + * - When nbWorkers>=1, function is non-blocking : it copies a portion of input, distributes jobs to internal worker threads, flush to output whatever is available, + * and then immediately returns, just indicating that there is some data remaining to be flushed. + * The function nonetheless guarantees forward progress : it will return only after it reads or write at least 1+ byte. + * - Exception : if the first call requests a ZSTD_e_end directive and provides enough dstCapacity, the function delegates to ZSTD_compress2() which is always blocking. + * - @return provides a minimum amount of data remaining to be flushed from internal buffers + * or an error code, which can be tested using ZSTD_isError(). + * if @return != 0, flush is not fully completed, there is still some data left within internal buffers. + * This is useful for ZSTD_e_flush, since in this case more flushes are necessary to empty all buffers. + * For ZSTD_e_end, @return == 0 when internal buffers are fully flushed and frame is completed. + * - after a ZSTD_e_end directive, if internal buffer is not fully flushed (@return != 0), + * only ZSTD_e_end or ZSTD_e_flush operations are allowed. + * Before starting a new compression job, or changing compression parameters, + * it is required to fully flush internal buffers. + */ +ZSTDLIB_API size_t ZSTD_compressStream2( ZSTD_CCtx* cctx, + ZSTD_outBuffer* output, + ZSTD_inBuffer* input, + ZSTD_EndDirective endOp); + + +/* These buffer sizes are softly recommended. + * They are not required : ZSTD_compressStream*() happily accepts any buffer size, for both input and output. + * Respecting the recommended size just makes it a bit easier for ZSTD_compressStream*(), + * reducing the amount of memory shuffling and buffering, resulting in minor performance savings. + * + * However, note that these recommendations are from the perspective of a C caller program. + * If the streaming interface is invoked from some other language, + * especially managed ones such as Java or Go, through a foreign function interface such as jni or cgo, + * a major performance rule is to reduce crossing such interface to an absolute minimum. + * It's not rare that performance ends being spent more into the interface, rather than compression itself. + * In which cases, prefer using large buffers, as large as practical, + * for both input and output, to reduce the nb of roundtrips. + */ +ZSTDLIB_API size_t ZSTD_CStreamInSize(void); /**< recommended size for input buffer */ +ZSTDLIB_API size_t ZSTD_CStreamOutSize(void); /**< recommended size for output buffer. Guarantee to successfully flush at least one complete compressed block. */ + + +/* ***************************************************************************** + * This following is a legacy streaming API, available since v1.0+ . + * It can be replaced by ZSTD_CCtx_reset() and ZSTD_compressStream2(). + * It is redundant, but remains fully supported. + * Streaming in combination with advanced parameters and dictionary compression + * can only be used through the new API. + ******************************************************************************/ + +/*! + * Equivalent to: + * + * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); + * ZSTD_CCtx_refCDict(zcs, NULL); // clear the dictionary (if any) + * ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel); + */ +ZSTDLIB_API size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel); +/*! + * Alternative for ZSTD_compressStream2(zcs, output, input, ZSTD_e_continue). + * NOTE: The return value is different. ZSTD_compressStream() returns a hint for + * the next read size (if non-zero and not an error). ZSTD_compressStream2() + * returns the minimum nb of bytes left to flush (if non-zero and not an error). + */ +ZSTDLIB_API size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input); +/*! Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_flush). */ +ZSTDLIB_API size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output); +/*! Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_end). */ +ZSTDLIB_API size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output); + + +/*-*************************************************************************** +* Streaming decompression - HowTo +* +* A ZSTD_DStream object is required to track streaming operations. +* Use ZSTD_createDStream() and ZSTD_freeDStream() to create/release resources. +* ZSTD_DStream objects can be re-used multiple times. +* +* Use ZSTD_initDStream() to start a new decompression operation. +* @return : recommended first input size +* Alternatively, use advanced API to set specific properties. +* +* Use ZSTD_decompressStream() repetitively to consume your input. +* The function will update both `pos` fields. +* If `input.pos < input.size`, some input has not been consumed. +* It's up to the caller to present again remaining data. +* The function tries to flush all data decoded immediately, respecting output buffer size. +* If `output.pos < output.size`, decoder has flushed everything it could. +* But if `output.pos == output.size`, there might be some data left within internal buffers., +* In which case, call ZSTD_decompressStream() again to flush whatever remains in the buffer. +* Note : with no additional input provided, amount of data flushed is necessarily <= ZSTD_BLOCKSIZE_MAX. +* @return : 0 when a frame is completely decoded and fully flushed, +* or an error code, which can be tested using ZSTD_isError(), +* or any other value > 0, which means there is still some decoding or flushing to do to complete current frame : +* the return value is a suggested next input size (just a hint for better latency) +* that will never request more than the remaining frame size. +* *******************************************************************************/ + +typedef ZSTD_DCtx ZSTD_DStream; /**< DCtx and DStream are now effectively same object (>= v1.3.0) */ + /* For compatibility with versions <= v1.2.0, prefer differentiating them. */ +/*===== ZSTD_DStream management functions =====*/ +ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream(void); +ZSTDLIB_API size_t ZSTD_freeDStream(ZSTD_DStream* zds); /* accept NULL pointer */ + +/*===== Streaming decompression functions =====*/ + +/* This function is redundant with the advanced API and equivalent to: + * + * ZSTD_DCtx_reset(zds, ZSTD_reset_session_only); + * ZSTD_DCtx_refDDict(zds, NULL); + */ +ZSTDLIB_API size_t ZSTD_initDStream(ZSTD_DStream* zds); + +ZSTDLIB_API size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input); + +ZSTDLIB_API size_t ZSTD_DStreamInSize(void); /*!< recommended size for input buffer */ +ZSTDLIB_API size_t ZSTD_DStreamOutSize(void); /*!< recommended size for output buffer. Guarantee to successfully flush at least one complete block in all circumstances. */ + + +/************************** +* Simple dictionary API +***************************/ +/*! ZSTD_compress_usingDict() : + * Compression at an explicit compression level using a Dictionary. + * A dictionary can be any arbitrary data segment (also called a prefix), + * or a buffer with specified information (see zdict.h). + * Note : This function loads the dictionary, resulting in significant startup delay. + * It's intended for a dictionary used only once. + * Note 2 : When `dict == NULL || dictSize < 8` no dictionary is used. */ +ZSTDLIB_API size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const void* dict,size_t dictSize, + int compressionLevel); + +/*! ZSTD_decompress_usingDict() : + * Decompression using a known Dictionary. + * Dictionary must be identical to the one used during compression. + * Note : This function loads the dictionary, resulting in significant startup delay. + * It's intended for a dictionary used only once. + * Note : When `dict == NULL || dictSize < 8` no dictionary is used. */ +ZSTDLIB_API size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const void* dict,size_t dictSize); + + +/*********************************** + * Bulk processing dictionary API + **********************************/ +typedef struct ZSTD_CDict_s ZSTD_CDict; + +/*! ZSTD_createCDict() : + * When compressing multiple messages or blocks using the same dictionary, + * it's recommended to digest the dictionary only once, since it's a costly operation. + * ZSTD_createCDict() will create a state from digesting a dictionary. + * The resulting state can be used for future compression operations with very limited startup cost. + * ZSTD_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only. + * @dictBuffer can be released after ZSTD_CDict creation, because its content is copied within CDict. + * Note 1 : Consider experimental function `ZSTD_createCDict_byReference()` if you prefer to not duplicate @dictBuffer content. + * Note 2 : A ZSTD_CDict can be created from an empty @dictBuffer, + * in which case the only thing that it transports is the @compressionLevel. + * This can be useful in a pipeline featuring ZSTD_compress_usingCDict() exclusively, + * expecting a ZSTD_CDict parameter with any data, including those without a known dictionary. */ +ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict(const void* dictBuffer, size_t dictSize, + int compressionLevel); + +/*! ZSTD_freeCDict() : + * Function frees memory allocated by ZSTD_createCDict(). + * If a NULL pointer is passed, no operation is performed. */ +ZSTDLIB_API size_t ZSTD_freeCDict(ZSTD_CDict* CDict); + +/*! ZSTD_compress_usingCDict() : + * Compression using a digested Dictionary. + * Recommended when same dictionary is used multiple times. + * Note : compression level is _decided at dictionary creation time_, + * and frame parameters are hardcoded (dictID=yes, contentSize=yes, checksum=no) */ +ZSTDLIB_API size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const ZSTD_CDict* cdict); + + +typedef struct ZSTD_DDict_s ZSTD_DDict; + +/*! ZSTD_createDDict() : + * Create a digested dictionary, ready to start decompression operation without startup delay. + * dictBuffer can be released after DDict creation, as its content is copied inside DDict. */ +ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict(const void* dictBuffer, size_t dictSize); + +/*! ZSTD_freeDDict() : + * Function frees memory allocated with ZSTD_createDDict() + * If a NULL pointer is passed, no operation is performed. */ +ZSTDLIB_API size_t ZSTD_freeDDict(ZSTD_DDict* ddict); + +/*! ZSTD_decompress_usingDDict() : + * Decompression using a digested Dictionary. + * Recommended when same dictionary is used multiple times. */ +ZSTDLIB_API size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const ZSTD_DDict* ddict); + + +/******************************** + * Dictionary helper functions + *******************************/ + +/*! ZSTD_getDictID_fromDict() : Requires v1.4.0+ + * Provides the dictID stored within dictionary. + * if @return == 0, the dictionary is not conformant with Zstandard specification. + * It can still be loaded, but as a content-only dictionary. */ +ZSTDLIB_API unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize); + +/*! ZSTD_getDictID_fromCDict() : Requires v1.5.0+ + * Provides the dictID of the dictionary loaded into `cdict`. + * If @return == 0, the dictionary is not conformant to Zstandard specification, or empty. + * Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */ +ZSTDLIB_API unsigned ZSTD_getDictID_fromCDict(const ZSTD_CDict* cdict); + +/*! ZSTD_getDictID_fromDDict() : Requires v1.4.0+ + * Provides the dictID of the dictionary loaded into `ddict`. + * If @return == 0, the dictionary is not conformant to Zstandard specification, or empty. + * Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */ +ZSTDLIB_API unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict); + +/*! ZSTD_getDictID_fromFrame() : Requires v1.4.0+ + * Provides the dictID required to decompressed the frame stored within `src`. + * If @return == 0, the dictID could not be decoded. + * This could for one of the following reasons : + * - The frame does not require a dictionary to be decoded (most common case). + * - The frame was built with dictID intentionally removed. Whatever dictionary is necessary is a hidden information. + * Note : this use case also happens when using a non-conformant dictionary. + * - `srcSize` is too small, and as a result, the frame header could not be decoded (only possible if `srcSize < ZSTD_FRAMEHEADERSIZE_MAX`). + * - This is not a Zstandard frame. + * When identifying the exact failure cause, it's possible to use ZSTD_getFrameHeader(), which will provide a more precise error code. */ +ZSTDLIB_API unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize); + + +/******************************************************************************* + * Advanced dictionary and prefix API (Requires v1.4.0+) + * + * This API allows dictionaries to be used with ZSTD_compress2(), + * ZSTD_compressStream2(), and ZSTD_decompressDCtx(). Dictionaries are sticky, and + * only reset with the context is reset with ZSTD_reset_parameters or + * ZSTD_reset_session_and_parameters. Prefixes are single-use. + ******************************************************************************/ + + +/*! ZSTD_CCtx_loadDictionary() : Requires v1.4.0+ + * Create an internal CDict from `dict` buffer. + * Decompression will have to use same dictionary. + * @result : 0, or an error code (which can be tested with ZSTD_isError()). + * Special: Loading a NULL (or 0-size) dictionary invalidates previous dictionary, + * meaning "return to no-dictionary mode". + * Note 1 : Dictionary is sticky, it will be used for all future compressed frames. + * To return to "no-dictionary" situation, load a NULL dictionary (or reset parameters). + * Note 2 : Loading a dictionary involves building tables. + * It's also a CPU consuming operation, with non-negligible impact on latency. + * Tables are dependent on compression parameters, and for this reason, + * compression parameters can no longer be changed after loading a dictionary. + * Note 3 :`dict` content will be copied internally. + * Use experimental ZSTD_CCtx_loadDictionary_byReference() to reference content instead. + * In such a case, dictionary buffer must outlive its users. + * Note 4 : Use ZSTD_CCtx_loadDictionary_advanced() + * to precisely select how dictionary content must be interpreted. */ +ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize); + +/*! ZSTD_CCtx_refCDict() : Requires v1.4.0+ + * Reference a prepared dictionary, to be used for all next compressed frames. + * Note that compression parameters are enforced from within CDict, + * and supersede any compression parameter previously set within CCtx. + * The parameters ignored are labelled as "superseded-by-cdict" in the ZSTD_cParameter enum docs. + * The ignored parameters will be used again if the CCtx is returned to no-dictionary mode. + * The dictionary will remain valid for future compressed frames using same CCtx. + * @result : 0, or an error code (which can be tested with ZSTD_isError()). + * Special : Referencing a NULL CDict means "return to no-dictionary mode". + * Note 1 : Currently, only one dictionary can be managed. + * Referencing a new dictionary effectively "discards" any previous one. + * Note 2 : CDict is just referenced, its lifetime must outlive its usage within CCtx. */ +ZSTDLIB_API size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); + +/*! ZSTD_CCtx_refPrefix() : Requires v1.4.0+ + * Reference a prefix (single-usage dictionary) for next compressed frame. + * A prefix is **only used once**. Tables are discarded at end of frame (ZSTD_e_end). + * Decompression will need same prefix to properly regenerate data. + * Compressing with a prefix is similar in outcome as performing a diff and compressing it, + * but performs much faster, especially during decompression (compression speed is tunable with compression level). + * @result : 0, or an error code (which can be tested with ZSTD_isError()). + * Special: Adding any prefix (including NULL) invalidates any previous prefix or dictionary + * Note 1 : Prefix buffer is referenced. It **must** outlive compression. + * Its content must remain unmodified during compression. + * Note 2 : If the intention is to diff some large src data blob with some prior version of itself, + * ensure that the window size is large enough to contain the entire source. + * See ZSTD_c_windowLog. + * Note 3 : Referencing a prefix involves building tables, which are dependent on compression parameters. + * It's a CPU consuming operation, with non-negligible impact on latency. + * If there is a need to use the same prefix multiple times, consider loadDictionary instead. + * Note 4 : By default, the prefix is interpreted as raw content (ZSTD_dct_rawContent). + * Use experimental ZSTD_CCtx_refPrefix_advanced() to alter dictionary interpretation. */ +ZSTDLIB_API size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx, + const void* prefix, size_t prefixSize); + +/*! ZSTD_DCtx_loadDictionary() : Requires v1.4.0+ + * Create an internal DDict from dict buffer, + * to be used to decompress next frames. + * The dictionary remains valid for all future frames, until explicitly invalidated. + * @result : 0, or an error code (which can be tested with ZSTD_isError()). + * Special : Adding a NULL (or 0-size) dictionary invalidates any previous dictionary, + * meaning "return to no-dictionary mode". + * Note 1 : Loading a dictionary involves building tables, + * which has a non-negligible impact on CPU usage and latency. + * It's recommended to "load once, use many times", to amortize the cost + * Note 2 :`dict` content will be copied internally, so `dict` can be released after loading. + * Use ZSTD_DCtx_loadDictionary_byReference() to reference dictionary content instead. + * Note 3 : Use ZSTD_DCtx_loadDictionary_advanced() to take control of + * how dictionary content is loaded and interpreted. + */ +ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize); + +/*! ZSTD_DCtx_refDDict() : Requires v1.4.0+ + * Reference a prepared dictionary, to be used to decompress next frames. + * The dictionary remains active for decompression of future frames using same DCtx. + * + * If called with ZSTD_d_refMultipleDDicts enabled, repeated calls of this function + * will store the DDict references in a table, and the DDict used for decompression + * will be determined at decompression time, as per the dict ID in the frame. + * The memory for the table is allocated on the first call to refDDict, and can be + * freed with ZSTD_freeDCtx(). + * + * @result : 0, or an error code (which can be tested with ZSTD_isError()). + * Note 1 : Currently, only one dictionary can be managed. + * Referencing a new dictionary effectively "discards" any previous one. + * Special: referencing a NULL DDict means "return to no-dictionary mode". + * Note 2 : DDict is just referenced, its lifetime must outlive its usage from DCtx. + */ +ZSTDLIB_API size_t ZSTD_DCtx_refDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict); + +/*! ZSTD_DCtx_refPrefix() : Requires v1.4.0+ + * Reference a prefix (single-usage dictionary) to decompress next frame. + * This is the reverse operation of ZSTD_CCtx_refPrefix(), + * and must use the same prefix as the one used during compression. + * Prefix is **only used once**. Reference is discarded at end of frame. + * End of frame is reached when ZSTD_decompressStream() returns 0. + * @result : 0, or an error code (which can be tested with ZSTD_isError()). + * Note 1 : Adding any prefix (including NULL) invalidates any previously set prefix or dictionary + * Note 2 : Prefix buffer is referenced. It **must** outlive decompression. + * Prefix buffer must remain unmodified up to the end of frame, + * reached when ZSTD_decompressStream() returns 0. + * Note 3 : By default, the prefix is treated as raw content (ZSTD_dct_rawContent). + * Use ZSTD_CCtx_refPrefix_advanced() to alter dictMode (Experimental section) + * Note 4 : Referencing a raw content prefix has almost no cpu nor memory cost. + * A full dictionary is more costly, as it requires building tables. + */ +ZSTDLIB_API size_t ZSTD_DCtx_refPrefix(ZSTD_DCtx* dctx, + const void* prefix, size_t prefixSize); + +/* === Memory management === */ + +/*! ZSTD_sizeof_*() : Requires v1.4.0+ + * These functions give the _current_ memory usage of selected object. + * Note that object memory usage can evolve (increase or decrease) over time. */ +ZSTDLIB_API size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx); +ZSTDLIB_API size_t ZSTD_sizeof_DCtx(const ZSTD_DCtx* dctx); +ZSTDLIB_API size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs); +ZSTDLIB_API size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds); +ZSTDLIB_API size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict); +ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict); + +#endif /* ZSTD_H_235446 */ + + +/* ************************************************************************************** + * ADVANCED AND EXPERIMENTAL FUNCTIONS + **************************************************************************************** + * The definitions in the following section are considered experimental. + * They are provided for advanced scenarios. + * They should never be used with a dynamic library, as prototypes may change in the future. + * Use them only in association with static linking. + * ***************************************************************************************/ + +#if defined(ZSTD_STATIC_LINKING_ONLY) && !defined(ZSTD_H_ZSTD_STATIC_LINKING_ONLY) +#define ZSTD_H_ZSTD_STATIC_LINKING_ONLY + +/* This can be overridden externally to hide static symbols. */ +#ifndef ZSTDLIB_STATIC_API +# if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1) +# define ZSTDLIB_STATIC_API __declspec(dllexport) ZSTDLIB_VISIBLE +# elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1) +# define ZSTDLIB_STATIC_API __declspec(dllimport) ZSTDLIB_VISIBLE +# else +# define ZSTDLIB_STATIC_API ZSTDLIB_VISIBLE +# endif +#endif + +/* Deprecation warnings : + * Should these warnings be a problem, it is generally possible to disable them, + * typically with -Wno-deprecated-declarations for gcc or _CRT_SECURE_NO_WARNINGS in Visual. + * Otherwise, it's also possible to define ZSTD_DISABLE_DEPRECATE_WARNINGS. + */ +#ifdef ZSTD_DISABLE_DEPRECATE_WARNINGS +# define ZSTD_DEPRECATED(message) ZSTDLIB_STATIC_API /* disable deprecation warnings */ +#else +# if defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */ +# define ZSTD_DEPRECATED(message) [[deprecated(message)]] ZSTDLIB_STATIC_API +# elif (defined(GNUC) && (GNUC > 4 || (GNUC == 4 && GNUC_MINOR >= 5))) || defined(__clang__) +# define ZSTD_DEPRECATED(message) ZSTDLIB_STATIC_API __attribute__((deprecated(message))) +# elif defined(__GNUC__) && (__GNUC__ >= 3) +# define ZSTD_DEPRECATED(message) ZSTDLIB_STATIC_API __attribute__((deprecated)) +# elif defined(_MSC_VER) +# define ZSTD_DEPRECATED(message) ZSTDLIB_STATIC_API __declspec(deprecated(message)) +# else +# pragma message("WARNING: You need to implement ZSTD_DEPRECATED for this compiler") +# define ZSTD_DEPRECATED(message) ZSTDLIB_STATIC_API +# endif +#endif /* ZSTD_DISABLE_DEPRECATE_WARNINGS */ + +/**************************************************************************************** + * experimental API (static linking only) + **************************************************************************************** + * The following symbols and constants + * are not planned to join "stable API" status in the near future. + * They can still change in future versions. + * Some of them are planned to remain in the static_only section indefinitely. + * Some of them might be removed in the future (especially when redundant with existing stable functions) + * ***************************************************************************************/ + +#define ZSTD_FRAMEHEADERSIZE_PREFIX(format) ((format) == ZSTD_f_zstd1 ? 5 : 1) /* minimum input size required to query frame header size */ +#define ZSTD_FRAMEHEADERSIZE_MIN(format) ((format) == ZSTD_f_zstd1 ? 6 : 2) +#define ZSTD_FRAMEHEADERSIZE_MAX 18 /* can be useful for static allocation */ +#define ZSTD_SKIPPABLEHEADERSIZE 8 + +/* compression parameter bounds */ +#define ZSTD_WINDOWLOG_MAX_32 30 +#define ZSTD_WINDOWLOG_MAX_64 31 +#define ZSTD_WINDOWLOG_MAX ((int)(sizeof(size_t) == 4 ? ZSTD_WINDOWLOG_MAX_32 : ZSTD_WINDOWLOG_MAX_64)) +#define ZSTD_WINDOWLOG_MIN 10 +#define ZSTD_HASHLOG_MAX ((ZSTD_WINDOWLOG_MAX < 30) ? ZSTD_WINDOWLOG_MAX : 30) +#define ZSTD_HASHLOG_MIN 6 +#define ZSTD_CHAINLOG_MAX_32 29 +#define ZSTD_CHAINLOG_MAX_64 30 +#define ZSTD_CHAINLOG_MAX ((int)(sizeof(size_t) == 4 ? ZSTD_CHAINLOG_MAX_32 : ZSTD_CHAINLOG_MAX_64)) +#define ZSTD_CHAINLOG_MIN ZSTD_HASHLOG_MIN +#define ZSTD_SEARCHLOG_MAX (ZSTD_WINDOWLOG_MAX-1) +#define ZSTD_SEARCHLOG_MIN 1 +#define ZSTD_MINMATCH_MAX 7 /* only for ZSTD_fast, other strategies are limited to 6 */ +#define ZSTD_MINMATCH_MIN 3 /* only for ZSTD_btopt+, faster strategies are limited to 4 */ +#define ZSTD_TARGETLENGTH_MAX ZSTD_BLOCKSIZE_MAX +#define ZSTD_TARGETLENGTH_MIN 0 /* note : comparing this constant to an unsigned results in a tautological test */ +#define ZSTD_STRATEGY_MIN ZSTD_fast +#define ZSTD_STRATEGY_MAX ZSTD_btultra2 + + +#define ZSTD_OVERLAPLOG_MIN 0 +#define ZSTD_OVERLAPLOG_MAX 9 + +#define ZSTD_WINDOWLOG_LIMIT_DEFAULT 27 /* by default, the streaming decoder will refuse any frame + * requiring larger than (1< 0: + * If litLength != 0: + * rep == 1 --> offset == repeat_offset_1 + * rep == 2 --> offset == repeat_offset_2 + * rep == 3 --> offset == repeat_offset_3 + * If litLength == 0: + * rep == 1 --> offset == repeat_offset_2 + * rep == 2 --> offset == repeat_offset_3 + * rep == 3 --> offset == repeat_offset_1 - 1 + * + * Note: This field is optional. ZSTD_generateSequences() will calculate the value of + * 'rep', but repeat offsets do not necessarily need to be calculated from an external + * sequence provider's perspective. For example, ZSTD_compressSequences() does not + * use this 'rep' field at all (as of now). + */ +} ZSTD_Sequence; + +typedef struct { + unsigned windowLog; /**< largest match distance : larger == more compression, more memory needed during decompression */ + unsigned chainLog; /**< fully searched segment : larger == more compression, slower, more memory (useless for fast) */ + unsigned hashLog; /**< dispatch table : larger == faster, more memory */ + unsigned searchLog; /**< nb of searches : larger == more compression, slower */ + unsigned minMatch; /**< match length searched : larger == faster decompression, sometimes less compression */ + unsigned targetLength; /**< acceptable match size for optimal parser (only) : larger == more compression, slower */ + ZSTD_strategy strategy; /**< see ZSTD_strategy definition above */ +} ZSTD_compressionParameters; + +typedef struct { + int contentSizeFlag; /**< 1: content size will be in frame header (when known) */ + int checksumFlag; /**< 1: generate a 32-bits checksum using XXH64 algorithm at end of frame, for error detection */ + int noDictIDFlag; /**< 1: no dictID will be saved into frame header (dictID is only useful for dictionary compression) */ +} ZSTD_frameParameters; + +typedef struct { + ZSTD_compressionParameters cParams; + ZSTD_frameParameters fParams; +} ZSTD_parameters; + +typedef enum { + ZSTD_dct_auto = 0, /* dictionary is "full" when starting with ZSTD_MAGIC_DICTIONARY, otherwise it is "rawContent" */ + ZSTD_dct_rawContent = 1, /* ensures dictionary is always loaded as rawContent, even if it starts with ZSTD_MAGIC_DICTIONARY */ + ZSTD_dct_fullDict = 2 /* refuses to load a dictionary if it does not respect Zstandard's specification, starting with ZSTD_MAGIC_DICTIONARY */ +} ZSTD_dictContentType_e; + +typedef enum { + ZSTD_dlm_byCopy = 0, /**< Copy dictionary content internally */ + ZSTD_dlm_byRef = 1 /**< Reference dictionary content -- the dictionary buffer must outlive its users. */ +} ZSTD_dictLoadMethod_e; + +typedef enum { + ZSTD_f_zstd1 = 0, /* zstd frame format, specified in zstd_compression_format.md (default) */ + ZSTD_f_zstd1_magicless = 1 /* Variant of zstd frame format, without initial 4-bytes magic number. + * Useful to save 4 bytes per generated frame. + * Decoder cannot recognise automatically this format, requiring this instruction. */ +} ZSTD_format_e; + +typedef enum { + /* Note: this enum controls ZSTD_d_forceIgnoreChecksum */ + ZSTD_d_validateChecksum = 0, + ZSTD_d_ignoreChecksum = 1 +} ZSTD_forceIgnoreChecksum_e; + +typedef enum { + /* Note: this enum controls ZSTD_d_refMultipleDDicts */ + ZSTD_rmd_refSingleDDict = 0, + ZSTD_rmd_refMultipleDDicts = 1 +} ZSTD_refMultipleDDicts_e; + +typedef enum { + /* Note: this enum and the behavior it controls are effectively internal + * implementation details of the compressor. They are expected to continue + * to evolve and should be considered only in the context of extremely + * advanced performance tuning. + * + * Zstd currently supports the use of a CDict in three ways: + * + * - The contents of the CDict can be copied into the working context. This + * means that the compression can search both the dictionary and input + * while operating on a single set of internal tables. This makes + * the compression faster per-byte of input. However, the initial copy of + * the CDict's tables incurs a fixed cost at the beginning of the + * compression. For small compressions (< 8 KB), that copy can dominate + * the cost of the compression. + * + * - The CDict's tables can be used in-place. In this model, compression is + * slower per input byte, because the compressor has to search two sets of + * tables. However, this model incurs no start-up cost (as long as the + * working context's tables can be reused). For small inputs, this can be + * faster than copying the CDict's tables. + * + * - The CDict's tables are not used at all, and instead we use the working + * context alone to reload the dictionary and use params based on the source + * size. See ZSTD_compress_insertDictionary() and ZSTD_compress_usingDict(). + * This method is effective when the dictionary sizes are very small relative + * to the input size, and the input size is fairly large to begin with. + * + * Zstd has a simple internal heuristic that selects which strategy to use + * at the beginning of a compression. However, if experimentation shows that + * Zstd is making poor choices, it is possible to override that choice with + * this enum. + */ + ZSTD_dictDefaultAttach = 0, /* Use the default heuristic. */ + ZSTD_dictForceAttach = 1, /* Never copy the dictionary. */ + ZSTD_dictForceCopy = 2, /* Always copy the dictionary. */ + ZSTD_dictForceLoad = 3 /* Always reload the dictionary */ +} ZSTD_dictAttachPref_e; + +typedef enum { + ZSTD_lcm_auto = 0, /**< Automatically determine the compression mode based on the compression level. + * Negative compression levels will be uncompressed, and positive compression + * levels will be compressed. */ + ZSTD_lcm_huffman = 1, /**< Always attempt Huffman compression. Uncompressed literals will still be + * emitted if Huffman compression is not profitable. */ + ZSTD_lcm_uncompressed = 2 /**< Always emit uncompressed literals. */ +} ZSTD_literalCompressionMode_e; + +typedef enum { + /* Note: This enum controls features which are conditionally beneficial. Zstd typically will make a final + * decision on whether or not to enable the feature (ZSTD_ps_auto), but setting the switch to ZSTD_ps_enable + * or ZSTD_ps_disable allow for a force enable/disable the feature. + */ + ZSTD_ps_auto = 0, /* Let the library automatically determine whether the feature shall be enabled */ + ZSTD_ps_enable = 1, /* Force-enable the feature */ + ZSTD_ps_disable = 2 /* Do not use the feature */ +} ZSTD_paramSwitch_e; + +/*************************************** +* Frame size functions +***************************************/ + +/*! ZSTD_findDecompressedSize() : + * `src` should point to the start of a series of ZSTD encoded and/or skippable frames + * `srcSize` must be the _exact_ size of this series + * (i.e. there should be a frame boundary at `src + srcSize`) + * @return : - decompressed size of all data in all successive frames + * - if the decompressed size cannot be determined: ZSTD_CONTENTSIZE_UNKNOWN + * - if an error occurred: ZSTD_CONTENTSIZE_ERROR + * + * note 1 : decompressed size is an optional field, that may not be present, especially in streaming mode. + * When `return==ZSTD_CONTENTSIZE_UNKNOWN`, data to decompress could be any size. + * In which case, it's necessary to use streaming mode to decompress data. + * note 2 : decompressed size is always present when compression is done with ZSTD_compress() + * note 3 : decompressed size can be very large (64-bits value), + * potentially larger than what local system can handle as a single memory segment. + * In which case, it's necessary to use streaming mode to decompress data. + * note 4 : If source is untrusted, decompressed size could be wrong or intentionally modified. + * Always ensure result fits within application's authorized limits. + * Each application can set its own limits. + * note 5 : ZSTD_findDecompressedSize handles multiple frames, and so it must traverse the input to + * read each contained frame header. This is fast as most of the data is skipped, + * however it does mean that all frame data must be present and valid. */ +ZSTDLIB_STATIC_API unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize); + +/*! ZSTD_decompressBound() : + * `src` should point to the start of a series of ZSTD encoded and/or skippable frames + * `srcSize` must be the _exact_ size of this series + * (i.e. there should be a frame boundary at `src + srcSize`) + * @return : - upper-bound for the decompressed size of all data in all successive frames + * - if an error occurred: ZSTD_CONTENTSIZE_ERROR + * + * note 1 : an error can occur if `src` contains an invalid or incorrectly formatted frame. + * note 2 : the upper-bound is exact when the decompressed size field is available in every ZSTD encoded frame of `src`. + * in this case, `ZSTD_findDecompressedSize` and `ZSTD_decompressBound` return the same value. + * note 3 : when the decompressed size field isn't available, the upper-bound for that frame is calculated by: + * upper-bound = # blocks * min(128 KB, Window_Size) + */ +ZSTDLIB_STATIC_API unsigned long long ZSTD_decompressBound(const void* src, size_t srcSize); + +/*! ZSTD_frameHeaderSize() : + * srcSize must be >= ZSTD_FRAMEHEADERSIZE_PREFIX. + * @return : size of the Frame Header, + * or an error code (if srcSize is too small) */ +ZSTDLIB_STATIC_API size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize); + +typedef enum { + ZSTD_sf_noBlockDelimiters = 0, /* Representation of ZSTD_Sequence has no block delimiters, sequences only */ + ZSTD_sf_explicitBlockDelimiters = 1 /* Representation of ZSTD_Sequence contains explicit block delimiters */ +} ZSTD_sequenceFormat_e; + +/*! ZSTD_generateSequences() : + * Generate sequences using ZSTD_compress2, given a source buffer. + * + * Each block will end with a dummy sequence + * with offset == 0, matchLength == 0, and litLength == length of last literals. + * litLength may be == 0, and if so, then the sequence of (of: 0 ml: 0 ll: 0) + * simply acts as a block delimiter. + * + * zc can be used to insert custom compression params. + * This function invokes ZSTD_compress2 + * + * The output of this function can be fed into ZSTD_compressSequences() with CCtx + * setting of ZSTD_c_blockDelimiters as ZSTD_sf_explicitBlockDelimiters + * @return : number of sequences generated + */ + +ZSTDLIB_STATIC_API size_t ZSTD_generateSequences(ZSTD_CCtx* zc, ZSTD_Sequence* outSeqs, + size_t outSeqsSize, const void* src, size_t srcSize); + +/*! ZSTD_mergeBlockDelimiters() : + * Given an array of ZSTD_Sequence, remove all sequences that represent block delimiters/last literals + * by merging them into into the literals of the next sequence. + * + * As such, the final generated result has no explicit representation of block boundaries, + * and the final last literals segment is not represented in the sequences. + * + * The output of this function can be fed into ZSTD_compressSequences() with CCtx + * setting of ZSTD_c_blockDelimiters as ZSTD_sf_noBlockDelimiters + * @return : number of sequences left after merging + */ +ZSTDLIB_STATIC_API size_t ZSTD_mergeBlockDelimiters(ZSTD_Sequence* sequences, size_t seqsSize); + +/*! ZSTD_compressSequences() : + * Compress an array of ZSTD_Sequence, generated from the original source buffer, into dst. + * If a dictionary is included, then the cctx should reference the dict. (see: ZSTD_CCtx_refCDict(), ZSTD_CCtx_loadDictionary(), etc.) + * The entire source is compressed into a single frame. + * + * The compression behavior changes based on cctx params. In particular: + * If ZSTD_c_blockDelimiters == ZSTD_sf_noBlockDelimiters, the array of ZSTD_Sequence is expected to contain + * no block delimiters (defined in ZSTD_Sequence). Block boundaries are roughly determined based on + * the block size derived from the cctx, and sequences may be split. This is the default setting. + * + * If ZSTD_c_blockDelimiters == ZSTD_sf_explicitBlockDelimiters, the array of ZSTD_Sequence is expected to contain + * block delimiters (defined in ZSTD_Sequence). Behavior is undefined if no block delimiters are provided. + * + * If ZSTD_c_validateSequences == 0, this function will blindly accept the sequences provided. Invalid sequences cause undefined + * behavior. If ZSTD_c_validateSequences == 1, then if sequence is invalid (see doc/zstd_compression_format.md for + * specifics regarding offset/matchlength requirements) then the function will bail out and return an error. + * + * In addition to the two adjustable experimental params, there are other important cctx params. + * - ZSTD_c_minMatch MUST be set as less than or equal to the smallest match generated by the match finder. It has a minimum value of ZSTD_MINMATCH_MIN. + * - ZSTD_c_compressionLevel accordingly adjusts the strength of the entropy coder, as it would in typical compression. + * - ZSTD_c_windowLog affects offset validation: this function will return an error at higher debug levels if a provided offset + * is larger than what the spec allows for a given window log and dictionary (if present). See: doc/zstd_compression_format.md + * + * Note: Repcodes are, as of now, always re-calculated within this function, so ZSTD_Sequence::rep is unused. + * Note 2: Once we integrate ability to ingest repcodes, the explicit block delims mode must respect those repcodes exactly, + * and cannot emit an RLE block that disagrees with the repcode history + * @return : final compressed size or a ZSTD error. + */ +ZSTDLIB_STATIC_API size_t ZSTD_compressSequences(ZSTD_CCtx* const cctx, void* dst, size_t dstSize, + const ZSTD_Sequence* inSeqs, size_t inSeqsSize, + const void* src, size_t srcSize); + + +/*! ZSTD_writeSkippableFrame() : + * Generates a zstd skippable frame containing data given by src, and writes it to dst buffer. + * + * Skippable frames begin with a a 4-byte magic number. There are 16 possible choices of magic number, + * ranging from ZSTD_MAGIC_SKIPPABLE_START to ZSTD_MAGIC_SKIPPABLE_START+15. + * As such, the parameter magicVariant controls the exact skippable frame magic number variant used, so + * the magic number used will be ZSTD_MAGIC_SKIPPABLE_START + magicVariant. + * + * Returns an error if destination buffer is not large enough, if the source size is not representable + * with a 4-byte unsigned int, or if the parameter magicVariant is greater than 15 (and therefore invalid). + * + * @return : number of bytes written or a ZSTD error. + */ +ZSTDLIB_STATIC_API size_t ZSTD_writeSkippableFrame(void* dst, size_t dstCapacity, + const void* src, size_t srcSize, unsigned magicVariant); + +/*! ZSTD_readSkippableFrame() : + * Retrieves a zstd skippable frame containing data given by src, and writes it to dst buffer. + * + * The parameter magicVariant will receive the magicVariant that was supplied when the frame was written, + * i.e. magicNumber - ZSTD_MAGIC_SKIPPABLE_START. This can be NULL if the caller is not interested + * in the magicVariant. + * + * Returns an error if destination buffer is not large enough, or if the frame is not skippable. + * + * @return : number of bytes written or a ZSTD error. + */ +ZSTDLIB_API size_t ZSTD_readSkippableFrame(void* dst, size_t dstCapacity, unsigned* magicVariant, + const void* src, size_t srcSize); + +/*! ZSTD_isSkippableFrame() : + * Tells if the content of `buffer` starts with a valid Frame Identifier for a skippable frame. + */ +ZSTDLIB_API unsigned ZSTD_isSkippableFrame(const void* buffer, size_t size); + + + +/*************************************** +* Memory management +***************************************/ + +/*! ZSTD_estimate*() : + * These functions make it possible to estimate memory usage + * of a future {D,C}Ctx, before its creation. + * + * ZSTD_estimateCCtxSize() will provide a memory budget large enough + * for any compression level up to selected one. + * Note : Unlike ZSTD_estimateCStreamSize*(), this estimate + * does not include space for a window buffer. + * Therefore, the estimation is only guaranteed for single-shot compressions, not streaming. + * The estimate will assume the input may be arbitrarily large, + * which is the worst case. + * + * When srcSize can be bound by a known and rather "small" value, + * this fact can be used to provide a tighter estimation + * because the CCtx compression context will need less memory. + * This tighter estimation can be provided by more advanced functions + * ZSTD_estimateCCtxSize_usingCParams(), which can be used in tandem with ZSTD_getCParams(), + * and ZSTD_estimateCCtxSize_usingCCtxParams(), which can be used in tandem with ZSTD_CCtxParams_setParameter(). + * Both can be used to estimate memory using custom compression parameters and arbitrary srcSize limits. + * + * Note 2 : only single-threaded compression is supported. + * ZSTD_estimateCCtxSize_usingCCtxParams() will return an error code if ZSTD_c_nbWorkers is >= 1. + */ +ZSTDLIB_STATIC_API size_t ZSTD_estimateCCtxSize(int compressionLevel); +ZSTDLIB_STATIC_API size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams); +ZSTDLIB_STATIC_API size_t ZSTD_estimateCCtxSize_usingCCtxParams(const ZSTD_CCtx_params* params); +ZSTDLIB_STATIC_API size_t ZSTD_estimateDCtxSize(void); + +/*! ZSTD_estimateCStreamSize() : + * ZSTD_estimateCStreamSize() will provide a budget large enough for any compression level up to selected one. + * It will also consider src size to be arbitrarily "large", which is worst case. + * If srcSize is known to always be small, ZSTD_estimateCStreamSize_usingCParams() can provide a tighter estimation. + * ZSTD_estimateCStreamSize_usingCParams() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel. + * ZSTD_estimateCStreamSize_usingCCtxParams() can be used in tandem with ZSTD_CCtxParams_setParameter(). Only single-threaded compression is supported. This function will return an error code if ZSTD_c_nbWorkers is >= 1. + * Note : CStream size estimation is only correct for single-threaded compression. + * ZSTD_DStream memory budget depends on window Size. + * This information can be passed manually, using ZSTD_estimateDStreamSize, + * or deducted from a valid frame Header, using ZSTD_estimateDStreamSize_fromFrame(); + * Note : if streaming is init with function ZSTD_init?Stream_usingDict(), + * an internal ?Dict will be created, which additional size is not estimated here. + * In this case, get total size by adding ZSTD_estimate?DictSize */ +ZSTDLIB_STATIC_API size_t ZSTD_estimateCStreamSize(int compressionLevel); +ZSTDLIB_STATIC_API size_t ZSTD_estimateCStreamSize_usingCParams(ZSTD_compressionParameters cParams); +ZSTDLIB_STATIC_API size_t ZSTD_estimateCStreamSize_usingCCtxParams(const ZSTD_CCtx_params* params); +ZSTDLIB_STATIC_API size_t ZSTD_estimateDStreamSize(size_t windowSize); +ZSTDLIB_STATIC_API size_t ZSTD_estimateDStreamSize_fromFrame(const void* src, size_t srcSize); + +/*! ZSTD_estimate?DictSize() : + * ZSTD_estimateCDictSize() will bet that src size is relatively "small", and content is copied, like ZSTD_createCDict(). + * ZSTD_estimateCDictSize_advanced() makes it possible to control compression parameters precisely, like ZSTD_createCDict_advanced(). + * Note : dictionaries created by reference (`ZSTD_dlm_byRef`) are logically smaller. + */ +ZSTDLIB_STATIC_API size_t ZSTD_estimateCDictSize(size_t dictSize, int compressionLevel); +ZSTDLIB_STATIC_API size_t ZSTD_estimateCDictSize_advanced(size_t dictSize, ZSTD_compressionParameters cParams, ZSTD_dictLoadMethod_e dictLoadMethod); +ZSTDLIB_STATIC_API size_t ZSTD_estimateDDictSize(size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod); + +/*! ZSTD_initStatic*() : + * Initialize an object using a pre-allocated fixed-size buffer. + * workspace: The memory area to emplace the object into. + * Provided pointer *must be 8-bytes aligned*. + * Buffer must outlive object. + * workspaceSize: Use ZSTD_estimate*Size() to determine + * how large workspace must be to support target scenario. + * @return : pointer to object (same address as workspace, just different type), + * or NULL if error (size too small, incorrect alignment, etc.) + * Note : zstd will never resize nor malloc() when using a static buffer. + * If the object requires more memory than available, + * zstd will just error out (typically ZSTD_error_memory_allocation). + * Note 2 : there is no corresponding "free" function. + * Since workspace is allocated externally, it must be freed externally too. + * Note 3 : cParams : use ZSTD_getCParams() to convert a compression level + * into its associated cParams. + * Limitation 1 : currently not compatible with internal dictionary creation, triggered by + * ZSTD_CCtx_loadDictionary(), ZSTD_initCStream_usingDict() or ZSTD_initDStream_usingDict(). + * Limitation 2 : static cctx currently not compatible with multi-threading. + * Limitation 3 : static dctx is incompatible with legacy support. + */ +ZSTDLIB_STATIC_API ZSTD_CCtx* ZSTD_initStaticCCtx(void* workspace, size_t workspaceSize); +ZSTDLIB_STATIC_API ZSTD_CStream* ZSTD_initStaticCStream(void* workspace, size_t workspaceSize); /**< same as ZSTD_initStaticCCtx() */ + +ZSTDLIB_STATIC_API ZSTD_DCtx* ZSTD_initStaticDCtx(void* workspace, size_t workspaceSize); +ZSTDLIB_STATIC_API ZSTD_DStream* ZSTD_initStaticDStream(void* workspace, size_t workspaceSize); /**< same as ZSTD_initStaticDCtx() */ + +ZSTDLIB_STATIC_API const ZSTD_CDict* ZSTD_initStaticCDict( + void* workspace, size_t workspaceSize, + const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType, + ZSTD_compressionParameters cParams); + +ZSTDLIB_STATIC_API const ZSTD_DDict* ZSTD_initStaticDDict( + void* workspace, size_t workspaceSize, + const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType); + + +/*! Custom memory allocation : + * These prototypes make it possible to pass your own allocation/free functions. + * ZSTD_customMem is provided at creation time, using ZSTD_create*_advanced() variants listed below. + * All allocation/free operations will be completed using these custom variants instead of regular ones. + */ +typedef void* (*ZSTD_allocFunction) (void* opaque, size_t size); +typedef void (*ZSTD_freeFunction) (void* opaque, void* address); +typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; void* opaque; } ZSTD_customMem; +static +#ifdef __GNUC__ +__attribute__((__unused__)) +#endif +ZSTD_customMem const ZSTD_defaultCMem = { NULL, NULL, NULL }; /**< this constant defers to stdlib's functions */ + +ZSTDLIB_STATIC_API ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem); +ZSTDLIB_STATIC_API ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem); +ZSTDLIB_STATIC_API ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem); +ZSTDLIB_STATIC_API ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem); + +ZSTDLIB_STATIC_API ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType, + ZSTD_compressionParameters cParams, + ZSTD_customMem customMem); + +/*! Thread pool : + * These prototypes make it possible to share a thread pool among multiple compression contexts. + * This can limit resources for applications with multiple threads where each one uses + * a threaded compression mode (via ZSTD_c_nbWorkers parameter). + * ZSTD_createThreadPool creates a new thread pool with a given number of threads. + * Note that the lifetime of such pool must exist while being used. + * ZSTD_CCtx_refThreadPool assigns a thread pool to a context (use NULL argument value + * to use an internal thread pool). + * ZSTD_freeThreadPool frees a thread pool, accepts NULL pointer. + */ +typedef struct POOL_ctx_s ZSTD_threadPool; +ZSTDLIB_STATIC_API ZSTD_threadPool* ZSTD_createThreadPool(size_t numThreads); +ZSTDLIB_STATIC_API void ZSTD_freeThreadPool (ZSTD_threadPool* pool); /* accept NULL pointer */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtx_refThreadPool(ZSTD_CCtx* cctx, ZSTD_threadPool* pool); + + +/* + * This API is temporary and is expected to change or disappear in the future! + */ +ZSTDLIB_STATIC_API ZSTD_CDict* ZSTD_createCDict_advanced2( + const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType, + const ZSTD_CCtx_params* cctxParams, + ZSTD_customMem customMem); + +ZSTDLIB_STATIC_API ZSTD_DDict* ZSTD_createDDict_advanced( + const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType, + ZSTD_customMem customMem); + + +/*************************************** +* Advanced compression functions +***************************************/ + +/*! ZSTD_createCDict_byReference() : + * Create a digested dictionary for compression + * Dictionary content is just referenced, not duplicated. + * As a consequence, `dictBuffer` **must** outlive CDict, + * and its content must remain unmodified throughout the lifetime of CDict. + * note: equivalent to ZSTD_createCDict_advanced(), with dictLoadMethod==ZSTD_dlm_byRef */ +ZSTDLIB_STATIC_API ZSTD_CDict* ZSTD_createCDict_byReference(const void* dictBuffer, size_t dictSize, int compressionLevel); + +/*! ZSTD_getCParams() : + * @return ZSTD_compressionParameters structure for a selected compression level and estimated srcSize. + * `estimatedSrcSize` value is optional, select 0 if not known */ +ZSTDLIB_STATIC_API ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize); + +/*! ZSTD_getParams() : + * same as ZSTD_getCParams(), but @return a full `ZSTD_parameters` object instead of sub-component `ZSTD_compressionParameters`. + * All fields of `ZSTD_frameParameters` are set to default : contentSize=1, checksum=0, noDictID=0 */ +ZSTDLIB_STATIC_API ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize); + +/*! ZSTD_checkCParams() : + * Ensure param values remain within authorized range. + * @return 0 on success, or an error code (can be checked with ZSTD_isError()) */ +ZSTDLIB_STATIC_API size_t ZSTD_checkCParams(ZSTD_compressionParameters params); + +/*! ZSTD_adjustCParams() : + * optimize params for a given `srcSize` and `dictSize`. + * `srcSize` can be unknown, in which case use ZSTD_CONTENTSIZE_UNKNOWN. + * `dictSize` must be `0` when there is no dictionary. + * cPar can be invalid : all parameters will be clamped within valid range in the @return struct. + * This function never fails (wide contract) */ +ZSTDLIB_STATIC_API ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize); + +/*! ZSTD_compress_advanced() : + * Note : this function is now DEPRECATED. + * It can be replaced by ZSTD_compress2(), in combination with ZSTD_CCtx_setParameter() and other parameter setters. + * This prototype will generate compilation warnings. */ +ZSTD_DEPRECATED("use ZSTD_compress2") +size_t ZSTD_compress_advanced(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const void* dict,size_t dictSize, + ZSTD_parameters params); + +/*! ZSTD_compress_usingCDict_advanced() : + * Note : this function is now DEPRECATED. + * It can be replaced by ZSTD_compress2(), in combination with ZSTD_CCtx_loadDictionary() and other parameter setters. + * This prototype will generate compilation warnings. */ +ZSTD_DEPRECATED("use ZSTD_compress2 with ZSTD_CCtx_loadDictionary") +size_t ZSTD_compress_usingCDict_advanced(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const ZSTD_CDict* cdict, + ZSTD_frameParameters fParams); + + +/*! ZSTD_CCtx_loadDictionary_byReference() : + * Same as ZSTD_CCtx_loadDictionary(), but dictionary content is referenced, instead of being copied into CCtx. + * It saves some memory, but also requires that `dict` outlives its usage within `cctx` */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtx_loadDictionary_byReference(ZSTD_CCtx* cctx, const void* dict, size_t dictSize); + +/*! ZSTD_CCtx_loadDictionary_advanced() : + * Same as ZSTD_CCtx_loadDictionary(), but gives finer control over + * how to load the dictionary (by copy ? by reference ?) + * and how to interpret it (automatic ? force raw mode ? full mode only ?) */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtx_loadDictionary_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType); + +/*! ZSTD_CCtx_refPrefix_advanced() : + * Same as ZSTD_CCtx_refPrefix(), but gives finer control over + * how to interpret prefix content (automatic ? force raw mode (default) ? full mode only ?) */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType); + +/* === experimental parameters === */ +/* these parameters can be used with ZSTD_setParameter() + * they are not guaranteed to remain supported in the future */ + + /* Enables rsyncable mode, + * which makes compressed files more rsync friendly + * by adding periodic synchronization points to the compressed data. + * The target average block size is ZSTD_c_jobSize / 2. + * It's possible to modify the job size to increase or decrease + * the granularity of the synchronization point. + * Once the jobSize is smaller than the window size, + * it will result in compression ratio degradation. + * NOTE 1: rsyncable mode only works when multithreading is enabled. + * NOTE 2: rsyncable performs poorly in combination with long range mode, + * since it will decrease the effectiveness of synchronization points, + * though mileage may vary. + * NOTE 3: Rsyncable mode limits maximum compression speed to ~400 MB/s. + * If the selected compression level is already running significantly slower, + * the overall speed won't be significantly impacted. + */ + #define ZSTD_c_rsyncable ZSTD_c_experimentalParam1 + +/* Select a compression format. + * The value must be of type ZSTD_format_e. + * See ZSTD_format_e enum definition for details */ +#define ZSTD_c_format ZSTD_c_experimentalParam2 + +/* Force back-reference distances to remain < windowSize, + * even when referencing into Dictionary content (default:0) */ +#define ZSTD_c_forceMaxWindow ZSTD_c_experimentalParam3 + +/* Controls whether the contents of a CDict + * are used in place, or copied into the working context. + * Accepts values from the ZSTD_dictAttachPref_e enum. + * See the comments on that enum for an explanation of the feature. */ +#define ZSTD_c_forceAttachDict ZSTD_c_experimentalParam4 + +/* Controlled with ZSTD_paramSwitch_e enum. + * Default is ZSTD_ps_auto. + * Set to ZSTD_ps_disable to never compress literals. + * Set to ZSTD_ps_enable to always compress literals. (Note: uncompressed literals + * may still be emitted if huffman is not beneficial to use.) + * + * By default, in ZSTD_ps_auto, the library will decide at runtime whether to use + * literals compression based on the compression parameters - specifically, + * negative compression levels do not use literal compression. + */ +#define ZSTD_c_literalCompressionMode ZSTD_c_experimentalParam5 + +/* Tries to fit compressed block size to be around targetCBlockSize. + * No target when targetCBlockSize == 0. + * There is no guarantee on compressed block size (default:0) */ +#define ZSTD_c_targetCBlockSize ZSTD_c_experimentalParam6 + +/* User's best guess of source size. + * Hint is not valid when srcSizeHint == 0. + * There is no guarantee that hint is close to actual source size, + * but compression ratio may regress significantly if guess considerably underestimates */ +#define ZSTD_c_srcSizeHint ZSTD_c_experimentalParam7 + +/* Controls whether the new and experimental "dedicated dictionary search + * structure" can be used. This feature is still rough around the edges, be + * prepared for surprising behavior! + * + * How to use it: + * + * When using a CDict, whether to use this feature or not is controlled at + * CDict creation, and it must be set in a CCtxParams set passed into that + * construction (via ZSTD_createCDict_advanced2()). A compression will then + * use the feature or not based on how the CDict was constructed; the value of + * this param, set in the CCtx, will have no effect. + * + * However, when a dictionary buffer is passed into a CCtx, such as via + * ZSTD_CCtx_loadDictionary(), this param can be set on the CCtx to control + * whether the CDict that is created internally can use the feature or not. + * + * What it does: + * + * Normally, the internal data structures of the CDict are analogous to what + * would be stored in a CCtx after compressing the contents of a dictionary. + * To an approximation, a compression using a dictionary can then use those + * data structures to simply continue what is effectively a streaming + * compression where the simulated compression of the dictionary left off. + * Which is to say, the search structures in the CDict are normally the same + * format as in the CCtx. + * + * It is possible to do better, since the CDict is not like a CCtx: the search + * structures are written once during CDict creation, and then are only read + * after that, while the search structures in the CCtx are both read and + * written as the compression goes along. This means we can choose a search + * structure for the dictionary that is read-optimized. + * + * This feature enables the use of that different structure. + * + * Note that some of the members of the ZSTD_compressionParameters struct have + * different semantics and constraints in the dedicated search structure. It is + * highly recommended that you simply set a compression level in the CCtxParams + * you pass into the CDict creation call, and avoid messing with the cParams + * directly. + * + * Effects: + * + * This will only have any effect when the selected ZSTD_strategy + * implementation supports this feature. Currently, that's limited to + * ZSTD_greedy, ZSTD_lazy, and ZSTD_lazy2. + * + * Note that this means that the CDict tables can no longer be copied into the + * CCtx, so the dict attachment mode ZSTD_dictForceCopy will no longer be + * usable. The dictionary can only be attached or reloaded. + * + * In general, you should expect compression to be faster--sometimes very much + * so--and CDict creation to be slightly slower. Eventually, we will probably + * make this mode the default. + */ +#define ZSTD_c_enableDedicatedDictSearch ZSTD_c_experimentalParam8 + +/* ZSTD_c_stableInBuffer + * Experimental parameter. + * Default is 0 == disabled. Set to 1 to enable. + * + * Tells the compressor that the ZSTD_inBuffer will ALWAYS be the same + * between calls, except for the modifications that zstd makes to pos (the + * caller must not modify pos). This is checked by the compressor, and + * compression will fail if it ever changes. This means the only flush + * mode that makes sense is ZSTD_e_end, so zstd will error if ZSTD_e_end + * is not used. The data in the ZSTD_inBuffer in the range [src, src + pos) + * MUST not be modified during compression or you will get data corruption. + * + * When this flag is enabled zstd won't allocate an input window buffer, + * because the user guarantees it can reference the ZSTD_inBuffer until + * the frame is complete. But, it will still allocate an output buffer + * large enough to fit a block (see ZSTD_c_stableOutBuffer). This will also + * avoid the memcpy() from the input buffer to the input window buffer. + * + * NOTE: ZSTD_compressStream2() will error if ZSTD_e_end is not used. + * That means this flag cannot be used with ZSTD_compressStream(). + * + * NOTE: So long as the ZSTD_inBuffer always points to valid memory, using + * this flag is ALWAYS memory safe, and will never access out-of-bounds + * memory. However, compression WILL fail if you violate the preconditions. + * + * WARNING: The data in the ZSTD_inBuffer in the range [dst, dst + pos) MUST + * not be modified during compression or you will get data corruption. This + * is because zstd needs to reference data in the ZSTD_inBuffer to find + * matches. Normally zstd maintains its own window buffer for this purpose, + * but passing this flag tells zstd to use the user provided buffer. + */ +#define ZSTD_c_stableInBuffer ZSTD_c_experimentalParam9 + +/* ZSTD_c_stableOutBuffer + * Experimental parameter. + * Default is 0 == disabled. Set to 1 to enable. + * + * Tells he compressor that the ZSTD_outBuffer will not be resized between + * calls. Specifically: (out.size - out.pos) will never grow. This gives the + * compressor the freedom to say: If the compressed data doesn't fit in the + * output buffer then return ZSTD_error_dstSizeTooSmall. This allows us to + * always decompress directly into the output buffer, instead of decompressing + * into an internal buffer and copying to the output buffer. + * + * When this flag is enabled zstd won't allocate an output buffer, because + * it can write directly to the ZSTD_outBuffer. It will still allocate the + * input window buffer (see ZSTD_c_stableInBuffer). + * + * Zstd will check that (out.size - out.pos) never grows and return an error + * if it does. While not strictly necessary, this should prevent surprises. + */ +#define ZSTD_c_stableOutBuffer ZSTD_c_experimentalParam10 + +/* ZSTD_c_blockDelimiters + * Default is 0 == ZSTD_sf_noBlockDelimiters. + * + * For use with sequence compression API: ZSTD_compressSequences(). + * + * Designates whether or not the given array of ZSTD_Sequence contains block delimiters + * and last literals, which are defined as sequences with offset == 0 and matchLength == 0. + * See the definition of ZSTD_Sequence for more specifics. + */ +#define ZSTD_c_blockDelimiters ZSTD_c_experimentalParam11 + +/* ZSTD_c_validateSequences + * Default is 0 == disabled. Set to 1 to enable sequence validation. + * + * For use with sequence compression API: ZSTD_compressSequences(). + * Designates whether or not we validate sequences provided to ZSTD_compressSequences() + * during function execution. + * + * Without validation, providing a sequence that does not conform to the zstd spec will cause + * undefined behavior, and may produce a corrupted block. + * + * With validation enabled, a if sequence is invalid (see doc/zstd_compression_format.md for + * specifics regarding offset/matchlength requirements) then the function will bail out and + * return an error. + * + */ +#define ZSTD_c_validateSequences ZSTD_c_experimentalParam12 + +/* ZSTD_c_useBlockSplitter + * Controlled with ZSTD_paramSwitch_e enum. + * Default is ZSTD_ps_auto. + * Set to ZSTD_ps_disable to never use block splitter. + * Set to ZSTD_ps_enable to always use block splitter. + * + * By default, in ZSTD_ps_auto, the library will decide at runtime whether to use + * block splitting based on the compression parameters. + */ +#define ZSTD_c_useBlockSplitter ZSTD_c_experimentalParam13 + +/* ZSTD_c_useRowMatchFinder + * Controlled with ZSTD_paramSwitch_e enum. + * Default is ZSTD_ps_auto. + * Set to ZSTD_ps_disable to never use row-based matchfinder. + * Set to ZSTD_ps_enable to force usage of row-based matchfinder. + * + * By default, in ZSTD_ps_auto, the library will decide at runtime whether to use + * the row-based matchfinder based on support for SIMD instructions and the window log. + * Note that this only pertains to compression strategies: greedy, lazy, and lazy2 + */ +#define ZSTD_c_useRowMatchFinder ZSTD_c_experimentalParam14 + +/* ZSTD_c_deterministicRefPrefix + * Default is 0 == disabled. Set to 1 to enable. + * + * Zstd produces different results for prefix compression when the prefix is + * directly adjacent to the data about to be compressed vs. when it isn't. + * This is because zstd detects that the two buffers are contiguous and it can + * use a more efficient match finding algorithm. However, this produces different + * results than when the two buffers are non-contiguous. This flag forces zstd + * to always load the prefix in non-contiguous mode, even if it happens to be + * adjacent to the data, to guarantee determinism. + * + * If you really care about determinism when using a dictionary or prefix, + * like when doing delta compression, you should select this option. It comes + * at a speed penalty of about ~2.5% if the dictionary and data happened to be + * contiguous, and is free if they weren't contiguous. We don't expect that + * intentionally making the dictionary and data contiguous will be worth the + * cost to memcpy() the data. + */ +#define ZSTD_c_deterministicRefPrefix ZSTD_c_experimentalParam15 + +/*! ZSTD_CCtx_getParameter() : + * Get the requested compression parameter value, selected by enum ZSTD_cParameter, + * and store it into int* value. + * @return : 0, or an error code (which can be tested with ZSTD_isError()). + */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtx_getParameter(const ZSTD_CCtx* cctx, ZSTD_cParameter param, int* value); + + +/*! ZSTD_CCtx_params : + * Quick howto : + * - ZSTD_createCCtxParams() : Create a ZSTD_CCtx_params structure + * - ZSTD_CCtxParams_setParameter() : Push parameters one by one into + * an existing ZSTD_CCtx_params structure. + * This is similar to + * ZSTD_CCtx_setParameter(). + * - ZSTD_CCtx_setParametersUsingCCtxParams() : Apply parameters to + * an existing CCtx. + * These parameters will be applied to + * all subsequent frames. + * - ZSTD_compressStream2() : Do compression using the CCtx. + * - ZSTD_freeCCtxParams() : Free the memory, accept NULL pointer. + * + * This can be used with ZSTD_estimateCCtxSize_advanced_usingCCtxParams() + * for static allocation of CCtx for single-threaded compression. + */ +ZSTDLIB_STATIC_API ZSTD_CCtx_params* ZSTD_createCCtxParams(void); +ZSTDLIB_STATIC_API size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params); /* accept NULL pointer */ + +/*! ZSTD_CCtxParams_reset() : + * Reset params to default values. + */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtxParams_reset(ZSTD_CCtx_params* params); + +/*! ZSTD_CCtxParams_init() : + * Initializes the compression parameters of cctxParams according to + * compression level. All other parameters are reset to their default values. + */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtxParams_init(ZSTD_CCtx_params* cctxParams, int compressionLevel); + +/*! ZSTD_CCtxParams_init_advanced() : + * Initializes the compression and frame parameters of cctxParams according to + * params. All other parameters are reset to their default values. + */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtxParams_init_advanced(ZSTD_CCtx_params* cctxParams, ZSTD_parameters params); + +/*! ZSTD_CCtxParams_setParameter() : Requires v1.4.0+ + * Similar to ZSTD_CCtx_setParameter. + * Set one compression parameter, selected by enum ZSTD_cParameter. + * Parameters must be applied to a ZSTD_CCtx using + * ZSTD_CCtx_setParametersUsingCCtxParams(). + * @result : a code representing success or failure (which can be tested with + * ZSTD_isError()). + */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* params, ZSTD_cParameter param, int value); + +/*! ZSTD_CCtxParams_getParameter() : + * Similar to ZSTD_CCtx_getParameter. + * Get the requested value of one compression parameter, selected by enum ZSTD_cParameter. + * @result : 0, or an error code (which can be tested with ZSTD_isError()). + */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtxParams_getParameter(const ZSTD_CCtx_params* params, ZSTD_cParameter param, int* value); + +/*! ZSTD_CCtx_setParametersUsingCCtxParams() : + * Apply a set of ZSTD_CCtx_params to the compression context. + * This can be done even after compression is started, + * if nbWorkers==0, this will have no impact until a new compression is started. + * if nbWorkers>=1, new parameters will be picked up at next job, + * with a few restrictions (windowLog, pledgedSrcSize, nbWorkers, jobSize, and overlapLog are not updated). + */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtx_setParametersUsingCCtxParams( + ZSTD_CCtx* cctx, const ZSTD_CCtx_params* params); + +/*! ZSTD_compressStream2_simpleArgs() : + * Same as ZSTD_compressStream2(), + * but using only integral types as arguments. + * This variant might be helpful for binders from dynamic languages + * which have troubles handling structures containing memory pointers. + */ +ZSTDLIB_STATIC_API size_t ZSTD_compressStream2_simpleArgs ( + ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, size_t* dstPos, + const void* src, size_t srcSize, size_t* srcPos, + ZSTD_EndDirective endOp); + + +/*************************************** +* Advanced decompression functions +***************************************/ + +/*! ZSTD_isFrame() : + * Tells if the content of `buffer` starts with a valid Frame Identifier. + * Note : Frame Identifier is 4 bytes. If `size < 4`, @return will always be 0. + * Note 2 : Legacy Frame Identifiers are considered valid only if Legacy Support is enabled. + * Note 3 : Skippable Frame Identifiers are considered valid. */ +ZSTDLIB_STATIC_API unsigned ZSTD_isFrame(const void* buffer, size_t size); + +/*! ZSTD_createDDict_byReference() : + * Create a digested dictionary, ready to start decompression operation without startup delay. + * Dictionary content is referenced, and therefore stays in dictBuffer. + * It is important that dictBuffer outlives DDict, + * it must remain read accessible throughout the lifetime of DDict */ +ZSTDLIB_STATIC_API ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize); + +/*! ZSTD_DCtx_loadDictionary_byReference() : + * Same as ZSTD_DCtx_loadDictionary(), + * but references `dict` content instead of copying it into `dctx`. + * This saves memory if `dict` remains around., + * However, it's imperative that `dict` remains accessible (and unmodified) while being used, so it must outlive decompression. */ +ZSTDLIB_STATIC_API size_t ZSTD_DCtx_loadDictionary_byReference(ZSTD_DCtx* dctx, const void* dict, size_t dictSize); + +/*! ZSTD_DCtx_loadDictionary_advanced() : + * Same as ZSTD_DCtx_loadDictionary(), + * but gives direct control over + * how to load the dictionary (by copy ? by reference ?) + * and how to interpret it (automatic ? force raw mode ? full mode only ?). */ +ZSTDLIB_STATIC_API size_t ZSTD_DCtx_loadDictionary_advanced(ZSTD_DCtx* dctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType); + +/*! ZSTD_DCtx_refPrefix_advanced() : + * Same as ZSTD_DCtx_refPrefix(), but gives finer control over + * how to interpret prefix content (automatic ? force raw mode (default) ? full mode only ?) */ +ZSTDLIB_STATIC_API size_t ZSTD_DCtx_refPrefix_advanced(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType); + +/*! ZSTD_DCtx_setMaxWindowSize() : + * Refuses allocating internal buffers for frames requiring a window size larger than provided limit. + * This protects a decoder context from reserving too much memory for itself (potential attack scenario). + * This parameter is only useful in streaming mode, since no internal buffer is allocated in single-pass mode. + * By default, a decompression context accepts all window sizes <= (1 << ZSTD_WINDOWLOG_LIMIT_DEFAULT) + * @return : 0, or an error code (which can be tested using ZSTD_isError()). + */ +ZSTDLIB_STATIC_API size_t ZSTD_DCtx_setMaxWindowSize(ZSTD_DCtx* dctx, size_t maxWindowSize); + +/*! ZSTD_DCtx_getParameter() : + * Get the requested decompression parameter value, selected by enum ZSTD_dParameter, + * and store it into int* value. + * @return : 0, or an error code (which can be tested with ZSTD_isError()). + */ +ZSTDLIB_STATIC_API size_t ZSTD_DCtx_getParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param, int* value); + +/* ZSTD_d_format + * experimental parameter, + * allowing selection between ZSTD_format_e input compression formats + */ +#define ZSTD_d_format ZSTD_d_experimentalParam1 +/* ZSTD_d_stableOutBuffer + * Experimental parameter. + * Default is 0 == disabled. Set to 1 to enable. + * + * Tells the decompressor that the ZSTD_outBuffer will ALWAYS be the same + * between calls, except for the modifications that zstd makes to pos (the + * caller must not modify pos). This is checked by the decompressor, and + * decompression will fail if it ever changes. Therefore the ZSTD_outBuffer + * MUST be large enough to fit the entire decompressed frame. This will be + * checked when the frame content size is known. The data in the ZSTD_outBuffer + * in the range [dst, dst + pos) MUST not be modified during decompression + * or you will get data corruption. + * + * When this flags is enabled zstd won't allocate an output buffer, because + * it can write directly to the ZSTD_outBuffer, but it will still allocate + * an input buffer large enough to fit any compressed block. This will also + * avoid the memcpy() from the internal output buffer to the ZSTD_outBuffer. + * If you need to avoid the input buffer allocation use the buffer-less + * streaming API. + * + * NOTE: So long as the ZSTD_outBuffer always points to valid memory, using + * this flag is ALWAYS memory safe, and will never access out-of-bounds + * memory. However, decompression WILL fail if you violate the preconditions. + * + * WARNING: The data in the ZSTD_outBuffer in the range [dst, dst + pos) MUST + * not be modified during decompression or you will get data corruption. This + * is because zstd needs to reference data in the ZSTD_outBuffer to regenerate + * matches. Normally zstd maintains its own buffer for this purpose, but passing + * this flag tells zstd to use the user provided buffer. + */ +#define ZSTD_d_stableOutBuffer ZSTD_d_experimentalParam2 + +/* ZSTD_d_forceIgnoreChecksum + * Experimental parameter. + * Default is 0 == disabled. Set to 1 to enable + * + * Tells the decompressor to skip checksum validation during decompression, regardless + * of whether checksumming was specified during compression. This offers some + * slight performance benefits, and may be useful for debugging. + * Param has values of type ZSTD_forceIgnoreChecksum_e + */ +#define ZSTD_d_forceIgnoreChecksum ZSTD_d_experimentalParam3 + +/* ZSTD_d_refMultipleDDicts + * Experimental parameter. + * Default is 0 == disabled. Set to 1 to enable + * + * If enabled and dctx is allocated on the heap, then additional memory will be allocated + * to store references to multiple ZSTD_DDict. That is, multiple calls of ZSTD_refDDict() + * using a given ZSTD_DCtx, rather than overwriting the previous DDict reference, will instead + * store all references. At decompression time, the appropriate dictID is selected + * from the set of DDicts based on the dictID in the frame. + * + * Usage is simply calling ZSTD_refDDict() on multiple dict buffers. + * + * Param has values of byte ZSTD_refMultipleDDicts_e + * + * WARNING: Enabling this parameter and calling ZSTD_DCtx_refDDict(), will trigger memory + * allocation for the hash table. ZSTD_freeDCtx() also frees this memory. + * Memory is allocated as per ZSTD_DCtx::customMem. + * + * Although this function allocates memory for the table, the user is still responsible for + * memory management of the underlying ZSTD_DDict* themselves. + */ +#define ZSTD_d_refMultipleDDicts ZSTD_d_experimentalParam4 + + +/*! ZSTD_DCtx_setFormat() : + * This function is REDUNDANT. Prefer ZSTD_DCtx_setParameter(). + * Instruct the decoder context about what kind of data to decode next. + * This instruction is mandatory to decode data without a fully-formed header, + * such ZSTD_f_zstd1_magicless for example. + * @return : 0, or an error code (which can be tested using ZSTD_isError()). */ +ZSTD_DEPRECATED("use ZSTD_DCtx_setParameter() instead") +size_t ZSTD_DCtx_setFormat(ZSTD_DCtx* dctx, ZSTD_format_e format); + +/*! ZSTD_decompressStream_simpleArgs() : + * Same as ZSTD_decompressStream(), + * but using only integral types as arguments. + * This can be helpful for binders from dynamic languages + * which have troubles handling structures containing memory pointers. + */ +ZSTDLIB_STATIC_API size_t ZSTD_decompressStream_simpleArgs ( + ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, size_t* dstPos, + const void* src, size_t srcSize, size_t* srcPos); + + +/******************************************************************** +* Advanced streaming functions +* Warning : most of these functions are now redundant with the Advanced API. +* Once Advanced API reaches "stable" status, +* redundant functions will be deprecated, and then at some point removed. +********************************************************************/ + +/*===== Advanced Streaming compression functions =====*/ + +/*! ZSTD_initCStream_srcSize() : + * This function is DEPRECATED, and equivalent to: + * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); + * ZSTD_CCtx_refCDict(zcs, NULL); // clear the dictionary (if any) + * ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel); + * ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize); + * + * pledgedSrcSize must be correct. If it is not known at init time, use + * ZSTD_CONTENTSIZE_UNKNOWN. Note that, for compatibility with older programs, + * "0" also disables frame content size field. It may be enabled in the future. + * This prototype will generate compilation warnings. + */ +ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions") +size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, + int compressionLevel, + unsigned long long pledgedSrcSize); + +/*! ZSTD_initCStream_usingDict() : + * This function is DEPRECATED, and is equivalent to: + * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); + * ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel); + * ZSTD_CCtx_loadDictionary(zcs, dict, dictSize); + * + * Creates of an internal CDict (incompatible with static CCtx), except if + * dict == NULL or dictSize < 8, in which case no dict is used. + * Note: dict is loaded with ZSTD_dct_auto (treated as a full zstd dictionary if + * it begins with ZSTD_MAGIC_DICTIONARY, else as raw content) and ZSTD_dlm_byCopy. + * This prototype will generate compilation warnings. + */ +ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions") +size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, + const void* dict, size_t dictSize, + int compressionLevel); + +/*! ZSTD_initCStream_advanced() : + * This function is DEPRECATED, and is approximately equivalent to: + * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); + * // Pseudocode: Set each zstd parameter and leave the rest as-is. + * for ((param, value) : params) { + * ZSTD_CCtx_setParameter(zcs, param, value); + * } + * ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize); + * ZSTD_CCtx_loadDictionary(zcs, dict, dictSize); + * + * dict is loaded with ZSTD_dct_auto and ZSTD_dlm_byCopy. + * pledgedSrcSize must be correct. + * If srcSize is not known at init time, use value ZSTD_CONTENTSIZE_UNKNOWN. + * This prototype will generate compilation warnings. + */ +ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions") +size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, + const void* dict, size_t dictSize, + ZSTD_parameters params, + unsigned long long pledgedSrcSize); + +/*! ZSTD_initCStream_usingCDict() : + * This function is DEPRECATED, and equivalent to: + * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); + * ZSTD_CCtx_refCDict(zcs, cdict); + * + * note : cdict will just be referenced, and must outlive compression session + * This prototype will generate compilation warnings. + */ +ZSTD_DEPRECATED("use ZSTD_CCtx_reset and ZSTD_CCtx_refCDict, see zstd.h for detailed instructions") +size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict); + +/*! ZSTD_initCStream_usingCDict_advanced() : + * This function is DEPRECATED, and is approximately equivalent to: + * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); + * // Pseudocode: Set each zstd frame parameter and leave the rest as-is. + * for ((fParam, value) : fParams) { + * ZSTD_CCtx_setParameter(zcs, fParam, value); + * } + * ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize); + * ZSTD_CCtx_refCDict(zcs, cdict); + * + * same as ZSTD_initCStream_usingCDict(), with control over frame parameters. + * pledgedSrcSize must be correct. If srcSize is not known at init time, use + * value ZSTD_CONTENTSIZE_UNKNOWN. + * This prototype will generate compilation warnings. + */ +ZSTD_DEPRECATED("use ZSTD_CCtx_reset and ZSTD_CCtx_refCDict, see zstd.h for detailed instructions") +size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, + const ZSTD_CDict* cdict, + ZSTD_frameParameters fParams, + unsigned long long pledgedSrcSize); + +/*! ZSTD_resetCStream() : + * This function is DEPRECATED, and is equivalent to: + * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); + * ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize); + * Note: ZSTD_resetCStream() interprets pledgedSrcSize == 0 as ZSTD_CONTENTSIZE_UNKNOWN, but + * ZSTD_CCtx_setPledgedSrcSize() does not do the same, so ZSTD_CONTENTSIZE_UNKNOWN must be + * explicitly specified. + * + * start a new frame, using same parameters from previous frame. + * This is typically useful to skip dictionary loading stage, since it will re-use it in-place. + * Note that zcs must be init at least once before using ZSTD_resetCStream(). + * If pledgedSrcSize is not known at reset time, use macro ZSTD_CONTENTSIZE_UNKNOWN. + * If pledgedSrcSize > 0, its value must be correct, as it will be written in header, and controlled at the end. + * For the time being, pledgedSrcSize==0 is interpreted as "srcSize unknown" for compatibility with older programs, + * but it will change to mean "empty" in future version, so use macro ZSTD_CONTENTSIZE_UNKNOWN instead. + * @return : 0, or an error code (which can be tested using ZSTD_isError()) + * This prototype will generate compilation warnings. + */ +ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions") +size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize); + + +typedef struct { + unsigned long long ingested; /* nb input bytes read and buffered */ + unsigned long long consumed; /* nb input bytes actually compressed */ + unsigned long long produced; /* nb of compressed bytes generated and buffered */ + unsigned long long flushed; /* nb of compressed bytes flushed : not provided; can be tracked from caller side */ + unsigned currentJobID; /* MT only : latest started job nb */ + unsigned nbActiveWorkers; /* MT only : nb of workers actively compressing at probe time */ +} ZSTD_frameProgression; + +/* ZSTD_getFrameProgression() : + * tells how much data has been ingested (read from input) + * consumed (input actually compressed) and produced (output) for current frame. + * Note : (ingested - consumed) is amount of input data buffered internally, not yet compressed. + * Aggregates progression inside active worker threads. + */ +ZSTDLIB_STATIC_API ZSTD_frameProgression ZSTD_getFrameProgression(const ZSTD_CCtx* cctx); + +/*! ZSTD_toFlushNow() : + * Tell how many bytes are ready to be flushed immediately. + * Useful for multithreading scenarios (nbWorkers >= 1). + * Probe the oldest active job, defined as oldest job not yet entirely flushed, + * and check its output buffer. + * @return : amount of data stored in oldest job and ready to be flushed immediately. + * if @return == 0, it means either : + * + there is no active job (could be checked with ZSTD_frameProgression()), or + * + oldest job is still actively compressing data, + * but everything it has produced has also been flushed so far, + * therefore flush speed is limited by production speed of oldest job + * irrespective of the speed of concurrent (and newer) jobs. + */ +ZSTDLIB_STATIC_API size_t ZSTD_toFlushNow(ZSTD_CCtx* cctx); + + +/*===== Advanced Streaming decompression functions =====*/ + +/*! + * This function is deprecated, and is equivalent to: + * + * ZSTD_DCtx_reset(zds, ZSTD_reset_session_only); + * ZSTD_DCtx_loadDictionary(zds, dict, dictSize); + * + * note: no dictionary will be used if dict == NULL or dictSize < 8 + * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x + */ +ZSTDLIB_STATIC_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize); + +/*! + * This function is deprecated, and is equivalent to: + * + * ZSTD_DCtx_reset(zds, ZSTD_reset_session_only); + * ZSTD_DCtx_refDDict(zds, ddict); + * + * note : ddict is referenced, it must outlive decompression session + * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x + */ +ZSTDLIB_STATIC_API size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict); + +/*! + * This function is deprecated, and is equivalent to: + * + * ZSTD_DCtx_reset(zds, ZSTD_reset_session_only); + * + * re-use decompression parameters from previous init; saves dictionary loading + * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x + */ +ZSTDLIB_STATIC_API size_t ZSTD_resetDStream(ZSTD_DStream* zds); + + +/********************************************************************* +* Buffer-less and synchronous inner streaming functions +* +* This is an advanced API, giving full control over buffer management, for users which need direct control over memory. +* But it's also a complex one, with several restrictions, documented below. +* Prefer normal streaming API for an easier experience. +********************************************************************* */ + +/** + Buffer-less streaming compression (synchronous mode) + + A ZSTD_CCtx object is required to track streaming operations. + Use ZSTD_createCCtx() / ZSTD_freeCCtx() to manage resource. + ZSTD_CCtx object can be re-used multiple times within successive compression operations. + + Start by initializing a context. + Use ZSTD_compressBegin(), or ZSTD_compressBegin_usingDict() for dictionary compression. + It's also possible to duplicate a reference context which has already been initialized, using ZSTD_copyCCtx() + + Then, consume your input using ZSTD_compressContinue(). + There are some important considerations to keep in mind when using this advanced function : + - ZSTD_compressContinue() has no internal buffer. It uses externally provided buffers only. + - Interface is synchronous : input is consumed entirely and produces 1+ compressed blocks. + - Caller must ensure there is enough space in `dst` to store compressed data under worst case scenario. + Worst case evaluation is provided by ZSTD_compressBound(). + ZSTD_compressContinue() doesn't guarantee recover after a failed compression. + - ZSTD_compressContinue() presumes prior input ***is still accessible and unmodified*** (up to maximum distance size, see WindowLog). + It remembers all previous contiguous blocks, plus one separated memory segment (which can itself consists of multiple contiguous blocks) + - ZSTD_compressContinue() detects that prior input has been overwritten when `src` buffer overlaps. + In which case, it will "discard" the relevant memory section from its history. + + Finish a frame with ZSTD_compressEnd(), which will write the last block(s) and optional checksum. + It's possible to use srcSize==0, in which case, it will write a final empty block to end the frame. + Without last block mark, frames are considered unfinished (hence corrupted) by compliant decoders. + + `ZSTD_CCtx` object can be re-used (ZSTD_compressBegin()) to compress again. +*/ + +/*===== Buffer-less streaming compression functions =====*/ +ZSTDLIB_STATIC_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel); +ZSTDLIB_STATIC_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel); +ZSTDLIB_STATIC_API size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); /**< note: fails if cdict==NULL */ +ZSTDLIB_STATIC_API size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); /**< note: if pledgedSrcSize is not known, use ZSTD_CONTENTSIZE_UNKNOWN */ + +ZSTDLIB_STATIC_API size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); +ZSTDLIB_STATIC_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); + +/* The ZSTD_compressBegin_advanced() and ZSTD_compressBegin_usingCDict_advanced() are now DEPRECATED and will generate a compiler warning */ +ZSTD_DEPRECATED("use advanced API to access custom parameters") +size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize : If srcSize is not known at init time, use ZSTD_CONTENTSIZE_UNKNOWN */ +ZSTD_DEPRECATED("use advanced API to access custom parameters") +size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize); /* compression parameters are already set within cdict. pledgedSrcSize must be correct. If srcSize is not known, use macro ZSTD_CONTENTSIZE_UNKNOWN */ +/** + Buffer-less streaming decompression (synchronous mode) + + A ZSTD_DCtx object is required to track streaming operations. + Use ZSTD_createDCtx() / ZSTD_freeDCtx() to manage it. + A ZSTD_DCtx object can be re-used multiple times. + + First typical operation is to retrieve frame parameters, using ZSTD_getFrameHeader(). + Frame header is extracted from the beginning of compressed frame, so providing only the frame's beginning is enough. + Data fragment must be large enough to ensure successful decoding. + `ZSTD_frameHeaderSize_max` bytes is guaranteed to always be large enough. + @result : 0 : successful decoding, the `ZSTD_frameHeader` structure is correctly filled. + >0 : `srcSize` is too small, please provide at least @result bytes on next attempt. + errorCode, which can be tested using ZSTD_isError(). + + It fills a ZSTD_frameHeader structure with important information to correctly decode the frame, + such as the dictionary ID, content size, or maximum back-reference distance (`windowSize`). + Note that these values could be wrong, either because of data corruption, or because a 3rd party deliberately spoofs false information. + As a consequence, check that values remain within valid application range. + For example, do not allocate memory blindly, check that `windowSize` is within expectation. + Each application can set its own limits, depending on local restrictions. + For extended interoperability, it is recommended to support `windowSize` of at least 8 MB. + + ZSTD_decompressContinue() needs previous data blocks during decompression, up to `windowSize` bytes. + ZSTD_decompressContinue() is very sensitive to contiguity, + if 2 blocks don't follow each other, make sure that either the compressor breaks contiguity at the same place, + or that previous contiguous segment is large enough to properly handle maximum back-reference distance. + There are multiple ways to guarantee this condition. + + The most memory efficient way is to use a round buffer of sufficient size. + Sufficient size is determined by invoking ZSTD_decodingBufferSize_min(), + which can @return an error code if required value is too large for current system (in 32-bits mode). + In a round buffer methodology, ZSTD_decompressContinue() decompresses each block next to previous one, + up to the moment there is not enough room left in the buffer to guarantee decoding another full block, + which maximum size is provided in `ZSTD_frameHeader` structure, field `blockSizeMax`. + At which point, decoding can resume from the beginning of the buffer. + Note that already decoded data stored in the buffer should be flushed before being overwritten. + + There are alternatives possible, for example using two or more buffers of size `windowSize` each, though they consume more memory. + + Finally, if you control the compression process, you can also ignore all buffer size rules, + as long as the encoder and decoder progress in "lock-step", + aka use exactly the same buffer sizes, break contiguity at the same place, etc. + + Once buffers are setup, start decompression, with ZSTD_decompressBegin(). + If decompression requires a dictionary, use ZSTD_decompressBegin_usingDict() or ZSTD_decompressBegin_usingDDict(). + + Then use ZSTD_nextSrcSizeToDecompress() and ZSTD_decompressContinue() alternatively. + ZSTD_nextSrcSizeToDecompress() tells how many bytes to provide as 'srcSize' to ZSTD_decompressContinue(). + ZSTD_decompressContinue() requires this _exact_ amount of bytes, or it will fail. + + @result of ZSTD_decompressContinue() is the number of bytes regenerated within 'dst' (necessarily <= dstCapacity). + It can be zero : it just means ZSTD_decompressContinue() has decoded some metadata item. + It can also be an error code, which can be tested with ZSTD_isError(). + + A frame is fully decoded when ZSTD_nextSrcSizeToDecompress() returns zero. + Context can then be reset to start a new decompression. + + Note : it's possible to know if next input to present is a header or a block, using ZSTD_nextInputType(). + This information is not required to properly decode a frame. + + == Special case : skippable frames == + + Skippable frames allow integration of user-defined data into a flow of concatenated frames. + Skippable frames will be ignored (skipped) by decompressor. + The format of skippable frames is as follows : + a) Skippable frame ID - 4 Bytes, Little endian format, any value from 0x184D2A50 to 0x184D2A5F + b) Frame Size - 4 Bytes, Little endian format, unsigned 32-bits + c) Frame Content - any content (User Data) of length equal to Frame Size + For skippable frames ZSTD_getFrameHeader() returns zfhPtr->frameType==ZSTD_skippableFrame. + For skippable frames ZSTD_decompressContinue() always returns 0 : it only skips the content. +*/ + +/*===== Buffer-less streaming decompression functions =====*/ +typedef enum { ZSTD_frame, ZSTD_skippableFrame } ZSTD_frameType_e; +typedef struct { + unsigned long long frameContentSize; /* if == ZSTD_CONTENTSIZE_UNKNOWN, it means this field is not available. 0 means "empty" */ + unsigned long long windowSize; /* can be very large, up to <= frameContentSize */ + unsigned blockSizeMax; + ZSTD_frameType_e frameType; /* if == ZSTD_skippableFrame, frameContentSize is the size of skippable content */ + unsigned headerSize; + unsigned dictID; + unsigned checksumFlag; +} ZSTD_frameHeader; + +/*! ZSTD_getFrameHeader() : + * decode Frame Header, or requires larger `srcSize`. + * @return : 0, `zfhPtr` is correctly filled, + * >0, `srcSize` is too small, value is wanted `srcSize` amount, + * or an error code, which can be tested using ZSTD_isError() */ +ZSTDLIB_STATIC_API size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize); /**< doesn't consume input */ +/*! ZSTD_getFrameHeader_advanced() : + * same as ZSTD_getFrameHeader(), + * with added capability to select a format (like ZSTD_f_zstd1_magicless) */ +ZSTDLIB_STATIC_API size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize, ZSTD_format_e format); +ZSTDLIB_STATIC_API size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long long frameContentSize); /**< when frame content size is not known, pass in frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN */ + +ZSTDLIB_STATIC_API size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx); +ZSTDLIB_STATIC_API size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize); +ZSTDLIB_STATIC_API size_t ZSTD_decompressBegin_usingDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict); + +ZSTDLIB_STATIC_API size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx); +ZSTDLIB_STATIC_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); + +/* misc */ +ZSTDLIB_STATIC_API void ZSTD_copyDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx); +typedef enum { ZSTDnit_frameHeader, ZSTDnit_blockHeader, ZSTDnit_block, ZSTDnit_lastBlock, ZSTDnit_checksum, ZSTDnit_skippableFrame } ZSTD_nextInputType_e; +ZSTDLIB_STATIC_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx); + + + + +/* ============================ */ +/** Block level API */ +/* ============================ */ + +/*! + Block functions produce and decode raw zstd blocks, without frame metadata. + Frame metadata cost is typically ~12 bytes, which can be non-negligible for very small blocks (< 100 bytes). + But users will have to take in charge needed metadata to regenerate data, such as compressed and content sizes. + + A few rules to respect : + - Compressing and decompressing require a context structure + + Use ZSTD_createCCtx() and ZSTD_createDCtx() + - It is necessary to init context before starting + + compression : any ZSTD_compressBegin*() variant, including with dictionary + + decompression : any ZSTD_decompressBegin*() variant, including with dictionary + + copyCCtx() and copyDCtx() can be used too + - Block size is limited, it must be <= ZSTD_getBlockSize() <= ZSTD_BLOCKSIZE_MAX == 128 KB + + If input is larger than a block size, it's necessary to split input data into multiple blocks + + For inputs larger than a single block, consider using regular ZSTD_compress() instead. + Frame metadata is not that costly, and quickly becomes negligible as source size grows larger than a block. + - When a block is considered not compressible enough, ZSTD_compressBlock() result will be 0 (zero) ! + ===> In which case, nothing is produced into `dst` ! + + User __must__ test for such outcome and deal directly with uncompressed data + + A block cannot be declared incompressible if ZSTD_compressBlock() return value was != 0. + Doing so would mess up with statistics history, leading to potential data corruption. + + ZSTD_decompressBlock() _doesn't accept uncompressed data as input_ !! + + In case of multiple successive blocks, should some of them be uncompressed, + decoder must be informed of their existence in order to follow proper history. + Use ZSTD_insertBlock() for such a case. +*/ + +/*===== Raw zstd block functions =====*/ +ZSTDLIB_STATIC_API size_t ZSTD_getBlockSize (const ZSTD_CCtx* cctx); +ZSTDLIB_STATIC_API size_t ZSTD_compressBlock (ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); +ZSTDLIB_STATIC_API size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); +ZSTDLIB_STATIC_API size_t ZSTD_insertBlock (ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize); /**< insert uncompressed block into `dctx` history. Useful for multi-blocks decompression. */ + + +#endif /* ZSTD_H_ZSTD_STATIC_LINKING_ONLY */ + +#if defined (__cplusplus) +} +#endif diff --git a/src/blosc2/internal-complibs/zstd-1.5.2/zstd_errors.h b/src/blosc2/internal-complibs/zstd-1.5.2/zstd_errors.h new file mode 100644 index 0000000..fa3686b --- /dev/null +++ b/src/blosc2/internal-complibs/zstd-1.5.2/zstd_errors.h @@ -0,0 +1,95 @@ +/* + * Copyright (c) Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_ERRORS_H_398273423 +#define ZSTD_ERRORS_H_398273423 + +#if defined (__cplusplus) +extern "C" { +#endif + +/*===== dependency =====*/ +#include /* size_t */ + + +/* ===== ZSTDERRORLIB_API : control library symbols visibility ===== */ +#ifndef ZSTDERRORLIB_VISIBILITY +# if defined(__GNUC__) && (__GNUC__ >= 4) +# define ZSTDERRORLIB_VISIBILITY __attribute__ ((visibility ("default"))) +# else +# define ZSTDERRORLIB_VISIBILITY +# endif +#endif +#if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1) +# define ZSTDERRORLIB_API __declspec(dllexport) ZSTDERRORLIB_VISIBILITY +#elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1) +# define ZSTDERRORLIB_API __declspec(dllimport) ZSTDERRORLIB_VISIBILITY /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/ +#else +# define ZSTDERRORLIB_API ZSTDERRORLIB_VISIBILITY +#endif + +/*-********************************************* + * Error codes list + *-********************************************* + * Error codes _values_ are pinned down since v1.3.1 only. + * Therefore, don't rely on values if you may link to any version < v1.3.1. + * + * Only values < 100 are considered stable. + * + * note 1 : this API shall be used with static linking only. + * dynamic linking is not yet officially supported. + * note 2 : Prefer relying on the enum than on its value whenever possible + * This is the only supported way to use the error list < v1.3.1 + * note 3 : ZSTD_isError() is always correct, whatever the library version. + **********************************************/ +typedef enum { + ZSTD_error_no_error = 0, + ZSTD_error_GENERIC = 1, + ZSTD_error_prefix_unknown = 10, + ZSTD_error_version_unsupported = 12, + ZSTD_error_frameParameter_unsupported = 14, + ZSTD_error_frameParameter_windowTooLarge = 16, + ZSTD_error_corruption_detected = 20, + ZSTD_error_checksum_wrong = 22, + ZSTD_error_dictionary_corrupted = 30, + ZSTD_error_dictionary_wrong = 32, + ZSTD_error_dictionaryCreation_failed = 34, + ZSTD_error_parameter_unsupported = 40, + ZSTD_error_parameter_outOfBound = 42, + ZSTD_error_tableLog_tooLarge = 44, + ZSTD_error_maxSymbolValue_tooLarge = 46, + ZSTD_error_maxSymbolValue_tooSmall = 48, + ZSTD_error_stage_wrong = 60, + ZSTD_error_init_missing = 62, + ZSTD_error_memory_allocation = 64, + ZSTD_error_workSpace_tooSmall= 66, + ZSTD_error_dstSize_tooSmall = 70, + ZSTD_error_srcSize_wrong = 72, + ZSTD_error_dstBuffer_null = 74, + /* following error codes are __NOT STABLE__, they can be removed or changed in future versions */ + ZSTD_error_frameIndex_tooLarge = 100, + ZSTD_error_seekableIO = 102, + ZSTD_error_dstBuffer_wrong = 104, + ZSTD_error_srcBuffer_wrong = 105, + ZSTD_error_maxCode = 120 /* never EVER use this value directly, it can change in future versions! Use ZSTD_isError() instead */ +} ZSTD_ErrorCode; + +/*! ZSTD_getErrorCode() : + convert a `size_t` function result into a `ZSTD_ErrorCode` enum type, + which can be used to compare with enum list published above */ +ZSTDERRORLIB_API ZSTD_ErrorCode ZSTD_getErrorCode(size_t functionResult); +ZSTDERRORLIB_API const char* ZSTD_getErrorString(ZSTD_ErrorCode code); /**< Same as ZSTD_getErrorName, but using a `ZSTD_ErrorCode` enum argument */ + + +#if defined (__cplusplus) +} +#endif + +#endif /* ZSTD_ERRORS_H_398273423 */ diff --git a/src/blosc2/plugins/CMakeLists.txt b/src/blosc2/plugins/CMakeLists.txt new file mode 100644 index 0000000..62c1f95 --- /dev/null +++ b/src/blosc2/plugins/CMakeLists.txt @@ -0,0 +1,4 @@ +add_subdirectory(codecs) +add_subdirectory(filters) + +set(SOURCES ${SOURCES} ../plugins/plugin_utils.c ../plugins/plugin_utils.h PARENT_SCOPE) diff --git a/src/blosc2/plugins/README.md b/src/blosc2/plugins/README.md new file mode 100644 index 0000000..380d1fc --- /dev/null +++ b/src/blosc2/plugins/README.md @@ -0,0 +1,144 @@ +Plugins registry for Blosc users +================================ + +Blosc has a tradition of supporting different filters and codecs for compressing data, +and it was up to the user to choose one or another depending on his needs. +However, it is clear that there will always be scenarios where a more richer variety +of them could be useful. So the Blosc Development Team has set new goals: + +1) Implement a way for users to locally register filters and codecs so that they can use + them in their setup at will. + +2) Setup a central registry so that *other* users can make use of these filters and codecs + without intefering with other ones that have been created by other users. + +As a bonus, those codecs and filters accepted in the central registry and meeting the quality standards +defined in these guidelines will be distributed *inside* the C-Blosc2 library, +allowing a much easier way for others to use them: install C-Blosc2 library and you are all set. +Of course, to achieve such a status, plugins will require a careful testing process described below. + + +Plugin types +-------------- + +The plugins that are registered in the repository can be codecs or filters. + +A **codec** is a program able to compress and decompress a digital data stream +with the objective of reduce dataset size to enable a faster transmission +of data. +Some of the codecs used by Blosc are e.g. *BLOSCLZ*, *LZ4* and *ZSTANDARD*. + +A **filter** is a program that reorders the data without +changing its size, so that the initial and final size are equal. +A filter consists of encoder and decoder. Filter encoder is applied before +using the codec compressor (or codec encoder) in order to make data easier to compress +and filter decoder is used after codec decompressor (or codec decoder) to recover +the original data arrangement. +Some filters actually used by Blosc are e.g. *SHUFFLE*, which rearranges data +based on the typesize, or *TRUNC*, which zeroes mantissa bits so as to reduce +the precision of (floating point) data, and hence, increase the compression ratio. + +Here it is an example on how the compression process goes: + + + -------------------- filter encoder ------------------- codec encoder ------- + | src | -----------> | tmp | ----------> | c_src | + -------------------- ------------------- ------- + +And the decompression process: + + -------- codec decoder ------------------- filter decoder ------------------- + | c_src | -----------> | tmp | ----------> | src | + -------- ------------------- ------------------- + +Moreover, during the pipeline process you can use even 6 different +filters ordered as you prefer. + + +Blosc global registered plugins vs user registered plugins +---------------------------------------------------------- + +**Blosc global registered plugins** are official Blosc plugins that have passed through a selection process +and have been recognised by the Blosc Development Team. These plugins are available for +everybody in the C-Blosc2 GitHub repository and users can install them anytime. + +**User registered plugins** are plugins that users register locally and they can use them +in the same way as in the examples `urcodecs.c` and `urfilters.c`. + +If you only want to use a plugin on your own devices you can just register it as a user registered +plugin with an ID between *BLOSC2_USER_REGISTERED_FILTERS_START* and *BLOSC2_USER_REGISTERED_FILTERS_STOP*. +Otherwise, if you think that your plugin could be useful for the community you can apply for +registering it as an official Blosc plugin following the next steps. + + +Requirements for registering plugins +------------------------------------ + +For users wanting to register a new codec or filter, there are some requirements +that their code must satisfy: + +- First, the plugin code must be **developed in C**, have a relatively small footprint + and meet decent quality code standards. + +- Second, users must develop a test suite which prove that the plugin works correctly. + +Finally, even if these requirements are completely satisfied, it is not +guaranteed that the plugin will be useful or contribute something +different than the existing ones, so the Blosc development team has the final +say and will decide if a plugin is to be accepted or not. + + +Steps +----- + +1. First, tests must be provided and be passing. + + **It is completely mandatory and necessary to add these lines to `main()` in each test to make plugins machinery work:** + - `blosc2_init()` at the beginning + - `blosc2_destroy()` in the end + + +2. Then, the user must make a fork of the C-Blosc2 Github repository, + adding a new folder within the plugin sources to the path `plugins/codecs` or + `plugins/filters` depending on the plugin type. + +3. Furthermore, a text file named `README.rst` must be provided where it is explained: + + * The plugin motivation, why and for what purpose was the plugin created. + + * How to use the plugin. + + * What does the plugin do and how it works. + + * The advantages and disadvantages of the plugin compared to the rest. + +4. To register a plugin the user must choose a plugin ID between *BLOSC2_GLOBAL_REGISTERED_FILTERS_START* and *BLOSC2_GLOBAL_REGISTERED_FILTERS_STOP* and + write it at `include/blosc2/codecs-registry.h` + or `include/blosc2/filters-registry.h` depending on the plugin type. Then, you have to edit `include/blosc2/codecs-registry.c`or + + `include/blosc2/filters-registry.c` in the next way: + + At the top it must be added `#include "plugin_folder/plugin_header.h"`, + + and into the register function you must follow the same steps that were done for the existing plugins. + +5. Finally, the Blosc development team will carry out the evaluation process + (probably via a votation process, with the BDFL having the last say in case of the team is undecided) + so as to decide whether the plugin is useful and hence, candidate to be integrated into the C-Blosc2 + source code distribution. In case of a negative decision, the original author will be informed, + together with a series of advices for starting a new iteration if desired. + + +Examples +-------- + +In the `plugins/` directory there can be found different examples of codecs and filters +available as plugins that can be used in the compression process, and that +can be used as an example on how to implement plugins that can make into C-Blosc2. +Some of these examples are `ndlz`, `ndcell` or `ndmean`. + + +Thanks +------ + +We would like to express our gratitude to the NumFOCUS Foundation so as to provide the funds to implement this functionality. diff --git a/src/blosc2/plugins/codecs/CMakeLists.txt b/src/blosc2/plugins/codecs/CMakeLists.txt new file mode 100644 index 0000000..e2fdcfa --- /dev/null +++ b/src/blosc2/plugins/codecs/CMakeLists.txt @@ -0,0 +1,4 @@ +add_subdirectory(ndlz) +add_subdirectory(zfp) + +set(SOURCES ${SOURCES} ../plugins/codecs/codecs-registry.c PARENT_SCOPE) \ No newline at end of file diff --git a/src/blosc2/plugins/codecs/codecs-registry.c b/src/blosc2/plugins/codecs/codecs-registry.c new file mode 100644 index 0000000..751cc4f --- /dev/null +++ b/src/blosc2/plugins/codecs/codecs-registry.c @@ -0,0 +1,49 @@ +/* + Copyright (C) 2021 The Blosc Developers + http://blosc.org + License: BSD 3-Clause (see LICENSE.txt) +*/ + +#include +#include "blosc2/codecs-registry.h" +#include "ndlz/ndlz.h" +#include "zfp/blosc2-zfp.h" + +void register_codecs(void) { + + blosc2_codec ndlz; + ndlz.compcode = BLOSC_CODEC_NDLZ; + ndlz.compver = 1; + ndlz.complib = BLOSC_CODEC_NDLZ; + ndlz.encoder = ndlz_compress; + ndlz.decoder = ndlz_decompress; + ndlz.compname = "ndlz"; + register_codec_private(&ndlz); + + blosc2_codec zfp_acc; + zfp_acc.compcode = BLOSC_CODEC_ZFP_FIXED_ACCURACY; + zfp_acc.compver = 1; + zfp_acc.complib = BLOSC_CODEC_ZFP_FIXED_ACCURACY; + zfp_acc.encoder = zfp_acc_compress; + zfp_acc.decoder = zfp_acc_decompress; + zfp_acc.compname = "zfp_acc"; + register_codec_private(&zfp_acc); + + blosc2_codec zfp_prec; + zfp_prec.compcode = BLOSC_CODEC_ZFP_FIXED_PRECISION; + zfp_prec.compver = 1; + zfp_prec.complib = BLOSC_CODEC_ZFP_FIXED_PRECISION; + zfp_prec.encoder = zfp_prec_compress; + zfp_prec.decoder = zfp_prec_decompress; + zfp_prec.compname = "zfp_prec"; + register_codec_private(&zfp_prec); + + blosc2_codec zfp_rate; + zfp_rate.compcode = BLOSC_CODEC_ZFP_FIXED_RATE; + zfp_rate.compver = 1; + zfp_rate.complib = BLOSC_CODEC_ZFP_FIXED_RATE; + zfp_rate.encoder = zfp_rate_compress; + zfp_rate.decoder = zfp_rate_decompress; + zfp_rate.compname = "zfp_rate"; + register_codec_private(&zfp_rate); +} diff --git a/src/blosc2/plugins/codecs/ndlz/CMakeLists.txt b/src/blosc2/plugins/codecs/ndlz/CMakeLists.txt new file mode 100644 index 0000000..da7ce92 --- /dev/null +++ b/src/blosc2/plugins/codecs/ndlz/CMakeLists.txt @@ -0,0 +1,27 @@ +# sources +set(SOURCES ${SOURCES} ../plugins/codecs/ndlz/ndlz.c ../plugins/codecs/ndlz/ndlz.h ../plugins/codecs/ndlz/ndlz-private.h + ../plugins/codecs/ndlz/ndlz4x4.c ../plugins/codecs/ndlz/ndlz4x4.h ../plugins/codecs/ndlz/ndlz8x8.c + ../plugins/codecs/ndlz/ndlz8x8.h ../plugins/codecs/ndlz/xxhash.c ../plugins/codecs/ndlz/xxhash.h PARENT_SCOPE) + +# targets +if(BUILD_TESTS) + add_executable(test_ndlz test_ndlz.c) + # Define the BLOSC_TESTING symbol so normally-hidden functions + # aren't hidden from the view of the test programs. + set_property( + TARGET test_ndlz + APPEND PROPERTY COMPILE_DEFINITIONS BLOSC_TESTING) + + target_link_libraries(test_ndlz blosc_testing) + + # tests + add_test(NAME test_plugin_test_ndlz + COMMAND ${CMAKE_CROSSCOMPILING_EMULATOR} $) + + # Copy test files + file(GLOB TESTS_DATA ../../test_data/example_s*.caterva) + foreach (data ${TESTS_DATA}) + file(COPY ${data} + DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/) + endforeach(data) +endif() diff --git a/src/blosc2/plugins/codecs/ndlz/README.md b/src/blosc2/plugins/codecs/ndlz/README.md new file mode 100644 index 0000000..de2174f --- /dev/null +++ b/src/blosc2/plugins/codecs/ndlz/README.md @@ -0,0 +1,68 @@ +NDLZ: a multidimensional lossless codec +============================================================================= + +Given a 2-dim array or matrix, *NDLZ* is a compressor based on the Lempel-Ziv algorithm of lossless data compression. + +Plugin motivation +-------------------- + +*NDLZ* was created in order to search for patterns repetitions in multidimensional cells using the Caterva blocking machinery. + +Plugin usage +------------------- + +The codec consists of an encoder called *ndlz_compress()* to codify data and +a decoder called *ndlz_decompress()* to recover the original data. + +The parameters used by *NDLZ* are the ones specified in the *blosc2_codec* +structure of *blosc2.h*. +Furthermore, since *NDLZ* goes through dataset blocks dividing them into fixed size cells, +user must specify the parameter meta as 4 to use cells of size 4x4 or +8 to use 8x8 cells. If user tries to use other value for meta, the codec +will return an error value. + +NDLZ only works for 2-dim datasets of 1 byte items (typesize = 1), +so if you want to use it for a dataset with bigger typesize then you +must activate SHUFFLE filter and splitting mode. + +Plugin behaviour +------------------- + +This codec is meant to leverage multidimensionality for getting +better compression ratios. The idea is to look for similarities +in places that are closer in a euclidean metric, not the typical +linear one. + +First *NDLZ* goes through dataset blocks dividing them into fixed size cells. +Then, for each cell the codec searches for data coincidences with previous +cells in order to copy only references to those cells instead of copying +the full current cell. + +To understand how the compressor and decompressor work it is important to +learn about the compressed block format. An *NDLZ* compressed block is +composed of a not-compressed byte called token and some 2 bytes values +called offsets. + +The token is divided in two fields. The first field is composed of the 2 first bits of the token and gives important +information about the cells and rows couples matches. +The high-bit of the field is activated when there exists repeated information (there are matches) and there exists offset. +If it is activated, the other field indicates special patterns of matches, and if not we have to look at the second bit. +If it is activated (token = 01000000), this means that the whole cell is composed of the same element, and if not +(token = 00000000) there is not repeated information and the whole cell is literally copied. + +The offsets are references to previous literal copies that match with the +data that is being evaluated at the moment. + +Otherwise, it is important to know that there exist different hash tables which store the references to the literal copies of cells and rows in their hash position. + +Advantages and disadvantages +------------------------------ + +The main advantage of *NDLZ* in front of most of the codecs is that this one +considers dataset multidimensionality and takes advantage of it instead of +processing all data as serial. + +The main disadvantage of *NDLZ* is that it is only useful for 2-dim datasets +and at the moment other more developed +codecs that do not consider multidimensionality obtain better results +(times and ratios) for 2-dim datasets. diff --git a/src/blosc2/plugins/codecs/ndlz/ndlz-private.h b/src/blosc2/plugins/codecs/ndlz/ndlz-private.h new file mode 100644 index 0000000..4c70a4b --- /dev/null +++ b/src/blosc2/plugins/codecs/ndlz/ndlz-private.h @@ -0,0 +1,34 @@ +/********************************************************************* + Blosc - Blocked Shuffling and Compression Library + + Copyright (C) 2021 The Blosc Developers + https://blosc.org + License: BSD 3-Clause (see LICENSE.txt) + + See LICENSE.txt for details about copyright and rights to use. +**********************************************************************/ + + + +#ifndef NDLZ_PRIVATE_H +#define NDLZ_PRIVATE_H +#include "context.h" + +#if defined (__cplusplus) +extern "C" { +#endif +#define XXH_INLINE_ALL + +#define NDLZ_ERROR_NULL(pointer) \ + do { \ + if ((pointer) == NULL) { \ + return 0; \ + } \ + } while (0) + + +#if defined (__cplusplus) +} +#endif + +#endif /* NDLZ_PRIVATE_H */ diff --git a/src/blosc2/plugins/codecs/ndlz/ndlz.c b/src/blosc2/plugins/codecs/ndlz/ndlz.c new file mode 100644 index 0000000..86c7c08 --- /dev/null +++ b/src/blosc2/plugins/codecs/ndlz/ndlz.c @@ -0,0 +1,65 @@ +/********************************************************************* + Blosc - Blocked Shuffling and Compression Library + + Author: Francesc Alted + Author: Oscar Griñón + Author: Aleix Alcacer + Creation date: 2020-06-12 + + See LICENSE.txt for details about copyright and rights to use. +**********************************************************************/ + +/********************************************************************* + This codec is meant to leverage multidimensionality for getting + better compression ratios. The idea is to look for similarities + in places that are closer in a euclidean metric, not the typical + linear one. +**********************************************************************/ + + +#include +#include "ndlz.h" +#include "ndlz-private.h" +#include "ndlz4x4.h" +#include "ndlz8x8.h" + +int ndlz_compress(const uint8_t *input, int32_t input_len, uint8_t *output, int32_t output_len, + uint8_t meta, blosc2_cparams *cparams, const void* chunk) { + NDLZ_ERROR_NULL(input); + NDLZ_ERROR_NULL(output); + NDLZ_ERROR_NULL(cparams); + BLOSC_UNUSED_PARAM(chunk); + + switch (meta) { + case 4: + return ndlz4_compress(input, input_len, output, output_len, meta, cparams); + case 8: + return ndlz8_compress(input, input_len, output, output_len, meta, cparams); + default: + printf("\n NDLZ is not available for this cellsize \n"); + return 0; + } +} + +int ndlz_decompress(const uint8_t *input, int32_t input_len, uint8_t *output, int32_t output_len, + uint8_t meta, blosc2_dparams *dparams, const void* chunk) { + NDLZ_ERROR_NULL(input); + NDLZ_ERROR_NULL(output); + NDLZ_ERROR_NULL(dparams); + BLOSC_UNUSED_PARAM(chunk); + + switch (meta) { + case 4: + return ndlz4_decompress(input, input_len, output, output_len, meta, dparams); + case 8: + return ndlz8_decompress(input, input_len, output, output_len, meta, dparams); + default: + printf("\n NDLZ is not available for this cellsize \n"); + return 0; + } +} + + + + + diff --git a/src/blosc2/plugins/codecs/ndlz/ndlz.h b/src/blosc2/plugins/codecs/ndlz/ndlz.h new file mode 100644 index 0000000..a9934d8 --- /dev/null +++ b/src/blosc2/plugins/codecs/ndlz/ndlz.h @@ -0,0 +1,31 @@ +/********************************************************************* + Blosc - Blocked Shuffling and Compression Library + + Copyright (C) 2021 The Blosc Developers + https://blosc.org + License: BSD 3-Clause (see LICENSE.txt) + + See LICENSE.txt for details about copyright and rights to use. +**********************************************************************/ + + + +#ifndef NDLZ_H +#define NDLZ_H +#include "context.h" + +#if defined (__cplusplus) +extern "C" { +#endif + +int ndlz_compress(const uint8_t *input, int32_t input_len, uint8_t *output, int32_t output_len, + uint8_t meta, blosc2_cparams *cparams, const void* chunk); + +int ndlz_decompress(const uint8_t *input, int32_t input_len, uint8_t *output, int32_t output_len, + uint8_t meta, blosc2_dparams *dparams, const void* chunk); + +#if defined (__cplusplus) +} +#endif + +#endif /* NDLZ_H */ diff --git a/src/blosc2/plugins/codecs/ndlz/ndlz4x4.c b/src/blosc2/plugins/codecs/ndlz/ndlz4x4.c new file mode 100644 index 0000000..e75c61d --- /dev/null +++ b/src/blosc2/plugins/codecs/ndlz/ndlz4x4.c @@ -0,0 +1,701 @@ +/********************************************************************* + Blosc - Blocked Shuffling and Compression Library + + Author: Francesc Alted + Author: Oscar Griñón + Author: Aleix Alcacer + Creation date: 2020-06-12 + + See LICENSE.txt for details about copyright and rights to use. +**********************************************************************/ + +/********************************************************************* + This codec is meant to leverage multidimensionality for getting + better compression ratios. The idea is to look for similarities + in places that are closer in a euclidean metric, not the typical + linear one. +**********************************************************************/ + + +#include +#include "ndlz4x4.h" +#include "ndlz.h" +#include "xxhash.h" +#include "../plugins/plugin_utils.h" + + +/* + * Give hints to the compiler for branch prediction optimization. + */ +#if defined(__GNUC__) && (__GNUC__ > 2) +#define NDLZ_EXPECT_CONDITIONAL(c) (__builtin_expect((c), 1)) +#define NDLZ_UNEXPECT_CONDITIONAL(c) (__builtin_expect((c), 0)) +#else +#define NDLZ_EXPECT_CONDITIONAL(c) (c) +#define NDLZ_UNEXPECT_CONDITIONAL(c) (c) +#endif + +/* + * Use inlined functions for supported systems. + */ +#if defined(_MSC_VER) && !defined(__cplusplus) /* Visual Studio */ +#define inline __inline /* Visual C is not C99, but supports some kind of inline */ +#endif + +#define MAX_COPY 32U +#define MAX_DISTANCE 65535 + + +#ifdef BLOSC_STRICT_ALIGN + #define NDLZ_READU16(p) ((p)[0] | (p)[1]<<8) + #define NDLZ_READU32(p) ((p)[0] | (p)[1]<<8 | (p)[2]<<16 | (p)[3]<<24) +#else + #define NDLZ_READU16(p) *((const uint16_t*)(p)) + #define NDLZ_READU32(p) *((const uint32_t*)(p)) +#endif + +#define HASH_LOG (12) + + +int ndlz4_compress(const uint8_t *input, int32_t input_len, uint8_t *output, int32_t output_len, + uint8_t meta, blosc2_cparams *cparams) { + BLOSC_UNUSED_PARAM(meta); + + int8_t ndim; + int64_t* shape = malloc(8 * sizeof(int64_t)); + int32_t* chunkshape = malloc(8 * sizeof(int32_t)); + int32_t* blockshape = malloc(8 * sizeof(int32_t)); + uint8_t* smeta; + int32_t smeta_len; + if (blosc2_meta_get(cparams->schunk, "caterva", &smeta, &smeta_len) < 0) { + printf("Blosc error"); + return -1; + } + deserialize_meta(smeta, smeta_len, &ndim, shape, chunkshape, blockshape); + free(smeta); + + if (ndim != 2) { + fprintf(stderr, "This codec only works for ndim = 2"); + return -1; + } + + if (input_len != (blockshape[0] * blockshape[1])) { + printf("Length not equal to blocksize \n"); + return -1; + } + + if (NDLZ_UNEXPECT_CONDITIONAL(output_len < (int) (1 + ndim * sizeof(int32_t)))) { + printf("Output too small \n"); + return -1; + } + + uint8_t* ip = (uint8_t *) input; + uint8_t* op = (uint8_t *) output; + uint8_t* op_limit; + uint32_t hval, hash_cell; + uint32_t hash_triple[2] = {0}; + uint32_t hash_pair[3] = {0}; + uint8_t bufarea[16]; + uint8_t* buf_cell = bufarea; + uint8_t buf_triple[12]; + uint8_t buf_pair[8]; + uint8_t* buf_aux; + uint32_t tab_cell[1U << 12U] = {0}; + uint32_t tab_triple[1U << 12U] = {0}; + uint32_t tab_pair[1U << 12U] = {0}; + uint32_t update_triple[2] = {0}; + uint32_t update_pair[3] = {0}; + + // Minimum cratios before issuing and _early giveup_ + // Remind that ndlz is not meant for cratios <= 2 (too costly to decompress) + + op_limit = op + output_len; + + // Initialize the hash table to distances of 0 + for (unsigned i = 0; i < (1U << 12U); i++) { + tab_cell[i] = 0; + } + + /* input and output buffer cannot be less than 16 and 66 bytes or we can get into trouble */ + int overhead = 17 + (blockshape[0] * blockshape[1] / 16 - 1) * 2; + if (input_len < 16 || output_len < overhead) { + printf("Incorrect length or maxout"); + return 0; + } + + uint8_t* obase = op; + + /* we start with literal copy */ + *op++ = ndim; + memcpy(op, &blockshape[0], 4); + op += 4; + memcpy(op, &blockshape[1], 4); + op += 4; + + uint32_t i_stop[2]; + for (int i = 0; i < 2; ++i) { + i_stop[i] = (blockshape[i] + 3) / 4; + } + + /* main loop */ + uint32_t padding[2]; + uint32_t ii[2]; + for (ii[0] = 0; ii[0] < i_stop[0]; ++ii[0]) { + for (ii[1] = 0; ii[1] < i_stop[1]; ++ii[1]) { // for each cell + uint8_t token; + for (int h = 0; h < 2; h++) { // new cell -> new possible references + update_triple[h] = 0; + update_pair[h] = 0; + } + update_pair[2] = 0; + + if (NDLZ_UNEXPECT_CONDITIONAL(op + 16 + 1 > op_limit)) { + free(shape); + free(chunkshape); + free(blockshape); + return 0; + } + + uint32_t orig = ii[0] * 4 * blockshape[1] + ii[1] * 4; + if (((blockshape[0] % 4 != 0) && (ii[0] == i_stop[0] - 1)) || ((blockshape[1] % 4 != 0) && (ii[1] == i_stop[1] - 1))) { + token = 0; // padding -> literal copy + *op++ = token; + if (ii[0] == i_stop[0] - 1) { + padding[0] = (blockshape[0] % 4 == 0) ? 4 : blockshape[0] % 4; + } else { + padding[0] = 4; + } + if (ii[1] == i_stop[1] - 1) { + padding[1] = (blockshape[1] % 4 == 0) ? 4 : blockshape[1] % 4; + } else { + padding[1] = 4; + } + for (uint32_t i = 0; i < padding[0]; i++) { + memcpy(op, &ip[orig + i * blockshape[1]], padding[1]); + op += padding[1]; + } + } + else { + for (uint64_t i = 0; i < 4; i++) { // fill cell buffer + uint64_t ind = orig + i * blockshape[1]; + memcpy(buf_cell, &ip[ind], 4); + buf_cell += 4; + } + buf_cell -= 16; + + const uint8_t* ref; + uint32_t distance; + uint8_t* anchor = op; /* comparison starting-point */ + + /* find potential match */ + hash_cell = XXH32(buf_cell, 16, 1); // calculate cell hash + hash_cell >>= 32U - 12U; + ref = obase + tab_cell[hash_cell]; + + /* calculate distance to the match */ + if (tab_cell[hash_cell] == 0) { + distance = 0; + } else { + bool same = true; + buf_aux = obase + tab_cell[hash_cell]; + for(int i = 0; i < 16; i++){ + if (buf_cell[i] != buf_aux[i]) { + same = false; + break; + } + } + if (same) { + distance = (int32_t) (anchor - ref); + } else { + distance = 0; + } + } + + bool alleq = true; + for (int i = 1; i < 16; i++) { + if (buf_cell[i] != buf_cell[0]) { + alleq = false; + break; + } + } + if (alleq) { // all elements of the cell equal + token = (uint8_t) (1U << 6U); + *op++ = token; + *op++ = buf_cell[0]; + + } else if (distance == 0 || (distance >= MAX_DISTANCE)) { // no cell match + bool literal = true; + + // 2 rows pairs matches + for (int j = 1; j < 4; j++) { + memcpy(buf_pair, buf_cell, 4); + memcpy(&buf_pair[4], &buf_cell[j * 4], 4); + hval = XXH32(buf_pair, 8, 1); // calculate rows pair hash + hval >>= 32U - 12U; + ref = obase + tab_pair[hval]; + /* calculate distance to the match */ + bool same = true; + uint16_t offset; + if (tab_pair[hval] != 0) { + buf_aux = obase + tab_pair[hval]; + for (int k = 0; k < 8; k++) { + if (buf_pair[k] != buf_aux[k]) { + same = false; + break; + } + } + offset = (uint16_t) (anchor - obase - tab_pair[hval]); + } else { + same = false; + } + if (same) { + distance = (int32_t) (anchor - ref); + } else { + distance = 0; + } + if ((distance != 0) && (distance < MAX_DISTANCE)) { /* rows pair match */ + int k, m, l = -1; + for (k = 1; k < 4; k++) { + if (k != j) { + if (l == -1) { + l = k; + } else { + m = k; + } + } + } + memcpy(buf_pair, &buf_cell[l * 4], 4); + memcpy(&buf_pair[4], &buf_cell[m * 4], 4); + hval = XXH32(buf_pair, 8, 1); // calculate rows pair hash + hval >>= 32U - 12U; + ref = obase + tab_pair[hval]; + same = true; + if (tab_pair[hval] != 0) { + buf_aux = obase + tab_pair[hval]; + for (k = 0; k < 8; k++) { + if (buf_pair[k] != buf_aux[k]) { + same = false; + break; + } + } + } else { + same = false; + } + if (same) { + distance = (int32_t) (anchor + l * 4 - ref); + } else { + distance = 0; + } + if ((distance != 0) && (distance < MAX_DISTANCE)) { /* 2 pair matches */ + literal = false; + token = (uint8_t) ((1U << 5U) | (j << 3U)); + *op++ = token; + uint16_t offset_2 = (uint16_t) (anchor - obase - tab_pair[hval]); + *(uint16_t *) op = offset; + op += sizeof(offset); + *(uint16_t *) op = offset_2; + op += sizeof(offset_2); + goto match; + } + } + } + + // rows triples + for(int i = 0; i < 2; i++) { + memcpy(buf_triple, &buf_cell[i * 4], 4); + for (int j = i + 1; j < 3; j++) { + memcpy(&buf_triple[4], &buf_cell[j * 4], 4); + for (int k = j + 1; k < 4; k++) { + memcpy(&buf_triple[8], &buf_cell[k * 4], 4); + hval = XXH32(buf_triple, 12, 1); // calculate triple hash + hval >>= 32U - 12U; + /* calculate distance to the match */ + bool same = true; + uint16_t offset; + if (tab_triple[hval] != 0) { + buf_aux = obase + tab_triple[hval]; + for (int l = 0; l < 12; l++) { + if (buf_triple[l] != buf_aux[l]) { + same = false; + break; + } + } + offset = (uint16_t) (anchor - obase - tab_triple[hval]); + } else { + same = false; + if ((j - i == 1) && (k - j == 1)) { + update_triple[i] = (uint32_t) (anchor + 1 + i * 4 - obase); /* update hash table */ + hash_triple[i] = hval; + } + } + ref = obase + tab_triple[hval]; + + if (same) { + distance = (int32_t) (anchor + i * 4 - ref); + } else { + distance = 0; + } + if ((distance != 0) && (distance < MAX_DISTANCE)) { + literal = false; + if (i == 1) { + token = (uint8_t) (7U << 5U); + } else { + token = (uint8_t) ((7U << 5U) | ((j + k - 2) << 3U)); + } + *op++ = token; + memcpy(op, &offset, 2); + op += 2; + for (int l = 0; l < 4; l++) { + if ((l != i) && (l != j) && (l != k)) { + memcpy(op, &buf_cell[4 * l], 4); + op += 4; + goto match; + } + } + } + } + } + } + + // rows pairs + for(int i = 0; i < 3; i++) { + memcpy(buf_pair, &buf_cell[i * 4], 4); + for (int j = i + 1; j < 4; j++) { + memcpy(&buf_pair[4], &buf_cell[j * 4], 4); + hval = XXH32(buf_pair, 8, 1); // calculate rows pair hash + hval >>= 32U - 12U; + ref = obase + tab_pair[hval]; + /* calculate distance to the match */ + bool same = true; + uint16_t offset; + if (tab_pair[hval] != 0) { + buf_aux = obase + tab_pair[hval]; + for(int k = 0; k < 8; k++){ + if(buf_pair[k] != buf_aux[k]) { + same = false; + break; + } + } + offset = (uint16_t) (anchor - obase - tab_pair[hval]); + } else { + same = false; + if (j - i == 1) { + update_pair[i] = (uint32_t) (anchor + 1 + i * 4 - obase); /* update hash table */ + hash_pair[i] = hval; + } + } + if (same) { + distance = (int32_t) (anchor + i * 4 - ref); + } else { + distance = 0; + } + if ((distance != 0) && (distance < MAX_DISTANCE)) { /* rows pair match */ + literal = false; + if (i == 2) { + token = (uint8_t) (1U << 7U); + } else { + token = (uint8_t) ((1U << 7U) | (i << 5U) | (j << 3U)); + } + *op++ = token; + memcpy(op, &offset, 2); + op += 2; + for (int k = 0; k < 4; k++) { + if ((k != i) && (k != j)) { + memcpy(op, &buf_cell[4 * k], 4); + op += 4; + } + } + goto match; + } + } + } + + match: + if (literal) { + tab_cell[hash_cell] = (uint32_t) (anchor + 1 - obase); /* update hash tables */ + if (update_triple[0] != 0) { + for (int h = 0; h < 2; h++) { + tab_triple[hash_triple[h]] = update_triple[h]; + } + } + if (update_pair[0] != 0) { + for (int h = 0; h < 3; h++) { + tab_pair[hash_pair[h]] = update_pair[h]; + } + } + token = 0; + *op++ = token; + memcpy(op, buf_cell, 16); + op += 16; + } + + } else { // cell match + token = (uint8_t )((1U << 7U) | (1U << 6U)); + *op++ = token; + uint16_t offset = (uint16_t) (anchor - obase - tab_cell[hash_cell]); + memcpy(op, &offset, 2); + op += 2; + } + + } + if((op - obase) > input_len) { + printf("Compressed data is bigger than input! \n"); + return 0; + } + } + } + + free(shape); + free(chunkshape); + free(blockshape); + + return (int)(op - obase); +} + + +// See https://habr.com/en/company/yandex/blog/457612/ +#ifdef __AVX2__ + +#if defined(_MSC_VER) +#define ALIGNED_(x) __declspec(align(x)) +#else +#if defined(__GNUC__) +#define ALIGNED_(x) __attribute__ ((aligned(x))) +#endif +#endif +#define ALIGNED_TYPE_(t, x) t ALIGNED_(x) + +static unsigned char* copy_match_16(unsigned char *op, const unsigned char *match, int32_t len) +{ + size_t offset = op - match; + while (len >= 16) { + + static const ALIGNED_TYPE_(uint8_t, 16) masks[] = + { + 0, 1, 2, 1, 4, 1, 4, 2, 8, 7, 6, 5, 4, 3, 2, 1, // offset = 0, not used as mask, but for shift + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // offset = 1 + 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, + 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, + 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, + 0, 1, 2, 3, 4, 0, 1, 2, 3, 4, 0, 1, 2, 3, 4, 0, + 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, + 0, 1, 2, 3, 4, 5, 6, 0, 1, 2, 3, 4, 5, 6, 0, 1, + 0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 0, 1, 2, 3, 4, 5, 6, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 0, 1, 2, 3, 4, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 0, 1, 2, 3, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 0, 1, 2, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 0, 1, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 0, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, // offset = 16 + }; + + _mm_storeu_si128((__m128i *)(op), + _mm_shuffle_epi8(_mm_loadu_si128((const __m128i *)(match)), + _mm_load_si128((const __m128i *)(masks) + offset))); + + match += masks[offset]; + + op += 16; + len -= 16; + } + // Deal with remainders + for (; len > 0; len--) { + *op++ = *match++; + } + return op; +} +#endif + + +int ndlz4_decompress(const uint8_t *input, int32_t input_len, uint8_t *output, int32_t output_len, + uint8_t meta, blosc2_dparams *dparams) { + BLOSC_UNUSED_PARAM(meta); + BLOSC_UNUSED_PARAM(dparams); + + uint8_t* ip = (uint8_t*)input; + uint8_t* ip_limit = ip + input_len; + uint8_t* op = (uint8_t*)output; + uint8_t ndim; + uint32_t blockshape[2]; + uint32_t eshape[2]; + uint8_t* buffercpy; + uint8_t local_buffer[16]; + uint8_t token; + if (NDLZ_UNEXPECT_CONDITIONAL(input_len < 8)) { + return 0; + } + + /* we start with literal copy */ + ndim = *ip; + ip ++; + if (ndim != 2) { + fprintf(stderr, "This codec only works for ndim = 2"); + return -1; + } + memcpy(&blockshape[0], ip, 4); + ip += 4; + memcpy(&blockshape[1], ip, 4); + ip += 4; + eshape[0] = ((blockshape[0] + 3) / 4) * 4; + eshape[1] = ((blockshape[1] + 3) / 4) * 4; + + if (NDLZ_UNEXPECT_CONDITIONAL(output_len < (int32_t)(blockshape[0] * blockshape[1]))) { + return 0; + } + memset(op, 0, blockshape[0] * blockshape[1]); + + uint32_t i_stop[2]; + for (int i = 0; i < 2; ++i) { + i_stop[i] = eshape[i] / 4; + } + + + /* main loop */ + uint32_t ii[2]; + uint32_t padding[2] = {0}; + uint32_t ind = 0; + uint8_t cell_aux[16]; + for (ii[0] = 0; ii[0] < i_stop[0]; ++ii[0]) { + for (ii[1] = 0; ii[1] < i_stop[1]; ++ii[1]) { // for each cell + if (NDLZ_UNEXPECT_CONDITIONAL(ip > ip_limit)) { + printf("Literal copy \n"); + return 0; + } + if (ii[0] == i_stop[0] - 1) { + padding[0] = (blockshape[0] % 4 == 0) ? 4 : blockshape[0] % 4; + } else { + padding[0] = 4; + } + if (ii[1] == i_stop[1] - 1) { + padding[1] = (blockshape[1] % 4 == 0) ? 4 : blockshape[1] % 4; + } else { + padding[1] = 4; + } + token = *ip++; + if (token == 0){ // no match + buffercpy = ip; + ip += padding[0] * padding[1]; + } else if (token == (uint8_t)((1U << 7U) | (1U << 6U))) { // cell match + uint16_t offset = *((uint16_t*) ip); + buffercpy = ip - offset - 1; + ip += 2; + } else if (token == (uint8_t)(1U << 6U)) { // whole cell of same element + buffercpy = cell_aux; + memset(buffercpy, *ip, 16); + ip++; + } else if (token >= 224) { // three rows match + buffercpy = local_buffer; + uint16_t offset = *((uint16_t*) ip); + offset += 3; + ip += 2; + int i, j, k; + if ((token >> 3U) == 28) { + i = 1; + j = 2; + k = 3; + } else { + i = 0; + if ((token >> 3U) < 30) { + j = 1; + k = 2; + } else { + k = 3; + if ((token >> 3U) == 30) { + j = 1; + } else { + j = 2; + } + } + } + memcpy(&buffercpy[i * 4], ip - offset, 4); + memcpy(&buffercpy[j * 4], ip - offset + 4, 4); + memcpy(&buffercpy[k * 4], ip - offset + 8, 4); + for (int l = 0; l < 4; l++) { + if ((l != i) && (l != j) && (l != k)) { + memcpy(&buffercpy[l * 4], ip, 4); + ip += 4; + break; + } + } + + } else if ((token >= 128) && (token <= 191)){ // rows pair match + buffercpy = local_buffer; + uint16_t offset = *((uint16_t*) ip); + offset += 3; + ip += 2; + int i, j; + if (token == 128) { + i = 2; + j = 3; + } else { + i = (token - 128) >> 5U; + j = ((token - 128) >> 3U) - (i << 2U); + } + memcpy(&buffercpy[i * 4], ip - offset, 4); + memcpy(&buffercpy[j * 4], ip - offset + 4, 4); + for (int k = 0; k < 4; k++) { + if ((k != i) && (k != j)) { + memcpy(&buffercpy[k * 4], ip, 4); + ip += 4; + } + } + } else if ((token >= 40) && (token <= 63)) { // 2 rows pair matches + buffercpy = local_buffer; + uint16_t offset_1 = *((uint16_t*) ip); + offset_1 += 5; + ip += 2; + uint16_t offset_2 = *((uint16_t*) ip); + offset_2 += 5; + ip += 2; + int i, j, k, l, m; + i = 0; + j = ((token - 32) >> 3U); + l = -1; + for (k = 1; k < 4; k++) { + if ((k != i) && (k != j)) { + if (l == -1) { + l = k; + } else { + m = k; + } + } + } + memcpy(&buffercpy[i * 4], ip - offset_1, 4); + memcpy(&buffercpy[j * 4], ip - offset_1 + 4, 4); + memcpy(&buffercpy[l * 4], ip - offset_2, 4); + memcpy(&buffercpy[m * 4], ip - offset_2 + 4, 4); + + } else { + printf("Invalid token: %u at cell [%d, %d]\n", token, ii[0], ii[1]); + return 0; + } + // fill op with buffercpy + uint32_t orig = ii[0] * 4 * blockshape[1] + ii[1] * 4; + for (uint32_t i = 0; i < 4; i++) { + if (i < padding[0]) { + ind = orig + i * blockshape[1]; + memcpy(&op[ind], buffercpy, padding[1]); + } + buffercpy += padding[1]; + } + if (ind > (uint32_t) output_len) { + printf("Output size is bigger than max \n"); + return 0; + } + } + } + ind += padding[1]; + + if (ind != (blockshape[0] * blockshape[1])) { + printf("Output size is not compatible with embedded blockshape \n"); + return 0; + } + if (ind > (uint32_t) output_len) { + printf("Output size is bigger than max \n"); + return 0; + } + + return (int)ind; +} diff --git a/src/blosc2/plugins/codecs/ndlz/ndlz4x4.h b/src/blosc2/plugins/codecs/ndlz/ndlz4x4.h new file mode 100644 index 0000000..7a477d5 --- /dev/null +++ b/src/blosc2/plugins/codecs/ndlz/ndlz4x4.h @@ -0,0 +1,72 @@ +/********************************************************************* + Blosc - Blocked Shuffling and Compression Library + + Copyright (C) 2021 The Blosc Developers + https://blosc.org + License: BSD 3-Clause (see LICENSE.txt) + + See LICENSE.txt for details about copyright and rights to use. +**********************************************************************/ + + + +#ifndef NDLZ4_H +#define NDLZ4_H +#include "context.h" + +#if defined (__cplusplus) +extern "C" { +#endif +#include "ndlz.h" +#include "ndlz-private.h" +/* +#include +#include "blosc2/blosc2-common.h" +#include "fastcopy.h" +*/ + + + +/** + Compress a block of data in the input buffer and returns the size of + compressed block. The size of input buffer is specified by + length. The minimum input buffer size is 16. + + The output buffer must be at least 5% larger than the input buffer + and can not be smaller than 66 bytes. + + If the input is not compressible, or output does not fit in maxout + bytes, the return value will be 0 and you will have to discard the + output buffer. + + The acceleration parameter is related with the frequency for + updating the internal hash. An acceleration of 1 means that the + internal hash is updated at full rate. A value < 1 is not allowed + and will be silently set to 1. + + The input buffer and the output buffer can not overlap. +*/ + +int ndlz4_compress(const uint8_t *input, int32_t input_len, uint8_t *output, int32_t output_len, + uint8_t meta, blosc2_cparams *cparams); + +/** + Decompress a block of compressed data and returns the size of the + decompressed block. If error occurs, e.g. the compressed data is + corrupted or the output buffer is not large enough, then 0 (zero) + will be returned instead. + + The input buffer and the output buffer can not overlap. + + Decompression is memory safe and guaranteed not to write the output buffer + more than what is specified in maxout. + */ + +int ndlz4_decompress(const uint8_t *input, int32_t input_len, uint8_t *output, int32_t output_len, + uint8_t meta, blosc2_dparams *dparams); + +#if defined (__cplusplus) +} +#endif + +#endif /* NDLZ4_H */ diff --git a/src/blosc2/plugins/codecs/ndlz/ndlz8x8.c b/src/blosc2/plugins/codecs/ndlz/ndlz8x8.c new file mode 100644 index 0000000..06d0e86 --- /dev/null +++ b/src/blosc2/plugins/codecs/ndlz/ndlz8x8.c @@ -0,0 +1,578 @@ +/********************************************************************* + Blosc - Blocked Shuffling and Compression Library + + Author: Francesc Alted + Author: Oscar Griñón + Author: Aleix Alcacer + Creation date: 2020-06-12 + + See LICENSE.txt for details about copyright and rights to use. +**********************************************************************/ + +/********************************************************************* + This codec is meant to leverage multidimensionality for getting + better compression ratios. The idea is to look for similarities + in places that are closer in a euclidean metric, not the typical + linear one. +**********************************************************************/ + + +#include +#include "ndlz8x8.h" +#include "ndlz.h" +#include "xxhash.h" +#include "../plugins/plugin_utils.h" + + +/* + * Give hints to the compiler for branch prediction optimization. + */ +#if defined(__GNUC__) && (__GNUC__ > 2) +#define NDLZ_EXPECT_CONDITIONAL(c) (__builtin_expect((c), 1)) +#define NDLZ_UNEXPECT_CONDITIONAL(c) (__builtin_expect((c), 0)) +#else +#define NDLZ_EXPECT_CONDITIONAL(c) (c) +#define NDLZ_UNEXPECT_CONDITIONAL(c) (c) +#endif + +/* + * Use inlined functions for supported systems. + */ +#if defined(_MSC_VER) && !defined(__cplusplus) /* Visual Studio */ +#define inline __inline /* Visual C is not C99, but supports some kind of inline */ +#endif + +#define MAX_COPY 32U +#define MAX_DISTANCE 65535 + + +#ifdef BLOSC_STRICT_ALIGN + #define NDLZ_READU16(p) ((p)[0] | (p)[1]<<8) + #define NDLZ_READU32(p) ((p)[0] | (p)[1]<<8 | (p)[2]<<16 | (p)[3]<<24) +#else + #define NDLZ_READU16(p) *((const uint16_t*)(p)) + #define NDLZ_READU32(p) *((const uint32_t*)(p)) +#endif + +#define HASH_LOG (12) + + +int ndlz8_compress(const uint8_t *input, int32_t input_len, uint8_t *output, int32_t output_len, + uint8_t meta, blosc2_cparams *cparams) { + BLOSC_UNUSED_PARAM(meta); + + const int cell_shape = 8; + const int cell_size = 64; + int8_t ndim; + int64_t* shape = malloc(8 * sizeof(int64_t)); + int32_t* chunkshape = malloc(8 * sizeof(int32_t)); + int32_t* blockshape = malloc(8 * sizeof(int32_t)); + uint8_t* smeta; + int32_t smeta_len; + if (blosc2_meta_get(cparams->schunk, "caterva", &smeta, &smeta_len) < 0) { + printf("Blosc error"); + return 0; + } + deserialize_meta(smeta, smeta_len, &ndim, shape, chunkshape, blockshape); + free(smeta); + + if (ndim != 2) { + fprintf(stderr, "This codec only works for ndim = 2"); + return -1; + } + + if (input_len != (blockshape[0] * blockshape[1])) { + printf("Length not equal to blocksize \n"); + return -1; + } + + if (NDLZ_UNEXPECT_CONDITIONAL(output_len < (int) (1 + ndim * sizeof(int32_t)))) { + printf("Output too small \n"); + return -1; + } + + uint8_t* ip = (uint8_t *) input; + uint8_t* op = (uint8_t *) output; + uint8_t* op_limit; + uint32_t hval, hash_cell; + uint32_t hash_triple[6] = {0}; + uint32_t hash_pair[7] = {0}; + uint8_t* bufarea = malloc(cell_size); + uint8_t* buf_cell = bufarea; + uint8_t* buf_aux; + uint32_t tab_cell[1U << 12U] = {0}; + uint32_t tab_triple[1U << 12U] = {0}; + uint32_t tab_pair[1U << 12U] = {0}; + uint32_t update_triple[6] = {0}; + uint32_t update_pair[7] = {0}; + + // Minimum cratios before issuing and _early giveup_ + // Remind that ndlz is not meant for cratios <= 2 (too costly to decompress) + + op_limit = op + output_len; + + // Initialize the hash table to distances of 0 + for (unsigned i = 0; i < (1U << 12U); i++) { + tab_cell[i] = 0; + tab_triple[i] = 0; + tab_pair[i] = 0; + } + + /* input and output buffer cannot be less than 64 (cells are 8x8) */ + int overhead = 17 + (blockshape[0] * blockshape[1] / cell_size - 1) * 2; + if (input_len < cell_size || output_len < overhead) { + printf("Incorrect length or maxout"); + return 0; + } + + uint8_t* obase = op; + + /* we start with literal copy */ + *op++ = ndim; + memcpy(op, &blockshape[0], 4); + op += 4; + memcpy(op, &blockshape[1], 4); + op += 4; + + uint32_t i_stop[2]; + for (int i = 0; i < 2; ++i) { + i_stop[i] = (blockshape[i] + cell_shape - 1) / cell_shape; + } + + + /* main loop */ + uint32_t padding[2]; + uint32_t ii[2]; + for (ii[0] = 0; ii[0] < i_stop[0]; ++ii[0]) { + for (ii[1] = 0; ii[1] < i_stop[1]; ++ii[1]) { // for each cell + for (int h = 0; h < 7; h++) { // new cell -> new possible references + update_pair[h] = 0; + if (h != 6) { + update_triple[h] = 0; + } + } + + if (NDLZ_UNEXPECT_CONDITIONAL(op + cell_size + 1 > op_limit)) { + free(shape); + free(chunkshape); + free(blockshape); + free(bufarea); + return 0; + } + + uint32_t orig = ii[0] * cell_shape * blockshape[1] + ii[1] * cell_shape; + if (((blockshape[0] % cell_shape != 0) && (ii[0] == i_stop[0] - 1)) || + ((blockshape[1] % cell_shape != 0) && (ii[1] == i_stop[1] - 1))) { + uint8_t token = 0; // padding -> literal copy + *op++ = token; + if (ii[0] == i_stop[0] - 1) { + padding[0] = (blockshape[0] % cell_shape == 0) ? cell_shape : blockshape[0] % cell_shape; + } else { + padding[0] = cell_shape; + } + if (ii[1] == i_stop[1] - 1) { + padding[1] = (blockshape[1] % cell_shape == 0) ? cell_shape : blockshape[1] % cell_shape; + } else { + padding[1] = cell_shape; + } + for (uint32_t i = 0; i < padding[0]; i++) { + memcpy(op, &ip[orig + i * blockshape[1]], padding[1]); + op += padding[1]; + } + } + else { + for (uint64_t i = 0; i < (uint64_t) cell_shape; i++) { // fill cell buffer + uint64_t ind = orig + i * blockshape[1]; + memcpy(buf_cell, &ip[ind], cell_shape); + buf_cell += cell_shape; + } + buf_cell -= cell_size; + + const uint8_t* ref; + uint32_t distance; + uint8_t* anchor = op; /* comparison starting-point */ + + /* find potential match */ + hash_cell = XXH32(buf_cell, cell_size, 1); // calculate cell hash + hash_cell >>= 32U - 12U; + ref = obase + tab_cell[hash_cell]; + + /* calculate distance to the match */ + if (tab_cell[hash_cell] == 0) { + distance = 0; + } else { + bool same = true; + buf_aux = obase + tab_cell[hash_cell]; + for(int i = 0; i < cell_size; i++){ + if (buf_cell[i] != buf_aux[i]) { + same = false; + break; + } + } + if (same) { + distance = (int32_t) (anchor - ref); + } else { + distance = 0; + } + } + + bool alleq = true; + for (int i = 1; i < cell_size; i++) { + if (buf_cell[i] != buf_cell[0]) { + alleq = false; + break; + } + } + if (alleq) { // all elements of the cell equal + uint8_t token = (uint8_t) (1U << 6U); + *op++ = token; + *op++ = buf_cell[0]; + + } else if (distance == 0 || (distance >= MAX_DISTANCE)) { // no cell match + bool literal = true; + + // rows triples matches + for (int i = 0; i < 6; i++) { + int triple_start = i * cell_shape; + hval = XXH32(&buf_cell[triple_start], 24, 1); // calculate triple hash + hval >>= 32U - 12U; + /* calculate distance to the match */ + bool same = true; + uint16_t offset; + if (tab_triple[hval] != 0) { + buf_aux = obase + tab_triple[hval]; + for (int l = 0; l < 24; l++) { + if (buf_cell[triple_start + l] != buf_aux[l]) { + same = false; + break; + } + } + offset = (uint16_t) (anchor - obase - tab_triple[hval]); + } else { + same = false; + update_triple[i] = (uint32_t) (anchor + 1 + triple_start - obase); /* update hash table */ + hash_triple[i] = hval; + } + ref = obase + tab_triple[hval]; + if (same) { + distance = (int32_t) (anchor + triple_start - ref); + } else { + distance = 0; + } + if ((distance != 0) && (distance < MAX_DISTANCE)) { // 3 rows match + literal = false; + uint8_t token = (uint8_t) ((21 << 3U) | i); + *op++ = token; + memcpy(op, &offset, 2); + op += 2; + for (int l = 0; l < 8; l++) { + if ((l < i) || (l > i + 2)) { + memcpy(op, &buf_cell[l * cell_shape], cell_shape); + op += cell_shape; + } + } + goto match; + } + } + + // rows pairs matches + for (int i = 0; i < 7; i++) { + int pair_start = i * cell_shape; + hval = XXH32(&buf_cell[pair_start], 16, 1); // calculate rows pair hash + hval >>= 32U - 12U; + ref = obase + tab_pair[hval]; + /* calculate distance to the match */ + bool same = true; + uint16_t offset; + if (tab_pair[hval] != 0) { + buf_aux = obase + tab_pair[hval]; + for (int k = 0; k < 16; k++) { + if (buf_cell[pair_start + k] != buf_aux[k]) { + same = false; + break; + } + } + offset = (uint16_t) (anchor - obase - tab_pair[hval]); + } else { + same = false; + update_pair[i] = (uint32_t) (anchor + 1 + pair_start - obase); /* update hash table */ + hash_pair[i] = hval; + } + if (same) { + distance = (int32_t) (anchor + pair_start - ref); + } else { + distance = 0; + } + if ((distance != 0) && (distance < MAX_DISTANCE)) { /* 1 rows pair match */ + literal = false; + uint8_t token = (uint8_t) ((17 << 3U) | i); + *op++ = token; + offset = (uint16_t) (anchor - obase - tab_pair[hval]); + memcpy(op, &offset, 2); + op += 2; + for (int l = 0; l < 8; l++) { + if ((l < i) || (l > i + 1)) { + memcpy(op, &buf_cell[l * cell_shape], cell_shape); + op += cell_shape; + } + } + goto match; + } + } + + match: + if (literal) { + tab_cell[hash_cell] = (uint32_t) (anchor + 1 - obase); /* update hash tables */ + + if (update_triple[0] != 0) { + for (int h = 0; h < 6; h++) { + tab_triple[hash_triple[h]] = update_triple[h]; + } + } + if (update_pair[0] != 0) { + for (int h = 0; h < 7; h++) { + tab_pair[hash_pair[h]] = update_pair[h]; + } + } + uint8_t token = 0; + *op++ = token; + memcpy(op, buf_cell, cell_size); + op += cell_size; + + } + + } else { // cell match + uint8_t token = (uint8_t)((1U << 7U) | (1U << 6U)); + *op++ = token; + uint16_t offset = (uint16_t) (anchor - obase - tab_cell[hash_cell]); + memcpy(op, &offset, 2); + op += 2; + + } + + } + if((op - obase) > input_len) { + free(shape); + free(chunkshape); + free(blockshape); + free(bufarea); + return 0; + } + } + } + + free(shape); + free(chunkshape); + free(blockshape); + free(bufarea); + + return (int)(op - obase); +} + + +// See https://habr.com/en/company/yandex/blog/457612/ +#ifdef __AVX2__ + +#if defined(_MSC_VER) +#define ALIGNED_(x) __declspec(align(x)) +#else +#if defined(__GNUC__) +#define ALIGNED_(x) __attribute__ ((aligned(x))) +#endif +#endif +#define ALIGNED_TYPE_(t, x) t ALIGNED_(x) + +static unsigned char* copy_match_16(unsigned char *op, const unsigned char *match, int32_t len) +{ + size_t offset = op - match; + while (len >= 16) { + + static const ALIGNED_TYPE_(uint8_t, 16) masks[] = + { + 0, 1, 2, 1, 4, 1, 4, 2, 8, 7, 6, 5, 4, 3, 2, 1, // offset = 0, not used as mask, but for shift + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // offset = 1 + 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, + 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, + 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, + 0, 1, 2, 3, 4, 0, 1, 2, 3, 4, 0, 1, 2, 3, 4, 0, + 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, + 0, 1, 2, 3, 4, 5, 6, 0, 1, 2, 3, 4, 5, 6, 0, 1, + 0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 0, 1, 2, 3, 4, 5, 6, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 0, 1, 2, 3, 4, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 0, 1, 2, 3, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 0, 1, 2, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 0, 1, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 0, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, // offset = 16 + }; + + _mm_storeu_si128((__m128i *)(op), + _mm_shuffle_epi8(_mm_loadu_si128((const __m128i *)(match)), + _mm_load_si128((const __m128i *)(masks) + offset))); + + match += masks[offset]; + + op += 16; + len -= 16; + } + // Deal with remainders + for (; len > 0; len--) { + *op++ = *match++; + } + return op; +} +#endif + + +int ndlz8_decompress(const uint8_t *input, int32_t input_len, uint8_t *output, int32_t output_len, + uint8_t meta, blosc2_dparams *dparams) { + BLOSC_UNUSED_PARAM(meta); + BLOSC_UNUSED_PARAM(dparams); + + const int cell_shape = 8; + const int cell_size = 64; + uint8_t* ip = (uint8_t*)input; + uint8_t* ip_limit = ip + input_len; + uint8_t* op = (uint8_t*)output; + uint8_t ndim; + int32_t blockshape[2]; + int32_t eshape[2]; + uint8_t* buffercpy; + uint8_t token; + if (NDLZ_UNEXPECT_CONDITIONAL(input_len < 8)) { + return 0; + } + + /* we start with literal copy */ + ndim = *ip; + ip ++; + if (ndim != 2) { + fprintf(stderr, "This codec only works for ndim = 2"); + return -1; + } + memcpy(&blockshape[0], ip, 4); + ip += 4; + memcpy(&blockshape[1], ip, 4); + ip += 4; + eshape[0] = ((blockshape[0] + 7) / cell_shape) * cell_shape; + eshape[1] = ((blockshape[1] + 7) / cell_shape) * cell_shape; + + if (NDLZ_UNEXPECT_CONDITIONAL(output_len < blockshape[0] * blockshape[1])) { + return 0; + } + memset(op, 0, blockshape[0] * blockshape[1]); + + int32_t i_stop[2]; + for (int i = 0; i < 2; ++i) { + i_stop[i] = eshape[i] / cell_shape; + } + + + /* main loop */ + int32_t ii[2]; + int32_t padding[2] = {0}; + int32_t ind = 0; + uint8_t* local_buffer = malloc(cell_size); + uint8_t* cell_aux = malloc(cell_size); + for (ii[0] = 0; ii[0] < i_stop[0]; ++ii[0]) { + for (ii[1] = 0; ii[1] < i_stop[1]; ++ii[1]) { // for each cell + if (NDLZ_UNEXPECT_CONDITIONAL(ip > ip_limit)) { + printf("Literal copy \n"); + free(local_buffer); + free(cell_aux); + return 0; + } + if (ii[0] == i_stop[0] - 1) { + padding[0] = (blockshape[0] % cell_shape == 0) ? cell_shape : blockshape[0] % cell_shape; + } else { + padding[0] = cell_shape; + } + if (ii[1] == i_stop[1] - 1) { + padding[1] = (blockshape[1] % cell_shape == 0) ? cell_shape : blockshape[1] % cell_shape; + } else { + padding[1] = cell_shape; + } + token = *ip++; + uint8_t match_type = (token >> 3U); + if (token == 0){ // no match + buffercpy = ip; + ip += padding[0] * padding[1]; + } else if (token == (uint8_t)((1U << 7U) | (1U << 6U))) { // cell match + uint16_t offset = *((uint16_t*) ip); + buffercpy = ip - offset - 1; + ip += 2; + } else if (token == (uint8_t)(1U << 6U)) { // whole cell of same element + buffercpy = cell_aux; + memset(buffercpy, *ip, cell_size); + ip++; + } else if (match_type == 21) { // triple match + buffercpy = local_buffer; + int row = (int) (token & 7); + uint16_t offset = *((uint16_t*) ip); + ip += 2; + for (int l = 0; l < 3; l++) { + memcpy(&buffercpy[(row + l) * cell_shape], + ip - sizeof(token) - sizeof(offset) - offset + l * cell_shape, cell_shape); + } + for (int l = 0; l < cell_shape; l++) { + if ((l < row) || (l > row + 2)) { + memcpy(&buffercpy[l * cell_shape], ip, cell_shape); + ip += cell_shape; + } + } + } else if (match_type == 17) { // pair match + buffercpy = local_buffer; + int row = (int) (token & 7); + uint16_t offset = *((uint16_t*) ip); + ip += 2; + for (int l = 0; l < 2; l++) { + memcpy(&buffercpy[(row + l) * cell_shape], + ip - sizeof(token) - sizeof(offset) - offset + l * cell_shape, cell_shape); + } + for (int l = 0; l < cell_shape; l++) { + if ((l < row) || (l > row + 1)) { + memcpy(&buffercpy[l * cell_shape], ip, cell_shape); + ip += cell_shape; + } + } + } else { + printf("Invalid token: %u at cell [%d, %d]\n", token, ii[0], ii[1]); + free(local_buffer); + free(cell_aux); + return 0; + } + + uint32_t orig = ii[0] * cell_shape * blockshape[1] + ii[1] * cell_shape; + for (int32_t i = 0; i < (int32_t) cell_shape; i++) { + if (i < padding[0]) { + ind = orig + i * blockshape[1]; + memcpy(&op[ind], buffercpy, padding[1]); + } + buffercpy += padding[1]; + } + if (ind > output_len) { + printf("Output size is bigger than max \n"); + free(local_buffer); + free(cell_aux); + return 0; + } + } + } + ind += padding[1]; + + free(cell_aux); + free(local_buffer); + + if (ind != (blockshape[0] * blockshape[1])) { + printf("Output size is not compatible with embedded blockshape \n"); + return 0; + } + if (ind > output_len) { + printf("Output size is bigger than max \n"); + return 0; + } + + return (int)ind; +} diff --git a/src/blosc2/plugins/codecs/ndlz/ndlz8x8.h b/src/blosc2/plugins/codecs/ndlz/ndlz8x8.h new file mode 100644 index 0000000..f7a98ac --- /dev/null +++ b/src/blosc2/plugins/codecs/ndlz/ndlz8x8.h @@ -0,0 +1,65 @@ +/********************************************************************* + Blosc - Blocked Shuffling and Compression Library + + Copyright (C) 2021 The Blosc Developers + https://blosc.org + License: BSD 3-Clause (see LICENSE.txt) + + See LICENSE.txt for details about copyright and rights to use. +**********************************************************************/ + + + +#ifndef NDLZ8_H +#define NDLZ8_H +#include "context.h" + +#if defined (__cplusplus) +extern "C" { +#endif +#include "ndlz.h" +#include "ndlz-private.h" + +/** + Compress a block of data in the input buffer and returns the size of + compressed block. The size of input buffer is specified by + length. The minimum input buffer size is 16. + + The output buffer must be at least 5% larger than the input buffer + and can not be smaller than 66 bytes. + + If the input is not compressible, or output does not fit in maxout + bytes, the return value will be 0 and you will have to discard the + output buffer. + + The acceleration parameter is related with the frequency for + updating the internal hash. An acceleration of 1 means that the + internal hash is updated at full rate. A value < 1 is not allowed + and will be silently set to 1. + + The input buffer and the output buffer can not overlap. +*/ + +int ndlz8_compress(const uint8_t *input, int32_t input_len, uint8_t *output, int32_t output_len, + uint8_t meta, blosc2_cparams *cparams); + +/** + Decompress a block of compressed data and returns the size of the + decompressed block. If error occurs, e.g. the compressed data is + corrupted or the output buffer is not large enough, then 0 (zero) + will be returned instead. + + The input buffer and the output buffer can not overlap. + + Decompression is memory safe and guaranteed not to write the output buffer + more than what is specified in maxout. + */ + +int ndlz8_decompress(const uint8_t *input, int32_t input_len, uint8_t *output, int32_t output_len, + uint8_t meta, blosc2_dparams *dparams); + +#if defined (__cplusplus) +} +#endif + +#endif /* NDLZ8_H */ diff --git a/src/blosc2/plugins/codecs/ndlz/test_ndlz.c b/src/blosc2/plugins/codecs/ndlz/test_ndlz.c new file mode 100644 index 0000000..3484c1f --- /dev/null +++ b/src/blosc2/plugins/codecs/ndlz/test_ndlz.c @@ -0,0 +1,239 @@ +/********************************************************************* + Blosc - Blocked Shuffling and Compression Library + + Copyright (C) 2021 The Blosc Developers + https://blosc.org + License: BSD 3-Clause (see LICENSE.txt) + + See LICENSE.txt for details about copyright and rights to use. + + Test program demonstrating use of the Blosc filter from C code. + To compile this program: + + $ gcc -O test_ndlz.c -o test_ndlz -lblosc2 + + To run: + + $ ./test_ndlz + Blosc version info: 2.0.0a6.dev ($Date:: 2018-05-18 #$) + Successful roundtrip! + Compression: 1792 -> 1630 (1.1x) + Successful roundtrip! + Compression: 1792 -> 1749 (1.0x) + same_cells: 43 obtained + + Successful roundtrip! + Compression: 16128 -> 2579 (6.3x) + Successful roundtrip! + Compression: 16128 -> 3829 (4.2x) + some_matches: 12299 obtained + +**********************************************************************/ + +#include +#include "blosc2.h" +#include "blosc2/codecs-registry.h" +#include + +static int test_ndlz_4(blosc2_schunk* schunk) { + + int64_t nchunks = schunk->nchunks; + int32_t chunksize = schunk->chunksize; + uint8_t *data_in = malloc(chunksize); + int decompressed; + int64_t csize; + int64_t dsize; + int64_t csize_f = 0; + uint8_t *data_out = malloc(chunksize + BLOSC2_MAX_OVERHEAD); + uint8_t *data_dest = malloc(chunksize); + + /* Create a context for compression */ + blosc2_cparams cparams = BLOSC2_CPARAMS_DEFAULTS; + cparams.splitmode = BLOSC_ALWAYS_SPLIT; + cparams.typesize = schunk->typesize; + cparams.compcode = BLOSC_CODEC_NDLZ; + cparams.compcode_meta = 4; + cparams.filters[BLOSC2_MAX_FILTERS - 1] = BLOSC_SHUFFLE; + cparams.clevel = 5; + cparams.nthreads = 1; + cparams.blocksize = schunk->blocksize; + cparams.schunk = schunk; + blosc2_context *cctx; + cctx = blosc2_create_cctx(cparams); + + blosc2_dparams dparams = BLOSC2_DPARAMS_DEFAULTS; + dparams.nthreads = 1; + blosc2_context *dctx; + dctx = blosc2_create_dctx(dparams); + + for (int ci = 0; ci < nchunks; ci++) { + + decompressed = blosc2_schunk_decompress_chunk(schunk, ci, data_in, chunksize); + if (decompressed < 0) { + printf("Error decompressing chunk \n"); + return -1; + } + + /* Compress with clevel=5 and shuffle active */ + csize = blosc2_compress_ctx(cctx, data_in, chunksize, data_out, chunksize + BLOSC2_MAX_OVERHEAD); + if (csize == 0) { + printf("Buffer is uncompressible. Giving up.\n"); + return 0; + } else if (csize < 0) { + printf("Compression error. Error code: %" PRId64 "\n", csize); + return (int) csize; + } + csize_f += csize; + + + /* Decompress */ + dsize = blosc2_decompress_ctx(dctx, data_out, chunksize + BLOSC2_MAX_OVERHEAD, data_dest, chunksize); + if (dsize <= 0) { + printf("Decompression error. Error code: %" PRId64 "\n", dsize); + return (int) dsize; + } + + for (int i = 0; i < chunksize; i++) { + if (data_in[i] != data_dest[i]) { + printf("i: %d, data %u, dest %u", i, data_in[i], data_dest[i]); + printf("\n Decompressed data differs from original!\n"); + return -1; + } + } + } + csize_f = csize_f / nchunks; + + free(data_in); + free(data_out); + free(data_dest); + blosc2_free_ctx(cctx); + blosc2_free_ctx(dctx); + + printf("Successful roundtrip!\n"); + printf("Compression: %d -> %" PRId64 " (%.1fx)\n", chunksize, csize_f, (1. * chunksize) / (double)csize_f); + return (int) (chunksize - csize_f); +} + +static int test_ndlz_8(blosc2_schunk* schunk) { + + int64_t nchunks = schunk->nchunks; + int32_t chunksize = (int32_t) (schunk->chunksize); + // int isize = (int) array->extchunknitems * typesize; + uint8_t *data_in = malloc(chunksize); + int decompressed; + int64_t csize; + int64_t dsize; + int64_t csize_f = 0; + uint8_t *data_out = malloc(chunksize + BLOSC2_MAX_OVERHEAD); + uint8_t *data_dest = malloc(chunksize); + + /* Create a context for compression */ + blosc2_cparams cparams = BLOSC2_CPARAMS_DEFAULTS; + cparams.splitmode = BLOSC_ALWAYS_SPLIT; + cparams.typesize = schunk->typesize; + cparams.compcode = BLOSC_CODEC_NDLZ; + cparams.compcode_meta = 8; + cparams.filters[BLOSC2_MAX_FILTERS - 1] = BLOSC_SHUFFLE; + cparams.clevel = 5; + cparams.nthreads = 1; + cparams.blocksize = schunk->blocksize; + cparams.schunk = schunk; + blosc2_context *cctx; + cctx = blosc2_create_cctx(cparams); + + blosc2_dparams dparams = BLOSC2_DPARAMS_DEFAULTS; + dparams.nthreads = 1; + blosc2_context *dctx; + dctx = blosc2_create_dctx(dparams); + + for (int ci = 0; ci < nchunks; ci++) { + + decompressed = blosc2_schunk_decompress_chunk(schunk, ci, data_in, chunksize); + if (decompressed < 0) { + printf("Error decompressing chunk \n"); + return -1; + } + + /* Compress with clevel=5 and shuffle active */ + csize = blosc2_compress_ctx(cctx, data_in, chunksize, data_out, chunksize + BLOSC2_MAX_OVERHEAD); + if (csize == 0) { + printf("Buffer is uncompressible. Giving up.\n"); + return 0; + } else if (csize < 0) { + printf("Compression error. Error code: %" PRId64 "\n", csize); + return (int) csize; + } + csize_f += csize; + + /* Decompress */ + dsize = blosc2_decompress_ctx(dctx, data_out, chunksize + BLOSC2_MAX_OVERHEAD, data_dest, chunksize); + if (dsize <= 0) { + printf("Decompression error. Error code: %" PRId64 "\n", dsize); + return (int) dsize; + } + + for (int i = 0; i < chunksize; i++) { + if (data_in[i] != data_dest[i]) { + printf("i: %d, data %u, dest %u", i, data_in[i], data_dest[i]); + printf("\n Decompressed data differs from original!\n"); + return -1; + } + } + } + csize_f = csize_f / nchunks; + + free(data_in); + free(data_out); + free(data_dest); + blosc2_free_ctx(cctx); + blosc2_free_ctx(dctx); + + printf("Successful roundtrip!\n"); + printf("Compression: %d -> %" PRId64 " (%.1fx)\n", chunksize, csize_f, (1. * chunksize) / (double)csize_f); + return (int) (chunksize - csize_f); +} + + +int same_cells() { + blosc2_schunk *schunk = blosc2_schunk_open("example_same_cells.caterva"); + + /* Run the test. */ + int result = test_ndlz_4(schunk); + if (result < 0) { + blosc2_schunk_free(schunk); + return result; + } + result = test_ndlz_8(schunk); + blosc2_schunk_free(schunk); + + return result; +} + +int some_matches() { + blosc2_schunk *schunk = blosc2_schunk_open("example_some_matches.caterva"); + + /* Run the test. */ + int result = test_ndlz_4(schunk); + if (result < 0) { + blosc2_schunk_free(schunk); + return result; + } + result = test_ndlz_8(schunk); + blosc2_schunk_free(schunk); + + return result; +} + + +int main(void) { + + int result; + blosc2_init(); // this is mandatory for initiallizing the plugin mechanism + result = same_cells(); + printf("same_cells: %d obtained \n \n", result); + result = some_matches(); + printf("some_matches: %d obtained \n \n", result); + blosc2_destroy(); + + return BLOSC2_ERROR_SUCCESS; +} diff --git a/src/blosc2/plugins/codecs/ndlz/xxhash.c b/src/blosc2/plugins/codecs/ndlz/xxhash.c new file mode 100644 index 0000000..0fae88c --- /dev/null +++ b/src/blosc2/plugins/codecs/ndlz/xxhash.c @@ -0,0 +1,43 @@ +/* + * xxHash - Extremely Fast Hash algorithm + * Copyright (C) 2012-2020 Yann Collet + * + * BSD 2-Clause License (https://www.opensource.org/licenses/bsd-license.php) + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * You can contact the author at: + * - xxHash homepage: https://www.xxhash.com + * - xxHash source repository: https://github.com/Cyan4973/xxHash + */ + + +/* + * xxhash.c instantiates functions defined in xxhash.h + */ + +#define XXH_STATIC_LINKING_ONLY /* access advanced declarations */ +#define XXH_IMPLEMENTATION /* access definitions */ + +#include "xxhash.h" diff --git a/src/blosc2/plugins/codecs/ndlz/xxhash.h b/src/blosc2/plugins/codecs/ndlz/xxhash.h new file mode 100644 index 0000000..253ed6b --- /dev/null +++ b/src/blosc2/plugins/codecs/ndlz/xxhash.h @@ -0,0 +1,5328 @@ +/* + * xxHash - Extremely Fast Hash algorithm + * Header File + * Copyright (C) 2012-2020 Yann Collet + * + * BSD 2-Clause License (https://www.opensource.org/licenses/bsd-license.php) + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * You can contact the author at: + * - xxHash homepage: https://www.xxhash.com + * - xxHash source repository: https://github.com/Cyan4973/xxHash + */ +/*! + * @mainpage xxHash + * + * @file xxhash.h + * xxHash prototypes and implementation + */ +/* TODO: update */ +/* Notice extracted from xxHash homepage: + +xxHash is an extremely fast hash algorithm, running at RAM speed limits. +It also successfully passes all tests from the SMHasher suite. + +Comparison (single thread, Windows Seven 32 bits, using SMHasher on a Core 2 Duo @3GHz) + +Name Speed Q.Score Author +xxHash 5.4 GB/s 10 +CrapWow 3.2 GB/s 2 Andrew +MurmurHash 3a 2.7 GB/s 10 Austin Appleby +SpookyHash 2.0 GB/s 10 Bob Jenkins +SBox 1.4 GB/s 9 Bret Mulvey +Lookup3 1.2 GB/s 9 Bob Jenkins +SuperFastHash 1.2 GB/s 1 Paul Hsieh +CityHash64 1.05 GB/s 10 Pike & Alakuijala +FNV 0.55 GB/s 5 Fowler, Noll, Vo +CRC32 0.43 GB/s 9 +MD5-32 0.33 GB/s 10 Ronald L. Rivest +SHA1-32 0.28 GB/s 10 + +Q.Score is a measure of quality of the hash function. +It depends on successfully passing SMHasher test set. +10 is a perfect score. + +Note: SMHasher's CRC32 implementation is not the fastest one. +Other speed-oriented implementations can be faster, +especially in combination with PCLMUL instruction: +https://fastcompression.blogspot.com/2019/03/presenting-xxh3.html?showComment=1552696407071#c3490092340461170735 + +A 64-bit version, named XXH64, is available since r35. +It offers much better speed, but for 64-bit applications only. +Name Speed on 64 bits Speed on 32 bits +XXH64 13.8 GB/s 1.9 GB/s +XXH32 6.8 GB/s 6.0 GB/s +*/ + +#if defined (__cplusplus) +extern "C" { +#endif + +/* **************************** + * INLINE mode + ******************************/ +/*! + * XXH_INLINE_ALL (and XXH_PRIVATE_API) + * Use these build macros to inline xxhash into the target unit. + * Inlining improves performance on small inputs, especially when the length is + * expressed as a compile-time constant: + * + * https://fastcompression.blogspot.com/2018/03/xxhash-for-small-keys-impressive-power.html + * + * It also keeps xxHash symbols private to the unit, so they are not exported. + * + * Usage: + * #define XXH_INLINE_ALL + * #include "xxhash.h" + * + * Do not compile and link xxhash.o as a separate object, as it is not useful. + */ +#if (defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API)) \ + && !defined(XXH_INLINE_ALL_31684351384) + /* this section should be traversed only once */ +# define XXH_INLINE_ALL_31684351384 + /* give access to the advanced API, required to compile implementations */ +# undef XXH_STATIC_LINKING_ONLY /* avoid macro redef */ +# define XXH_STATIC_LINKING_ONLY + /* make all functions private */ +# undef XXH_PUBLIC_API +# if defined(__GNUC__) +# define XXH_PUBLIC_API static __inline __attribute__((unused)) +# elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) +# define XXH_PUBLIC_API static inline +# elif defined(_MSC_VER) +# define XXH_PUBLIC_API static __inline +# else + /* note: this version may generate warnings for unused static functions */ +# define XXH_PUBLIC_API static +# endif + + /* + * This part deals with the special case where a unit wants to inline xxHash, + * but "xxhash.h" has previously been included without XXH_INLINE_ALL, such + * as part of some previously included *.h header file. + * Without further action, the new include would just be ignored, + * and functions would effectively _not_ be inlined (silent failure). + * The following macros solve this situation by prefixing all inlined names, + * avoiding naming collision with previous inclusions. + */ +# ifdef XXH_NAMESPACE +# error "XXH_INLINE_ALL with XXH_NAMESPACE is not supported" + /* + * Note: Alternative: #undef all symbols (it's a pretty large list). + * Without #error: it compiles, but functions are actually not inlined. + */ +# endif +# define XXH_NAMESPACE XXH_INLINE_ + /* + * Some identifiers (enums, type names) are not symbols, but they must + * still be renamed to avoid redeclaration. + * Alternative solution: do not redeclare them. + * However, this requires some #ifdefs, and is a more dispersed action. + * Meanwhile, renaming can be achieved in a single block + */ +# define XXH_IPREF(Id) XXH_INLINE_ ## Id +# define XXH_OK XXH_IPREF(XXH_OK) +# define XXH_ERROR XXH_IPREF(XXH_ERROR) +# define XXH_errorcode XXH_IPREF(XXH_errorcode) +# define XXH32_canonical_t XXH_IPREF(XXH32_canonical_t) +# define XXH64_canonical_t XXH_IPREF(XXH64_canonical_t) +# define XXH128_canonical_t XXH_IPREF(XXH128_canonical_t) +# define XXH32_state_s XXH_IPREF(XXH32_state_s) +# define XXH32_state_t XXH_IPREF(XXH32_state_t) +# define XXH64_state_s XXH_IPREF(XXH64_state_s) +# define XXH64_state_t XXH_IPREF(XXH64_state_t) +# define XXH3_state_s XXH_IPREF(XXH3_state_s) +# define XXH3_state_t XXH_IPREF(XXH3_state_t) +# define XXH128_hash_t XXH_IPREF(XXH128_hash_t) + /* Ensure the header is parsed again, even if it was previously included */ +# undef XXHASH_H_5627135585666179 +# undef XXHASH_H_STATIC_13879238742 +#endif /* XXH_INLINE_ALL || XXH_PRIVATE_API */ + + + +/* **************************************************************** + * Stable API + *****************************************************************/ +#ifndef XXHASH_H_5627135585666179 +#define XXHASH_H_5627135585666179 1 + + +/*! + * @defgroup public Public API + * Contains details on the public xxHash functions. + * @{ + */ +/* specific declaration modes for Windows */ +#if !defined(XXH_INLINE_ALL) && !defined(XXH_PRIVATE_API) +# if defined(WIN32) && defined(_MSC_VER) && (defined(XXH_IMPORT) || defined(XXH_EXPORT)) +# ifdef XXH_EXPORT +# define XXH_PUBLIC_API __declspec(dllexport) +# elif XXH_IMPORT +# define XXH_PUBLIC_API __declspec(dllimport) +# endif +# else +# define XXH_PUBLIC_API /* do nothing */ +# endif +#endif + +#ifdef XXH_DOXYGEN +/*! + * @brief Emulate a namespace by transparently prefixing all symbols. + * + * If you want to include _and expose_ xxHash functions from within your own + * library, but also want to avoid symbol collisions with other libraries which + * may also include xxHash, you can use XXH_NAMESPACE to automatically prefix + * any public symbol from xxhash library with the value of XXH_NAMESPACE + * (therefore, avoid empty or numeric values). + * + * Note that no change is required within the calling program as long as it + * includes `xxhash.h`: Regular symbol names will be automatically translated + * by this header. + */ +# define XXH_NAMESPACE /* YOUR NAME HERE */ +# undef XXH_NAMESPACE +#endif + +#undef XXH_NAMESPACE +# define XXH_NAMESPACE NDLZ + +#ifdef XXH_NAMESPACE +# define XXH_CAT(A,B) A##B +# define XXH_NAME2(A,B) XXH_CAT(A,B) +# define XXH_versionNumber XXH_NAME2(XXH_NAMESPACE, XXH_versionNumber) +/* XXH32 */ +# define XXH32 XXH_NAME2(XXH_NAMESPACE, XXH32) +# define XXH32_createState XXH_NAME2(XXH_NAMESPACE, XXH32_createState) +# define XXH32_freeState XXH_NAME2(XXH_NAMESPACE, XXH32_freeState) +# define XXH32_reset XXH_NAME2(XXH_NAMESPACE, XXH32_reset) +# define XXH32_update XXH_NAME2(XXH_NAMESPACE, XXH32_update) +# define XXH32_digest XXH_NAME2(XXH_NAMESPACE, XXH32_digest) +# define XXH32_copyState XXH_NAME2(XXH_NAMESPACE, XXH32_copyState) +# define XXH32_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH32_canonicalFromHash) +# define XXH32_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH32_hashFromCanonical) +/* XXH64 */ +# define XXH64 XXH_NAME2(XXH_NAMESPACE, XXH64) +# define XXH64_createState XXH_NAME2(XXH_NAMESPACE, XXH64_createState) +# define XXH64_freeState XXH_NAME2(XXH_NAMESPACE, XXH64_freeState) +# define XXH64_reset XXH_NAME2(XXH_NAMESPACE, XXH64_reset) +# define XXH64_update XXH_NAME2(XXH_NAMESPACE, XXH64_update) +# define XXH64_digest XXH_NAME2(XXH_NAMESPACE, XXH64_digest) +# define XXH64_copyState XXH_NAME2(XXH_NAMESPACE, XXH64_copyState) +# define XXH64_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH64_canonicalFromHash) +# define XXH64_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH64_hashFromCanonical) +/* XXH3_64bits */ +# define XXH3_64bits XXH_NAME2(XXH_NAMESPACE, XXH3_64bits) +# define XXH3_64bits_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_withSecret) +# define XXH3_64bits_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_withSeed) +# define XXH3_createState XXH_NAME2(XXH_NAMESPACE, XXH3_createState) +# define XXH3_freeState XXH_NAME2(XXH_NAMESPACE, XXH3_freeState) +# define XXH3_copyState XXH_NAME2(XXH_NAMESPACE, XXH3_copyState) +# define XXH3_64bits_reset XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset) +# define XXH3_64bits_reset_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset_withSeed) +# define XXH3_64bits_reset_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset_withSecret) +# define XXH3_64bits_update XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_update) +# define XXH3_64bits_digest XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_digest) +# define XXH3_generateSecret XXH_NAME2(XXH_NAMESPACE, XXH3_generateSecret) +/* XXH3_128bits */ +# define XXH128 XXH_NAME2(XXH_NAMESPACE, XXH128) +# define XXH3_128bits XXH_NAME2(XXH_NAMESPACE, XXH3_128bits) +# define XXH3_128bits_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_withSeed) +# define XXH3_128bits_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_withSecret) +# define XXH3_128bits_reset XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset) +# define XXH3_128bits_reset_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset_withSeed) +# define XXH3_128bits_reset_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset_withSecret) +# define XXH3_128bits_update XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_update) +# define XXH3_128bits_digest XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_digest) +# define XXH128_isEqual XXH_NAME2(XXH_NAMESPACE, XXH128_isEqual) +# define XXH128_cmp XXH_NAME2(XXH_NAMESPACE, XXH128_cmp) +# define XXH128_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH128_canonicalFromHash) +# define XXH128_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH128_hashFromCanonical) +#endif + + +/* ************************************* +* Version +***************************************/ +#define XXH_VERSION_MAJOR 0 +#define XXH_VERSION_MINOR 8 +#define XXH_VERSION_RELEASE 0 +#define XXH_VERSION_NUMBER (XXH_VERSION_MAJOR *100*100 + XXH_VERSION_MINOR *100 + XXH_VERSION_RELEASE) + +/*! + * @brief Obtains the xxHash version. + * + * This is only useful when xxHash is compiled as a shared library, as it is + * independent of the version defined in the header. + * + * @return `XXH_VERSION_NUMBER` as of when the function was compiled. + */ +XXH_PUBLIC_API unsigned XXH_versionNumber (void); + + +/* **************************** +* Definitions +******************************/ +#include /* size_t */ +typedef enum { XXH_OK=0, XXH_ERROR } XXH_errorcode; + + +/*-********************************************************************** +* 32-bit hash +************************************************************************/ +#if defined(XXH_DOXYGEN) /* Don't show include */ +/*! + * @brief An unsigned 32-bit integer. + * + * Not necessarily defined to `uint32_t` but functionally equivalent. + */ +typedef uint32_t XXH32_hash_t; +#elif !defined (__VMS) \ + && (defined (__cplusplus) \ + || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) +# include + typedef uint32_t XXH32_hash_t; +#else +# include +# if UINT_MAX == 0xFFFFFFFFUL + typedef unsigned int XXH32_hash_t; +# else +# if ULONG_MAX == 0xFFFFFFFFUL + typedef unsigned long XXH32_hash_t; +# else +# error "unsupported platform: need a 32-bit type" +# endif +# endif +#endif + +/*! + * @} + * + * @defgroup xxh32_family XXH32 family + * @ingroup public + * Contains functions used in the classic 32-bit xxHash algorithm. + * + * @note + * XXH32 is considered rather weak by today's standards. + * The @ref xxh3_family provides competitive speed for both 32-bit and 64-bit + * systems, and offers true 64/128 bit hash results. It provides a superior + * level of dispersion, and greatly reduces the risks of collisions. + * + * @see @ref xxh64_family, @ref xxh3_family : Other xxHash families + * @see @ref xxh32_impl for implementation details + * @{ + */ + +/*! + * @brief Calculates the 32-bit hash of @p input using xxHash32. + * + * Speed on Core 2 Duo @ 3 GHz (single thread, SMHasher benchmark): 5.4 GB/s + * + * @param input The block of data to be hashed, at least @p length bytes in size. + * @param length The length of @p input, in bytes. + * @param seed The 32-bit seed to alter the hash's output predictably. + * + * @pre + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * @return The calculated 32-bit hash value. + * + * @see + * XXH64(), XXH3_64bits_withSeed(), XXH3_128bits_withSeed(), XXH128(): + * Direct equivalents for the other variants of xxHash. + * @see + * XXH32_createState(), XXH32_update(), XXH32_digest(): Streaming version. + */ +XXH_PUBLIC_API XXH32_hash_t XXH32 (const void* input, size_t length, XXH32_hash_t seed); + +/*! + * Streaming functions generate the xxHash value from an incremental input. + * This method is slower than single-call functions, due to state management. + * For small inputs, prefer `XXH32()` and `XXH64()`, which are better optimized. + * + * An XXH state must first be allocated using `XXH*_createState()`. + * + * Start a new hash by initializing the state with a seed using `XXH*_reset()`. + * + * Then, feed the hash state by calling `XXH*_update()` as many times as necessary. + * + * The function returns an error code, with 0 meaning OK, and any other value + * meaning there is an error. + * + * Finally, a hash value can be produced anytime, by using `XXH*_digest()`. + * This function returns the nn-bits hash as an int or long long. + * + * It's still possible to continue inserting input into the hash state after a + * digest, and generate new hash values later on by invoking `XXH*_digest()`. + * + * When done, release the state using `XXH*_freeState()`. + * + * Example code for incrementally hashing a file: + * @code{.c} + * #include + * #include + * #define BUFFER_SIZE 256 + * + * // Note: XXH64 and XXH3 use the same interface. + * XXH32_hash_t + * hashFile(FILE* stream) + * { + * XXH32_state_t* state; + * unsigned char buf[BUFFER_SIZE]; + * size_t amt; + * XXH32_hash_t hash; + * + * state = XXH32_createState(); // Create a state + * assert(state != NULL); // Error check here + * XXH32_reset(state, 0xbaad5eed); // Reset state with our seed + * while ((amt = fread(buf, 1, sizeof(buf), stream)) != 0) { + * XXH32_update(state, buf, amt); // Hash the file in chunks + * } + * hash = XXH32_digest(state); // Finalize the hash + * XXH32_freeState(state); // Clean up + * return hash; + * } + * @endcode + */ + +/*! + * @typedef struct XXH32_state_s XXH32_state_t + * @brief The opaque state struct for the XXH32 streaming API. + * + * @see XXH32_state_s for details. + */ +typedef struct XXH32_state_s XXH32_state_t; + +/*! + * @brief Allocates an @ref XXH32_state_t. + * + * Must be freed with XXH32_freeState(). + * @return An allocated XXH32_state_t on success, `NULL` on failure. + */ +XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void); +/*! + * @brief Frees an @ref XXH32_state_t. + * + * Must be allocated with XXH32_createState(). + * @param statePtr A pointer to an @ref XXH32_state_t allocated with @ref XXH32_createState(). + * @return XXH_OK. + */ +XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr); +/*! + * @brief Copies one @ref XXH32_state_t to another. + * + * @param dst_state The state to copy to. + * @param src_state The state to copy from. + * @pre + * @p dst_state and @p src_state must not be `NULL` and must not overlap. + */ +XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* dst_state, const XXH32_state_t* src_state); + +/*! + * @brief Resets an @ref XXH32_state_t to begin a new hash. + * + * This function resets and seeds a state. Call it before @ref XXH32_update(). + * + * @param statePtr The state struct to reset. + * @param seed The 32-bit seed to alter the hash result predictably. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return @ref XXH_OK on success, @ref XXH_ERROR on failure. + */ +XXH_PUBLIC_API XXH_errorcode XXH32_reset (XXH32_state_t* statePtr, XXH32_hash_t seed); + +/*! + * @brief Consumes a block of @p input to an @ref XXH32_state_t. + * + * Call this to incrementally consume blocks of data. + * + * @param statePtr The state struct to update. + * @param input The block of data to be hashed, at least @p length bytes in size. + * @param length The length of @p input, in bytes. + * + * @pre + * @p statePtr must not be `NULL`. + * @pre + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * @return @ref XXH_OK on success, @ref XXH_ERROR on failure. + */ +XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* statePtr, const void* input, size_t length); + +/*! + * @brief Returns the calculated hash value from an @ref XXH32_state_t. + * + * @note + * Calling XXH32_digest() will not affect @p statePtr, so you can update, + * digest, and update again. + * + * @param statePtr The state struct to calculate the hash from. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return The calculated xxHash32 value from that state. + */ +XXH_PUBLIC_API XXH32_hash_t XXH32_digest (const XXH32_state_t* statePtr); + +/******* Canonical representation *******/ + +/* + * The default return values from XXH functions are unsigned 32 and 64 bit + * integers. + * This the simplest and fastest format for further post-processing. + * + * However, this leaves open the question of what is the order on the byte level, + * since little and big endian conventions will store the same number differently. + * + * The canonical representation settles this issue by mandating big-endian + * convention, the same convention as human-readable numbers (large digits first). + * + * When writing hash values to storage, sending them over a network, or printing + * them, it's highly recommended to use the canonical representation to ensure + * portability across a wider range of systems, present and future. + * + * The following functions allow transformation of hash values to and from + * canonical format. + */ + +/*! + * @brief Canonical (big endian) representation of @ref XXH32_hash_t. + */ +typedef struct { + unsigned char digest[4]; /*!< Hash bytes, big endian */ +} XXH32_canonical_t; + +/*! + * @brief Converts an @ref XXH32_hash_t to a big endian @ref XXH32_canonical_t. + * + * @param dst The @ref XXH32_canonical_t pointer to be stored to. + * @param hash The @ref XXH32_hash_t to be converted. + * + * @pre + * @p dst must not be `NULL`. + */ +XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash); + +/*! + * @brief Converts an @ref XXH32_canonical_t to a native @ref XXH32_hash_t. + * + * @param src The @ref XXH32_canonical_t to convert. + * + * @pre + * @p src must not be `NULL`. + * + * @return The converted hash. + */ +XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src); + + +/*! + * @} + * @ingroup public + * @{ + */ + +#ifndef XXH_NO_LONG_LONG +/*-********************************************************************** +* 64-bit hash +************************************************************************/ +#if defined(XXH_DOXYGEN) /* don't include */ +/*! + * @brief An unsigned 64-bit integer. + * + * Not necessarily defined to `uint64_t` but functionally equivalent. + */ +typedef uint64_t XXH64_hash_t; +#elif !defined (__VMS) \ + && (defined (__cplusplus) \ + || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) +# include + typedef uint64_t XXH64_hash_t; +#else +# include +# if defined(__LP64__) && ULONG_MAX == 0xFFFFFFFFFFFFFFFFULL + /* LP64 ABI says uint64_t is unsigned long */ + typedef unsigned long XXH64_hash_t; +# else + /* the following type must have a width of 64-bit */ + typedef unsigned long long XXH64_hash_t; +# endif +#endif + +/*! + * @} + * + * @defgroup xxh64_family XXH64 family + * @ingroup public + * @{ + * Contains functions used in the classic 64-bit xxHash algorithm. + * + * @note + * XXH3 provides competitive speed for both 32-bit and 64-bit systems, + * and offers true 64/128 bit hash results. It provides a superior level of + * dispersion, and greatly reduces the risks of collisions. + */ + + +/*! + * @brief Calculates the 64-bit hash of @p input using xxHash64. + * + * This function usually runs faster on 64-bit systems, but slower on 32-bit + * systems (see benchmark). + * + * @param input The block of data to be hashed, at least @p length bytes in size. + * @param length The length of @p input, in bytes. + * @param seed The 64-bit seed to alter the hash's output predictably. + * + * @pre + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * @return The calculated 64-bit hash. + * + * @see + * XXH32(), XXH3_64bits_withSeed(), XXH3_128bits_withSeed(), XXH128(): + * Direct equivalents for the other variants of xxHash. + * @see + * XXH64_createState(), XXH64_update(), XXH64_digest(): Streaming version. + */ +XXH_PUBLIC_API XXH64_hash_t XXH64(const void* input, size_t length, XXH64_hash_t seed); + +/******* Streaming *******/ +/*! + * @brief The opaque state struct for the XXH64 streaming API. + * + * @see XXH64_state_s for details. + */ +typedef struct XXH64_state_s XXH64_state_t; /* incomplete type */ +XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void); +XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr); +XXH_PUBLIC_API void XXH64_copyState(XXH64_state_t* dst_state, const XXH64_state_t* src_state); + +XXH_PUBLIC_API XXH_errorcode XXH64_reset (XXH64_state_t* statePtr, XXH64_hash_t seed); +XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH64_state_t* statePtr, const void* input, size_t length); +XXH_PUBLIC_API XXH64_hash_t XXH64_digest (const XXH64_state_t* statePtr); + +/******* Canonical representation *******/ +typedef struct { unsigned char digest[sizeof(XXH64_hash_t)]; } XXH64_canonical_t; +XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t* dst, XXH64_hash_t hash); +XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src); + +/*! + * @} + * ************************************************************************ + * @defgroup xxh3_family XXH3 family + * @ingroup public + * @{ + * + * XXH3 is a more recent hash algorithm featuring: + * - Improved speed for both small and large inputs + * - True 64-bit and 128-bit outputs + * - SIMD acceleration + * - Improved 32-bit viability + * + * Speed analysis methodology is explained here: + * + * https://fastcompression.blogspot.com/2019/03/presenting-xxh3.html + * + * Compared to XXH64, expect XXH3 to run approximately + * ~2x faster on large inputs and >3x faster on small ones, + * exact differences vary depending on platform. + * + * XXH3's speed benefits greatly from SIMD and 64-bit arithmetic, + * but does not require it. + * Any 32-bit and 64-bit targets that can run XXH32 smoothly + * can run XXH3 at competitive speeds, even without vector support. + * Further details are explained in the implementation. + * + * Optimized implementations are provided for AVX512, AVX2, SSE2, NEON, POWER8, + * ZVector and scalar targets. This can be controlled via the XXH_VECTOR macro. + * + * XXH3 implementation is portable: + * it has a generic C90 formulation that can be compiled on any platform, + * all implementations generage exactly the same hash value on all platforms. + * Starting from v0.8.0, it's also labelled "stable", meaning that + * any future version will also generate the same hash value. + * + * XXH3 offers 2 variants, _64bits and _128bits. + * + * When only 64 bits are needed, prefer invoking the _64bits variant, as it + * reduces the amount of mixing, resulting in faster speed on small inputs. + * It's also generally simpler to manipulate a scalar return type than a struct. + * + * The API supports one-shot hashing, streaming mode, and custom secrets. + */ + +/*-********************************************************************** +* XXH3 64-bit variant +************************************************************************/ + +/* XXH3_64bits(): + * default 64-bit variant, using default secret and default seed of 0. + * It's the fastest variant. */ +XXH_PUBLIC_API XXH64_hash_t XXH3_64bits(const void* data, size_t len); + +/* + * XXH3_64bits_withSeed(): + * This variant generates a custom secret on the fly + * based on default secret altered using the `seed` value. + * While this operation is decently fast, note that it's not completely free. + * Note: seed==0 produces the same results as XXH3_64bits(). + */ +XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_withSeed(const void* data, size_t len, XXH64_hash_t seed); + +/*! + * The bare minimum size for a custom secret. + * + * @see + * XXH3_64bits_withSecret(), XXH3_64bits_reset_withSecret(), + * XXH3_128bits_withSecret(), XXH3_128bits_reset_withSecret(). + */ +#define XXH3_SECRET_SIZE_MIN 136 + +/* + * XXH3_64bits_withSecret(): + * It's possible to provide any blob of bytes as a "secret" to generate the hash. + * This makes it more difficult for an external actor to prepare an intentional collision. + * The main condition is that secretSize *must* be large enough (>= XXH3_SECRET_SIZE_MIN). + * However, the quality of produced hash values depends on secret's entropy. + * Technically, the secret must look like a bunch of random bytes. + * Avoid "trivial" or structured data such as repeated sequences or a text document. + * Whenever unsure about the "randomness" of the blob of bytes, + * consider relabelling it as a "custom seed" instead, + * and employ "XXH3_generateSecret()" (see below) + * to generate a high entropy secret derived from the custom seed. + */ +XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_withSecret(const void* data, size_t len, const void* secret, size_t secretSize); + + +/******* Streaming *******/ +/* + * Streaming requires state maintenance. + * This operation costs memory and CPU. + * As a consequence, streaming is slower than one-shot hashing. + * For better performance, prefer one-shot functions whenever applicable. + */ + +/*! + * @brief The state struct for the XXH3 streaming API. + * + * @see XXH3_state_s for details. + */ +typedef struct XXH3_state_s XXH3_state_t; +XXH_PUBLIC_API XXH3_state_t* XXH3_createState(void); +XXH_PUBLIC_API XXH_errorcode XXH3_freeState(XXH3_state_t* statePtr); +XXH_PUBLIC_API void XXH3_copyState(XXH3_state_t* dst_state, const XXH3_state_t* src_state); + +/* + * XXH3_64bits_reset(): + * Initialize with default parameters. + * digest will be equivalent to `XXH3_64bits()`. + */ +XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset(XXH3_state_t* statePtr); +/* + * XXH3_64bits_reset_withSeed(): + * Generate a custom secret from `seed`, and store it into `statePtr`. + * digest will be equivalent to `XXH3_64bits_withSeed()`. + */ +XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset_withSeed(XXH3_state_t* statePtr, XXH64_hash_t seed); +/* + * XXH3_64bits_reset_withSecret(): + * `secret` is referenced, it _must outlive_ the hash streaming session. + * Similar to one-shot API, `secretSize` must be >= `XXH3_SECRET_SIZE_MIN`, + * and the quality of produced hash values depends on secret's entropy + * (secret's content should look like a bunch of random bytes). + * When in doubt about the randomness of a candidate `secret`, + * consider employing `XXH3_generateSecret()` instead (see below). + */ +XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset_withSecret(XXH3_state_t* statePtr, const void* secret, size_t secretSize); + +XXH_PUBLIC_API XXH_errorcode XXH3_64bits_update (XXH3_state_t* statePtr, const void* input, size_t length); +XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_digest (const XXH3_state_t* statePtr); + +/* note : canonical representation of XXH3 is the same as XXH64 + * since they both produce XXH64_hash_t values */ + + +/*-********************************************************************** +* XXH3 128-bit variant +************************************************************************/ + +/*! + * @brief The return value from 128-bit hashes. + * + * Stored in little endian order, although the fields themselves are in native + * endianness. + */ +typedef struct { + XXH64_hash_t low64; /*!< `value & 0xFFFFFFFFFFFFFFFF` */ + XXH64_hash_t high64; /*!< `value >> 64` */ +} XXH128_hash_t; + +XXH_PUBLIC_API XXH128_hash_t XXH3_128bits(const void* data, size_t len); +XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_withSeed(const void* data, size_t len, XXH64_hash_t seed); +XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_withSecret(const void* data, size_t len, const void* secret, size_t secretSize); + +/******* Streaming *******/ +/* + * Streaming requires state maintenance. + * This operation costs memory and CPU. + * As a consequence, streaming is slower than one-shot hashing. + * For better performance, prefer one-shot functions whenever applicable. + * + * XXH3_128bits uses the same XXH3_state_t as XXH3_64bits(). + * Use already declared XXH3_createState() and XXH3_freeState(). + * + * All reset and streaming functions have same meaning as their 64-bit counterpart. + */ + +XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset(XXH3_state_t* statePtr); +XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset_withSeed(XXH3_state_t* statePtr, XXH64_hash_t seed); +XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset_withSecret(XXH3_state_t* statePtr, const void* secret, size_t secretSize); + +XXH_PUBLIC_API XXH_errorcode XXH3_128bits_update (XXH3_state_t* statePtr, const void* input, size_t length); +XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_digest (const XXH3_state_t* statePtr); + +/* Following helper functions make it possible to compare XXH128_hast_t values. + * Since XXH128_hash_t is a structure, this capability is not offered by the language. + * Note: For better performance, these functions can be inlined using XXH_INLINE_ALL */ + +/*! + * XXH128_isEqual(): + * Return: 1 if `h1` and `h2` are equal, 0 if they are not. + */ +XXH_PUBLIC_API int XXH128_isEqual(XXH128_hash_t h1, XXH128_hash_t h2); + +/*! + * XXH128_cmp(): + * + * This comparator is compatible with stdlib's `qsort()`/`bsearch()`. + * + * return: >0 if *h128_1 > *h128_2 + * =0 if *h128_1 == *h128_2 + * <0 if *h128_1 < *h128_2 + */ +XXH_PUBLIC_API int XXH128_cmp(const void* h128_1, const void* h128_2); + + +/******* Canonical representation *******/ +typedef struct { unsigned char digest[sizeof(XXH128_hash_t)]; } XXH128_canonical_t; +XXH_PUBLIC_API void XXH128_canonicalFromHash(XXH128_canonical_t* dst, XXH128_hash_t hash); +XXH_PUBLIC_API XXH128_hash_t XXH128_hashFromCanonical(const XXH128_canonical_t* src); + + +#endif /* XXH_NO_LONG_LONG */ + +/*! + * @} + */ +#endif /* XXHASH_H_5627135585666179 */ + + + +#if defined(XXH_STATIC_LINKING_ONLY) && !defined(XXHASH_H_STATIC_13879238742) +#define XXHASH_H_STATIC_13879238742 +/* **************************************************************************** + * This section contains declarations which are not guaranteed to remain stable. + * They may change in future versions, becoming incompatible with a different + * version of the library. + * These declarations should only be used with static linking. + * Never use them in association with dynamic linking! + ***************************************************************************** */ + +/* + * These definitions are only present to allow static allocation + * of XXH states, on stack or in a struct, for example. + * Never **ever** access their members directly. + */ + +/*! + * @internal + * @brief Structure for XXH32 streaming API. + * + * @note This is only defined when @ref XXH_STATIC_LINKING_ONLY, + * @ref XXH_INLINE_ALL, or @ref XXH_IMPLEMENTATION is defined. Otherwise it is + * an opaque type. This allows fields to safely be changed. + * + * Typedef'd to @ref XXH32_state_t. + * Do not access the members of this struct directly. + * @see XXH64_state_s, XXH3_state_s + */ +struct XXH32_state_s { + XXH32_hash_t total_len_32; /*!< Total length hashed, modulo 2^32 */ + XXH32_hash_t large_len; /*!< Whether the hash is >= 16 (handles @ref total_len_32 overflow) */ + XXH32_hash_t v1; /*!< First accumulator lane */ + XXH32_hash_t v2; /*!< Second accumulator lane */ + XXH32_hash_t v3; /*!< Third accumulator lane */ + XXH32_hash_t v4; /*!< Fourth accumulator lane */ + XXH32_hash_t mem32[4]; /*!< Internal buffer for partial reads. Treated as unsigned char[16]. */ + XXH32_hash_t memsize; /*!< Amount of data in @ref mem32 */ + XXH32_hash_t reserved; /*!< Reserved field. Do not read or write to it, it may be removed. */ +}; /* typedef'd to XXH32_state_t */ + + +#ifndef XXH_NO_LONG_LONG /* defined when there is no 64-bit support */ + +/*! + * @internal + * @brief Structure for XXH64 streaming API. + * + * @note This is only defined when @ref XXH_STATIC_LINKING_ONLY, + * @ref XXH_INLINE_ALL, or @ref XXH_IMPLEMENTATION is defined. Otherwise it is + * an opaque type. This allows fields to safely be changed. + * + * Typedef'd to @ref XXH64_state_t. + * Do not access the members of this struct directly. + * @see XXH32_state_s, XXH3_state_s + */ +struct XXH64_state_s { + XXH64_hash_t total_len; /*!< Total length hashed. This is always 64-bit. */ + XXH64_hash_t v1; /*!< First accumulator lane */ + XXH64_hash_t v2; /*!< Second accumulator lane */ + XXH64_hash_t v3; /*!< Third accumulator lane */ + XXH64_hash_t v4; /*!< Fourth accumulator lane */ + XXH64_hash_t mem64[4]; /*!< Internal buffer for partial reads. Treated as unsigned char[32]. */ + XXH32_hash_t memsize; /*!< Amount of data in @ref mem64 */ + XXH32_hash_t reserved32; /*!< Reserved field, needed for padding anyways*/ + XXH64_hash_t reserved64; /*!< Reserved field. Do not read or write to it, it may be removed. */ +}; /* typedef'd to XXH64_state_t */ + +#if defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* C11+ */ +# include +# define XXH_ALIGN(n) alignas(n) +#elif defined(__GNUC__) +# define XXH_ALIGN(n) __attribute__ ((aligned(n))) +#elif defined(_MSC_VER) +# define XXH_ALIGN(n) __declspec(align(n)) +#else +# define XXH_ALIGN(n) /* disabled */ +#endif + +/* Old GCC versions only accept the attribute after the type in structures. */ +#if !(defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L)) /* C11+ */ \ + && defined(__GNUC__) +# define XXH_ALIGN_MEMBER(align, type) type XXH_ALIGN(align) +#else +# define XXH_ALIGN_MEMBER(align, type) XXH_ALIGN(align) type +#endif + +/*! + * @brief The size of the internal XXH3 buffer. + * + * This is the optimal update size for incremental hashing. + * + * @see XXH3_64b_update(), XXH3_128b_update(). + */ +#define XXH3_INTERNALBUFFER_SIZE 256 + +/*! + * @brief Default size of the secret buffer (and @ref XXH3_kSecret). + * + * This is the size used in @ref XXH3_kSecret and the seeded functions. + * + * Not to be confused with @ref XXH3_SECRET_SIZE_MIN. + */ +#define XXH3_SECRET_DEFAULT_SIZE 192 + +/*! + * @internal + * @brief Structure for XXH3 streaming API. + * + * @note This is only defined when @ref XXH_STATIC_LINKING_ONLY, + * @ref XXH_INLINE_ALL, or @ref XXH_IMPLEMENTATION is defined. Otherwise it is + * an opaque type. This allows fields to safely be changed. + * + * @note **This structure has a strict alignment requirement of 64 bytes.** Do + * not allocate this with `malloc()` or `new`, it will not be sufficiently + * aligned. Use @ref XXH3_createState() and @ref XXH3_freeState(), or stack + * allocation. + * + * Typedef'd to @ref XXH3_state_t. + * Do not access the members of this struct directly. + * + * @see XXH3_INITSTATE() for stack initialization. + * @see XXH3_createState(), XXH3_freeState(). + * @see XXH32_state_s, XXH64_state_s + */ +struct XXH3_state_s { + XXH_ALIGN_MEMBER(64, XXH64_hash_t acc[8]); + /*!< The 8 accumulators. Similar to `vN` in @ref XXH32_state_s::v1 and @ref XXH64_state_s */ + XXH_ALIGN_MEMBER(64, unsigned char customSecret[XXH3_SECRET_DEFAULT_SIZE]); + /*!< Used to store a custom secret generated from a seed. */ + XXH_ALIGN_MEMBER(64, unsigned char buffer[XXH3_INTERNALBUFFER_SIZE]); + /*!< The internal buffer. @see XXH32_state_s::mem32 */ + XXH32_hash_t bufferedSize; + /*!< The amount of memory in @ref buffer, @see XXH32_state_s::memsize */ + XXH32_hash_t reserved32; + /*!< Reserved field. Needed for padding on 64-bit. */ + size_t nbStripesSoFar; + /*!< Number or stripes processed. */ + XXH64_hash_t totalLen; + /*!< Total length hashed. 64-bit even on 32-bit targets. */ + size_t nbStripesPerBlock; + /*!< Number of stripes per block. */ + size_t secretLimit; + /*!< Size of @ref customSecret or @ref extSecret */ + XXH64_hash_t seed; + /*!< Seed for _withSeed variants. Must be zero otherwise, @see XXH3_INITSTATE() */ + XXH64_hash_t reserved64; + /*!< Reserved field. */ + const unsigned char* extSecret; + /*!< Reference to an external secret for the _withSecret variants, NULL + * for other variants. */ + /* note: there may be some padding at the end due to alignment on 64 bytes */ +}; /* typedef'd to XXH3_state_t */ + +#undef XXH_ALIGN_MEMBER + +/*! + * @brief Initializes a stack-allocated `XXH3_state_s`. + * + * When the @ref XXH3_state_t structure is merely emplaced on stack, + * it should be initialized with XXH3_INITSTATE() or a memset() + * in case its first reset uses XXH3_NNbits_reset_withSeed(). + * This init can be omitted if the first reset uses default or _withSecret mode. + * This operation isn't necessary when the state is created with XXH3_createState(). + * Note that this doesn't prepare the state for a streaming operation, + * it's still necessary to use XXH3_NNbits_reset*() afterwards. + */ +#define XXH3_INITSTATE(XXH3_state_ptr) { (XXH3_state_ptr)->seed = 0; } + + +/* === Experimental API === */ +/* Symbols defined below must be considered tied to a specific library version. */ + +/* + * XXH3_generateSecret(): + * + * Derive a high-entropy secret from any user-defined content, named customSeed. + * The generated secret can be used in combination with `*_withSecret()` functions. + * The `_withSecret()` variants are useful to provide a higher level of protection than 64-bit seed, + * as it becomes much more difficult for an external actor to guess how to impact the calculation logic. + * + * The function accepts as input a custom seed of any length and any content, + * and derives from it a high-entropy secret of length XXH3_SECRET_DEFAULT_SIZE + * into an already allocated buffer secretBuffer. + * The generated secret is _always_ XXH_SECRET_DEFAULT_SIZE bytes long. + * + * The generated secret can then be used with any `*_withSecret()` variant. + * Functions `XXH3_128bits_withSecret()`, `XXH3_64bits_withSecret()`, + * `XXH3_128bits_reset_withSecret()` and `XXH3_64bits_reset_withSecret()` + * are part of this list. They all accept a `secret` parameter + * which must be very long for implementation reasons (>= XXH3_SECRET_SIZE_MIN) + * _and_ feature very high entropy (consist of random-looking bytes). + * These conditions can be a high bar to meet, so + * this function can be used to generate a secret of proper quality. + * + * customSeed can be anything. It can have any size, even small ones, + * and its content can be anything, even stupidly "low entropy" source such as a bunch of zeroes. + * The resulting `secret` will nonetheless provide all expected qualities. + * + * Supplying NULL as the customSeed copies the default secret into `secretBuffer`. + * When customSeedSize > 0, supplying NULL as customSeed is undefined behavior. + */ +XXH_PUBLIC_API void XXH3_generateSecret(void* secretBuffer, const void* customSeed, size_t customSeedSize); + + +/* simple short-cut to pre-selected XXH3_128bits variant */ +XXH_PUBLIC_API XXH128_hash_t XXH128(const void* data, size_t len, XXH64_hash_t seed); + + +#endif /* XXH_NO_LONG_LONG */ +#if defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API) +# define XXH_IMPLEMENTATION +#endif + +#endif /* defined(XXH_STATIC_LINKING_ONLY) && !defined(XXHASH_H_STATIC_13879238742) */ + + +/* ======================================================================== */ +/* ======================================================================== */ +/* ======================================================================== */ + + +/*-********************************************************************** + * xxHash implementation + *-********************************************************************** + * xxHash's implementation used to be hosted inside xxhash.c. + * + * However, inlining requires implementation to be visible to the compiler, + * hence be included alongside the header. + * Previously, implementation was hosted inside xxhash.c, + * which was then #included when inlining was activated. + * This construction created issues with a few build and install systems, + * as it required xxhash.c to be stored in /include directory. + * + * xxHash implementation is now directly integrated within xxhash.h. + * As a consequence, xxhash.c is no longer needed in /include. + * + * xxhash.c is still available and is still useful. + * In a "normal" setup, when xxhash is not inlined, + * xxhash.h only exposes the prototypes and public symbols, + * while xxhash.c can be built into an object file xxhash.o + * which can then be linked into the final binary. + ************************************************************************/ + +#if ( defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API) \ + || defined(XXH_IMPLEMENTATION) ) && !defined(XXH_IMPLEM_13a8737387) +# define XXH_IMPLEM_13a8737387 + +/* ************************************* +* Tuning parameters +***************************************/ + +/*! + * @defgroup tuning Tuning parameters + * @{ + * + * Various macros to control xxHash's behavior. + */ +#ifdef XXH_DOXYGEN +/*! + * @brief Define this to disable 64-bit code. + * + * Useful if only using the @ref xxh32_family and you have a strict C90 compiler. + */ +# define XXH_NO_LONG_LONG +# undef XXH_NO_LONG_LONG /* don't actually */ +/*! + * @brief Controls how unaligned memory is accessed. + * + * By default, access to unaligned memory is controlled by `memcpy()`, which is + * safe and portable. + * + * Unfortunately, on some target/compiler combinations, the generated assembly + * is sub-optimal. + * + * The below switch allow selection of a different access method + * in the search for improved performance. + * + * @par Possible options: + * + * - `XXH_FORCE_MEMORY_ACCESS=0` (default): `memcpy` + * @par + * Use `memcpy()`. Safe and portable. Note that most modern compilers will + * eliminate the function call and treat it as an unaligned access. + * + * - `XXH_FORCE_MEMORY_ACCESS=1`: `__attribute__((packed))` + * @par + * Depends on compiler extensions and is therefore not portable. + * This method is safe _if_ your compiler supports it, + * and *generally* as fast or faster than `memcpy`. + * + * - `XXH_FORCE_MEMORY_ACCESS=2`: Direct cast + * @par + * Casts directly and dereferences. This method doesn't depend on the + * compiler, but it violates the C standard as it directly dereferences an + * unaligned pointer. It can generate buggy code on targets which do not + * support unaligned memory accesses, but in some circumstances, it's the + * only known way to get the most performance. + * + * - `XXH_FORCE_MEMORY_ACCESS=3`: Byteshift + * @par + * Also portable. This can generate the best code on old compilers which don't + * inline small `memcpy()` calls, and it might also be faster on big-endian + * systems which lack a native byteswap instruction. However, some compilers + * will emit literal byteshifts even if the target supports unaligned access. + * . + * + * @warning + * Methods 1 and 2 rely on implementation-defined behavior. Use these with + * care, as what works on one compiler/platform/optimization level may cause + * another to read garbage data or even crash. + * + * See https://stackoverflow.com/a/32095106/646947 for details. + * + * Prefer these methods in priority order (0 > 3 > 1 > 2) + */ +# define XXH_FORCE_MEMORY_ACCESS 0 +/*! + * @def XXH_ACCEPT_NULL_INPUT_POINTER + * @brief Whether to add explicit `NULL` checks. + * + * If the input pointer is `NULL` and the length is non-zero, xxHash's default + * behavior is to dereference it, triggering a segfault. + * + * When this macro is enabled, xxHash actively checks the input for a null pointer. + * If it is, the result for null input pointers is the same as a zero-length input. + */ +# define XXH_ACCEPT_NULL_INPUT_POINTER 0 +/*! + * @def XXH_FORCE_ALIGN_CHECK + * @brief If defined to non-zero, adds a special path for aligned inputs (XXH32() + * and XXH64() only). + * + * This is an important performance trick for architectures without decent + * unaligned memory access performance. + * + * It checks for input alignment, and when conditions are met, uses a "fast + * path" employing direct 32-bit/64-bit reads, resulting in _dramatically + * faster_ read speed. + * + * The check costs one initial branch per hash, which is generally negligible, + * but not zero. + * + * Moreover, it's not useful to generate an additional code path if memory + * access uses the same instruction for both aligned and unaligned + * addresses (e.g. x86 and aarch64). + * + * In these cases, the alignment check can be removed by setting this macro to 0. + * Then the code will always use unaligned memory access. + * Align check is automatically disabled on x86, x64 & arm64, + * which are platforms known to offer good unaligned memory accesses performance. + * + * This option does not affect XXH3 (only XXH32 and XXH64). + */ +# define XXH_FORCE_ALIGN_CHECK 0 + +/*! + * @def XXH_NO_INLINE_HINTS + * @brief When non-zero, sets all functions to `static`. + * + * By default, xxHash tries to force the compiler to inline almost all internal + * functions. + * + * This can usually improve performance due to reduced jumping and improved + * constant folding, but significantly increases the size of the binary which + * might not be favorable. + * + * Additionally, sometimes the forced inlining can be detrimental to performance, + * depending on the architecture. + * + * XXH_NO_INLINE_HINTS marks all internal functions as static, giving the + * compiler full control on whether to inline or not. + * + * When not optimizing (-O0), optimizing for size (-Os, -Oz), or using + * -fno-inline with GCC or Clang, this will automatically be defined. + */ +# define XXH_NO_INLINE_HINTS 0 + +/*! + * @def XXH_REROLL + * @brief Whether to reroll `XXH32_finalize` and `XXH64_finalize`. + * + * For performance, `XXH32_finalize` and `XXH64_finalize` use an unrolled loop + * in the form of a switch statement. + * + * This is not always desirable, as it generates larger code, and depending on + * the architecture, may even be slower + * + * This is automatically defined with `-Os`/`-Oz` on GCC and Clang. + */ +# define XXH_REROLL 0 + +/*! + * @internal + * @brief Redefines old internal names. + * + * For compatibility with code that uses xxHash's internals before the names + * were changed to improve namespacing. There is no other reason to use this. + */ +# define XXH_OLD_NAMES +# undef XXH_OLD_NAMES /* don't actually use, it is ugly. */ +#endif /* XXH_DOXYGEN */ +/*! + * @} + */ + +#ifndef XXH_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */ + /* prefer __packed__ structures (method 1) for gcc on armv7 and armv8 */ +# if !defined(__clang__) && ( \ + (defined(__INTEL_COMPILER) && !defined(_WIN32)) || \ + (defined(__GNUC__) && (defined(__ARM_ARCH) && __ARM_ARCH >= 7)) ) +# define XXH_FORCE_MEMORY_ACCESS 1 +# endif +#endif + +#ifndef XXH_ACCEPT_NULL_INPUT_POINTER /* can be defined externally */ +# define XXH_ACCEPT_NULL_INPUT_POINTER 0 +#endif + +#ifndef XXH_FORCE_ALIGN_CHECK /* can be defined externally */ +# if defined(__i386) || defined(__x86_64__) || defined(__aarch64__) \ + || defined(_M_IX86) || defined(_M_X64) || defined(_M_ARM64) /* visual */ +# define XXH_FORCE_ALIGN_CHECK 0 +# else +# define XXH_FORCE_ALIGN_CHECK 1 +# endif +#endif + +#ifndef XXH_NO_INLINE_HINTS +# if defined(__OPTIMIZE_SIZE__) /* -Os, -Oz */ \ + || defined(__NO_INLINE__) /* -O0, -fno-inline */ +# define XXH_NO_INLINE_HINTS 1 +# else +# define XXH_NO_INLINE_HINTS 0 +# endif +#endif + +#ifndef XXH_REROLL +# if defined(__OPTIMIZE_SIZE__) +# define XXH_REROLL 1 +# else +# define XXH_REROLL 0 +# endif +#endif + +/*! + * @defgroup impl Implementation + * @{ + */ + + +/* ************************************* +* Includes & Memory related functions +***************************************/ +/* + * Modify the local functions below should you wish to use + * different memory routines for malloc() and free() + */ +#include + +/*! + * @internal + * @brief Modify this function to use a different routine than malloc(). + */ +static void* XXH_malloc(size_t s) { return malloc(s); } + +/*! + * @internal + * @brief Modify this function to use a different routine than free(). + */ +static void XXH_free(void* p) { free(p); } + +#include + +/*! + * @internal + * @brief Modify this function to use a different routine than memcpy(). + */ +static void* XXH_memcpy(void* dest, const void* src, size_t size) +{ + return memcpy(dest,src,size); +} + +#include /* ULLONG_MAX */ + + +/* ************************************* +* Compiler Specific Options +***************************************/ +#ifdef _MSC_VER /* Visual Studio warning fix */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +#endif + +#if XXH_NO_INLINE_HINTS /* disable inlining hints */ +# if defined(__GNUC__) +# define XXH_FORCE_INLINE static __attribute__((unused)) +# else +# define XXH_FORCE_INLINE static +# endif +# define XXH_NO_INLINE static +/* enable inlining hints */ +#elif defined(_MSC_VER) /* Visual Studio */ +# define XXH_FORCE_INLINE static __forceinline +# define XXH_NO_INLINE static __declspec(noinline) +#elif defined(__GNUC__) +# define XXH_FORCE_INLINE static __inline__ __attribute__((always_inline, unused)) +# define XXH_NO_INLINE static __attribute__((noinline)) +#elif defined (__cplusplus) \ + || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)) /* C99 */ +# define XXH_FORCE_INLINE static inline +# define XXH_NO_INLINE static +#else +# define XXH_FORCE_INLINE static +# define XXH_NO_INLINE static +#endif + + + +/* ************************************* +* Debug +***************************************/ +/*! + * @ingroup tuning + * @def XXH_DEBUGLEVEL + * @brief Sets the debugging level. + * + * XXH_DEBUGLEVEL is expected to be defined externally, typically via the + * compiler's command line options. The value must be a number. + */ +#ifndef XXH_DEBUGLEVEL +# ifdef DEBUGLEVEL /* backwards compat */ +# define XXH_DEBUGLEVEL DEBUGLEVEL +# else +# define XXH_DEBUGLEVEL 0 +# endif +#endif + +#if (XXH_DEBUGLEVEL>=1) +# include /* note: can still be disabled with NDEBUG */ +# define XXH_ASSERT(c) assert(c) +#else +# define XXH_ASSERT(c) ((void)0) +#endif + +/* note: use after variable declarations */ +#define XXH_STATIC_ASSERT(c) do { enum { XXH_sa = 1/(int)(!!(c)) }; } while (0) + +/*! + * @internal + * @def XXH_COMPILER_GUARD(var) + * @brief Used to prevent unwanted optimizations for @p var. + * + * It uses an empty GCC inline assembly statement with a register constraint + * which forces @p var into a general purpose register (eg eax, ebx, ecx + * on x86) and marks it as modified. + * + * This is used in a few places to avoid unwanted autovectorization (e.g. + * XXH32_round()). All vectorization we want is explicit via intrinsics, + * and _usually_ isn't wanted elsewhere. + * + * We also use it to prevent unwanted constant folding for AArch64 in + * XXH3_initCustomSecret_scalar(). + */ +#ifdef __GNUC__ +# define XXH_COMPILER_GUARD(var) __asm__ __volatile__("" : "+r" (var)) +#else +# define XXH_COMPILER_GUARD(var) ((void)0) +#endif + +/* ************************************* +* Basic Types +***************************************/ +#if !defined (__VMS) \ + && (defined (__cplusplus) \ + || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) +# include + typedef uint8_t xxh_u8; +#else + typedef unsigned char xxh_u8; +#endif +typedef XXH32_hash_t xxh_u32; + +#ifdef XXH_OLD_NAMES +# define BYTE xxh_u8 +# define U8 xxh_u8 +# define U32 xxh_u32 +#endif + +/* *** Memory access *** */ + +/*! + * @internal + * @fn xxh_u32 XXH_read32(const void* ptr) + * @brief Reads an unaligned 32-bit integer from @p ptr in native endianness. + * + * Affected by @ref XXH_FORCE_MEMORY_ACCESS. + * + * @param ptr The pointer to read from. + * @return The 32-bit native endian integer from the bytes at @p ptr. + */ + +/*! + * @internal + * @fn xxh_u32 XXH_readLE32(const void* ptr) + * @brief Reads an unaligned 32-bit little endian integer from @p ptr. + * + * Affected by @ref XXH_FORCE_MEMORY_ACCESS. + * + * @param ptr The pointer to read from. + * @return The 32-bit little endian integer from the bytes at @p ptr. + */ + +/*! + * @internal + * @fn xxh_u32 XXH_readBE32(const void* ptr) + * @brief Reads an unaligned 32-bit big endian integer from @p ptr. + * + * Affected by @ref XXH_FORCE_MEMORY_ACCESS. + * + * @param ptr The pointer to read from. + * @return The 32-bit big endian integer from the bytes at @p ptr. + */ + +/*! + * @internal + * @fn xxh_u32 XXH_readLE32_align(const void* ptr, XXH_alignment align) + * @brief Like @ref XXH_readLE32(), but has an option for aligned reads. + * + * Affected by @ref XXH_FORCE_MEMORY_ACCESS. + * Note that when @ref XXH_FORCE_ALIGN_CHECK == 0, the @p align parameter is + * always @ref XXH_alignment::XXH_unaligned. + * + * @param ptr The pointer to read from. + * @param align Whether @p ptr is aligned. + * @pre + * If @p align == @ref XXH_alignment::XXH_aligned, @p ptr must be 4 byte + * aligned. + * @return The 32-bit little endian integer from the bytes at @p ptr. + */ + +#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3)) +/* + * Manual byteshift. Best for old compilers which don't inline memcpy. + * We actually directly use XXH_readLE32 and XXH_readBE32. + */ +#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2)) + +/* + * Force direct memory access. Only works on CPU which support unaligned memory + * access in hardware. + */ +static xxh_u32 XXH_read32(const void* memPtr) { return *(const xxh_u32*) memPtr; } + +#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1)) + +/* + * __pack instructions are safer but compiler specific, hence potentially + * problematic for some compilers. + * + * Currently only defined for GCC and ICC. + */ +#ifdef XXH_OLD_NAMES +typedef union { xxh_u32 u32; } __attribute__((packed)) unalign; +#endif +static xxh_u32 XXH_read32(const void* ptr) +{ + typedef union { xxh_u32 u32; } __attribute__((packed)) xxh_unalign; + return ((const xxh_unalign*)ptr)->u32; +} + +#else + +/* + * Portable and safe solution. Generally efficient. + * see: https://stackoverflow.com/a/32095106/646947 + */ +static xxh_u32 XXH_read32(const void* memPtr) +{ + xxh_u32 val; + memcpy(&val, memPtr, sizeof(val)); + return val; +} + +#endif /* XXH_FORCE_DIRECT_MEMORY_ACCESS */ + + +/* *** Endianness *** */ +typedef enum { XXH_bigEndian=0, XXH_littleEndian=1 } XXH_endianess; + +/*! + * @ingroup tuning + * @def XXH_CPU_LITTLE_ENDIAN + * @brief Whether the target is little endian. + * + * Defined to 1 if the target is little endian, or 0 if it is big endian. + * It can be defined externally, for example on the compiler command line. + * + * If it is not defined, a runtime check (which is usually constant folded) + * is used instead. + * + * @note + * This is not necessarily defined to an integer constant. + * + * @see XXH_isLittleEndian() for the runtime check. + */ +#ifndef XXH_CPU_LITTLE_ENDIAN +/* + * Try to detect endianness automatically, to avoid the nonstandard behavior + * in `XXH_isLittleEndian()` + */ +# if defined(_WIN32) /* Windows is always little endian */ \ + || defined(__LITTLE_ENDIAN__) \ + || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) +# define XXH_CPU_LITTLE_ENDIAN 1 +# elif defined(__BIG_ENDIAN__) \ + || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) +# define XXH_CPU_LITTLE_ENDIAN 0 +# else +/*! + * @internal + * @brief Runtime check for @ref XXH_CPU_LITTLE_ENDIAN. + * + * Most compilers will constant fold this. + */ +static int XXH_isLittleEndian(void) +{ + /* + * Portable and well-defined behavior. + * Don't use static: it is detrimental to performance. + */ + const union { xxh_u32 u; xxh_u8 c[4]; } one = { 1 }; + return one.c[0]; +} +# define XXH_CPU_LITTLE_ENDIAN XXH_isLittleEndian() +# endif +#endif + + + + +/* **************************************** +* Compiler-specific Functions and Macros +******************************************/ +#define XXH_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) + +#ifdef __has_builtin +# define XXH_HAS_BUILTIN(x) __has_builtin(x) +#else +# define XXH_HAS_BUILTIN(x) 0 +#endif + +/*! + * @internal + * @def XXH_rotl32(x,r) + * @brief 32-bit rotate left. + * + * @param x The 32-bit integer to be rotated. + * @param r The number of bits to rotate. + * @pre + * @p r > 0 && @p r < 32 + * @note + * @p x and @p r may be evaluated multiple times. + * @return The rotated result. + */ +#if !defined(NO_CLANG_BUILTIN) && XXH_HAS_BUILTIN(__builtin_rotateleft32) \ + && XXH_HAS_BUILTIN(__builtin_rotateleft64) +# define XXH_rotl32 __builtin_rotateleft32 +# define XXH_rotl64 __builtin_rotateleft64 +/* Note: although _rotl exists for minGW (GCC under windows), performance seems poor */ +#elif defined(_MSC_VER) +# define XXH_rotl32(x,r) _rotl(x,r) +# define XXH_rotl64(x,r) _rotl64(x,r) +#else +# define XXH_rotl32(x,r) (((x) << (r)) | ((x) >> (32 - (r)))) +# define XXH_rotl64(x,r) (((x) << (r)) | ((x) >> (64 - (r)))) +#endif + +/*! + * @internal + * @fn xxh_u32 XXH_swap32(xxh_u32 x) + * @brief A 32-bit byteswap. + * + * @param x The 32-bit integer to byteswap. + * @return @p x, byteswapped. + */ +#if defined(_MSC_VER) /* Visual Studio */ +# define XXH_swap32 _byteswap_ulong +#elif XXH_GCC_VERSION >= 403 +# define XXH_swap32 __builtin_bswap32 +#else +static xxh_u32 XXH_swap32 (xxh_u32 x) +{ + return ((x << 24) & 0xff000000 ) | + ((x << 8) & 0x00ff0000 ) | + ((x >> 8) & 0x0000ff00 ) | + ((x >> 24) & 0x000000ff ); +} +#endif + + +/* *************************** +* Memory reads +*****************************/ + +/*! + * @internal + * @brief Enum to indicate whether a pointer is aligned. + */ +typedef enum { + XXH_aligned, /*!< Aligned */ + XXH_unaligned /*!< Possibly unaligned */ +} XXH_alignment; + +/* + * XXH_FORCE_MEMORY_ACCESS==3 is an endian-independent byteshift load. + * + * This is ideal for older compilers which don't inline memcpy. + */ +#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3)) + +XXH_FORCE_INLINE xxh_u32 XXH_readLE32(const void* memPtr) +{ + const xxh_u8* bytePtr = (const xxh_u8 *)memPtr; + return bytePtr[0] + | ((xxh_u32)bytePtr[1] << 8) + | ((xxh_u32)bytePtr[2] << 16) + | ((xxh_u32)bytePtr[3] << 24); +} + +XXH_FORCE_INLINE xxh_u32 XXH_readBE32(const void* memPtr) +{ + const xxh_u8* bytePtr = (const xxh_u8 *)memPtr; + return bytePtr[3] + | ((xxh_u32)bytePtr[2] << 8) + | ((xxh_u32)bytePtr[1] << 16) + | ((xxh_u32)bytePtr[0] << 24); +} + +#else +XXH_FORCE_INLINE xxh_u32 XXH_readLE32(const void* ptr) +{ + return XXH_CPU_LITTLE_ENDIAN ? XXH_read32(ptr) : XXH_swap32(XXH_read32(ptr)); +} + +static xxh_u32 XXH_readBE32(const void* ptr) +{ + return XXH_CPU_LITTLE_ENDIAN ? XXH_swap32(XXH_read32(ptr)) : XXH_read32(ptr); +} +#endif + +XXH_FORCE_INLINE xxh_u32 +XXH_readLE32_align(const void* ptr, XXH_alignment align) +{ + if (align==XXH_unaligned) { + return XXH_readLE32(ptr); + } else { + return XXH_CPU_LITTLE_ENDIAN ? *(const xxh_u32*)ptr : XXH_swap32(*(const xxh_u32*)ptr); + } +} + + +/* ************************************* +* Misc +***************************************/ +/*! @ingroup public */ +XXH_PUBLIC_API unsigned XXH_versionNumber (void) { return XXH_VERSION_NUMBER; } + + +/* ******************************************************************* +* 32-bit hash functions +*********************************************************************/ +/*! + * @} + * @defgroup xxh32_impl XXH32 implementation + * @ingroup impl + * @{ + */ + /* #define instead of static const, to be used as initializers */ +#define XXH_PRIME32_1 0x9E3779B1U /*!< 0b10011110001101110111100110110001 */ +#define XXH_PRIME32_2 0x85EBCA77U /*!< 0b10000101111010111100101001110111 */ +#define XXH_PRIME32_3 0xC2B2AE3DU /*!< 0b11000010101100101010111000111101 */ +#define XXH_PRIME32_4 0x27D4EB2FU /*!< 0b00100111110101001110101100101111 */ +#define XXH_PRIME32_5 0x165667B1U /*!< 0b00010110010101100110011110110001 */ + +#ifdef XXH_OLD_NAMES +# define PRIME32_1 XXH_PRIME32_1 +# define PRIME32_2 XXH_PRIME32_2 +# define PRIME32_3 XXH_PRIME32_3 +# define PRIME32_4 XXH_PRIME32_4 +# define PRIME32_5 XXH_PRIME32_5 +#endif + +/*! + * @internal + * @brief Normal stripe processing routine. + * + * This shuffles the bits so that any bit from @p input impacts several bits in + * @p acc. + * + * @param acc The accumulator lane. + * @param input The stripe of input to mix. + * @return The mixed accumulator lane. + */ +static xxh_u32 XXH32_round(xxh_u32 acc, xxh_u32 input) +{ + acc += input * XXH_PRIME32_2; + acc = XXH_rotl32(acc, 13); + acc *= XXH_PRIME32_1; +#if (defined(__SSE4_1__) || defined(__aarch64__)) && !defined(XXH_ENABLE_AUTOVECTORIZE) + /* + * UGLY HACK: + * A compiler fence is the only thing that prevents GCC and Clang from + * autovectorizing the XXH32 loop (pragmas and attributes don't work for some + * reason) without globally disabling SSE4.1. + * + * The reason we want to avoid vectorization is because despite working on + * 4 integers at a time, there are multiple factors slowing XXH32 down on + * SSE4: + * - There's a ridiculous amount of lag from pmulld (10 cycles of latency on + * newer chips!) making it slightly slower to multiply four integers at + * once compared to four integers independently. Even when pmulld was + * fastest, Sandy/Ivy Bridge, it is still not worth it to go into SSE + * just to multiply unless doing a long operation. + * + * - Four instructions are required to rotate, + * movqda tmp, v // not required with VEX encoding + * pslld tmp, 13 // tmp <<= 13 + * psrld v, 19 // x >>= 19 + * por v, tmp // x |= tmp + * compared to one for scalar: + * roll v, 13 // reliably fast across the board + * shldl v, v, 13 // Sandy Bridge and later prefer this for some reason + * + * - Instruction level parallelism is actually more beneficial here because + * the SIMD actually serializes this operation: While v1 is rotating, v2 + * can load data, while v3 can multiply. SSE forces them to operate + * together. + * + * This is also enabled on AArch64, as Clang autovectorizes it incorrectly + * and it is pointless writing a NEON implementation that is basically the + * same speed as scalar for XXH32. + */ + XXH_COMPILER_GUARD(acc); +#endif + return acc; +} + +/*! + * @internal + * @brief Mixes all bits to finalize the hash. + * + * The final mix ensures that all input bits have a chance to impact any bit in + * the output digest, resulting in an unbiased distribution. + * + * @param h32 The hash to avalanche. + * @return The avalanched hash. + */ +static xxh_u32 XXH32_avalanche(xxh_u32 h32) +{ + h32 ^= h32 >> 15; + h32 *= XXH_PRIME32_2; + h32 ^= h32 >> 13; + h32 *= XXH_PRIME32_3; + h32 ^= h32 >> 16; + return(h32); +} + +#define XXH_get32bits(p) XXH_readLE32_align(p, align) + +/*! + * @internal + * @brief Processes the last 0-15 bytes of @p ptr. + * + * There may be up to 15 bytes remaining to consume from the input. + * This final stage will digest them to ensure that all input bytes are present + * in the final mix. + * + * @param h32 The hash to finalize. + * @param ptr The pointer to the remaining input. + * @param len The remaining length, modulo 16. + * @param align Whether @p ptr is aligned. + * @return The finalized hash. + */ +static xxh_u32 +XXH32_finalize(xxh_u32 h32, const xxh_u8* ptr, size_t len, XXH_alignment align) +{ +#define XXH_PROCESS1 do { \ + h32 += (*ptr++) * XXH_PRIME32_5; \ + h32 = XXH_rotl32(h32, 11) * XXH_PRIME32_1; \ +} while (0) + +#define XXH_PROCESS4 do { \ + h32 += XXH_get32bits(ptr) * XXH_PRIME32_3; \ + ptr += 4; \ + h32 = XXH_rotl32(h32, 17) * XXH_PRIME32_4; \ +} while (0) + + /* Compact rerolled version */ + if (XXH_REROLL) { + len &= 15; + while (len >= 4) { + XXH_PROCESS4; + len -= 4; + } + while (len > 0) { + XXH_PROCESS1; + --len; + } + return XXH32_avalanche(h32); + } else { + switch(len&15) /* or switch(bEnd - p) */ { + case 12: XXH_PROCESS4; + /* fallthrough */ + case 8: XXH_PROCESS4; + /* fallthrough */ + case 4: XXH_PROCESS4; + return XXH32_avalanche(h32); + + case 13: XXH_PROCESS4; + /* fallthrough */ + case 9: XXH_PROCESS4; + /* fallthrough */ + case 5: XXH_PROCESS4; + XXH_PROCESS1; + return XXH32_avalanche(h32); + + case 14: XXH_PROCESS4; + /* fallthrough */ + case 10: XXH_PROCESS4; + /* fallthrough */ + case 6: XXH_PROCESS4; + XXH_PROCESS1; + XXH_PROCESS1; + return XXH32_avalanche(h32); + + case 15: XXH_PROCESS4; + /* fallthrough */ + case 11: XXH_PROCESS4; + /* fallthrough */ + case 7: XXH_PROCESS4; + /* fallthrough */ + case 3: XXH_PROCESS1; + /* fallthrough */ + case 2: XXH_PROCESS1; + /* fallthrough */ + case 1: XXH_PROCESS1; + /* fallthrough */ + case 0: return XXH32_avalanche(h32); + } + XXH_ASSERT(0); + return h32; /* reaching this point is deemed impossible */ + } +} + +#ifdef XXH_OLD_NAMES +# define PROCESS1 XXH_PROCESS1 +# define PROCESS4 XXH_PROCESS4 +#else +# undef XXH_PROCESS1 +# undef XXH_PROCESS4 +#endif + +/*! + * @internal + * @brief The implementation for @ref XXH32(). + * + * @param input, len, seed Directly passed from @ref XXH32(). + * @param align Whether @p input is aligned. + * @return The calculated hash. + */ +XXH_FORCE_INLINE xxh_u32 +XXH32_endian_align(const xxh_u8* input, size_t len, xxh_u32 seed, XXH_alignment align) +{ + const xxh_u8* bEnd = input ? input + len : NULL; + xxh_u32 h32; + +#if defined(XXH_ACCEPT_NULL_INPUT_POINTER) && (XXH_ACCEPT_NULL_INPUT_POINTER>=1) + if (input==NULL) { + len=0; + bEnd=input=(const xxh_u8*)(size_t)16; + } +#endif + + if (len>=16) { + const xxh_u8* const limit = bEnd - 15; + xxh_u32 v1 = seed + XXH_PRIME32_1 + XXH_PRIME32_2; + xxh_u32 v2 = seed + XXH_PRIME32_2; + xxh_u32 v3 = seed + 0; + xxh_u32 v4 = seed - XXH_PRIME32_1; + + do { + v1 = XXH32_round(v1, XXH_get32bits(input)); input += 4; + v2 = XXH32_round(v2, XXH_get32bits(input)); input += 4; + v3 = XXH32_round(v3, XXH_get32bits(input)); input += 4; + v4 = XXH32_round(v4, XXH_get32bits(input)); input += 4; + } while (input < limit); + + h32 = XXH_rotl32(v1, 1) + XXH_rotl32(v2, 7) + + XXH_rotl32(v3, 12) + XXH_rotl32(v4, 18); + } else { + h32 = seed + XXH_PRIME32_5; + } + + h32 += (xxh_u32)len; + + return XXH32_finalize(h32, input, len&15, align); +} + +/*! @ingroup xxh32_family */ +XXH_PUBLIC_API XXH32_hash_t XXH32 (const void* input, size_t len, XXH32_hash_t seed) +{ +#if 0 + /* Simple version, good for code maintenance, but unfortunately slow for small inputs */ + XXH32_state_t state; + XXH32_reset(&state, seed); + XXH32_update(&state, (const xxh_u8*)input, len); + return XXH32_digest(&state); +#else + if (XXH_FORCE_ALIGN_CHECK) { + if ((((size_t)input) & 3) == 0) { /* Input is 4-bytes aligned, leverage the speed benefit */ + return XXH32_endian_align((const xxh_u8*)input, len, seed, XXH_aligned); + } } + + return XXH32_endian_align((const xxh_u8*)input, len, seed, XXH_unaligned); +#endif +} + + + +/******* Hash streaming *******/ +/*! + * @ingroup xxh32_family + */ +XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void) +{ + return (XXH32_state_t*)XXH_malloc(sizeof(XXH32_state_t)); +} +/*! @ingroup xxh32_family */ +XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr) +{ + XXH_free(statePtr); + return XXH_OK; +} + +/*! @ingroup xxh32_family */ +XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* dstState, const XXH32_state_t* srcState) +{ + memcpy(dstState, srcState, sizeof(*dstState)); +} + +/*! @ingroup xxh32_family */ +XXH_PUBLIC_API XXH_errorcode XXH32_reset(XXH32_state_t* statePtr, XXH32_hash_t seed) +{ + XXH32_state_t state; /* using a local state to memcpy() in order to avoid strict-aliasing warnings */ + memset(&state, 0, sizeof(state)); + state.v1 = seed + XXH_PRIME32_1 + XXH_PRIME32_2; + state.v2 = seed + XXH_PRIME32_2; + state.v3 = seed + 0; + state.v4 = seed - XXH_PRIME32_1; + /* do not write into reserved, planned to be removed in a future version */ + memcpy(statePtr, &state, sizeof(state) - sizeof(state.reserved)); + return XXH_OK; +} + + +/*! @ingroup xxh32_family */ +XXH_PUBLIC_API XXH_errorcode +XXH32_update(XXH32_state_t* state, const void* input, size_t len) +{ + if (input==NULL) +#if defined(XXH_ACCEPT_NULL_INPUT_POINTER) && (XXH_ACCEPT_NULL_INPUT_POINTER>=1) + return XXH_OK; +#else + return XXH_ERROR; +#endif + + { const xxh_u8* p = (const xxh_u8*)input; + const xxh_u8* const bEnd = p + len; + + state->total_len_32 += (XXH32_hash_t)len; + state->large_len |= (XXH32_hash_t)((len>=16) | (state->total_len_32>=16)); + + if (state->memsize + len < 16) { /* fill in tmp buffer */ + XXH_memcpy((xxh_u8*)(state->mem32) + state->memsize, input, len); + state->memsize += (XXH32_hash_t)len; + return XXH_OK; + } + + if (state->memsize) { /* some data left from previous update */ + XXH_memcpy((xxh_u8*)(state->mem32) + state->memsize, input, 16-state->memsize); + { const xxh_u32* p32 = state->mem32; + state->v1 = XXH32_round(state->v1, XXH_readLE32(p32)); p32++; + state->v2 = XXH32_round(state->v2, XXH_readLE32(p32)); p32++; + state->v3 = XXH32_round(state->v3, XXH_readLE32(p32)); p32++; + state->v4 = XXH32_round(state->v4, XXH_readLE32(p32)); + } + p += 16-state->memsize; + state->memsize = 0; + } + + if (p <= bEnd-16) { + const xxh_u8* const limit = bEnd - 16; + xxh_u32 v1 = state->v1; + xxh_u32 v2 = state->v2; + xxh_u32 v3 = state->v3; + xxh_u32 v4 = state->v4; + + do { + v1 = XXH32_round(v1, XXH_readLE32(p)); p+=4; + v2 = XXH32_round(v2, XXH_readLE32(p)); p+=4; + v3 = XXH32_round(v3, XXH_readLE32(p)); p+=4; + v4 = XXH32_round(v4, XXH_readLE32(p)); p+=4; + } while (p<=limit); + + state->v1 = v1; + state->v2 = v2; + state->v3 = v3; + state->v4 = v4; + } + + if (p < bEnd) { + XXH_memcpy(state->mem32, p, (size_t)(bEnd-p)); + state->memsize = (unsigned)(bEnd-p); + } + } + + return XXH_OK; +} + + +/*! @ingroup xxh32_family */ +XXH_PUBLIC_API XXH32_hash_t XXH32_digest(const XXH32_state_t* state) +{ + xxh_u32 h32; + + if (state->large_len) { + h32 = XXH_rotl32(state->v1, 1) + + XXH_rotl32(state->v2, 7) + + XXH_rotl32(state->v3, 12) + + XXH_rotl32(state->v4, 18); + } else { + h32 = state->v3 /* == seed */ + XXH_PRIME32_5; + } + + h32 += state->total_len_32; + + return XXH32_finalize(h32, (const xxh_u8*)state->mem32, state->memsize, XXH_aligned); +} + + +/******* Canonical representation *******/ + +/*! + * @ingroup xxh32_family + * The default return values from XXH functions are unsigned 32 and 64 bit + * integers. + * + * The canonical representation uses big endian convention, the same convention + * as human-readable numbers (large digits first). + * + * This way, hash values can be written into a file or buffer, remaining + * comparable across different systems. + * + * The following functions allow transformation of hash values to and from their + * canonical format. + */ +XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash) +{ + XXH_STATIC_ASSERT(sizeof(XXH32_canonical_t) == sizeof(XXH32_hash_t)); + if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap32(hash); + memcpy(dst, &hash, sizeof(*dst)); +} +/*! @ingroup xxh32_family */ +XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src) +{ + return XXH_readBE32(src); +} + + +#ifndef XXH_NO_LONG_LONG + +/* ******************************************************************* +* 64-bit hash functions +*********************************************************************/ +/*! + * @} + * @ingroup impl + * @{ + */ +/******* Memory access *******/ + +typedef XXH64_hash_t xxh_u64; + +#ifdef XXH_OLD_NAMES +# define U64 xxh_u64 +#endif + +#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3)) +/* + * Manual byteshift. Best for old compilers which don't inline memcpy. + * We actually directly use XXH_readLE64 and XXH_readBE64. + */ +#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2)) + +/* Force direct memory access. Only works on CPU which support unaligned memory access in hardware */ +static xxh_u64 XXH_read64(const void* memPtr) +{ + return *(const xxh_u64*) memPtr; +} + +#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1)) + +/* + * __pack instructions are safer, but compiler specific, hence potentially + * problematic for some compilers. + * + * Currently only defined for GCC and ICC. + */ +#ifdef XXH_OLD_NAMES +typedef union { xxh_u32 u32; xxh_u64 u64; } __attribute__((packed)) unalign64; +#endif +static xxh_u64 XXH_read64(const void* ptr) +{ + typedef union { xxh_u32 u32; xxh_u64 u64; } __attribute__((packed)) xxh_unalign64; + return ((const xxh_unalign64*)ptr)->u64; +} + +#else + +/* + * Portable and safe solution. Generally efficient. + * see: https://stackoverflow.com/a/32095106/646947 + */ +static xxh_u64 XXH_read64(const void* memPtr) +{ + xxh_u64 val; + memcpy(&val, memPtr, sizeof(val)); + return val; +} + +#endif /* XXH_FORCE_DIRECT_MEMORY_ACCESS */ + +#if defined(_MSC_VER) /* Visual Studio */ +# define XXH_swap64 _byteswap_uint64 +#elif XXH_GCC_VERSION >= 403 +# define XXH_swap64 __builtin_bswap64 +#else +static xxh_u64 XXH_swap64(xxh_u64 x) +{ + return ((x << 56) & 0xff00000000000000ULL) | + ((x << 40) & 0x00ff000000000000ULL) | + ((x << 24) & 0x0000ff0000000000ULL) | + ((x << 8) & 0x000000ff00000000ULL) | + ((x >> 8) & 0x00000000ff000000ULL) | + ((x >> 24) & 0x0000000000ff0000ULL) | + ((x >> 40) & 0x000000000000ff00ULL) | + ((x >> 56) & 0x00000000000000ffULL); +} +#endif + + +/* XXH_FORCE_MEMORY_ACCESS==3 is an endian-independent byteshift load. */ +#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3)) + +XXH_FORCE_INLINE xxh_u64 XXH_readLE64(const void* memPtr) +{ + const xxh_u8* bytePtr = (const xxh_u8 *)memPtr; + return bytePtr[0] + | ((xxh_u64)bytePtr[1] << 8) + | ((xxh_u64)bytePtr[2] << 16) + | ((xxh_u64)bytePtr[3] << 24) + | ((xxh_u64)bytePtr[4] << 32) + | ((xxh_u64)bytePtr[5] << 40) + | ((xxh_u64)bytePtr[6] << 48) + | ((xxh_u64)bytePtr[7] << 56); +} + +XXH_FORCE_INLINE xxh_u64 XXH_readBE64(const void* memPtr) +{ + const xxh_u8* bytePtr = (const xxh_u8 *)memPtr; + return bytePtr[7] + | ((xxh_u64)bytePtr[6] << 8) + | ((xxh_u64)bytePtr[5] << 16) + | ((xxh_u64)bytePtr[4] << 24) + | ((xxh_u64)bytePtr[3] << 32) + | ((xxh_u64)bytePtr[2] << 40) + | ((xxh_u64)bytePtr[1] << 48) + | ((xxh_u64)bytePtr[0] << 56); +} + +#else +XXH_FORCE_INLINE xxh_u64 XXH_readLE64(const void* ptr) +{ + return XXH_CPU_LITTLE_ENDIAN ? XXH_read64(ptr) : XXH_swap64(XXH_read64(ptr)); +} + +static xxh_u64 XXH_readBE64(const void* ptr) +{ + return XXH_CPU_LITTLE_ENDIAN ? XXH_swap64(XXH_read64(ptr)) : XXH_read64(ptr); +} +#endif + +XXH_FORCE_INLINE xxh_u64 +XXH_readLE64_align(const void* ptr, XXH_alignment align) +{ + if (align==XXH_unaligned) + return XXH_readLE64(ptr); + else + return XXH_CPU_LITTLE_ENDIAN ? *(const xxh_u64*)ptr : XXH_swap64(*(const xxh_u64*)ptr); +} + + +/******* xxh64 *******/ +/*! + * @} + * @defgroup xxh64_impl XXH64 implementation + * @ingroup impl + * @{ + */ +/* #define rather that static const, to be used as initializers */ +#define XXH_PRIME64_1 0x9E3779B185EBCA87ULL /*!< 0b1001111000110111011110011011000110000101111010111100101010000111 */ +#define XXH_PRIME64_2 0xC2B2AE3D27D4EB4FULL /*!< 0b1100001010110010101011100011110100100111110101001110101101001111 */ +#define XXH_PRIME64_3 0x165667B19E3779F9ULL /*!< 0b0001011001010110011001111011000110011110001101110111100111111001 */ +#define XXH_PRIME64_4 0x85EBCA77C2B2AE63ULL /*!< 0b1000010111101011110010100111011111000010101100101010111001100011 */ +#define XXH_PRIME64_5 0x27D4EB2F165667C5ULL /*!< 0b0010011111010100111010110010111100010110010101100110011111000101 */ + +#ifdef XXH_OLD_NAMES +# define PRIME64_1 XXH_PRIME64_1 +# define PRIME64_2 XXH_PRIME64_2 +# define PRIME64_3 XXH_PRIME64_3 +# define PRIME64_4 XXH_PRIME64_4 +# define PRIME64_5 XXH_PRIME64_5 +#endif + +static xxh_u64 XXH64_round(xxh_u64 acc, xxh_u64 input) +{ + acc += input * XXH_PRIME64_2; + acc = XXH_rotl64(acc, 31); + acc *= XXH_PRIME64_1; + return acc; +} + +static xxh_u64 XXH64_mergeRound(xxh_u64 acc, xxh_u64 val) +{ + val = XXH64_round(0, val); + acc ^= val; + acc = acc * XXH_PRIME64_1 + XXH_PRIME64_4; + return acc; +} + +static xxh_u64 XXH64_avalanche(xxh_u64 h64) +{ + h64 ^= h64 >> 33; + h64 *= XXH_PRIME64_2; + h64 ^= h64 >> 29; + h64 *= XXH_PRIME64_3; + h64 ^= h64 >> 32; + return h64; +} + + +#define XXH_get64bits(p) XXH_readLE64_align(p, align) + +static xxh_u64 +XXH64_finalize(xxh_u64 h64, const xxh_u8* ptr, size_t len, XXH_alignment align) +{ + len &= 31; + while (len >= 8) { + xxh_u64 const k1 = XXH64_round(0, XXH_get64bits(ptr)); + ptr += 8; + h64 ^= k1; + h64 = XXH_rotl64(h64,27) * XXH_PRIME64_1 + XXH_PRIME64_4; + len -= 8; + } + if (len >= 4) { + h64 ^= (xxh_u64)(XXH_get32bits(ptr)) * XXH_PRIME64_1; + ptr += 4; + h64 = XXH_rotl64(h64, 23) * XXH_PRIME64_2 + XXH_PRIME64_3; + len -= 4; + } + while (len > 0) { + h64 ^= (*ptr++) * XXH_PRIME64_5; + h64 = XXH_rotl64(h64, 11) * XXH_PRIME64_1; + --len; + } + return XXH64_avalanche(h64); +} + +#ifdef XXH_OLD_NAMES +# define PROCESS1_64 XXH_PROCESS1_64 +# define PROCESS4_64 XXH_PROCESS4_64 +# define PROCESS8_64 XXH_PROCESS8_64 +#else +# undef XXH_PROCESS1_64 +# undef XXH_PROCESS4_64 +# undef XXH_PROCESS8_64 +#endif + +XXH_FORCE_INLINE xxh_u64 +XXH64_endian_align(const xxh_u8* input, size_t len, xxh_u64 seed, XXH_alignment align) +{ + const xxh_u8* bEnd = input ? input + len : NULL; + xxh_u64 h64; + +#if defined(XXH_ACCEPT_NULL_INPUT_POINTER) && (XXH_ACCEPT_NULL_INPUT_POINTER>=1) + if (input==NULL) { + len=0; + bEnd=input=(const xxh_u8*)(size_t)32; + } +#endif + + if (len>=32) { + const xxh_u8* const limit = bEnd - 32; + xxh_u64 v1 = seed + XXH_PRIME64_1 + XXH_PRIME64_2; + xxh_u64 v2 = seed + XXH_PRIME64_2; + xxh_u64 v3 = seed + 0; + xxh_u64 v4 = seed - XXH_PRIME64_1; + + do { + v1 = XXH64_round(v1, XXH_get64bits(input)); input+=8; + v2 = XXH64_round(v2, XXH_get64bits(input)); input+=8; + v3 = XXH64_round(v3, XXH_get64bits(input)); input+=8; + v4 = XXH64_round(v4, XXH_get64bits(input)); input+=8; + } while (input<=limit); + + h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18); + h64 = XXH64_mergeRound(h64, v1); + h64 = XXH64_mergeRound(h64, v2); + h64 = XXH64_mergeRound(h64, v3); + h64 = XXH64_mergeRound(h64, v4); + + } else { + h64 = seed + XXH_PRIME64_5; + } + + h64 += (xxh_u64) len; + + return XXH64_finalize(h64, input, len, align); +} + + +/*! @ingroup xxh64_family */ +XXH_PUBLIC_API XXH64_hash_t XXH64 (const void* input, size_t len, XXH64_hash_t seed) +{ +#if 0 + /* Simple version, good for code maintenance, but unfortunately slow for small inputs */ + XXH64_state_t state; + XXH64_reset(&state, seed); + XXH64_update(&state, (const xxh_u8*)input, len); + return XXH64_digest(&state); +#else + if (XXH_FORCE_ALIGN_CHECK) { + if ((((size_t)input) & 7)==0) { /* Input is aligned, let's leverage the speed advantage */ + return XXH64_endian_align((const xxh_u8*)input, len, seed, XXH_aligned); + } } + + return XXH64_endian_align((const xxh_u8*)input, len, seed, XXH_unaligned); + +#endif +} + +/******* Hash Streaming *******/ + +/*! @ingroup xxh64_family*/ +XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void) +{ + return (XXH64_state_t*)XXH_malloc(sizeof(XXH64_state_t)); +} +/*! @ingroup xxh64_family */ +XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr) +{ + XXH_free(statePtr); + return XXH_OK; +} + +/*! @ingroup xxh64_family */ +XXH_PUBLIC_API void XXH64_copyState(XXH64_state_t* dstState, const XXH64_state_t* srcState) +{ + memcpy(dstState, srcState, sizeof(*dstState)); +} + +/*! @ingroup xxh64_family */ +XXH_PUBLIC_API XXH_errorcode XXH64_reset(XXH64_state_t* statePtr, XXH64_hash_t seed) +{ + XXH64_state_t state; /* use a local state to memcpy() in order to avoid strict-aliasing warnings */ + memset(&state, 0, sizeof(state)); + state.v1 = seed + XXH_PRIME64_1 + XXH_PRIME64_2; + state.v2 = seed + XXH_PRIME64_2; + state.v3 = seed + 0; + state.v4 = seed - XXH_PRIME64_1; + /* do not write into reserved64, might be removed in a future version */ + memcpy(statePtr, &state, sizeof(state) - sizeof(state.reserved64)); + return XXH_OK; +} + +/*! @ingroup xxh64_family */ +XXH_PUBLIC_API XXH_errorcode +XXH64_update (XXH64_state_t* state, const void* input, size_t len) +{ + if (input==NULL) +#if defined(XXH_ACCEPT_NULL_INPUT_POINTER) && (XXH_ACCEPT_NULL_INPUT_POINTER>=1) + return XXH_OK; +#else + return XXH_ERROR; +#endif + + { const xxh_u8* p = (const xxh_u8*)input; + const xxh_u8* const bEnd = p + len; + + state->total_len += len; + + if (state->memsize + len < 32) { /* fill in tmp buffer */ + XXH_memcpy(((xxh_u8*)state->mem64) + state->memsize, input, len); + state->memsize += (xxh_u32)len; + return XXH_OK; + } + + if (state->memsize) { /* tmp buffer is full */ + XXH_memcpy(((xxh_u8*)state->mem64) + state->memsize, input, 32-state->memsize); + state->v1 = XXH64_round(state->v1, XXH_readLE64(state->mem64+0)); + state->v2 = XXH64_round(state->v2, XXH_readLE64(state->mem64+1)); + state->v3 = XXH64_round(state->v3, XXH_readLE64(state->mem64+2)); + state->v4 = XXH64_round(state->v4, XXH_readLE64(state->mem64+3)); + p += 32 - state->memsize; + state->memsize = 0; + } + + if (p+32 <= bEnd) { + const xxh_u8* const limit = bEnd - 32; + xxh_u64 v1 = state->v1; + xxh_u64 v2 = state->v2; + xxh_u64 v3 = state->v3; + xxh_u64 v4 = state->v4; + + do { + v1 = XXH64_round(v1, XXH_readLE64(p)); p+=8; + v2 = XXH64_round(v2, XXH_readLE64(p)); p+=8; + v3 = XXH64_round(v3, XXH_readLE64(p)); p+=8; + v4 = XXH64_round(v4, XXH_readLE64(p)); p+=8; + } while (p<=limit); + + state->v1 = v1; + state->v2 = v2; + state->v3 = v3; + state->v4 = v4; + } + + if (p < bEnd) { + XXH_memcpy(state->mem64, p, (size_t)(bEnd-p)); + state->memsize = (unsigned)(bEnd-p); + } + } + + return XXH_OK; +} + + +/*! @ingroup xxh64_family */ +XXH_PUBLIC_API XXH64_hash_t XXH64_digest(const XXH64_state_t* state) +{ + xxh_u64 h64; + + if (state->total_len >= 32) { + xxh_u64 const v1 = state->v1; + xxh_u64 const v2 = state->v2; + xxh_u64 const v3 = state->v3; + xxh_u64 const v4 = state->v4; + + h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18); + h64 = XXH64_mergeRound(h64, v1); + h64 = XXH64_mergeRound(h64, v2); + h64 = XXH64_mergeRound(h64, v3); + h64 = XXH64_mergeRound(h64, v4); + } else { + h64 = state->v3 /*seed*/ + XXH_PRIME64_5; + } + + h64 += (xxh_u64) state->total_len; + + return XXH64_finalize(h64, (const xxh_u8*)state->mem64, (size_t)state->total_len, XXH_aligned); +} + + +/******* Canonical representation *******/ + +/*! @ingroup xxh64_family */ +XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t* dst, XXH64_hash_t hash) +{ + XXH_STATIC_ASSERT(sizeof(XXH64_canonical_t) == sizeof(XXH64_hash_t)); + if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap64(hash); + memcpy(dst, &hash, sizeof(*dst)); +} + +/*! @ingroup xxh64_family */ +XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src) +{ + return XXH_readBE64(src); +} + +#ifndef XXH_NO_XXH3 + +/* ********************************************************************* +* XXH3 +* New generation hash designed for speed on small keys and vectorization +************************************************************************ */ +/*! + * @} + * @defgroup xxh3_impl XXH3 implementation + * @ingroup impl + * @{ + */ + +/* === Compiler specifics === */ + +#if ((defined(sun) || defined(__sun)) && __cplusplus) /* Solaris includes __STDC_VERSION__ with C++. Tested with GCC 5.5 */ +# define XXH_RESTRICT /* disable */ +#elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* >= C99 */ +# define XXH_RESTRICT restrict +#else +/* Note: it might be useful to define __restrict or __restrict__ for some C++ compilers */ +# define XXH_RESTRICT /* disable */ +#endif + +#if (defined(__GNUC__) && (__GNUC__ >= 3)) \ + || (defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 800)) \ + || defined(__clang__) +# define XXH_likely(x) __builtin_expect(x, 1) +# define XXH_unlikely(x) __builtin_expect(x, 0) +#else +# define XXH_likely(x) (x) +# define XXH_unlikely(x) (x) +#endif + +#if defined(__GNUC__) +# if defined(__AVX2__) +# include +# elif defined(__SSE2__) +# include +# elif defined(__ARM_NEON__) || defined(__ARM_NEON) +# define inline __inline__ /* circumvent a clang bug */ +# include +# undef inline +# endif +#elif defined(_MSC_VER) +# include +#endif + +/* + * One goal of XXH3 is to make it fast on both 32-bit and 64-bit, while + * remaining a true 64-bit/128-bit hash function. + * + * This is done by prioritizing a subset of 64-bit operations that can be + * emulated without too many steps on the average 32-bit machine. + * + * For example, these two lines seem similar, and run equally fast on 64-bit: + * + * xxh_u64 x; + * x ^= (x >> 47); // good + * x ^= (x >> 13); // bad + * + * However, to a 32-bit machine, there is a major difference. + * + * x ^= (x >> 47) looks like this: + * + * x.lo ^= (x.hi >> (47 - 32)); + * + * while x ^= (x >> 13) looks like this: + * + * // note: funnel shifts are not usually cheap. + * x.lo ^= (x.lo >> 13) | (x.hi << (32 - 13)); + * x.hi ^= (x.hi >> 13); + * + * The first one is significantly faster than the second, simply because the + * shift is larger than 32. This means: + * - All the bits we need are in the upper 32 bits, so we can ignore the lower + * 32 bits in the shift. + * - The shift result will always fit in the lower 32 bits, and therefore, + * we can ignore the upper 32 bits in the xor. + * + * Thanks to this optimization, XXH3 only requires these features to be efficient: + * + * - Usable unaligned access + * - A 32-bit or 64-bit ALU + * - If 32-bit, a decent ADC instruction + * - A 32 or 64-bit multiply with a 64-bit result + * - For the 128-bit variant, a decent byteswap helps short inputs. + * + * The first two are already required by XXH32, and almost all 32-bit and 64-bit + * platforms which can run XXH32 can run XXH3 efficiently. + * + * Thumb-1, the classic 16-bit only subset of ARM's instruction set, is one + * notable exception. + * + * First of all, Thumb-1 lacks support for the UMULL instruction which + * performs the important long multiply. This means numerous __aeabi_lmul + * calls. + * + * Second of all, the 8 functional registers are just not enough. + * Setup for __aeabi_lmul, byteshift loads, pointers, and all arithmetic need + * Lo registers, and this shuffling results in thousands more MOVs than A32. + * + * A32 and T32 don't have this limitation. They can access all 14 registers, + * do a 32->64 multiply with UMULL, and the flexible operand allowing free + * shifts is helpful, too. + * + * Therefore, we do a quick sanity check. + * + * If compiling Thumb-1 for a target which supports ARM instructions, we will + * emit a warning, as it is not a "sane" platform to compile for. + * + * Usually, if this happens, it is because of an accident and you probably need + * to specify -march, as you likely meant to compile for a newer architecture. + * + * Credit: large sections of the vectorial and asm source code paths + * have been contributed by @easyaspi314 + */ +#if defined(__thumb__) && !defined(__thumb2__) && defined(__ARM_ARCH_ISA_ARM) +# warning "XXH3 is highly inefficient without ARM or Thumb-2." +#endif + +/* ========================================== + * Vectorization detection + * ========================================== */ + +#ifdef XXH_DOXYGEN +/*! + * @ingroup tuning + * @brief Overrides the vectorization implementation chosen for XXH3. + * + * Can be defined to 0 to disable SIMD or any of the values mentioned in + * @ref XXH_VECTOR_TYPE. + * + * If this is not defined, it uses predefined macros to determine the best + * implementation. + */ +# define XXH_VECTOR XXH_SCALAR +/*! + * @ingroup tuning + * @brief Possible values for @ref XXH_VECTOR. + * + * Note that these are actually implemented as macros. + * + * If this is not defined, it is detected automatically. + * @ref XXH_X86DISPATCH overrides this. + */ +enum XXH_VECTOR_TYPE /* fake enum */ { + XXH_SCALAR = 0, /*!< Portable scalar version */ + XXH_SSE2 = 1, /*!< + * SSE2 for Pentium 4, Opteron, all x86_64. + * + * @note SSE2 is also guaranteed on Windows 10, macOS, and + * Android x86. + */ + XXH_AVX2 = 2, /*!< AVX2 for Haswell and Bulldozer */ + XXH_AVX512 = 3, /*!< AVX512 for Skylake and Icelake */ + XXH_NEON = 4, /*!< NEON for most ARMv7-A and all AArch64 */ + XXH_VSX = 5, /*!< VSX and ZVector for POWER8/z13 (64-bit) */ +}; +/*! + * @ingroup tuning + * @brief Selects the minimum alignment for XXH3's accumulators. + * + * When using SIMD, this should match the alignment reqired for said vector + * type, so, for example, 32 for AVX2. + * + * Default: Auto detected. + */ +# define XXH_ACC_ALIGN 8 +#endif + +/* Actual definition */ +#ifndef XXH_DOXYGEN +# define XXH_SCALAR 0 +# define XXH_SSE2 1 +# define XXH_AVX2 2 +# define XXH_AVX512 3 +# define XXH_NEON 4 +# define XXH_VSX 5 +#endif + +#ifndef XXH_VECTOR /* can be defined on command line */ +# if defined(__AVX512F__) +# define XXH_VECTOR XXH_AVX512 +# elif defined(__AVX2__) +# define XXH_VECTOR XXH_AVX2 +# elif defined(__SSE2__) || defined(_M_AMD64) || defined(_M_X64) || (defined(_M_IX86_FP) && (_M_IX86_FP == 2)) +# define XXH_VECTOR XXH_SSE2 +# elif defined(__GNUC__) /* msvc support maybe later */ \ + && (defined(__ARM_NEON__) || defined(__ARM_NEON)) \ + && (defined(__LITTLE_ENDIAN__) /* We only support little endian NEON */ \ + || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)) +# define XXH_VECTOR XXH_NEON +# elif (defined(__PPC64__) && defined(__POWER8_VECTOR__)) \ + || (defined(__s390x__) && defined(__VEC__)) \ + && defined(__GNUC__) /* TODO: IBM XL */ +# define XXH_VECTOR XXH_VSX +# else +# define XXH_VECTOR XXH_SCALAR +# endif +#endif + +/* + * Controls the alignment of the accumulator, + * for compatibility with aligned vector loads, which are usually faster. + */ +#ifndef XXH_ACC_ALIGN +# if defined(XXH_X86DISPATCH) +# define XXH_ACC_ALIGN 64 /* for compatibility with avx512 */ +# elif XXH_VECTOR == XXH_SCALAR /* scalar */ +# define XXH_ACC_ALIGN 8 +# elif XXH_VECTOR == XXH_SSE2 /* sse2 */ +# define XXH_ACC_ALIGN 16 +# elif XXH_VECTOR == XXH_AVX2 /* avx2 */ +# define XXH_ACC_ALIGN 32 +# elif XXH_VECTOR == XXH_NEON /* neon */ +# define XXH_ACC_ALIGN 16 +# elif XXH_VECTOR == XXH_VSX /* vsx */ +# define XXH_ACC_ALIGN 16 +# elif XXH_VECTOR == XXH_AVX512 /* avx512 */ +# define XXH_ACC_ALIGN 64 +# endif +#endif + +#if defined(XXH_X86DISPATCH) || XXH_VECTOR == XXH_SSE2 \ + || XXH_VECTOR == XXH_AVX2 || XXH_VECTOR == XXH_AVX512 +# define XXH_SEC_ALIGN XXH_ACC_ALIGN +#else +# define XXH_SEC_ALIGN 8 +#endif + +/* + * UGLY HACK: + * GCC usually generates the best code with -O3 for xxHash. + * + * However, when targeting AVX2, it is overzealous in its unrolling resulting + * in code roughly 3/4 the speed of Clang. + * + * There are other issues, such as GCC splitting _mm256_loadu_si256 into + * _mm_loadu_si128 + _mm256_inserti128_si256. This is an optimization which + * only applies to Sandy and Ivy Bridge... which don't even support AVX2. + * + * That is why when compiling the AVX2 version, it is recommended to use either + * -O2 -mavx2 -march=haswell + * or + * -O2 -mavx2 -mno-avx256-split-unaligned-load + * for decent performance, or to use Clang instead. + * + * Fortunately, we can control the first one with a pragma that forces GCC into + * -O2, but the other one we can't control without "failed to inline always + * inline function due to target mismatch" warnings. + */ +#if XXH_VECTOR == XXH_AVX2 /* AVX2 */ \ + && defined(__GNUC__) && !defined(__clang__) /* GCC, not Clang */ \ + && defined(__OPTIMIZE__) && !defined(__OPTIMIZE_SIZE__) /* respect -O0 and -Os */ +# pragma GCC push_options +# pragma GCC optimize("-O2") +#endif + + +#if XXH_VECTOR == XXH_NEON +/* + * NEON's setup for vmlal_u32 is a little more complicated than it is on + * SSE2, AVX2, and VSX. + * + * While PMULUDQ and VMULEUW both perform a mask, VMLAL.U32 performs an upcast. + * + * To do the same operation, the 128-bit 'Q' register needs to be split into + * two 64-bit 'D' registers, performing this operation:: + * + * [ a | b ] + * | '---------. .--------' | + * | x | + * | .---------' '--------. | + * [ a & 0xFFFFFFFF | b & 0xFFFFFFFF ],[ a >> 32 | b >> 32 ] + * + * Due to significant changes in aarch64, the fastest method for aarch64 is + * completely different than the fastest method for ARMv7-A. + * + * ARMv7-A treats D registers as unions overlaying Q registers, so modifying + * D11 will modify the high half of Q5. This is similar to how modifying AH + * will only affect bits 8-15 of AX on x86. + * + * VZIP takes two registers, and puts even lanes in one register and odd lanes + * in the other. + * + * On ARMv7-A, this strangely modifies both parameters in place instead of + * taking the usual 3-operand form. + * + * Therefore, if we want to do this, we can simply use a D-form VZIP.32 on the + * lower and upper halves of the Q register to end up with the high and low + * halves where we want - all in one instruction. + * + * vzip.32 d10, d11 @ d10 = { d10[0], d11[0] }; d11 = { d10[1], d11[1] } + * + * Unfortunately we need inline assembly for this: Instructions modifying two + * registers at once is not possible in GCC or Clang's IR, and they have to + * create a copy. + * + * aarch64 requires a different approach. + * + * In order to make it easier to write a decent compiler for aarch64, many + * quirks were removed, such as conditional execution. + * + * NEON was also affected by this. + * + * aarch64 cannot access the high bits of a Q-form register, and writes to a + * D-form register zero the high bits, similar to how writes to W-form scalar + * registers (or DWORD registers on x86_64) work. + * + * The formerly free vget_high intrinsics now require a vext (with a few + * exceptions) + * + * Additionally, VZIP was replaced by ZIP1 and ZIP2, which are the equivalent + * of PUNPCKL* and PUNPCKH* in SSE, respectively, in order to only modify one + * operand. + * + * The equivalent of the VZIP.32 on the lower and upper halves would be this + * mess: + * + * ext v2.4s, v0.4s, v0.4s, #2 // v2 = { v0[2], v0[3], v0[0], v0[1] } + * zip1 v1.2s, v0.2s, v2.2s // v1 = { v0[0], v2[0] } + * zip2 v0.2s, v0.2s, v1.2s // v0 = { v0[1], v2[1] } + * + * Instead, we use a literal downcast, vmovn_u64 (XTN), and vshrn_n_u64 (SHRN): + * + * shrn v1.2s, v0.2d, #32 // v1 = (uint32x2_t)(v0 >> 32); + * xtn v0.2s, v0.2d // v0 = (uint32x2_t)(v0 & 0xFFFFFFFF); + * + * This is available on ARMv7-A, but is less efficient than a single VZIP.32. + */ + +/*! + * Function-like macro: + * void XXH_SPLIT_IN_PLACE(uint64x2_t &in, uint32x2_t &outLo, uint32x2_t &outHi) + * { + * outLo = (uint32x2_t)(in & 0xFFFFFFFF); + * outHi = (uint32x2_t)(in >> 32); + * in = UNDEFINED; + * } + */ +# if !defined(XXH_NO_VZIP_HACK) /* define to disable */ \ + && defined(__GNUC__) \ + && !defined(__aarch64__) && !defined(__arm64__) +# define XXH_SPLIT_IN_PLACE(in, outLo, outHi) \ + do { \ + /* Undocumented GCC/Clang operand modifier: %e0 = lower D half, %f0 = upper D half */ \ + /* https://github.com/gcc-mirror/gcc/blob/38cf91e5/gcc/config/arm/arm.c#L22486 */ \ + /* https://github.com/llvm-mirror/llvm/blob/2c4ca683/lib/Target/ARM/ARMAsmPrinter.cpp#L399 */ \ + __asm__("vzip.32 %e0, %f0" : "+w" (in)); \ + (outLo) = vget_low_u32 (vreinterpretq_u32_u64(in)); \ + (outHi) = vget_high_u32(vreinterpretq_u32_u64(in)); \ + } while (0) +# else +# define XXH_SPLIT_IN_PLACE(in, outLo, outHi) \ + do { \ + (outLo) = vmovn_u64 (in); \ + (outHi) = vshrn_n_u64 ((in), 32); \ + } while (0) +# endif +#endif /* XXH_VECTOR == XXH_NEON */ + +/* + * VSX and Z Vector helpers. + * + * This is very messy, and any pull requests to clean this up are welcome. + * + * There are a lot of problems with supporting VSX and s390x, due to + * inconsistent intrinsics, spotty coverage, and multiple endiannesses. + */ +#if XXH_VECTOR == XXH_VSX +# if defined(__s390x__) +# include +# else +/* gcc's altivec.h can have the unwanted consequence to unconditionally + * #define bool, vector, and pixel keywords, + * with bad consequences for programs already using these keywords for other purposes. + * The paragraph defining these macros is skipped when __APPLE_ALTIVEC__ is defined. + * __APPLE_ALTIVEC__ is _generally_ defined automatically by the compiler, + * but it seems that, in some cases, it isn't. + * Force the build macro to be defined, so that keywords are not altered. + */ +# if defined(__GNUC__) && !defined(__APPLE_ALTIVEC__) +# define __APPLE_ALTIVEC__ +# endif +# include +# endif + +typedef __vector unsigned long long xxh_u64x2; +typedef __vector unsigned char xxh_u8x16; +typedef __vector unsigned xxh_u32x4; + +# ifndef XXH_VSX_BE +# if defined(__BIG_ENDIAN__) \ + || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) +# define XXH_VSX_BE 1 +# elif defined(__VEC_ELEMENT_REG_ORDER__) && __VEC_ELEMENT_REG_ORDER__ == __ORDER_BIG_ENDIAN__ +# warning "-maltivec=be is not recommended. Please use native endianness." +# define XXH_VSX_BE 1 +# else +# define XXH_VSX_BE 0 +# endif +# endif /* !defined(XXH_VSX_BE) */ + +# if XXH_VSX_BE +# if defined(__POWER9_VECTOR__) || (defined(__clang__) && defined(__s390x__)) +# define XXH_vec_revb vec_revb +# else +/*! + * A polyfill for POWER9's vec_revb(). + */ +XXH_FORCE_INLINE xxh_u64x2 XXH_vec_revb(xxh_u64x2 val) +{ + xxh_u8x16 const vByteSwap = { 0x07, 0x06, 0x05, 0x04, 0x03, 0x02, 0x01, 0x00, + 0x0F, 0x0E, 0x0D, 0x0C, 0x0B, 0x0A, 0x09, 0x08 }; + return vec_perm(val, val, vByteSwap); +} +# endif +# endif /* XXH_VSX_BE */ + +/*! + * Performs an unaligned vector load and byte swaps it on big endian. + */ +XXH_FORCE_INLINE xxh_u64x2 XXH_vec_loadu(const void *ptr) +{ + xxh_u64x2 ret; + memcpy(&ret, ptr, sizeof(xxh_u64x2)); +# if XXH_VSX_BE + ret = XXH_vec_revb(ret); +# endif + return ret; +} + +/* + * vec_mulo and vec_mule are very problematic intrinsics on PowerPC + * + * These intrinsics weren't added until GCC 8, despite existing for a while, + * and they are endian dependent. Also, their meaning swap depending on version. + * */ +# if defined(__s390x__) + /* s390x is always big endian, no issue on this platform */ +# define XXH_vec_mulo vec_mulo +# define XXH_vec_mule vec_mule +# elif defined(__clang__) && XXH_HAS_BUILTIN(__builtin_altivec_vmuleuw) +/* Clang has a better way to control this, we can just use the builtin which doesn't swap. */ +# define XXH_vec_mulo __builtin_altivec_vmulouw +# define XXH_vec_mule __builtin_altivec_vmuleuw +# else +/* gcc needs inline assembly */ +/* Adapted from https://github.com/google/highwayhash/blob/master/highwayhash/hh_vsx.h. */ +XXH_FORCE_INLINE xxh_u64x2 XXH_vec_mulo(xxh_u32x4 a, xxh_u32x4 b) +{ + xxh_u64x2 result; + __asm__("vmulouw %0, %1, %2" : "=v" (result) : "v" (a), "v" (b)); + return result; +} +XXH_FORCE_INLINE xxh_u64x2 XXH_vec_mule(xxh_u32x4 a, xxh_u32x4 b) +{ + xxh_u64x2 result; + __asm__("vmuleuw %0, %1, %2" : "=v" (result) : "v" (a), "v" (b)); + return result; +} +# endif /* XXH_vec_mulo, XXH_vec_mule */ +#endif /* XXH_VECTOR == XXH_VSX */ + + +/* prefetch + * can be disabled, by declaring XXH_NO_PREFETCH build macro */ +#if defined(XXH_NO_PREFETCH) +# define XXH_PREFETCH(ptr) (void)(ptr) /* disabled */ +#else +# if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_IX86)) /* _mm_prefetch() not defined outside of x86/x64 */ +# include /* https://msdn.microsoft.com/fr-fr/library/84szxsww(v=vs.90).aspx */ +# define XXH_PREFETCH(ptr) _mm_prefetch((const char*)(ptr), _MM_HINT_T0) +# elif defined(__GNUC__) && ( (__GNUC__ >= 4) || ( (__GNUC__ == 3) && (__GNUC_MINOR__ >= 1) ) ) +# define XXH_PREFETCH(ptr) __builtin_prefetch((ptr), 0 /* rw==read */, 3 /* locality */) +# else +# define XXH_PREFETCH(ptr) (void)(ptr) /* disabled */ +# endif +#endif /* XXH_NO_PREFETCH */ + + +/* ========================================== + * XXH3 default settings + * ========================================== */ + +#define XXH_SECRET_DEFAULT_SIZE 192 /* minimum XXH3_SECRET_SIZE_MIN */ + +#if (XXH_SECRET_DEFAULT_SIZE < XXH3_SECRET_SIZE_MIN) +# error "default keyset is not large enough" +#endif + +/*! Pseudorandom secret taken directly from FARSH. */ +XXH_ALIGN(64) static const xxh_u8 XXH3_kSecret[XXH_SECRET_DEFAULT_SIZE] = { + 0xb8, 0xfe, 0x6c, 0x39, 0x23, 0xa4, 0x4b, 0xbe, 0x7c, 0x01, 0x81, 0x2c, 0xf7, 0x21, 0xad, 0x1c, + 0xde, 0xd4, 0x6d, 0xe9, 0x83, 0x90, 0x97, 0xdb, 0x72, 0x40, 0xa4, 0xa4, 0xb7, 0xb3, 0x67, 0x1f, + 0xcb, 0x79, 0xe6, 0x4e, 0xcc, 0xc0, 0xe5, 0x78, 0x82, 0x5a, 0xd0, 0x7d, 0xcc, 0xff, 0x72, 0x21, + 0xb8, 0x08, 0x46, 0x74, 0xf7, 0x43, 0x24, 0x8e, 0xe0, 0x35, 0x90, 0xe6, 0x81, 0x3a, 0x26, 0x4c, + 0x3c, 0x28, 0x52, 0xbb, 0x91, 0xc3, 0x00, 0xcb, 0x88, 0xd0, 0x65, 0x8b, 0x1b, 0x53, 0x2e, 0xa3, + 0x71, 0x64, 0x48, 0x97, 0xa2, 0x0d, 0xf9, 0x4e, 0x38, 0x19, 0xef, 0x46, 0xa9, 0xde, 0xac, 0xd8, + 0xa8, 0xfa, 0x76, 0x3f, 0xe3, 0x9c, 0x34, 0x3f, 0xf9, 0xdc, 0xbb, 0xc7, 0xc7, 0x0b, 0x4f, 0x1d, + 0x8a, 0x51, 0xe0, 0x4b, 0xcd, 0xb4, 0x59, 0x31, 0xc8, 0x9f, 0x7e, 0xc9, 0xd9, 0x78, 0x73, 0x64, + 0xea, 0xc5, 0xac, 0x83, 0x34, 0xd3, 0xeb, 0xc3, 0xc5, 0x81, 0xa0, 0xff, 0xfa, 0x13, 0x63, 0xeb, + 0x17, 0x0d, 0xdd, 0x51, 0xb7, 0xf0, 0xda, 0x49, 0xd3, 0x16, 0x55, 0x26, 0x29, 0xd4, 0x68, 0x9e, + 0x2b, 0x16, 0xbe, 0x58, 0x7d, 0x47, 0xa1, 0xfc, 0x8f, 0xf8, 0xb8, 0xd1, 0x7a, 0xd0, 0x31, 0xce, + 0x45, 0xcb, 0x3a, 0x8f, 0x95, 0x16, 0x04, 0x28, 0xaf, 0xd7, 0xfb, 0xca, 0xbb, 0x4b, 0x40, 0x7e, +}; + + +#ifdef XXH_OLD_NAMES +# define kSecret XXH3_kSecret +#endif + +#ifdef XXH_DOXYGEN +/*! + * @brief Calculates a 32-bit to 64-bit long multiply. + * + * Implemented as a macro. + * + * Wraps `__emulu` on MSVC x86 because it tends to call `__allmul` when it doesn't + * need to (but it shouldn't need to anyways, it is about 7 instructions to do + * a 64x64 multiply...). Since we know that this will _always_ emit `MULL`, we + * use that instead of the normal method. + * + * If you are compiling for platforms like Thumb-1 and don't have a better option, + * you may also want to write your own long multiply routine here. + * + * @param x, y Numbers to be multiplied + * @return 64-bit product of the low 32 bits of @p x and @p y. + */ +XXH_FORCE_INLINE xxh_u64 +XXH_mult32to64(xxh_u64 x, xxh_u64 y) +{ + return (x & 0xFFFFFFFF) * (y & 0xFFFFFFFF); +} +#elif defined(_MSC_VER) && defined(_M_IX86) +# include +# define XXH_mult32to64(x, y) __emulu((unsigned)(x), (unsigned)(y)) +#else +/* + * Downcast + upcast is usually better than masking on older compilers like + * GCC 4.2 (especially 32-bit ones), all without affecting newer compilers. + * + * The other method, (x & 0xFFFFFFFF) * (y & 0xFFFFFFFF), will AND both operands + * and perform a full 64x64 multiply -- entirely redundant on 32-bit. + */ +# define XXH_mult32to64(x, y) ((xxh_u64)(xxh_u32)(x) * (xxh_u64)(xxh_u32)(y)) +#endif + +/*! + * @brief Calculates a 64->128-bit long multiply. + * + * Uses `__uint128_t` and `_umul128` if available, otherwise uses a scalar + * version. + * + * @param lhs, rhs The 64-bit integers to be multiplied + * @return The 128-bit result represented in an @ref XXH128_hash_t. + */ +static XXH128_hash_t +XXH_mult64to128(xxh_u64 lhs, xxh_u64 rhs) +{ + /* + * GCC/Clang __uint128_t method. + * + * On most 64-bit targets, GCC and Clang define a __uint128_t type. + * This is usually the best way as it usually uses a native long 64-bit + * multiply, such as MULQ on x86_64 or MUL + UMULH on aarch64. + * + * Usually. + * + * Despite being a 32-bit platform, Clang (and emscripten) define this type + * despite not having the arithmetic for it. This results in a laggy + * compiler builtin call which calculates a full 128-bit multiply. + * In that case it is best to use the portable one. + * https://github.com/Cyan4973/xxHash/issues/211#issuecomment-515575677 + */ +#if defined(__GNUC__) && !defined(__wasm__) \ + && defined(__SIZEOF_INT128__) \ + || (defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128) + + __uint128_t const product = (__uint128_t)lhs * (__uint128_t)rhs; + XXH128_hash_t r128; + r128.low64 = (xxh_u64)(product); + r128.high64 = (xxh_u64)(product >> 64); + return r128; + + /* + * MSVC for x64's _umul128 method. + * + * xxh_u64 _umul128(xxh_u64 Multiplier, xxh_u64 Multiplicand, xxh_u64 *HighProduct); + * + * This compiles to single operand MUL on x64. + */ +#elif defined(_M_X64) || defined(_M_IA64) + +#ifndef _MSC_VER +# pragma intrinsic(_umul128) +#endif + xxh_u64 product_high; + xxh_u64 const product_low = _umul128(lhs, rhs, &product_high); + XXH128_hash_t r128; + r128.low64 = product_low; + r128.high64 = product_high; + return r128; + +#else + /* + * Portable scalar method. Optimized for 32-bit and 64-bit ALUs. + * + * This is a fast and simple grade school multiply, which is shown below + * with base 10 arithmetic instead of base 0x100000000. + * + * 9 3 // D2 lhs = 93 + * x 7 5 // D2 rhs = 75 + * ---------- + * 1 5 // D2 lo_lo = (93 % 10) * (75 % 10) = 15 + * 4 5 | // D2 hi_lo = (93 / 10) * (75 % 10) = 45 + * 2 1 | // D2 lo_hi = (93 % 10) * (75 / 10) = 21 + * + 6 3 | | // D2 hi_hi = (93 / 10) * (75 / 10) = 63 + * --------- + * 2 7 | // D2 cross = (15 / 10) + (45 % 10) + 21 = 27 + * + 6 7 | | // D2 upper = (27 / 10) + (45 / 10) + 63 = 67 + * --------- + * 6 9 7 5 // D4 res = (27 * 10) + (15 % 10) + (67 * 100) = 6975 + * + * The reasons for adding the products like this are: + * 1. It avoids manual carry tracking. Just like how + * (9 * 9) + 9 + 9 = 99, the same applies with this for UINT64_MAX. + * This avoids a lot of complexity. + * + * 2. It hints for, and on Clang, compiles to, the powerful UMAAL + * instruction available in ARM's Digital Signal Processing extension + * in 32-bit ARMv6 and later, which is shown below: + * + * void UMAAL(xxh_u32 *RdLo, xxh_u32 *RdHi, xxh_u32 Rn, xxh_u32 Rm) + * { + * xxh_u64 product = (xxh_u64)*RdLo * (xxh_u64)*RdHi + Rn + Rm; + * *RdLo = (xxh_u32)(product & 0xFFFFFFFF); + * *RdHi = (xxh_u32)(product >> 32); + * } + * + * This instruction was designed for efficient long multiplication, and + * allows this to be calculated in only 4 instructions at speeds + * comparable to some 64-bit ALUs. + * + * 3. It isn't terrible on other platforms. Usually this will be a couple + * of 32-bit ADD/ADCs. + */ + + /* First calculate all of the cross products. */ + xxh_u64 const lo_lo = XXH_mult32to64(lhs & 0xFFFFFFFF, rhs & 0xFFFFFFFF); + xxh_u64 const hi_lo = XXH_mult32to64(lhs >> 32, rhs & 0xFFFFFFFF); + xxh_u64 const lo_hi = XXH_mult32to64(lhs & 0xFFFFFFFF, rhs >> 32); + xxh_u64 const hi_hi = XXH_mult32to64(lhs >> 32, rhs >> 32); + + /* Now add the products together. These will never overflow. */ + xxh_u64 const cross = (lo_lo >> 32) + (hi_lo & 0xFFFFFFFF) + lo_hi; + xxh_u64 const upper = (hi_lo >> 32) + (cross >> 32) + hi_hi; + xxh_u64 const lower = (cross << 32) | (lo_lo & 0xFFFFFFFF); + + XXH128_hash_t r128; + r128.low64 = lower; + r128.high64 = upper; + return r128; +#endif +} + +/*! + * @brief Calculates a 64-bit to 128-bit multiply, then XOR folds it. + * + * The reason for the separate function is to prevent passing too many structs + * around by value. This will hopefully inline the multiply, but we don't force it. + * + * @param lhs, rhs The 64-bit integers to multiply + * @return The low 64 bits of the product XOR'd by the high 64 bits. + * @see XXH_mult64to128() + */ +static xxh_u64 +XXH3_mul128_fold64(xxh_u64 lhs, xxh_u64 rhs) +{ + XXH128_hash_t product = XXH_mult64to128(lhs, rhs); + return product.low64 ^ product.high64; +} + +/*! Seems to produce slightly better code on GCC for some reason. */ +XXH_FORCE_INLINE xxh_u64 XXH_xorshift64(xxh_u64 v64, int shift) +{ + XXH_ASSERT(0 <= shift && shift < 64); + return v64 ^ (v64 >> shift); +} + +/* + * This is a fast avalanche stage, + * suitable when input bits are already partially mixed + */ +static XXH64_hash_t XXH3_avalanche(xxh_u64 h64) +{ + h64 = XXH_xorshift64(h64, 37); + h64 *= 0x165667919E3779F9ULL; + h64 = XXH_xorshift64(h64, 32); + return h64; +} + +/* + * This is a stronger avalanche, + * inspired by Pelle Evensen's rrmxmx + * preferable when input has not been previously mixed + */ +static XXH64_hash_t XXH3_rrmxmx(xxh_u64 h64, xxh_u64 len) +{ + /* this mix is inspired by Pelle Evensen's rrmxmx */ + h64 ^= XXH_rotl64(h64, 49) ^ XXH_rotl64(h64, 24); + h64 *= 0x9FB21C651E98DF25ULL; + h64 ^= (h64 >> 35) + len ; + h64 *= 0x9FB21C651E98DF25ULL; + return XXH_xorshift64(h64, 28); +} + + +/* ========================================== + * Short keys + * ========================================== + * One of the shortcomings of XXH32 and XXH64 was that their performance was + * sub-optimal on short lengths. It used an iterative algorithm which strongly + * favored lengths that were a multiple of 4 or 8. + * + * Instead of iterating over individual inputs, we use a set of single shot + * functions which piece together a range of lengths and operate in constant time. + * + * Additionally, the number of multiplies has been significantly reduced. This + * reduces latency, especially when emulating 64-bit multiplies on 32-bit. + * + * Depending on the platform, this may or may not be faster than XXH32, but it + * is almost guaranteed to be faster than XXH64. + */ + +/* + * At very short lengths, there isn't enough input to fully hide secrets, or use + * the entire secret. + * + * There is also only a limited amount of mixing we can do before significantly + * impacting performance. + * + * Therefore, we use different sections of the secret and always mix two secret + * samples with an XOR. This should have no effect on performance on the + * seedless or withSeed variants because everything _should_ be constant folded + * by modern compilers. + * + * The XOR mixing hides individual parts of the secret and increases entropy. + * + * This adds an extra layer of strength for custom secrets. + */ +XXH_FORCE_INLINE XXH64_hash_t +XXH3_len_1to3_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(input != NULL); + XXH_ASSERT(1 <= len && len <= 3); + XXH_ASSERT(secret != NULL); + /* + * len = 1: combined = { input[0], 0x01, input[0], input[0] } + * len = 2: combined = { input[1], 0x02, input[0], input[1] } + * len = 3: combined = { input[2], 0x03, input[0], input[1] } + */ + { xxh_u8 const c1 = input[0]; + xxh_u8 const c2 = input[len >> 1]; + xxh_u8 const c3 = input[len - 1]; + xxh_u32 const combined = ((xxh_u32)c1 << 16) | ((xxh_u32)c2 << 24) + | ((xxh_u32)c3 << 0) | ((xxh_u32)len << 8); + xxh_u64 const bitflip = (XXH_readLE32(secret) ^ XXH_readLE32(secret+4)) + seed; + xxh_u64 const keyed = (xxh_u64)combined ^ bitflip; + return XXH64_avalanche(keyed); + } +} + +XXH_FORCE_INLINE XXH64_hash_t +XXH3_len_4to8_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(input != NULL); + XXH_ASSERT(secret != NULL); + XXH_ASSERT(4 <= len && len <= 8); + seed ^= (xxh_u64)XXH_swap32((xxh_u32)seed) << 32; + { xxh_u32 const input1 = XXH_readLE32(input); + xxh_u32 const input2 = XXH_readLE32(input + len - 4); + xxh_u64 const bitflip = (XXH_readLE64(secret+8) ^ XXH_readLE64(secret+16)) - seed; + xxh_u64 const input64 = input2 + (((xxh_u64)input1) << 32); + xxh_u64 const keyed = input64 ^ bitflip; + return XXH3_rrmxmx(keyed, len); + } +} + +XXH_FORCE_INLINE XXH64_hash_t +XXH3_len_9to16_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(input != NULL); + XXH_ASSERT(secret != NULL); + XXH_ASSERT(8 <= len && len <= 16); + { xxh_u64 const bitflip1 = (XXH_readLE64(secret+24) ^ XXH_readLE64(secret+32)) + seed; + xxh_u64 const bitflip2 = (XXH_readLE64(secret+40) ^ XXH_readLE64(secret+48)) - seed; + xxh_u64 const input_lo = XXH_readLE64(input) ^ bitflip1; + xxh_u64 const input_hi = XXH_readLE64(input + len - 8) ^ bitflip2; + xxh_u64 const acc = len + + XXH_swap64(input_lo) + input_hi + + XXH3_mul128_fold64(input_lo, input_hi); + return XXH3_avalanche(acc); + } +} + +XXH_FORCE_INLINE XXH64_hash_t +XXH3_len_0to16_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(len <= 16); + { if (XXH_likely(len > 8)) return XXH3_len_9to16_64b(input, len, secret, seed); + if (XXH_likely(len >= 4)) return XXH3_len_4to8_64b(input, len, secret, seed); + if (len) return XXH3_len_1to3_64b(input, len, secret, seed); + return XXH64_avalanche(seed ^ (XXH_readLE64(secret+56) ^ XXH_readLE64(secret+64))); + } +} + +/* + * DISCLAIMER: There are known *seed-dependent* multicollisions here due to + * multiplication by zero, affecting hashes of lengths 17 to 240. + * + * However, they are very unlikely. + * + * Keep this in mind when using the unseeded XXH3_64bits() variant: As with all + * unseeded non-cryptographic hashes, it does not attempt to defend itself + * against specially crafted inputs, only random inputs. + * + * Compared to classic UMAC where a 1 in 2^31 chance of 4 consecutive bytes + * cancelling out the secret is taken an arbitrary number of times (addressed + * in XXH3_accumulate_512), this collision is very unlikely with random inputs + * and/or proper seeding: + * + * This only has a 1 in 2^63 chance of 8 consecutive bytes cancelling out, in a + * function that is only called up to 16 times per hash with up to 240 bytes of + * input. + * + * This is not too bad for a non-cryptographic hash function, especially with + * only 64 bit outputs. + * + * The 128-bit variant (which trades some speed for strength) is NOT affected + * by this, although it is always a good idea to use a proper seed if you care + * about strength. + */ +XXH_FORCE_INLINE xxh_u64 XXH3_mix16B(const xxh_u8* XXH_RESTRICT input, + const xxh_u8* XXH_RESTRICT secret, xxh_u64 seed64) +{ +#if defined(__GNUC__) && !defined(__clang__) /* GCC, not Clang */ \ + && defined(__i386__) && defined(__SSE2__) /* x86 + SSE2 */ \ + && !defined(XXH_ENABLE_AUTOVECTORIZE) /* Define to disable like XXH32 hack */ + /* + * UGLY HACK: + * GCC for x86 tends to autovectorize the 128-bit multiply, resulting in + * slower code. + * + * By forcing seed64 into a register, we disrupt the cost model and + * cause it to scalarize. See `XXH32_round()` + * + * FIXME: Clang's output is still _much_ faster -- On an AMD Ryzen 3600, + * XXH3_64bits @ len=240 runs at 4.6 GB/s with Clang 9, but 3.3 GB/s on + * GCC 9.2, despite both emitting scalar code. + * + * GCC generates much better scalar code than Clang for the rest of XXH3, + * which is why finding a more optimal codepath is an interest. + */ + XXH_COMPILER_GUARD(seed64); +#endif + { xxh_u64 const input_lo = XXH_readLE64(input); + xxh_u64 const input_hi = XXH_readLE64(input+8); + return XXH3_mul128_fold64( + input_lo ^ (XXH_readLE64(secret) + seed64), + input_hi ^ (XXH_readLE64(secret+8) - seed64) + ); + } +} + +/* For mid range keys, XXH3 uses a Mum-hash variant. */ +XXH_FORCE_INLINE XXH64_hash_t +XXH3_len_17to128_64b(const xxh_u8* XXH_RESTRICT input, size_t len, + const xxh_u8* XXH_RESTRICT secret, size_t secretSize, + XXH64_hash_t seed) +{ + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize; + XXH_ASSERT(16 < len && len <= 128); + + { xxh_u64 acc = len * XXH_PRIME64_1; + if (len > 32) { + if (len > 64) { + if (len > 96) { + acc += XXH3_mix16B(input+48, secret+96, seed); + acc += XXH3_mix16B(input+len-64, secret+112, seed); + } + acc += XXH3_mix16B(input+32, secret+64, seed); + acc += XXH3_mix16B(input+len-48, secret+80, seed); + } + acc += XXH3_mix16B(input+16, secret+32, seed); + acc += XXH3_mix16B(input+len-32, secret+48, seed); + } + acc += XXH3_mix16B(input+0, secret+0, seed); + acc += XXH3_mix16B(input+len-16, secret+16, seed); + + return XXH3_avalanche(acc); + } +} + +#define XXH3_MIDSIZE_MAX 240 + +XXH_NO_INLINE XXH64_hash_t +XXH3_len_129to240_64b(const xxh_u8* XXH_RESTRICT input, size_t len, + const xxh_u8* XXH_RESTRICT secret, size_t secretSize, + XXH64_hash_t seed) +{ + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize; + XXH_ASSERT(128 < len && len <= XXH3_MIDSIZE_MAX); + + #define XXH3_MIDSIZE_STARTOFFSET 3 + #define XXH3_MIDSIZE_LASTOFFSET 17 + + { xxh_u64 acc = len * XXH_PRIME64_1; + int const nbRounds = (int)len / 16; + int i; + for (i=0; i<8; i++) { + acc += XXH3_mix16B(input+(16*i), secret+(16*i), seed); + } + acc = XXH3_avalanche(acc); + XXH_ASSERT(nbRounds >= 8); +#if defined(__clang__) /* Clang */ \ + && (defined(__ARM_NEON) || defined(__ARM_NEON__)) /* NEON */ \ + && !defined(XXH_ENABLE_AUTOVECTORIZE) /* Define to disable */ + /* + * UGLY HACK: + * Clang for ARMv7-A tries to vectorize this loop, similar to GCC x86. + * In everywhere else, it uses scalar code. + * + * For 64->128-bit multiplies, even if the NEON was 100% optimal, it + * would still be slower than UMAAL (see XXH_mult64to128). + * + * Unfortunately, Clang doesn't handle the long multiplies properly and + * converts them to the nonexistent "vmulq_u64" intrinsic, which is then + * scalarized into an ugly mess of VMOV.32 instructions. + * + * This mess is difficult to avoid without turning autovectorization + * off completely, but they are usually relatively minor and/or not + * worth it to fix. + * + * This loop is the easiest to fix, as unlike XXH32, this pragma + * _actually works_ because it is a loop vectorization instead of an + * SLP vectorization. + */ + #pragma clang loop vectorize(disable) +#endif + for (i=8 ; i < nbRounds; i++) { + acc += XXH3_mix16B(input+(16*i), secret+(16*(i-8)) + XXH3_MIDSIZE_STARTOFFSET, seed); + } + /* last bytes */ + acc += XXH3_mix16B(input + len - 16, secret + XXH3_SECRET_SIZE_MIN - XXH3_MIDSIZE_LASTOFFSET, seed); + return XXH3_avalanche(acc); + } +} + + +/* ======= Long Keys ======= */ + +#define XXH_STRIPE_LEN 64 +#define XXH_SECRET_CONSUME_RATE 8 /* nb of secret bytes consumed at each accumulation */ +#define XXH_ACC_NB (XXH_STRIPE_LEN / sizeof(xxh_u64)) + +#ifdef XXH_OLD_NAMES +# define STRIPE_LEN XXH_STRIPE_LEN +# define ACC_NB XXH_ACC_NB +#endif + +XXH_FORCE_INLINE void XXH_writeLE64(void* dst, xxh_u64 v64) +{ + if (!XXH_CPU_LITTLE_ENDIAN) v64 = XXH_swap64(v64); + memcpy(dst, &v64, sizeof(v64)); +} + +/* Several intrinsic functions below are supposed to accept __int64 as argument, + * as documented in https://software.intel.com/sites/landingpage/IntrinsicsGuide/ . + * However, several environments do not define __int64 type, + * requiring a workaround. + */ +#if !defined (__VMS) \ + && (defined (__cplusplus) \ + || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) + typedef int64_t xxh_i64; +#else + /* the following type must have a width of 64-bit */ + typedef long long xxh_i64; +#endif + +/* + * XXH3_accumulate_512 is the tightest loop for long inputs, and it is the most optimized. + * + * It is a hardened version of UMAC, based off of FARSH's implementation. + * + * This was chosen because it adapts quite well to 32-bit, 64-bit, and SIMD + * implementations, and it is ridiculously fast. + * + * We harden it by mixing the original input to the accumulators as well as the product. + * + * This means that in the (relatively likely) case of a multiply by zero, the + * original input is preserved. + * + * On 128-bit inputs, we swap 64-bit pairs when we add the input to improve + * cross-pollination, as otherwise the upper and lower halves would be + * essentially independent. + * + * This doesn't matter on 64-bit hashes since they all get merged together in + * the end, so we skip the extra step. + * + * Both XXH3_64bits and XXH3_128bits use this subroutine. + */ + +#if (XXH_VECTOR == XXH_AVX512) \ + || (defined(XXH_DISPATCH_AVX512) && XXH_DISPATCH_AVX512 != 0) + +#ifndef XXH_TARGET_AVX512 +# define XXH_TARGET_AVX512 /* disable attribute target */ +#endif + +XXH_FORCE_INLINE XXH_TARGET_AVX512 void +XXH3_accumulate_512_avx512(void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + XXH_ALIGN(64) __m512i* const xacc = (__m512i *) acc; + XXH_ASSERT((((size_t)acc) & 63) == 0); + XXH_STATIC_ASSERT(XXH_STRIPE_LEN == sizeof(__m512i)); + + { + /* data_vec = input[0]; */ + __m512i const data_vec = _mm512_loadu_si512 (input); + /* key_vec = secret[0]; */ + __m512i const key_vec = _mm512_loadu_si512 (secret); + /* data_key = data_vec ^ key_vec; */ + __m512i const data_key = _mm512_xor_si512 (data_vec, key_vec); + /* data_key_lo = data_key >> 32; */ + __m512i const data_key_lo = _mm512_shuffle_epi32 (data_key, (_MM_PERM_ENUM)_MM_SHUFFLE(0, 3, 0, 1)); + /* product = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */ + __m512i const product = _mm512_mul_epu32 (data_key, data_key_lo); + /* xacc[0] += swap(data_vec); */ + __m512i const data_swap = _mm512_shuffle_epi32(data_vec, (_MM_PERM_ENUM)_MM_SHUFFLE(1, 0, 3, 2)); + __m512i const sum = _mm512_add_epi64(*xacc, data_swap); + /* xacc[0] += product; */ + *xacc = _mm512_add_epi64(product, sum); + } +} + +/* + * XXH3_scrambleAcc: Scrambles the accumulators to improve mixing. + * + * Multiplication isn't perfect, as explained by Google in HighwayHash: + * + * // Multiplication mixes/scrambles bytes 0-7 of the 64-bit result to + * // varying degrees. In descending order of goodness, bytes + * // 3 4 2 5 1 6 0 7 have quality 228 224 164 160 100 96 36 32. + * // As expected, the upper and lower bytes are much worse. + * + * Source: https://github.com/google/highwayhash/blob/0aaf66b/highwayhash/hh_avx2.h#L291 + * + * Since our algorithm uses a pseudorandom secret to add some variance into the + * mix, we don't need to (or want to) mix as often or as much as HighwayHash does. + * + * This isn't as tight as XXH3_accumulate, but still written in SIMD to avoid + * extraction. + * + * Both XXH3_64bits and XXH3_128bits use this subroutine. + */ + +XXH_FORCE_INLINE XXH_TARGET_AVX512 void +XXH3_scrambleAcc_avx512(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 63) == 0); + XXH_STATIC_ASSERT(XXH_STRIPE_LEN == sizeof(__m512i)); + { XXH_ALIGN(64) __m512i* const xacc = (__m512i*) acc; + const __m512i prime32 = _mm512_set1_epi32((int)XXH_PRIME32_1); + + /* xacc[0] ^= (xacc[0] >> 47) */ + __m512i const acc_vec = *xacc; + __m512i const shifted = _mm512_srli_epi64 (acc_vec, 47); + __m512i const data_vec = _mm512_xor_si512 (acc_vec, shifted); + /* xacc[0] ^= secret; */ + __m512i const key_vec = _mm512_loadu_si512 (secret); + __m512i const data_key = _mm512_xor_si512 (data_vec, key_vec); + + /* xacc[0] *= XXH_PRIME32_1; */ + __m512i const data_key_hi = _mm512_shuffle_epi32 (data_key, (_MM_PERM_ENUM)_MM_SHUFFLE(0, 3, 0, 1)); + __m512i const prod_lo = _mm512_mul_epu32 (data_key, prime32); + __m512i const prod_hi = _mm512_mul_epu32 (data_key_hi, prime32); + *xacc = _mm512_add_epi64(prod_lo, _mm512_slli_epi64(prod_hi, 32)); + } +} + +XXH_FORCE_INLINE XXH_TARGET_AVX512 void +XXH3_initCustomSecret_avx512(void* XXH_RESTRICT customSecret, xxh_u64 seed64) +{ + XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 63) == 0); + XXH_STATIC_ASSERT(XXH_SEC_ALIGN == 64); + XXH_ASSERT(((size_t)customSecret & 63) == 0); + (void)(&XXH_writeLE64); + { int const nbRounds = XXH_SECRET_DEFAULT_SIZE / sizeof(__m512i); + __m512i const seed = _mm512_mask_set1_epi64(_mm512_set1_epi64((xxh_i64)seed64), 0xAA, -(xxh_i64)seed64); + + XXH_ALIGN(64) const __m512i* const src = (const __m512i*) XXH3_kSecret; + XXH_ALIGN(64) __m512i* const dest = ( __m512i*) customSecret; + int i; + for (i=0; i < nbRounds; ++i) { + /* GCC has a bug, _mm512_stream_load_si512 accepts 'void*', not 'void const*', + * this will warn "discards ‘const’ qualifier". */ + union { + XXH_ALIGN(64) const __m512i* cp; + XXH_ALIGN(64) void* p; + } remote_const_void; + remote_const_void.cp = src + i; + dest[i] = _mm512_add_epi64(_mm512_stream_load_si512(remote_const_void.p), seed); + } } +} + +#endif + +#if (XXH_VECTOR == XXH_AVX2) \ + || (defined(XXH_DISPATCH_AVX2) && XXH_DISPATCH_AVX2 != 0) + +#ifndef XXH_TARGET_AVX2 +# define XXH_TARGET_AVX2 /* disable attribute target */ +#endif + +XXH_FORCE_INLINE XXH_TARGET_AVX2 void +XXH3_accumulate_512_avx2( void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 31) == 0); + { XXH_ALIGN(32) __m256i* const xacc = (__m256i *) acc; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm256_loadu_si256 requires a const __m256i * pointer for some reason. */ + const __m256i* const xinput = (const __m256i *) input; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm256_loadu_si256 requires a const __m256i * pointer for some reason. */ + const __m256i* const xsecret = (const __m256i *) secret; + + size_t i; + for (i=0; i < XXH_STRIPE_LEN/sizeof(__m256i); i++) { + /* data_vec = xinput[i]; */ + __m256i const data_vec = _mm256_loadu_si256 (xinput+i); + /* key_vec = xsecret[i]; */ + __m256i const key_vec = _mm256_loadu_si256 (xsecret+i); + /* data_key = data_vec ^ key_vec; */ + __m256i const data_key = _mm256_xor_si256 (data_vec, key_vec); + /* data_key_lo = data_key >> 32; */ + __m256i const data_key_lo = _mm256_shuffle_epi32 (data_key, _MM_SHUFFLE(0, 3, 0, 1)); + /* product = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */ + __m256i const product = _mm256_mul_epu32 (data_key, data_key_lo); + /* xacc[i] += swap(data_vec); */ + __m256i const data_swap = _mm256_shuffle_epi32(data_vec, _MM_SHUFFLE(1, 0, 3, 2)); + __m256i const sum = _mm256_add_epi64(xacc[i], data_swap); + /* xacc[i] += product; */ + xacc[i] = _mm256_add_epi64(product, sum); + } } +} + +XXH_FORCE_INLINE XXH_TARGET_AVX2 void +XXH3_scrambleAcc_avx2(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 31) == 0); + { XXH_ALIGN(32) __m256i* const xacc = (__m256i*) acc; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm256_loadu_si256 requires a const __m256i * pointer for some reason. */ + const __m256i* const xsecret = (const __m256i *) secret; + const __m256i prime32 = _mm256_set1_epi32((int)XXH_PRIME32_1); + + size_t i; + for (i=0; i < XXH_STRIPE_LEN/sizeof(__m256i); i++) { + /* xacc[i] ^= (xacc[i] >> 47) */ + __m256i const acc_vec = xacc[i]; + __m256i const shifted = _mm256_srli_epi64 (acc_vec, 47); + __m256i const data_vec = _mm256_xor_si256 (acc_vec, shifted); + /* xacc[i] ^= xsecret; */ + __m256i const key_vec = _mm256_loadu_si256 (xsecret+i); + __m256i const data_key = _mm256_xor_si256 (data_vec, key_vec); + + /* xacc[i] *= XXH_PRIME32_1; */ + __m256i const data_key_hi = _mm256_shuffle_epi32 (data_key, _MM_SHUFFLE(0, 3, 0, 1)); + __m256i const prod_lo = _mm256_mul_epu32 (data_key, prime32); + __m256i const prod_hi = _mm256_mul_epu32 (data_key_hi, prime32); + xacc[i] = _mm256_add_epi64(prod_lo, _mm256_slli_epi64(prod_hi, 32)); + } + } +} + +XXH_FORCE_INLINE XXH_TARGET_AVX2 void XXH3_initCustomSecret_avx2(void* XXH_RESTRICT customSecret, xxh_u64 seed64) +{ + XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 31) == 0); + XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE / sizeof(__m256i)) == 6); + XXH_STATIC_ASSERT(XXH_SEC_ALIGN <= 64); + (void)(&XXH_writeLE64); + XXH_PREFETCH(customSecret); + { __m256i const seed = _mm256_set_epi64x(-(xxh_i64)seed64, (xxh_i64)seed64, -(xxh_i64)seed64, (xxh_i64)seed64); + + XXH_ALIGN(64) const __m256i* const src = (const __m256i*) XXH3_kSecret; + XXH_ALIGN(64) __m256i* dest = ( __m256i*) customSecret; + +# if defined(__GNUC__) || defined(__clang__) + /* + * On GCC & Clang, marking 'dest' as modified will cause the compiler: + * - do not extract the secret from sse registers in the internal loop + * - use less common registers, and avoid pushing these reg into stack + */ + XXH_COMPILER_GUARD(dest); +# endif + + /* GCC -O2 need unroll loop manually */ + dest[0] = _mm256_add_epi64(_mm256_stream_load_si256(src+0), seed); + dest[1] = _mm256_add_epi64(_mm256_stream_load_si256(src+1), seed); + dest[2] = _mm256_add_epi64(_mm256_stream_load_si256(src+2), seed); + dest[3] = _mm256_add_epi64(_mm256_stream_load_si256(src+3), seed); + dest[4] = _mm256_add_epi64(_mm256_stream_load_si256(src+4), seed); + dest[5] = _mm256_add_epi64(_mm256_stream_load_si256(src+5), seed); + } +} + +#endif + +/* x86dispatch always generates SSE2 */ +#if (XXH_VECTOR == XXH_SSE2) || defined(XXH_X86DISPATCH) + +#ifndef XXH_TARGET_SSE2 +# define XXH_TARGET_SSE2 /* disable attribute target */ +#endif + +XXH_FORCE_INLINE XXH_TARGET_SSE2 void +XXH3_accumulate_512_sse2( void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + /* SSE2 is just a half-scale version of the AVX2 version. */ + XXH_ASSERT((((size_t)acc) & 15) == 0); + { XXH_ALIGN(16) __m128i* const xacc = (__m128i *) acc; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm_loadu_si128 requires a const __m128i * pointer for some reason. */ + const __m128i* const xinput = (const __m128i *) input; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm_loadu_si128 requires a const __m128i * pointer for some reason. */ + const __m128i* const xsecret = (const __m128i *) secret; + + size_t i; + for (i=0; i < XXH_STRIPE_LEN/sizeof(__m128i); i++) { + /* data_vec = xinput[i]; */ + __m128i const data_vec = _mm_loadu_si128 (xinput+i); + /* key_vec = xsecret[i]; */ + __m128i const key_vec = _mm_loadu_si128 (xsecret+i); + /* data_key = data_vec ^ key_vec; */ + __m128i const data_key = _mm_xor_si128 (data_vec, key_vec); + /* data_key_lo = data_key >> 32; */ + __m128i const data_key_lo = _mm_shuffle_epi32 (data_key, _MM_SHUFFLE(0, 3, 0, 1)); + /* product = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */ + __m128i const product = _mm_mul_epu32 (data_key, data_key_lo); + /* xacc[i] += swap(data_vec); */ + __m128i const data_swap = _mm_shuffle_epi32(data_vec, _MM_SHUFFLE(1,0,3,2)); + __m128i const sum = _mm_add_epi64(xacc[i], data_swap); + /* xacc[i] += product; */ + xacc[i] = _mm_add_epi64(product, sum); + } } +} + +XXH_FORCE_INLINE XXH_TARGET_SSE2 void +XXH3_scrambleAcc_sse2(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 15) == 0); + { XXH_ALIGN(16) __m128i* const xacc = (__m128i*) acc; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm_loadu_si128 requires a const __m128i * pointer for some reason. */ + const __m128i* const xsecret = (const __m128i *) secret; + const __m128i prime32 = _mm_set1_epi32((int)XXH_PRIME32_1); + + size_t i; + for (i=0; i < XXH_STRIPE_LEN/sizeof(__m128i); i++) { + /* xacc[i] ^= (xacc[i] >> 47) */ + __m128i const acc_vec = xacc[i]; + __m128i const shifted = _mm_srli_epi64 (acc_vec, 47); + __m128i const data_vec = _mm_xor_si128 (acc_vec, shifted); + /* xacc[i] ^= xsecret[i]; */ + __m128i const key_vec = _mm_loadu_si128 (xsecret+i); + __m128i const data_key = _mm_xor_si128 (data_vec, key_vec); + + /* xacc[i] *= XXH_PRIME32_1; */ + __m128i const data_key_hi = _mm_shuffle_epi32 (data_key, _MM_SHUFFLE(0, 3, 0, 1)); + __m128i const prod_lo = _mm_mul_epu32 (data_key, prime32); + __m128i const prod_hi = _mm_mul_epu32 (data_key_hi, prime32); + xacc[i] = _mm_add_epi64(prod_lo, _mm_slli_epi64(prod_hi, 32)); + } + } +} + +XXH_FORCE_INLINE XXH_TARGET_SSE2 void XXH3_initCustomSecret_sse2(void* XXH_RESTRICT customSecret, xxh_u64 seed64) +{ + XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 15) == 0); + (void)(&XXH_writeLE64); + { int const nbRounds = XXH_SECRET_DEFAULT_SIZE / sizeof(__m128i); + +# if defined(_MSC_VER) && defined(_M_IX86) && _MSC_VER < 1900 + // MSVC 32bit mode does not support _mm_set_epi64x before 2015 + XXH_ALIGN(16) const xxh_i64 seed64x2[2] = { (xxh_i64)seed64, -(xxh_i64)seed64 }; + __m128i const seed = _mm_load_si128((__m128i const*)seed64x2); +# else + __m128i const seed = _mm_set_epi64x(-(xxh_i64)seed64, (xxh_i64)seed64); +# endif + int i; + + XXH_ALIGN(64) const float* const src = (float const*) XXH3_kSecret; + XXH_ALIGN(XXH_SEC_ALIGN) __m128i* dest = (__m128i*) customSecret; +# if defined(__GNUC__) || defined(__clang__) + /* + * On GCC & Clang, marking 'dest' as modified will cause the compiler: + * - do not extract the secret from sse registers in the internal loop + * - use less common registers, and avoid pushing these reg into stack + */ + XXH_COMPILER_GUARD(dest); +# endif + + for (i=0; i < nbRounds; ++i) { + dest[i] = _mm_add_epi64(_mm_castps_si128(_mm_load_ps(src+i*4)), seed); + } } +} + +#endif + +#if (XXH_VECTOR == XXH_NEON) + +XXH_FORCE_INLINE void +XXH3_accumulate_512_neon( void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 15) == 0); + { + XXH_ALIGN(16) uint64x2_t* const xacc = (uint64x2_t *) acc; + /* We don't use a uint32x4_t pointer because it causes bus errors on ARMv7. */ + uint8_t const* const xinput = (const uint8_t *) input; + uint8_t const* const xsecret = (const uint8_t *) secret; + + size_t i; + for (i=0; i < XXH_STRIPE_LEN / sizeof(uint64x2_t); i++) { + /* data_vec = xinput[i]; */ + uint8x16_t data_vec = vld1q_u8(xinput + (i * 16)); + /* key_vec = xsecret[i]; */ + uint8x16_t key_vec = vld1q_u8(xsecret + (i * 16)); + uint64x2_t data_key; + uint32x2_t data_key_lo, data_key_hi; + /* xacc[i] += swap(data_vec); */ + uint64x2_t const data64 = vreinterpretq_u64_u8(data_vec); + uint64x2_t const swapped = vextq_u64(data64, data64, 1); + xacc[i] = vaddq_u64 (xacc[i], swapped); + /* data_key = data_vec ^ key_vec; */ + data_key = vreinterpretq_u64_u8(veorq_u8(data_vec, key_vec)); + /* data_key_lo = (uint32x2_t) (data_key & 0xFFFFFFFF); + * data_key_hi = (uint32x2_t) (data_key >> 32); + * data_key = UNDEFINED; */ + XXH_SPLIT_IN_PLACE(data_key, data_key_lo, data_key_hi); + /* xacc[i] += (uint64x2_t) data_key_lo * (uint64x2_t) data_key_hi; */ + xacc[i] = vmlal_u32 (xacc[i], data_key_lo, data_key_hi); + + } + } +} + +XXH_FORCE_INLINE void +XXH3_scrambleAcc_neon(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 15) == 0); + + { uint64x2_t* xacc = (uint64x2_t*) acc; + uint8_t const* xsecret = (uint8_t const*) secret; + uint32x2_t prime = vdup_n_u32 (XXH_PRIME32_1); + + size_t i; + for (i=0; i < XXH_STRIPE_LEN/sizeof(uint64x2_t); i++) { + /* xacc[i] ^= (xacc[i] >> 47); */ + uint64x2_t acc_vec = xacc[i]; + uint64x2_t shifted = vshrq_n_u64 (acc_vec, 47); + uint64x2_t data_vec = veorq_u64 (acc_vec, shifted); + + /* xacc[i] ^= xsecret[i]; */ + uint8x16_t key_vec = vld1q_u8(xsecret + (i * 16)); + uint64x2_t data_key = veorq_u64(data_vec, vreinterpretq_u64_u8(key_vec)); + + /* xacc[i] *= XXH_PRIME32_1 */ + uint32x2_t data_key_lo, data_key_hi; + /* data_key_lo = (uint32x2_t) (xacc[i] & 0xFFFFFFFF); + * data_key_hi = (uint32x2_t) (xacc[i] >> 32); + * xacc[i] = UNDEFINED; */ + XXH_SPLIT_IN_PLACE(data_key, data_key_lo, data_key_hi); + { /* + * prod_hi = (data_key >> 32) * XXH_PRIME32_1; + * + * Avoid vmul_u32 + vshll_n_u32 since Clang 6 and 7 will + * incorrectly "optimize" this: + * tmp = vmul_u32(vmovn_u64(a), vmovn_u64(b)); + * shifted = vshll_n_u32(tmp, 32); + * to this: + * tmp = "vmulq_u64"(a, b); // no such thing! + * shifted = vshlq_n_u64(tmp, 32); + * + * However, unlike SSE, Clang lacks a 64-bit multiply routine + * for NEON, and it scalarizes two 64-bit multiplies instead. + * + * vmull_u32 has the same timing as vmul_u32, and it avoids + * this bug completely. + * See https://bugs.llvm.org/show_bug.cgi?id=39967 + */ + uint64x2_t prod_hi = vmull_u32 (data_key_hi, prime); + /* xacc[i] = prod_hi << 32; */ + xacc[i] = vshlq_n_u64(prod_hi, 32); + /* xacc[i] += (prod_hi & 0xFFFFFFFF) * XXH_PRIME32_1; */ + xacc[i] = vmlal_u32(xacc[i], data_key_lo, prime); + } + } } +} + +#endif + +#if (XXH_VECTOR == XXH_VSX) + +XXH_FORCE_INLINE void +XXH3_accumulate_512_vsx( void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + xxh_u64x2* const xacc = (xxh_u64x2*) acc; /* presumed aligned */ + xxh_u64x2 const* const xinput = (xxh_u64x2 const*) input; /* no alignment restriction */ + xxh_u64x2 const* const xsecret = (xxh_u64x2 const*) secret; /* no alignment restriction */ + xxh_u64x2 const v32 = { 32, 32 }; + size_t i; + for (i = 0; i < XXH_STRIPE_LEN / sizeof(xxh_u64x2); i++) { + /* data_vec = xinput[i]; */ + xxh_u64x2 const data_vec = XXH_vec_loadu(xinput + i); + /* key_vec = xsecret[i]; */ + xxh_u64x2 const key_vec = XXH_vec_loadu(xsecret + i); + xxh_u64x2 const data_key = data_vec ^ key_vec; + /* shuffled = (data_key << 32) | (data_key >> 32); */ + xxh_u32x4 const shuffled = (xxh_u32x4)vec_rl(data_key, v32); + /* product = ((xxh_u64x2)data_key & 0xFFFFFFFF) * ((xxh_u64x2)shuffled & 0xFFFFFFFF); */ + xxh_u64x2 const product = XXH_vec_mulo((xxh_u32x4)data_key, shuffled); + xacc[i] += product; + + /* swap high and low halves */ +#ifdef __s390x__ + xacc[i] += vec_permi(data_vec, data_vec, 2); +#else + xacc[i] += vec_xxpermdi(data_vec, data_vec, 2); +#endif + } +} + +XXH_FORCE_INLINE void +XXH3_scrambleAcc_vsx(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 15) == 0); + + { xxh_u64x2* const xacc = (xxh_u64x2*) acc; + const xxh_u64x2* const xsecret = (const xxh_u64x2*) secret; + /* constants */ + xxh_u64x2 const v32 = { 32, 32 }; + xxh_u64x2 const v47 = { 47, 47 }; + xxh_u32x4 const prime = { XXH_PRIME32_1, XXH_PRIME32_1, XXH_PRIME32_1, XXH_PRIME32_1 }; + size_t i; + for (i = 0; i < XXH_STRIPE_LEN / sizeof(xxh_u64x2); i++) { + /* xacc[i] ^= (xacc[i] >> 47); */ + xxh_u64x2 const acc_vec = xacc[i]; + xxh_u64x2 const data_vec = acc_vec ^ (acc_vec >> v47); + + /* xacc[i] ^= xsecret[i]; */ + xxh_u64x2 const key_vec = XXH_vec_loadu(xsecret + i); + xxh_u64x2 const data_key = data_vec ^ key_vec; + + /* xacc[i] *= XXH_PRIME32_1 */ + /* prod_lo = ((xxh_u64x2)data_key & 0xFFFFFFFF) * ((xxh_u64x2)prime & 0xFFFFFFFF); */ + xxh_u64x2 const prod_even = XXH_vec_mule((xxh_u32x4)data_key, prime); + /* prod_hi = ((xxh_u64x2)data_key >> 32) * ((xxh_u64x2)prime >> 32); */ + xxh_u64x2 const prod_odd = XXH_vec_mulo((xxh_u32x4)data_key, prime); + xacc[i] = prod_odd + (prod_even << v32); + } } +} + +#endif + +/* scalar variants - universal */ + +XXH_FORCE_INLINE void +XXH3_accumulate_512_scalar(void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64* const xacc = (xxh_u64*) acc; /* presumed aligned */ + const xxh_u8* const xinput = (const xxh_u8*) input; /* no alignment restriction */ + const xxh_u8* const xsecret = (const xxh_u8*) secret; /* no alignment restriction */ + size_t i; + XXH_ASSERT(((size_t)acc & (XXH_ACC_ALIGN-1)) == 0); + for (i=0; i < XXH_ACC_NB; i++) { + xxh_u64 const data_val = XXH_readLE64(xinput + 8*i); + xxh_u64 const data_key = data_val ^ XXH_readLE64(xsecret + i*8); + xacc[i ^ 1] += data_val; /* swap adjacent lanes */ + xacc[i] += XXH_mult32to64(data_key & 0xFFFFFFFF, data_key >> 32); + } +} + +XXH_FORCE_INLINE void +XXH3_scrambleAcc_scalar(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) +{ + XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64* const xacc = (xxh_u64*) acc; /* presumed aligned */ + const xxh_u8* const xsecret = (const xxh_u8*) secret; /* no alignment restriction */ + size_t i; + XXH_ASSERT((((size_t)acc) & (XXH_ACC_ALIGN-1)) == 0); + for (i=0; i < XXH_ACC_NB; i++) { + xxh_u64 const key64 = XXH_readLE64(xsecret + 8*i); + xxh_u64 acc64 = xacc[i]; + acc64 = XXH_xorshift64(acc64, 47); + acc64 ^= key64; + acc64 *= XXH_PRIME32_1; + xacc[i] = acc64; + } +} + +XXH_FORCE_INLINE void +XXH3_initCustomSecret_scalar(void* XXH_RESTRICT customSecret, xxh_u64 seed64) +{ + /* + * We need a separate pointer for the hack below, + * which requires a non-const pointer. + * Any decent compiler will optimize this out otherwise. + */ + const xxh_u8* kSecretPtr = XXH3_kSecret; + XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 15) == 0); + +#if defined(__clang__) && defined(__aarch64__) + /* + * UGLY HACK: + * Clang generates a bunch of MOV/MOVK pairs for aarch64, and they are + * placed sequentially, in order, at the top of the unrolled loop. + * + * While MOVK is great for generating constants (2 cycles for a 64-bit + * constant compared to 4 cycles for LDR), long MOVK chains stall the + * integer pipelines: + * I L S + * MOVK + * MOVK + * MOVK + * MOVK + * ADD + * SUB STR + * STR + * By forcing loads from memory (as the asm line causes Clang to assume + * that XXH3_kSecretPtr has been changed), the pipelines are used more + * efficiently: + * I L S + * LDR + * ADD LDR + * SUB STR + * STR + * XXH3_64bits_withSeed, len == 256, Snapdragon 835 + * without hack: 2654.4 MB/s + * with hack: 3202.9 MB/s + */ + XXH_COMPILER_GUARD(kSecretPtr); +#endif + /* + * Note: in debug mode, this overrides the asm optimization + * and Clang will emit MOVK chains again. + */ + XXH_ASSERT(kSecretPtr == XXH3_kSecret); + + { int const nbRounds = XXH_SECRET_DEFAULT_SIZE / 16; + int i; + for (i=0; i < nbRounds; i++) { + /* + * The asm hack causes Clang to assume that kSecretPtr aliases with + * customSecret, and on aarch64, this prevented LDP from merging two + * loads together for free. Putting the loads together before the stores + * properly generates LDP. + */ + xxh_u64 lo = XXH_readLE64(kSecretPtr + 16*i) + seed64; + xxh_u64 hi = XXH_readLE64(kSecretPtr + 16*i + 8) - seed64; + XXH_writeLE64((xxh_u8*)customSecret + 16*i, lo); + XXH_writeLE64((xxh_u8*)customSecret + 16*i + 8, hi); + } } +} + + +typedef void (*XXH3_f_accumulate_512)(void* XXH_RESTRICT, const void*, const void*); +typedef void (*XXH3_f_scrambleAcc)(void* XXH_RESTRICT, const void*); +typedef void (*XXH3_f_initCustomSecret)(void* XXH_RESTRICT, xxh_u64); + + +#if (XXH_VECTOR == XXH_AVX512) + +#define XXH3_accumulate_512 XXH3_accumulate_512_avx512 +#define XXH3_scrambleAcc XXH3_scrambleAcc_avx512 +#define XXH3_initCustomSecret XXH3_initCustomSecret_avx512 + +#elif (XXH_VECTOR == XXH_AVX2) + +#define XXH3_accumulate_512 XXH3_accumulate_512_avx2 +#define XXH3_scrambleAcc XXH3_scrambleAcc_avx2 +#define XXH3_initCustomSecret XXH3_initCustomSecret_avx2 + +#elif (XXH_VECTOR == XXH_SSE2) + +#define XXH3_accumulate_512 XXH3_accumulate_512_sse2 +#define XXH3_scrambleAcc XXH3_scrambleAcc_sse2 +#define XXH3_initCustomSecret XXH3_initCustomSecret_sse2 + +#elif (XXH_VECTOR == XXH_NEON) + +#define XXH3_accumulate_512 XXH3_accumulate_512_neon +#define XXH3_scrambleAcc XXH3_scrambleAcc_neon +#define XXH3_initCustomSecret XXH3_initCustomSecret_scalar + +#elif (XXH_VECTOR == XXH_VSX) + +#define XXH3_accumulate_512 XXH3_accumulate_512_vsx +#define XXH3_scrambleAcc XXH3_scrambleAcc_vsx +#define XXH3_initCustomSecret XXH3_initCustomSecret_scalar + +#else /* scalar */ + +#define XXH3_accumulate_512 XXH3_accumulate_512_scalar +#define XXH3_scrambleAcc XXH3_scrambleAcc_scalar +#define XXH3_initCustomSecret XXH3_initCustomSecret_scalar + +#endif + + + +#ifndef XXH_PREFETCH_DIST +# ifdef __clang__ +# define XXH_PREFETCH_DIST 320 +# else +# if (XXH_VECTOR == XXH_AVX512) +# define XXH_PREFETCH_DIST 512 +# else +# define XXH_PREFETCH_DIST 384 +# endif +# endif /* __clang__ */ +#endif /* XXH_PREFETCH_DIST */ + +/* + * XXH3_accumulate() + * Loops over XXH3_accumulate_512(). + * Assumption: nbStripes will not overflow the secret size + */ +XXH_FORCE_INLINE void +XXH3_accumulate( xxh_u64* XXH_RESTRICT acc, + const xxh_u8* XXH_RESTRICT input, + const xxh_u8* XXH_RESTRICT secret, + size_t nbStripes, + XXH3_f_accumulate_512 f_acc512) +{ + size_t n; + for (n = 0; n < nbStripes; n++ ) { + const xxh_u8* const in = input + n*XXH_STRIPE_LEN; + XXH_PREFETCH(in + XXH_PREFETCH_DIST); + f_acc512(acc, + in, + secret + n*XXH_SECRET_CONSUME_RATE); + } +} + +XXH_FORCE_INLINE void +XXH3_hashLong_internal_loop(xxh_u64* XXH_RESTRICT acc, + const xxh_u8* XXH_RESTRICT input, size_t len, + const xxh_u8* XXH_RESTRICT secret, size_t secretSize, + XXH3_f_accumulate_512 f_acc512, + XXH3_f_scrambleAcc f_scramble) +{ + size_t const nbStripesPerBlock = (secretSize - XXH_STRIPE_LEN) / XXH_SECRET_CONSUME_RATE; + size_t const block_len = XXH_STRIPE_LEN * nbStripesPerBlock; + size_t const nb_blocks = (len - 1) / block_len; + + size_t n; + + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); + + for (n = 0; n < nb_blocks; n++) { + XXH3_accumulate(acc, input + n*block_len, secret, nbStripesPerBlock, f_acc512); + f_scramble(acc, secret + secretSize - XXH_STRIPE_LEN); + } + + /* last partial block */ + XXH_ASSERT(len > XXH_STRIPE_LEN); + { size_t const nbStripes = ((len - 1) - (block_len * nb_blocks)) / XXH_STRIPE_LEN; + XXH_ASSERT(nbStripes <= (secretSize / XXH_SECRET_CONSUME_RATE)); + XXH3_accumulate(acc, input + nb_blocks*block_len, secret, nbStripes, f_acc512); + + /* last stripe */ + { const xxh_u8* const p = input + len - XXH_STRIPE_LEN; +#define XXH_SECRET_LASTACC_START 7 /* not aligned on 8, last secret is different from acc & scrambler */ + f_acc512(acc, p, secret + secretSize - XXH_STRIPE_LEN - XXH_SECRET_LASTACC_START); + } } +} + +XXH_FORCE_INLINE xxh_u64 +XXH3_mix2Accs(const xxh_u64* XXH_RESTRICT acc, const xxh_u8* XXH_RESTRICT secret) +{ + return XXH3_mul128_fold64( + acc[0] ^ XXH_readLE64(secret), + acc[1] ^ XXH_readLE64(secret+8) ); +} + +static XXH64_hash_t +XXH3_mergeAccs(const xxh_u64* XXH_RESTRICT acc, const xxh_u8* XXH_RESTRICT secret, xxh_u64 start) +{ + xxh_u64 result64 = start; + size_t i = 0; + + for (i = 0; i < 4; i++) { + result64 += XXH3_mix2Accs(acc+2*i, secret + 16*i); +#if defined(__clang__) /* Clang */ \ + && (defined(__arm__) || defined(__thumb__)) /* ARMv7 */ \ + && (defined(__ARM_NEON) || defined(__ARM_NEON__)) /* NEON */ \ + && !defined(XXH_ENABLE_AUTOVECTORIZE) /* Define to disable */ + /* + * UGLY HACK: + * Prevent autovectorization on Clang ARMv7-a. Exact same problem as + * the one in XXH3_len_129to240_64b. Speeds up shorter keys > 240b. + * XXH3_64bits, len == 256, Snapdragon 835: + * without hack: 2063.7 MB/s + * with hack: 2560.7 MB/s + */ + XXH_COMPILER_GUARD(result64); +#endif + } + + return XXH3_avalanche(result64); +} + +#define XXH3_INIT_ACC { XXH_PRIME32_3, XXH_PRIME64_1, XXH_PRIME64_2, XXH_PRIME64_3, \ + XXH_PRIME64_4, XXH_PRIME32_2, XXH_PRIME64_5, XXH_PRIME32_1 } + +XXH_FORCE_INLINE XXH64_hash_t +XXH3_hashLong_64b_internal(const void* XXH_RESTRICT input, size_t len, + const void* XXH_RESTRICT secret, size_t secretSize, + XXH3_f_accumulate_512 f_acc512, + XXH3_f_scrambleAcc f_scramble) +{ + XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64 acc[XXH_ACC_NB] = XXH3_INIT_ACC; + + XXH3_hashLong_internal_loop(acc, (const xxh_u8*)input, len, (const xxh_u8*)secret, secretSize, f_acc512, f_scramble); + + /* converge into final hash */ + XXH_STATIC_ASSERT(sizeof(acc) == 64); + /* do not align on 8, so that the secret is different from the accumulator */ +#define XXH_SECRET_MERGEACCS_START 11 + XXH_ASSERT(secretSize >= sizeof(acc) + XXH_SECRET_MERGEACCS_START); + return XXH3_mergeAccs(acc, (const xxh_u8*)secret + XXH_SECRET_MERGEACCS_START, (xxh_u64)len * XXH_PRIME64_1); +} + +/* + * It's important for performance that XXH3_hashLong is not inlined. + */ +XXH_NO_INLINE XXH64_hash_t +XXH3_hashLong_64b_withSecret(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, const xxh_u8* XXH_RESTRICT secret, size_t secretLen) +{ + (void)seed64; + return XXH3_hashLong_64b_internal(input, len, secret, secretLen, XXH3_accumulate_512, XXH3_scrambleAcc); +} + +/* + * It's important for performance that XXH3_hashLong is not inlined. + * Since the function is not inlined, the compiler may not be able to understand that, + * in some scenarios, its `secret` argument is actually a compile time constant. + * This variant enforces that the compiler can detect that, + * and uses this opportunity to streamline the generated code for better performance. + */ +XXH_NO_INLINE XXH64_hash_t +XXH3_hashLong_64b_default(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, const xxh_u8* XXH_RESTRICT secret, size_t secretLen) +{ + (void)seed64; (void)secret; (void)secretLen; + return XXH3_hashLong_64b_internal(input, len, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_accumulate_512, XXH3_scrambleAcc); +} + +/* + * XXH3_hashLong_64b_withSeed(): + * Generate a custom key based on alteration of default XXH3_kSecret with the seed, + * and then use this key for long mode hashing. + * + * This operation is decently fast but nonetheless costs a little bit of time. + * Try to avoid it whenever possible (typically when seed==0). + * + * It's important for performance that XXH3_hashLong is not inlined. Not sure + * why (uop cache maybe?), but the difference is large and easily measurable. + */ +XXH_FORCE_INLINE XXH64_hash_t +XXH3_hashLong_64b_withSeed_internal(const void* input, size_t len, + XXH64_hash_t seed, + XXH3_f_accumulate_512 f_acc512, + XXH3_f_scrambleAcc f_scramble, + XXH3_f_initCustomSecret f_initSec) +{ + if (seed == 0) + return XXH3_hashLong_64b_internal(input, len, + XXH3_kSecret, sizeof(XXH3_kSecret), + f_acc512, f_scramble); + { XXH_ALIGN(XXH_SEC_ALIGN) xxh_u8 secret[XXH_SECRET_DEFAULT_SIZE]; + f_initSec(secret, seed); + return XXH3_hashLong_64b_internal(input, len, secret, sizeof(secret), + f_acc512, f_scramble); + } +} + +/* + * It's important for performance that XXH3_hashLong is not inlined. + */ +XXH_NO_INLINE XXH64_hash_t +XXH3_hashLong_64b_withSeed(const void* input, size_t len, + XXH64_hash_t seed, const xxh_u8* secret, size_t secretLen) +{ + (void)secret; (void)secretLen; + return XXH3_hashLong_64b_withSeed_internal(input, len, seed, + XXH3_accumulate_512, XXH3_scrambleAcc, XXH3_initCustomSecret); +} + + +typedef XXH64_hash_t (*XXH3_hashLong64_f)(const void* XXH_RESTRICT, size_t, + XXH64_hash_t, const xxh_u8* XXH_RESTRICT, size_t); + +XXH_FORCE_INLINE XXH64_hash_t +XXH3_64bits_internal(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, const void* XXH_RESTRICT secret, size_t secretLen, + XXH3_hashLong64_f f_hashLong) +{ + XXH_ASSERT(secretLen >= XXH3_SECRET_SIZE_MIN); + /* + * If an action is to be taken if `secretLen` condition is not respected, + * it should be done here. + * For now, it's a contract pre-condition. + * Adding a check and a branch here would cost performance at every hash. + * Also, note that function signature doesn't offer room to return an error. + */ + if (len <= 16) + return XXH3_len_0to16_64b((const xxh_u8*)input, len, (const xxh_u8*)secret, seed64); + if (len <= 128) + return XXH3_len_17to128_64b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64); + if (len <= XXH3_MIDSIZE_MAX) + return XXH3_len_129to240_64b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64); + return f_hashLong(input, len, seed64, (const xxh_u8*)secret, secretLen); +} + + +/* === Public entry point === */ + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH64_hash_t XXH3_64bits(const void* input, size_t len) +{ + return XXH3_64bits_internal(input, len, 0, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_hashLong_64b_default); +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH64_hash_t +XXH3_64bits_withSecret(const void* input, size_t len, const void* secret, size_t secretSize) +{ + return XXH3_64bits_internal(input, len, 0, secret, secretSize, XXH3_hashLong_64b_withSecret); +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH64_hash_t +XXH3_64bits_withSeed(const void* input, size_t len, XXH64_hash_t seed) +{ + return XXH3_64bits_internal(input, len, seed, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_hashLong_64b_withSeed); +} + + +/* === XXH3 streaming === */ + +/* + * Malloc's a pointer that is always aligned to align. + * + * This must be freed with `XXH_alignedFree()`. + * + * malloc typically guarantees 16 byte alignment on 64-bit systems and 8 byte + * alignment on 32-bit. This isn't enough for the 32 byte aligned loads in AVX2 + * or on 32-bit, the 16 byte aligned loads in SSE2 and NEON. + * + * This underalignment previously caused a rather obvious crash which went + * completely unnoticed due to XXH3_createState() not actually being tested. + * Credit to RedSpah for noticing this bug. + * + * The alignment is done manually: Functions like posix_memalign or _mm_malloc + * are avoided: To maintain portability, we would have to write a fallback + * like this anyways, and besides, testing for the existence of library + * functions without relying on external build tools is impossible. + * + * The method is simple: Overallocate, manually align, and store the offset + * to the original behind the returned pointer. + * + * Align must be a power of 2 and 8 <= align <= 128. + */ +static void* XXH_alignedMalloc(size_t s, size_t align) +{ + XXH_ASSERT(align <= 128 && align >= 8); /* range check */ + XXH_ASSERT((align & (align-1)) == 0); /* power of 2 */ + XXH_ASSERT(s != 0 && s < (s + align)); /* empty/overflow */ + { /* Overallocate to make room for manual realignment and an offset byte */ + xxh_u8* base = (xxh_u8*)XXH_malloc(s + align); + if (base != NULL) { + /* + * Get the offset needed to align this pointer. + * + * Even if the returned pointer is aligned, there will always be + * at least one byte to store the offset to the original pointer. + */ + size_t offset = align - ((size_t)base & (align - 1)); /* base % align */ + /* Add the offset for the now-aligned pointer */ + xxh_u8* ptr = base + offset; + + XXH_ASSERT((size_t)ptr % align == 0); + + /* Store the offset immediately before the returned pointer. */ + ptr[-1] = (xxh_u8)offset; + return ptr; + } + return NULL; + } +} +/* + * Frees an aligned pointer allocated by XXH_alignedMalloc(). Don't pass + * normal malloc'd pointers, XXH_alignedMalloc has a specific data layout. + */ +static void XXH_alignedFree(void* p) +{ + if (p != NULL) { + xxh_u8* ptr = (xxh_u8*)p; + /* Get the offset byte we added in XXH_malloc. */ + xxh_u8 offset = ptr[-1]; + /* Free the original malloc'd pointer */ + xxh_u8* base = ptr - offset; + XXH_free(base); + } +} +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH3_state_t* XXH3_createState(void) +{ + XXH3_state_t* const state = (XXH3_state_t*)XXH_alignedMalloc(sizeof(XXH3_state_t), 64); + if (state==NULL) return NULL; + XXH3_INITSTATE(state); + return state; +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH_errorcode XXH3_freeState(XXH3_state_t* statePtr) +{ + XXH_alignedFree(statePtr); + return XXH_OK; +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API void +XXH3_copyState(XXH3_state_t* dst_state, const XXH3_state_t* src_state) +{ + memcpy(dst_state, src_state, sizeof(*dst_state)); +} + +static void +XXH3_reset_internal(XXH3_state_t* statePtr, + XXH64_hash_t seed, + const void* secret, size_t secretSize) +{ + size_t const initStart = offsetof(XXH3_state_t, bufferedSize); + size_t const initLength = offsetof(XXH3_state_t, nbStripesPerBlock) - initStart; + XXH_ASSERT(offsetof(XXH3_state_t, nbStripesPerBlock) > initStart); + XXH_ASSERT(statePtr != NULL); + /* set members from bufferedSize to nbStripesPerBlock (excluded) to 0 */ + memset((char*)statePtr + initStart, 0, initLength); + statePtr->acc[0] = XXH_PRIME32_3; + statePtr->acc[1] = XXH_PRIME64_1; + statePtr->acc[2] = XXH_PRIME64_2; + statePtr->acc[3] = XXH_PRIME64_3; + statePtr->acc[4] = XXH_PRIME64_4; + statePtr->acc[5] = XXH_PRIME32_2; + statePtr->acc[6] = XXH_PRIME64_5; + statePtr->acc[7] = XXH_PRIME32_1; + statePtr->seed = seed; + statePtr->extSecret = (const unsigned char*)secret; + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); + statePtr->secretLimit = secretSize - XXH_STRIPE_LEN; + statePtr->nbStripesPerBlock = statePtr->secretLimit / XXH_SECRET_CONSUME_RATE; +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_reset(XXH3_state_t* statePtr) +{ + if (statePtr == NULL) return XXH_ERROR; + XXH3_reset_internal(statePtr, 0, XXH3_kSecret, XXH_SECRET_DEFAULT_SIZE); + return XXH_OK; +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_reset_withSecret(XXH3_state_t* statePtr, const void* secret, size_t secretSize) +{ + if (statePtr == NULL) return XXH_ERROR; + XXH3_reset_internal(statePtr, 0, secret, secretSize); + if (secret == NULL) return XXH_ERROR; + if (secretSize < XXH3_SECRET_SIZE_MIN) return XXH_ERROR; + return XXH_OK; +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_reset_withSeed(XXH3_state_t* statePtr, XXH64_hash_t seed) +{ + if (statePtr == NULL) return XXH_ERROR; + if (seed==0) return XXH3_64bits_reset(statePtr); + if (seed != statePtr->seed) XXH3_initCustomSecret(statePtr->customSecret, seed); + XXH3_reset_internal(statePtr, seed, NULL, XXH_SECRET_DEFAULT_SIZE); + return XXH_OK; +} + +/* Note : when XXH3_consumeStripes() is invoked, + * there must be a guarantee that at least one more byte must be consumed from input + * so that the function can blindly consume all stripes using the "normal" secret segment */ +XXH_FORCE_INLINE void +XXH3_consumeStripes(xxh_u64* XXH_RESTRICT acc, + size_t* XXH_RESTRICT nbStripesSoFarPtr, size_t nbStripesPerBlock, + const xxh_u8* XXH_RESTRICT input, size_t nbStripes, + const xxh_u8* XXH_RESTRICT secret, size_t secretLimit, + XXH3_f_accumulate_512 f_acc512, + XXH3_f_scrambleAcc f_scramble) +{ + XXH_ASSERT(nbStripes <= nbStripesPerBlock); /* can handle max 1 scramble per invocation */ + XXH_ASSERT(*nbStripesSoFarPtr < nbStripesPerBlock); + if (nbStripesPerBlock - *nbStripesSoFarPtr <= nbStripes) { + /* need a scrambling operation */ + size_t const nbStripesToEndofBlock = nbStripesPerBlock - *nbStripesSoFarPtr; + size_t const nbStripesAfterBlock = nbStripes - nbStripesToEndofBlock; + XXH3_accumulate(acc, input, secret + nbStripesSoFarPtr[0] * XXH_SECRET_CONSUME_RATE, nbStripesToEndofBlock, f_acc512); + f_scramble(acc, secret + secretLimit); + XXH3_accumulate(acc, input + nbStripesToEndofBlock * XXH_STRIPE_LEN, secret, nbStripesAfterBlock, f_acc512); + *nbStripesSoFarPtr = nbStripesAfterBlock; + } else { + XXH3_accumulate(acc, input, secret + nbStripesSoFarPtr[0] * XXH_SECRET_CONSUME_RATE, nbStripes, f_acc512); + *nbStripesSoFarPtr += nbStripes; + } +} + +/* + * Both XXH3_64bits_update and XXH3_128bits_update use this routine. + */ +XXH_FORCE_INLINE XXH_errorcode +XXH3_update(XXH3_state_t* state, + const xxh_u8* input, size_t len, + XXH3_f_accumulate_512 f_acc512, + XXH3_f_scrambleAcc f_scramble) +{ + if (input==NULL) +#if defined(XXH_ACCEPT_NULL_INPUT_POINTER) && (XXH_ACCEPT_NULL_INPUT_POINTER>=1) + return XXH_OK; +#else + return XXH_ERROR; +#endif + + { const xxh_u8* const bEnd = input + len; + const unsigned char* const secret = (state->extSecret == NULL) ? state->customSecret : state->extSecret; + + state->totalLen += len; + XXH_ASSERT(state->bufferedSize <= XXH3_INTERNALBUFFER_SIZE); + + if (state->bufferedSize + len <= XXH3_INTERNALBUFFER_SIZE) { /* fill in tmp buffer */ + XXH_memcpy(state->buffer + state->bufferedSize, input, len); + state->bufferedSize += (XXH32_hash_t)len; + return XXH_OK; + } + /* total input is now > XXH3_INTERNALBUFFER_SIZE */ + + #define XXH3_INTERNALBUFFER_STRIPES (XXH3_INTERNALBUFFER_SIZE / XXH_STRIPE_LEN) + XXH_STATIC_ASSERT(XXH3_INTERNALBUFFER_SIZE % XXH_STRIPE_LEN == 0); /* clean multiple */ + + /* + * Internal buffer is partially filled (always, except at beginning) + * Complete it, then consume it. + */ + if (state->bufferedSize) { + size_t const loadSize = XXH3_INTERNALBUFFER_SIZE - state->bufferedSize; + XXH_memcpy(state->buffer + state->bufferedSize, input, loadSize); + input += loadSize; + XXH3_consumeStripes(state->acc, + &state->nbStripesSoFar, state->nbStripesPerBlock, + state->buffer, XXH3_INTERNALBUFFER_STRIPES, + secret, state->secretLimit, + f_acc512, f_scramble); + state->bufferedSize = 0; + } + XXH_ASSERT(input < bEnd); + + /* Consume input by a multiple of internal buffer size */ + if (input+XXH3_INTERNALBUFFER_SIZE < bEnd) { + const xxh_u8* const limit = bEnd - XXH3_INTERNALBUFFER_SIZE; + do { + XXH3_consumeStripes(state->acc, + &state->nbStripesSoFar, state->nbStripesPerBlock, + input, XXH3_INTERNALBUFFER_STRIPES, + secret, state->secretLimit, + f_acc512, f_scramble); + input += XXH3_INTERNALBUFFER_SIZE; + } while (inputbuffer + sizeof(state->buffer) - XXH_STRIPE_LEN, input - XXH_STRIPE_LEN, XXH_STRIPE_LEN); + } + XXH_ASSERT(input < bEnd); + + /* Some remaining input (always) : buffer it */ + XXH_memcpy(state->buffer, input, (size_t)(bEnd-input)); + state->bufferedSize = (XXH32_hash_t)(bEnd-input); + } + + return XXH_OK; +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_update(XXH3_state_t* state, const void* input, size_t len) +{ + return XXH3_update(state, (const xxh_u8*)input, len, + XXH3_accumulate_512, XXH3_scrambleAcc); +} + + +XXH_FORCE_INLINE void +XXH3_digest_long (XXH64_hash_t* acc, + const XXH3_state_t* state, + const unsigned char* secret) +{ + /* + * Digest on a local copy. This way, the state remains unaltered, and it can + * continue ingesting more input afterwards. + */ + memcpy(acc, state->acc, sizeof(state->acc)); + if (state->bufferedSize >= XXH_STRIPE_LEN) { + size_t const nbStripes = (state->bufferedSize - 1) / XXH_STRIPE_LEN; + size_t nbStripesSoFar = state->nbStripesSoFar; + XXH3_consumeStripes(acc, + &nbStripesSoFar, state->nbStripesPerBlock, + state->buffer, nbStripes, + secret, state->secretLimit, + XXH3_accumulate_512, XXH3_scrambleAcc); + /* last stripe */ + XXH3_accumulate_512(acc, + state->buffer + state->bufferedSize - XXH_STRIPE_LEN, + secret + state->secretLimit - XXH_SECRET_LASTACC_START); + } else { /* bufferedSize < XXH_STRIPE_LEN */ + xxh_u8 lastStripe[XXH_STRIPE_LEN]; + size_t const catchupSize = XXH_STRIPE_LEN - state->bufferedSize; + XXH_ASSERT(state->bufferedSize > 0); /* there is always some input buffered */ + memcpy(lastStripe, state->buffer + sizeof(state->buffer) - catchupSize, catchupSize); + memcpy(lastStripe + catchupSize, state->buffer, state->bufferedSize); + XXH3_accumulate_512(acc, + lastStripe, + secret + state->secretLimit - XXH_SECRET_LASTACC_START); + } +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_digest (const XXH3_state_t* state) +{ + const unsigned char* const secret = (state->extSecret == NULL) ? state->customSecret : state->extSecret; + if (state->totalLen > XXH3_MIDSIZE_MAX) { + XXH_ALIGN(XXH_ACC_ALIGN) XXH64_hash_t acc[XXH_ACC_NB]; + XXH3_digest_long(acc, state, secret); + return XXH3_mergeAccs(acc, + secret + XXH_SECRET_MERGEACCS_START, + (xxh_u64)state->totalLen * XXH_PRIME64_1); + } + /* totalLen <= XXH3_MIDSIZE_MAX: digesting a short input */ + if (state->seed) + return XXH3_64bits_withSeed(state->buffer, (size_t)state->totalLen, state->seed); + return XXH3_64bits_withSecret(state->buffer, (size_t)(state->totalLen), + secret, state->secretLimit + XXH_STRIPE_LEN); +} + + +#define XXH_MIN(x, y) (((x) > (y)) ? (y) : (x)) + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API void +XXH3_generateSecret(void* secretBuffer, const void* customSeed, size_t customSeedSize) +{ + XXH_ASSERT(secretBuffer != NULL); + if (customSeedSize == 0) { + memcpy(secretBuffer, XXH3_kSecret, XXH_SECRET_DEFAULT_SIZE); + return; + } + XXH_ASSERT(customSeed != NULL); + + { size_t const segmentSize = sizeof(XXH128_hash_t); + size_t const nbSegments = XXH_SECRET_DEFAULT_SIZE / segmentSize; + XXH128_canonical_t scrambler; + XXH64_hash_t seeds[12]; + size_t segnb; + XXH_ASSERT(nbSegments == 12); + XXH_ASSERT(segmentSize * nbSegments == XXH_SECRET_DEFAULT_SIZE); /* exact multiple */ + XXH128_canonicalFromHash(&scrambler, XXH128(customSeed, customSeedSize, 0)); + + /* + * Copy customSeed to seeds[], truncating or repeating as necessary. + */ + { size_t toFill = XXH_MIN(customSeedSize, sizeof(seeds)); + size_t filled = toFill; + memcpy(seeds, customSeed, toFill); + while (filled < sizeof(seeds)) { + toFill = XXH_MIN(filled, sizeof(seeds) - filled); + memcpy((char*)seeds + filled, seeds, toFill); + filled += toFill; + } } + + /* generate secret */ + memcpy(secretBuffer, &scrambler, sizeof(scrambler)); + for (segnb=1; segnb < nbSegments; segnb++) { + size_t const segmentStart = segnb * segmentSize; + XXH128_canonical_t segment; + XXH128_canonicalFromHash(&segment, + XXH128(&scrambler, sizeof(scrambler), XXH_readLE64(seeds + segnb) + segnb) ); + memcpy((char*)secretBuffer + segmentStart, &segment, sizeof(segment)); + } } +} + + +/* ========================================== + * XXH3 128 bits (a.k.a XXH128) + * ========================================== + * XXH3's 128-bit variant has better mixing and strength than the 64-bit variant, + * even without counting the significantly larger output size. + * + * For example, extra steps are taken to avoid the seed-dependent collisions + * in 17-240 byte inputs (See XXH3_mix16B and XXH128_mix32B). + * + * This strength naturally comes at the cost of some speed, especially on short + * lengths. Note that longer hashes are about as fast as the 64-bit version + * due to it using only a slight modification of the 64-bit loop. + * + * XXH128 is also more oriented towards 64-bit machines. It is still extremely + * fast for a _128-bit_ hash on 32-bit (it usually clears XXH64). + */ + +XXH_FORCE_INLINE XXH128_hash_t +XXH3_len_1to3_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + /* A doubled version of 1to3_64b with different constants. */ + XXH_ASSERT(input != NULL); + XXH_ASSERT(1 <= len && len <= 3); + XXH_ASSERT(secret != NULL); + /* + * len = 1: combinedl = { input[0], 0x01, input[0], input[0] } + * len = 2: combinedl = { input[1], 0x02, input[0], input[1] } + * len = 3: combinedl = { input[2], 0x03, input[0], input[1] } + */ + { xxh_u8 const c1 = input[0]; + xxh_u8 const c2 = input[len >> 1]; + xxh_u8 const c3 = input[len - 1]; + xxh_u32 const combinedl = ((xxh_u32)c1 <<16) | ((xxh_u32)c2 << 24) + | ((xxh_u32)c3 << 0) | ((xxh_u32)len << 8); + xxh_u32 const combinedh = XXH_rotl32(XXH_swap32(combinedl), 13); + xxh_u64 const bitflipl = (XXH_readLE32(secret) ^ XXH_readLE32(secret+4)) + seed; + xxh_u64 const bitfliph = (XXH_readLE32(secret+8) ^ XXH_readLE32(secret+12)) - seed; + xxh_u64 const keyed_lo = (xxh_u64)combinedl ^ bitflipl; + xxh_u64 const keyed_hi = (xxh_u64)combinedh ^ bitfliph; + XXH128_hash_t h128; + h128.low64 = XXH64_avalanche(keyed_lo); + h128.high64 = XXH64_avalanche(keyed_hi); + return h128; + } +} + +XXH_FORCE_INLINE XXH128_hash_t +XXH3_len_4to8_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(input != NULL); + XXH_ASSERT(secret != NULL); + XXH_ASSERT(4 <= len && len <= 8); + seed ^= (xxh_u64)XXH_swap32((xxh_u32)seed) << 32; + { xxh_u32 const input_lo = XXH_readLE32(input); + xxh_u32 const input_hi = XXH_readLE32(input + len - 4); + xxh_u64 const input_64 = input_lo + ((xxh_u64)input_hi << 32); + xxh_u64 const bitflip = (XXH_readLE64(secret+16) ^ XXH_readLE64(secret+24)) + seed; + xxh_u64 const keyed = input_64 ^ bitflip; + + /* Shift len to the left to ensure it is even, this avoids even multiplies. */ + XXH128_hash_t m128 = XXH_mult64to128(keyed, XXH_PRIME64_1 + (len << 2)); + + m128.high64 += (m128.low64 << 1); + m128.low64 ^= (m128.high64 >> 3); + + m128.low64 = XXH_xorshift64(m128.low64, 35); + m128.low64 *= 0x9FB21C651E98DF25ULL; + m128.low64 = XXH_xorshift64(m128.low64, 28); + m128.high64 = XXH3_avalanche(m128.high64); + return m128; + } +} + +XXH_FORCE_INLINE XXH128_hash_t +XXH3_len_9to16_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(input != NULL); + XXH_ASSERT(secret != NULL); + XXH_ASSERT(9 <= len && len <= 16); + { xxh_u64 const bitflipl = (XXH_readLE64(secret+32) ^ XXH_readLE64(secret+40)) - seed; + xxh_u64 const bitfliph = (XXH_readLE64(secret+48) ^ XXH_readLE64(secret+56)) + seed; + xxh_u64 const input_lo = XXH_readLE64(input); + xxh_u64 input_hi = XXH_readLE64(input + len - 8); + XXH128_hash_t m128 = XXH_mult64to128(input_lo ^ input_hi ^ bitflipl, XXH_PRIME64_1); + /* + * Put len in the middle of m128 to ensure that the length gets mixed to + * both the low and high bits in the 128x64 multiply below. + */ + m128.low64 += (xxh_u64)(len - 1) << 54; + input_hi ^= bitfliph; + /* + * Add the high 32 bits of input_hi to the high 32 bits of m128, then + * add the long product of the low 32 bits of input_hi and XXH_PRIME32_2 to + * the high 64 bits of m128. + * + * The best approach to this operation is different on 32-bit and 64-bit. + */ + if (sizeof(void *) < sizeof(xxh_u64)) { /* 32-bit */ + /* + * 32-bit optimized version, which is more readable. + * + * On 32-bit, it removes an ADC and delays a dependency between the two + * halves of m128.high64, but it generates an extra mask on 64-bit. + */ + m128.high64 += (input_hi & 0xFFFFFFFF00000000ULL) + XXH_mult32to64((xxh_u32)input_hi, XXH_PRIME32_2); + } else { + /* + * 64-bit optimized (albeit more confusing) version. + * + * Uses some properties of addition and multiplication to remove the mask: + * + * Let: + * a = input_hi.lo = (input_hi & 0x00000000FFFFFFFF) + * b = input_hi.hi = (input_hi & 0xFFFFFFFF00000000) + * c = XXH_PRIME32_2 + * + * a + (b * c) + * Inverse Property: x + y - x == y + * a + (b * (1 + c - 1)) + * Distributive Property: x * (y + z) == (x * y) + (x * z) + * a + (b * 1) + (b * (c - 1)) + * Identity Property: x * 1 == x + * a + b + (b * (c - 1)) + * + * Substitute a, b, and c: + * input_hi.hi + input_hi.lo + ((xxh_u64)input_hi.lo * (XXH_PRIME32_2 - 1)) + * + * Since input_hi.hi + input_hi.lo == input_hi, we get this: + * input_hi + ((xxh_u64)input_hi.lo * (XXH_PRIME32_2 - 1)) + */ + m128.high64 += input_hi + XXH_mult32to64((xxh_u32)input_hi, XXH_PRIME32_2 - 1); + } + /* m128 ^= XXH_swap64(m128 >> 64); */ + m128.low64 ^= XXH_swap64(m128.high64); + + { /* 128x64 multiply: h128 = m128 * XXH_PRIME64_2; */ + XXH128_hash_t h128 = XXH_mult64to128(m128.low64, XXH_PRIME64_2); + h128.high64 += m128.high64 * XXH_PRIME64_2; + + h128.low64 = XXH3_avalanche(h128.low64); + h128.high64 = XXH3_avalanche(h128.high64); + return h128; + } } +} + +/* + * Assumption: `secret` size is >= XXH3_SECRET_SIZE_MIN + */ +XXH_FORCE_INLINE XXH128_hash_t +XXH3_len_0to16_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(len <= 16); + { if (len > 8) return XXH3_len_9to16_128b(input, len, secret, seed); + if (len >= 4) return XXH3_len_4to8_128b(input, len, secret, seed); + if (len) return XXH3_len_1to3_128b(input, len, secret, seed); + { XXH128_hash_t h128; + xxh_u64 const bitflipl = XXH_readLE64(secret+64) ^ XXH_readLE64(secret+72); + xxh_u64 const bitfliph = XXH_readLE64(secret+80) ^ XXH_readLE64(secret+88); + h128.low64 = XXH64_avalanche(seed ^ bitflipl); + h128.high64 = XXH64_avalanche( seed ^ bitfliph); + return h128; + } } +} + +/* + * A bit slower than XXH3_mix16B, but handles multiply by zero better. + */ +XXH_FORCE_INLINE XXH128_hash_t +XXH128_mix32B(XXH128_hash_t acc, const xxh_u8* input_1, const xxh_u8* input_2, + const xxh_u8* secret, XXH64_hash_t seed) +{ + acc.low64 += XXH3_mix16B (input_1, secret+0, seed); + acc.low64 ^= XXH_readLE64(input_2) + XXH_readLE64(input_2 + 8); + acc.high64 += XXH3_mix16B (input_2, secret+16, seed); + acc.high64 ^= XXH_readLE64(input_1) + XXH_readLE64(input_1 + 8); + return acc; +} + + +XXH_FORCE_INLINE XXH128_hash_t +XXH3_len_17to128_128b(const xxh_u8* XXH_RESTRICT input, size_t len, + const xxh_u8* XXH_RESTRICT secret, size_t secretSize, + XXH64_hash_t seed) +{ + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize; + XXH_ASSERT(16 < len && len <= 128); + + { XXH128_hash_t acc; + acc.low64 = len * XXH_PRIME64_1; + acc.high64 = 0; + if (len > 32) { + if (len > 64) { + if (len > 96) { + acc = XXH128_mix32B(acc, input+48, input+len-64, secret+96, seed); + } + acc = XXH128_mix32B(acc, input+32, input+len-48, secret+64, seed); + } + acc = XXH128_mix32B(acc, input+16, input+len-32, secret+32, seed); + } + acc = XXH128_mix32B(acc, input, input+len-16, secret, seed); + { XXH128_hash_t h128; + h128.low64 = acc.low64 + acc.high64; + h128.high64 = (acc.low64 * XXH_PRIME64_1) + + (acc.high64 * XXH_PRIME64_4) + + ((len - seed) * XXH_PRIME64_2); + h128.low64 = XXH3_avalanche(h128.low64); + h128.high64 = (XXH64_hash_t)0 - XXH3_avalanche(h128.high64); + return h128; + } + } +} + +XXH_NO_INLINE XXH128_hash_t +XXH3_len_129to240_128b(const xxh_u8* XXH_RESTRICT input, size_t len, + const xxh_u8* XXH_RESTRICT secret, size_t secretSize, + XXH64_hash_t seed) +{ + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize; + XXH_ASSERT(128 < len && len <= XXH3_MIDSIZE_MAX); + + { XXH128_hash_t acc; + int const nbRounds = (int)len / 32; + int i; + acc.low64 = len * XXH_PRIME64_1; + acc.high64 = 0; + for (i=0; i<4; i++) { + acc = XXH128_mix32B(acc, + input + (32 * i), + input + (32 * i) + 16, + secret + (32 * i), + seed); + } + acc.low64 = XXH3_avalanche(acc.low64); + acc.high64 = XXH3_avalanche(acc.high64); + XXH_ASSERT(nbRounds >= 4); + for (i=4 ; i < nbRounds; i++) { + acc = XXH128_mix32B(acc, + input + (32 * i), + input + (32 * i) + 16, + secret + XXH3_MIDSIZE_STARTOFFSET + (32 * (i - 4)), + seed); + } + /* last bytes */ + acc = XXH128_mix32B(acc, + input + len - 16, + input + len - 32, + secret + XXH3_SECRET_SIZE_MIN - XXH3_MIDSIZE_LASTOFFSET - 16, + 0ULL - seed); + + { XXH128_hash_t h128; + h128.low64 = acc.low64 + acc.high64; + h128.high64 = (acc.low64 * XXH_PRIME64_1) + + (acc.high64 * XXH_PRIME64_4) + + ((len - seed) * XXH_PRIME64_2); + h128.low64 = XXH3_avalanche(h128.low64); + h128.high64 = (XXH64_hash_t)0 - XXH3_avalanche(h128.high64); + return h128; + } + } +} + +XXH_FORCE_INLINE XXH128_hash_t +XXH3_hashLong_128b_internal(const void* XXH_RESTRICT input, size_t len, + const xxh_u8* XXH_RESTRICT secret, size_t secretSize, + XXH3_f_accumulate_512 f_acc512, + XXH3_f_scrambleAcc f_scramble) +{ + XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64 acc[XXH_ACC_NB] = XXH3_INIT_ACC; + + XXH3_hashLong_internal_loop(acc, (const xxh_u8*)input, len, secret, secretSize, f_acc512, f_scramble); + + /* converge into final hash */ + XXH_STATIC_ASSERT(sizeof(acc) == 64); + XXH_ASSERT(secretSize >= sizeof(acc) + XXH_SECRET_MERGEACCS_START); + { XXH128_hash_t h128; + h128.low64 = XXH3_mergeAccs(acc, + secret + XXH_SECRET_MERGEACCS_START, + (xxh_u64)len * XXH_PRIME64_1); + h128.high64 = XXH3_mergeAccs(acc, + secret + secretSize + - sizeof(acc) - XXH_SECRET_MERGEACCS_START, + ~((xxh_u64)len * XXH_PRIME64_2)); + return h128; + } +} + +/* + * It's important for performance that XXH3_hashLong is not inlined. + */ +XXH_NO_INLINE XXH128_hash_t +XXH3_hashLong_128b_default(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, + const void* XXH_RESTRICT secret, size_t secretLen) +{ + (void)seed64; (void)secret; (void)secretLen; + return XXH3_hashLong_128b_internal(input, len, XXH3_kSecret, sizeof(XXH3_kSecret), + XXH3_accumulate_512, XXH3_scrambleAcc); +} + +/* + * It's important for performance that XXH3_hashLong is not inlined. + */ +XXH_NO_INLINE XXH128_hash_t +XXH3_hashLong_128b_withSecret(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, + const void* XXH_RESTRICT secret, size_t secretLen) +{ + (void)seed64; + return XXH3_hashLong_128b_internal(input, len, (const xxh_u8*)secret, secretLen, + XXH3_accumulate_512, XXH3_scrambleAcc); +} + +XXH_FORCE_INLINE XXH128_hash_t +XXH3_hashLong_128b_withSeed_internal(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, + XXH3_f_accumulate_512 f_acc512, + XXH3_f_scrambleAcc f_scramble, + XXH3_f_initCustomSecret f_initSec) +{ + if (seed64 == 0) + return XXH3_hashLong_128b_internal(input, len, + XXH3_kSecret, sizeof(XXH3_kSecret), + f_acc512, f_scramble); + { XXH_ALIGN(XXH_SEC_ALIGN) xxh_u8 secret[XXH_SECRET_DEFAULT_SIZE]; + f_initSec(secret, seed64); + return XXH3_hashLong_128b_internal(input, len, (const xxh_u8*)secret, sizeof(secret), + f_acc512, f_scramble); + } +} + +/* + * It's important for performance that XXH3_hashLong is not inlined. + */ +XXH_NO_INLINE XXH128_hash_t +XXH3_hashLong_128b_withSeed(const void* input, size_t len, + XXH64_hash_t seed64, const void* XXH_RESTRICT secret, size_t secretLen) +{ + (void)secret; (void)secretLen; + return XXH3_hashLong_128b_withSeed_internal(input, len, seed64, + XXH3_accumulate_512, XXH3_scrambleAcc, XXH3_initCustomSecret); +} + +typedef XXH128_hash_t (*XXH3_hashLong128_f)(const void* XXH_RESTRICT, size_t, + XXH64_hash_t, const void* XXH_RESTRICT, size_t); + +XXH_FORCE_INLINE XXH128_hash_t +XXH3_128bits_internal(const void* input, size_t len, + XXH64_hash_t seed64, const void* XXH_RESTRICT secret, size_t secretLen, + XXH3_hashLong128_f f_hl128) +{ + XXH_ASSERT(secretLen >= XXH3_SECRET_SIZE_MIN); + /* + * If an action is to be taken if `secret` conditions are not respected, + * it should be done here. + * For now, it's a contract pre-condition. + * Adding a check and a branch here would cost performance at every hash. + */ + if (len <= 16) + return XXH3_len_0to16_128b((const xxh_u8*)input, len, (const xxh_u8*)secret, seed64); + if (len <= 128) + return XXH3_len_17to128_128b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64); + if (len <= XXH3_MIDSIZE_MAX) + return XXH3_len_129to240_128b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64); + return f_hl128(input, len, seed64, secret, secretLen); +} + + +/* === Public XXH128 API === */ + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH128_hash_t XXH3_128bits(const void* input, size_t len) +{ + return XXH3_128bits_internal(input, len, 0, + XXH3_kSecret, sizeof(XXH3_kSecret), + XXH3_hashLong_128b_default); +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH128_hash_t +XXH3_128bits_withSecret(const void* input, size_t len, const void* secret, size_t secretSize) +{ + return XXH3_128bits_internal(input, len, 0, + (const xxh_u8*)secret, secretSize, + XXH3_hashLong_128b_withSecret); +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH128_hash_t +XXH3_128bits_withSeed(const void* input, size_t len, XXH64_hash_t seed) +{ + return XXH3_128bits_internal(input, len, seed, + XXH3_kSecret, sizeof(XXH3_kSecret), + XXH3_hashLong_128b_withSeed); +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH128_hash_t +XXH128(const void* input, size_t len, XXH64_hash_t seed) +{ + return XXH3_128bits_withSeed(input, len, seed); +} + + +/* === XXH3 128-bit streaming === */ + +/* + * All the functions are actually the same as for 64-bit streaming variant. + * The only difference is the finalization routine. + */ + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_128bits_reset(XXH3_state_t* statePtr) +{ + if (statePtr == NULL) return XXH_ERROR; + XXH3_reset_internal(statePtr, 0, XXH3_kSecret, XXH_SECRET_DEFAULT_SIZE); + return XXH_OK; +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_128bits_reset_withSecret(XXH3_state_t* statePtr, const void* secret, size_t secretSize) +{ + if (statePtr == NULL) return XXH_ERROR; + XXH3_reset_internal(statePtr, 0, secret, secretSize); + if (secret == NULL) return XXH_ERROR; + if (secretSize < XXH3_SECRET_SIZE_MIN) return XXH_ERROR; + return XXH_OK; +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_128bits_reset_withSeed(XXH3_state_t* statePtr, XXH64_hash_t seed) +{ + if (statePtr == NULL) return XXH_ERROR; + if (seed==0) return XXH3_128bits_reset(statePtr); + if (seed != statePtr->seed) XXH3_initCustomSecret(statePtr->customSecret, seed); + XXH3_reset_internal(statePtr, seed, NULL, XXH_SECRET_DEFAULT_SIZE); + return XXH_OK; +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_128bits_update(XXH3_state_t* state, const void* input, size_t len) +{ + return XXH3_update(state, (const xxh_u8*)input, len, + XXH3_accumulate_512, XXH3_scrambleAcc); +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_digest (const XXH3_state_t* state) +{ + const unsigned char* const secret = (state->extSecret == NULL) ? state->customSecret : state->extSecret; + if (state->totalLen > XXH3_MIDSIZE_MAX) { + XXH_ALIGN(XXH_ACC_ALIGN) XXH64_hash_t acc[XXH_ACC_NB]; + XXH3_digest_long(acc, state, secret); + XXH_ASSERT(state->secretLimit + XXH_STRIPE_LEN >= sizeof(acc) + XXH_SECRET_MERGEACCS_START); + { XXH128_hash_t h128; + h128.low64 = XXH3_mergeAccs(acc, + secret + XXH_SECRET_MERGEACCS_START, + (xxh_u64)state->totalLen * XXH_PRIME64_1); + h128.high64 = XXH3_mergeAccs(acc, + secret + state->secretLimit + XXH_STRIPE_LEN + - sizeof(acc) - XXH_SECRET_MERGEACCS_START, + ~((xxh_u64)state->totalLen * XXH_PRIME64_2)); + return h128; + } + } + /* len <= XXH3_MIDSIZE_MAX : short code */ + if (state->seed) + return XXH3_128bits_withSeed(state->buffer, (size_t)state->totalLen, state->seed); + return XXH3_128bits_withSecret(state->buffer, (size_t)(state->totalLen), + secret, state->secretLimit + XXH_STRIPE_LEN); +} + +/* 128-bit utility functions */ + +#include /* memcmp, memcpy */ + +/* return : 1 is equal, 0 if different */ +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API int XXH128_isEqual(XXH128_hash_t h1, XXH128_hash_t h2) +{ + /* note : XXH128_hash_t is compact, it has no padding byte */ + return !(memcmp(&h1, &h2, sizeof(h1))); +} + +/* This prototype is compatible with stdlib's qsort(). + * return : >0 if *h128_1 > *h128_2 + * <0 if *h128_1 < *h128_2 + * =0 if *h128_1 == *h128_2 */ +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API int XXH128_cmp(const void* h128_1, const void* h128_2) +{ + XXH128_hash_t const h1 = *(const XXH128_hash_t*)h128_1; + XXH128_hash_t const h2 = *(const XXH128_hash_t*)h128_2; + int const hcmp = (h1.high64 > h2.high64) - (h2.high64 > h1.high64); + /* note : bets that, in most cases, hash values are different */ + if (hcmp) return hcmp; + return (h1.low64 > h2.low64) - (h2.low64 > h1.low64); +} + + +/*====== Canonical representation ======*/ +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API void +XXH128_canonicalFromHash(XXH128_canonical_t* dst, XXH128_hash_t hash) +{ + XXH_STATIC_ASSERT(sizeof(XXH128_canonical_t) == sizeof(XXH128_hash_t)); + if (XXH_CPU_LITTLE_ENDIAN) { + hash.high64 = XXH_swap64(hash.high64); + hash.low64 = XXH_swap64(hash.low64); + } + memcpy(dst, &hash.high64, sizeof(hash.high64)); + memcpy((char*)dst + sizeof(hash.high64), &hash.low64, sizeof(hash.low64)); +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH128_hash_t +XXH128_hashFromCanonical(const XXH128_canonical_t* src) +{ + XXH128_hash_t h; + h.high64 = XXH_readBE64(src); + h.low64 = XXH_readBE64(src->digest + 8); + return h; +} + +/* Pop our optimization override from above */ +#if XXH_VECTOR == XXH_AVX2 /* AVX2 */ \ + && defined(__GNUC__) && !defined(__clang__) /* GCC, not Clang */ \ + && defined(__OPTIMIZE__) && !defined(__OPTIMIZE_SIZE__) /* respect -O0 and -Os */ +# pragma GCC pop_options +#endif + +#endif /* XXH_NO_LONG_LONG */ + +#endif /* XXH_NO_XXH3 */ + +/*! + * @} + */ +#endif /* XXH_IMPLEMENTATION */ + + +#if defined (__cplusplus) +} +#endif diff --git a/src/blosc2/plugins/codecs/zfp/CMakeLists.txt b/src/blosc2/plugins/codecs/zfp/CMakeLists.txt new file mode 100644 index 0000000..e7cd1a1 --- /dev/null +++ b/src/blosc2/plugins/codecs/zfp/CMakeLists.txt @@ -0,0 +1,58 @@ +get_filename_component(ZFP_LOCAL_DIR ./ ABSOLUTE) +file(GLOB ZFP_SRC_FILES ${ZFP_LOCAL_DIR}/blosc2-zfp.* ${ZFP_LOCAL_DIR}/zfp-private.h ${ZFP_LOCAL_DIR}/src/zfp.c + ${ZFP_LOCAL_DIR}/src/bitstream.c ${ZFP_LOCAL_DIR}/src/decode*.c ${ZFP_LOCAL_DIR}/src/encode*.c) +#message(status "--------------!!!!zfp:" ${ZFP_SRC_FILES}) +set(SOURCES ${SOURCES} ${ZFP_SRC_FILES} PARENT_SCOPE) + +# targets +if(BUILD_TESTS) + add_executable(test_zfp_acc_int test_zfp_acc_int.c) + add_executable(test_zfp_acc_float test_zfp_acc_float.c) + add_executable(test_zfp_prec_float test_zfp_prec_float.c) + add_executable(test_zfp_rate_float test_zfp_rate_float.c) + add_executable(test_zfp_rate_getitem test_zfp_rate_getitem.c) + # Define the BLOSC_TESTING symbol so normally-hidden functions + # aren't hidden from the view of the test programs. + set_property( + TARGET test_zfp_acc_int + APPEND PROPERTY COMPILE_DEFINITIONS BLOSC_TESTING) + set_property( + TARGET test_zfp_acc_float + APPEND PROPERTY COMPILE_DEFINITIONS BLOSC_TESTING) + set_property( + TARGET test_zfp_prec_float + APPEND PROPERTY COMPILE_DEFINITIONS BLOSC_TESTING) + set_property( + TARGET test_zfp_rate_float + APPEND PROPERTY COMPILE_DEFINITIONS BLOSC_TESTING) + set_property( + TARGET test_zfp_rate_getitem + APPEND PROPERTY COMPILE_DEFINITIONS BLOSC_TESTING) + + target_link_libraries(test_zfp_acc_int blosc_testing) + target_link_libraries(test_zfp_acc_float blosc_testing) + target_link_libraries(test_zfp_prec_float blosc_testing) + target_link_libraries(test_zfp_rate_float blosc_testing) + target_link_libraries(test_zfp_rate_getitem blosc_testing) + + # tests + add_test(NAME test_plugin_test_zfp_acc_int + COMMAND ${CMAKE_CROSSCOMPILING_EMULATOR} $) + add_test(NAME test_plugin_test_zfp_acc_float + COMMAND ${CMAKE_CROSSCOMPILING_EMULATOR} $) + add_test(NAME test_plugin_test_zfp_prec_float + COMMAND ${CMAKE_CROSSCOMPILING_EMULATOR} $) + add_test(NAME test_plugin_test_zfp_rate_float + COMMAND ${CMAKE_CROSSCOMPILING_EMULATOR} $) + add_test(NAME test_plugin_test_zfp_rate_getitem + COMMAND ${CMAKE_CROSSCOMPILING_EMULATOR} $) + + # Copy test files + file(GLOB TESTS_DATA ../../test_data/example_float_cyclic.caterva ../../test_data/example_double_same_cells.caterva + ../../test_data/example_day_month_temp.caterva ../../test_data/example_item_prices.caterva + ../../filters/ndmean/example_ndmean_r*.caterva) + foreach (data ${TESTS_DATA}) + file(COPY ${data} + DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/) + endforeach(data) +endif() diff --git a/src/blosc2/plugins/codecs/zfp/README.md b/src/blosc2/plugins/codecs/zfp/README.md new file mode 100644 index 0000000..a2af806 --- /dev/null +++ b/src/blosc2/plugins/codecs/zfp/README.md @@ -0,0 +1,61 @@ +ZFP: a multidimensional lossy codec +============================================================================= + +*ZFP* is a codec for lossy data compression, designed for being mainly used in multidimensional (up to 4-dim) floating point datasets. + +Plugin motivation +-------------------- + +A lossy codec like ZFP allows for much better compression ratios at the expense of losing some precision in floating point data. For a discussion on how it works and specially, how it performs, see our blog at: https://www.blosc.org/posts/support-lossy-zfp/. + +Plugin usage +------------------- + +The codec consists of different encoders to codify data and decoders to recover the original data, located at `blosc2-zfp.c`. + +This plugin only works with 1 to 4-dim datasets of floats or doubles, so if one tries to work with another type of dataset, it will return an error value. Also, the blocksize requires to be a minimum of 4 items for 1 dim, 4x4 for 2 dim, 4x4x4 for 3 dim and 4x4x4x4 for 4 dim. Finally, the ZFP codecs do interpret the *values*, so they are meant to be used *without any shuffle filter* (that would break byte/bit ordering, and hence, changing the values before they would reach the ZFP codec). + +The parameters used by *ZFP* are the ones specified in the `blosc2_codec` structure in the `blosc2.h` header. +Furthermore, *ZFP* allows to work in three modes, BLOSC_CODEC_ZFP_FIXED_ACCURACY, BLOSC_CODEC_ZFP_FIXED_PRECISION and BLOSC_CODEC_ZFP_FIXED_RATE, each of one can be fine-tuned via the `meta` parameter as follows: + +- BLOSC_CODEC_ZFP_FIXED_ACCURACY: `meta` is used as absolute error in truncation. For example, if meta = -2, each value loss must be less than or equal to 10^(-2) = 0,01. Then, if 23,0567 is a value of the original input, after compressing and decompressing this input with meta = -2, this value must be between 23,0467 and 23,0667. As `meta` is unsigned in the codec plugin interface, if user does not want the compiler to show a warning, a cast to a `uint8_t` must be done before calling the plugin. +- BLOSC_CODEC_ZFP_FIXED_PRECISION: `meta` must be a number between 1 and ZFP_MAX_PREC. It is called *precision* and represents the maximum number of bit planes encoded. This is, for each input value, the number of most significant bits that will be encoded. For more info, see: + https://zfp.readthedocs.io/en/latest/faq.html#q-relerr +- BLOSC_CODEC_ZFP_FIXED_RATE: `meta` must be a number between 1 and 100. It is called *ratio* and represents the size that the compressed cells must have based on the input cell size. For example, if the cell size is 2000 bytes and meta = 50, the output cell size will be 50% of 2000 = 1000 bytes. + +To choose one of the compression modes, user must use its corresponding identifier as `blosc2_cparams.compcode`. For example: + + cparams.compcode = BLOSC_CODEC_ZFP_FIXED_ACCURACY; + +or: + + cparams.compcode = BLOSC_CODEC_ZFP_FIXED_PRECISION; + +or: + + cparams.compcode = BLOSC_CODEC_ZFP_FIXED_RATE; + +Remember that all available codec IDs (including ZFP modes) are defined at: +https://github.com/Blosc/c-blosc2/blob/main/include/blosc2/codecs-registry.h + + +How ZFP works +------------------- + +In order to compress n-dimensional arrays of floating-point data, ZFP partitions them into cells of size 4^n so; for example, in a 3-dim dataset, the cellshape will be 4x4x4. +Then, depending on the compression mode, each block is compressed or decompressed individually into a fixed- or variable-length bit string, and these bit strings are concatenated into a single stream of bits. + +For more info you can see ZFP official documentation: +https://zfp.readthedocs.io/en/latest/ + +And the official repo: +https://github.com/LLNL/zfp + +Advantages and disadvantages +------------------------------ + +The main advantage of *ZFP* when compared with others is that *ZFP* makes use of the multidimensionality of datasets and takes advantage of this instead of processing all data as serial. + +For example, the difference between *ZFP* and *NDLZ* is that *ZFP* codec uses lossy data compression (which implies better ratios) and lets user to work with floating-point datasets. Furthermore, *ZFP* implements different compression modes that brings interesting new possibilities. + +The main disadvantage of *ZFP* is that it is mainly useful for 2 to 4-dim datasets only. It can also be used for 1-dim datasets, but then it loses most of its advantages (i.e. using spacial locality to better reduce data entropy). diff --git a/src/blosc2/plugins/codecs/zfp/blosc2-zfp.c b/src/blosc2/plugins/codecs/zfp/blosc2-zfp.c new file mode 100644 index 0000000..2f300ff --- /dev/null +++ b/src/blosc2/plugins/codecs/zfp/blosc2-zfp.c @@ -0,0 +1,876 @@ +#include "blosc2.h" +#include "frame.h" +#include "blosc2/codecs-registry.h" +#include "zfp.h" +#include "blosc2-zfp.h" +#include +#include "context.h" +#include "assert.h" +#include "../plugins/plugin_utils.h" + +int zfp_acc_compress(const uint8_t *input, int32_t input_len, uint8_t *output, + int32_t output_len, uint8_t meta, blosc2_cparams *cparams, const void* chunk) { + BLOSC_UNUSED_PARAM(chunk); + ZFP_ERROR_NULL(input); + ZFP_ERROR_NULL(output); + ZFP_ERROR_NULL(cparams); + + double tol = (int8_t) meta; + int8_t ndim; + int64_t *shape = malloc(8 * sizeof(int64_t)); + int32_t *chunkshape = malloc(8 * sizeof(int32_t)); + int32_t *blockshape = malloc(8 * sizeof(int32_t)); + uint8_t *smeta; + int32_t smeta_len; + if (blosc2_meta_get(cparams->schunk, "caterva", &smeta, &smeta_len) < 0) { + printf("Blosc error"); + free(shape); + free(chunkshape); + free(blockshape); + return -1; + } + deserialize_meta(smeta, smeta_len, &ndim, shape, chunkshape, blockshape); + free(smeta); + + zfp_type type; /* array scalar type */ + zfp_field *field; /* array meta data */ + zfp_stream *zfp; /* stream containing the real output buffer */ + zfp_stream *zfp_aux; /* auxiliary compressed stream */ + bitstream *stream; /* bit stream to write to or read from */ + bitstream *stream_aux; /* auxiliary bit stream to write to or read from */ + size_t zfpsize; /* byte size of compressed stream */ + double tolerance = pow(10, tol); + + int32_t typesize = cparams->typesize; + + switch (typesize) { + case sizeof(float): + type = zfp_type_float; + break; + case sizeof(double): + type = zfp_type_double; + break; + default: + printf("\n ZFP is not available for this typesize \n"); + free(shape); + free(chunkshape); + free(blockshape); + return 0; + } + + zfp = zfp_stream_open(NULL); + zfp_stream_set_accuracy(zfp, tolerance); + stream = stream_open(output, output_len); + zfp_stream_set_bit_stream(zfp, stream); + zfp_stream_rewind(zfp); + + switch (ndim) { + case 1: + field = zfp_field_1d((void *) input, type, blockshape[0]); + break; + case 2: + field = zfp_field_2d((void *) input, type, blockshape[1], blockshape[0]); + break; + case 3: + field = zfp_field_3d((void *) input, type, blockshape[2], blockshape[1], blockshape[0]); + break; + case 4: + field = zfp_field_4d((void *) input, type, blockshape[3], blockshape[2], blockshape[1], blockshape[0]); + break; + default: + printf("\n ZFP is not available for this number of dims \n"); + free(shape); + free(chunkshape); + free(blockshape); + return 0; + } + + int zfp_maxout = (int) zfp_stream_maximum_size(zfp, field); + zfp_stream_close(zfp); + stream_close(stream); + uint8_t *aux_out = malloc(zfp_maxout); + zfp_aux = zfp_stream_open(NULL); + zfp_stream_set_accuracy(zfp_aux, tolerance); + stream_aux = stream_open(aux_out, zfp_maxout); + zfp_stream_set_bit_stream(zfp_aux, stream_aux); + zfp_stream_rewind(zfp_aux); + + zfpsize = zfp_compress(zfp_aux, field); + + /* clean up */ + zfp_field_free(field); + zfp_stream_close(zfp_aux); + stream_close(stream_aux); + free(shape); + free(chunkshape); + free(blockshape); + + if (zfpsize == 0) { + BLOSC_TRACE_ERROR("\n ZFP: Compression failed\n"); + free(aux_out); + return (int) zfpsize; + } + if ((int32_t) zfpsize >= input_len) { + BLOSC_TRACE_ERROR("\n ZFP: Compressed data is bigger than input! \n"); + free(aux_out); + return 0; + } + + memcpy(output, aux_out, zfpsize); + free(aux_out); + + return (int) zfpsize; +} + +int zfp_acc_decompress(const uint8_t *input, int32_t input_len, uint8_t *output, + int32_t output_len, uint8_t meta, blosc2_dparams *dparams, const void* chunk) { + ZFP_ERROR_NULL(input); + ZFP_ERROR_NULL(output); + ZFP_ERROR_NULL(dparams); + + size_t typesize; + int flags; + blosc1_cbuffer_metainfo(chunk, &typesize, &flags); + + double tol = (int8_t) meta; + int8_t ndim; + int64_t *shape = malloc(8 * sizeof(int64_t)); + int32_t *chunkshape = malloc(8 * sizeof(int32_t)); + int32_t *blockshape = malloc(8 * sizeof(int32_t)); + uint8_t *smeta; + int32_t smeta_len; + if (blosc2_meta_get(dparams->schunk, "caterva", &smeta, &smeta_len) < 0) { + printf("Blosc error"); + free(shape); + free(chunkshape); + free(blockshape); + return -1; + } + deserialize_meta(smeta, smeta_len, &ndim, shape, chunkshape, blockshape); + free(smeta); + + zfp_type type; /* array scalar type */ + zfp_field *field; /* array meta data */ + zfp_stream *zfp; /* compressed stream */ + bitstream *stream; /* bit stream to write to or read from */ + size_t zfpsize; /* byte size of compressed stream */ + double tolerance = pow(10, tol); + + switch (typesize) { + case sizeof(float): + type = zfp_type_float; + break; + case sizeof(double): + type = zfp_type_double; + break; + default: + printf("\n ZFP is not available for this typesize \n"); + free(shape); + free(chunkshape); + free(blockshape); + return 0; + } + + zfp = zfp_stream_open(NULL); + zfp_stream_set_accuracy(zfp, tolerance); + stream = stream_open((void*) input, input_len); + zfp_stream_set_bit_stream(zfp, stream); + zfp_stream_rewind(zfp); + + switch (ndim) { + case 1: + field = zfp_field_1d((void *) output, type, blockshape[0]); + break; + case 2: + field = zfp_field_2d((void *) output, type, blockshape[1], blockshape[0]); + break; + case 3: + field = zfp_field_3d((void *) output, type, blockshape[2], blockshape[1], blockshape[0]); + break; + case 4: + field = zfp_field_4d((void *) output, type, blockshape[3], blockshape[2], blockshape[1], blockshape[0]); + break; + default: + printf("\n ZFP is not available for this number of dims \n"); + free(shape); + free(chunkshape); + free(blockshape); + return 0; + } + + zfpsize = zfp_decompress(zfp, field); + + /* clean up */ + zfp_field_free(field); + zfp_stream_close(zfp); + stream_close(stream); + free(shape); + free(chunkshape); + free(blockshape); + + if (zfpsize == 0) { + BLOSC_TRACE_ERROR("\n ZFP: Decompression failed\n"); + return (int) zfpsize; + } + + return (int) output_len; +} + +int zfp_prec_compress(const uint8_t *input, int32_t input_len, uint8_t *output, + int32_t output_len, uint8_t meta, blosc2_cparams *cparams, const void* chunk) { + BLOSC_UNUSED_PARAM(chunk); + ZFP_ERROR_NULL(input); + ZFP_ERROR_NULL(output); + ZFP_ERROR_NULL(cparams); + + int8_t ndim; + int64_t *shape = malloc(8 * sizeof(int64_t)); + int32_t *chunkshape = malloc(8 * sizeof(int32_t)); + int32_t *blockshape = malloc(8 * sizeof(int32_t)); + uint8_t *smeta; + int32_t smeta_len; + if (blosc2_meta_get(cparams->schunk, "caterva", &smeta, &smeta_len) < 0) { + printf("Blosc error"); + free(shape); + free(chunkshape); + free(blockshape); + return -1; + } + deserialize_meta(smeta, smeta_len, &ndim, shape, chunkshape, blockshape); + free(smeta); + + zfp_type type; /* array scalar type */ + zfp_field *field; /* array meta data */ + zfp_stream *zfp; /* stream containing the real output buffer */ + zfp_stream *zfp_aux; /* auxiliary compressed stream */ + bitstream *stream; /* bit stream to write to or read from */ + bitstream *stream_aux; /* auxiliary bit stream to write to or read from */ + size_t zfpsize; /* byte size of compressed stream */ + + uint prec; + switch (ndim) { + case 1: + prec = meta + 5; + break; + case 2: + prec = meta + 7; + break; + case 3: + prec = meta + 9; + break; + case 4: + prec = meta + 11; + break; + default: + printf("\n ZFP is not available for this ndim \n"); + free(shape); + free(chunkshape); + free(blockshape); + return 0; + } + + if(prec > ZFP_MAX_PREC) { + BLOSC_TRACE_ERROR("Max precision for this codecs is %d", ZFP_MAX_PREC); + prec = ZFP_MAX_PREC; + } + + int32_t typesize = cparams->typesize; + switch (typesize) { + case sizeof(float): + type = zfp_type_float; + break; + case sizeof(double): + type = zfp_type_double; + break; + default: + printf("\n ZFP is not available for this typesize \n"); + free(shape); + free(chunkshape); + free(blockshape); + return 0; + } + + zfp = zfp_stream_open(NULL); + zfp_stream_set_precision(zfp, prec); + stream = stream_open(output, output_len); + zfp_stream_set_bit_stream(zfp, stream); + zfp_stream_rewind(zfp); + + switch (ndim) { + case 1: + field = zfp_field_1d((void *) input, type, blockshape[0]); + break; + case 2: + field = zfp_field_2d((void *) input, type, blockshape[1], blockshape[0]); + break; + case 3: + field = zfp_field_3d((void *) input, type, blockshape[2], blockshape[1], blockshape[0]); + break; + case 4: + field = zfp_field_4d((void *) input, type, blockshape[3], blockshape[2], blockshape[1], blockshape[0]); + break; + default: + printf("\n ZFP is not available for this number of dims \n"); + free(shape); + free(chunkshape); + free(blockshape); + return 0; + } + + int zfp_maxout = (int) zfp_stream_maximum_size(zfp, field); + zfp_stream_close(zfp); + stream_close(stream); + uint8_t *aux_out = malloc(zfp_maxout); + zfp_aux = zfp_stream_open(NULL); + zfp_stream_set_precision(zfp_aux, prec); + stream_aux = stream_open(aux_out, zfp_maxout); + zfp_stream_set_bit_stream(zfp_aux, stream_aux); + zfp_stream_rewind(zfp_aux); + + zfpsize = zfp_compress(zfp_aux, field); + + /* clean up */ + zfp_field_free(field); + zfp_stream_close(zfp_aux); + stream_close(stream_aux); + free(shape); + free(chunkshape); + free(blockshape); + + if (zfpsize == 0) { + BLOSC_TRACE_ERROR("\n ZFP: Compression failed\n"); + free(aux_out); + return (int) zfpsize; + } + if ((int32_t) zfpsize >= input_len) { + BLOSC_TRACE_ERROR("\n ZFP: Compressed data is bigger than input! \n"); + free(aux_out); + return 0; + } + + memcpy(output, aux_out, zfpsize); + free(aux_out); + + return (int) zfpsize; +} + +int zfp_prec_decompress(const uint8_t *input, int32_t input_len, uint8_t *output, + int32_t output_len, uint8_t meta, blosc2_dparams *dparams, const void* chunk) { + ZFP_ERROR_NULL(input); + ZFP_ERROR_NULL(output); + ZFP_ERROR_NULL(dparams); + + size_t typesize; + int flags; + blosc1_cbuffer_metainfo(chunk, &typesize, &flags); + int8_t ndim; + int64_t *shape = malloc(8 * sizeof(int64_t)); + int32_t *chunkshape = malloc(8 * sizeof(int32_t)); + int32_t *blockshape = malloc(8 * sizeof(int32_t)); + uint8_t *smeta; + int32_t smeta_len; + if (blosc2_meta_get(dparams->schunk, "caterva", &smeta, &smeta_len) < 0) { + printf("Blosc error"); + free(shape); + free(chunkshape); + free(blockshape); + return -1; + } + deserialize_meta(smeta, smeta_len, &ndim, shape, chunkshape, blockshape); + free(smeta); + + zfp_type type; /* array scalar type */ + zfp_field *field; /* array meta data */ + zfp_stream *zfp; /* compressed stream */ + bitstream *stream; /* bit stream to write to or read from */ + size_t zfpsize; /* byte size of compressed stream */ + + uint prec; + switch (ndim) { + case 1: + prec = meta + 5; + break; + case 2: + prec = meta + 7; + break; + case 3: + prec = meta + 9; + break; + case 4: + prec = meta + 11; + break; + default: + printf("\n ZFP is not available for this ndim \n"); + free(shape); + free(chunkshape); + free(blockshape); + return 0; + } + + if(prec > ZFP_MAX_PREC) { + BLOSC_TRACE_ERROR("Max precision for this codecs is %d", ZFP_MAX_PREC); + prec = ZFP_MAX_PREC; + } + + switch (typesize) { + case sizeof(float): + type = zfp_type_float; + break; + case sizeof(double): + type = zfp_type_double; + break; + default: + printf("\n ZFP is not available for this typesize \n"); + free(shape); + free(chunkshape); + free(blockshape); + return 0; + } + + zfp = zfp_stream_open(NULL); + zfp_stream_set_precision(zfp, prec); + stream = stream_open((void*) input, input_len); + zfp_stream_set_bit_stream(zfp, stream); + zfp_stream_rewind(zfp); + + switch (ndim) { + case 1: + field = zfp_field_1d((void *) output, type, blockshape[0]); + break; + case 2: + field = zfp_field_2d((void *) output, type, blockshape[1], blockshape[0]); + break; + case 3: + field = zfp_field_3d((void *) output, type, blockshape[2], blockshape[1], blockshape[0]); + break; + case 4: + field = zfp_field_4d((void *) output, type, blockshape[3], blockshape[2], blockshape[1], blockshape[0]); + break; + default: + printf("\n ZFP is not available for this number of dims \n"); + free(shape); + free(chunkshape); + free(blockshape); + return 0; + } + + zfpsize = zfp_decompress(zfp, field); + + /* clean up */ + zfp_field_free(field); + zfp_stream_close(zfp); + stream_close(stream); + free(shape); + free(chunkshape); + free(blockshape); + + if (zfpsize == 0) { + BLOSC_TRACE_ERROR("\n ZFP: Decompression failed\n"); + return (int) zfpsize; + } + + return (int) output_len; +} + +int zfp_rate_compress(const uint8_t *input, int32_t input_len, uint8_t *output, + int32_t output_len, uint8_t meta, blosc2_cparams *cparams, const void* chunk) { + BLOSC_UNUSED_PARAM(chunk); + ZFP_ERROR_NULL(input); + ZFP_ERROR_NULL(output); + ZFP_ERROR_NULL(cparams); + + double ratio = (double) meta / 100.0; + int8_t ndim; + int64_t *shape = malloc(8 * sizeof(int64_t)); + int32_t *chunkshape = malloc(8 * sizeof(int32_t)); + int32_t *blockshape = malloc(8 * sizeof(int32_t)); + uint8_t *smeta; + int32_t smeta_len; + if (blosc2_meta_get(cparams->schunk, "caterva", &smeta, &smeta_len) < 0) { + printf("Blosc error"); + free(shape); + free(chunkshape); + free(blockshape); + return -1; + } + deserialize_meta(smeta, smeta_len, &ndim, shape, chunkshape, blockshape); + free(smeta); + + zfp_type type; /* array scalar type */ + zfp_field *field; /* array meta data */ + zfp_stream *zfp; /* stream containing the real output buffer */ + zfp_stream *zfp_aux; /* auxiliary compressed stream */ + bitstream *stream; /* bit stream to write to or read from */ + bitstream *stream_aux; /* auxiliary bit stream to write to or read from */ + size_t zfpsize; /* byte size of compressed stream */ + + int32_t typesize = cparams->typesize; + + switch (typesize) { + case sizeof(float): + type = zfp_type_float; + break; + case sizeof(double): + type = zfp_type_double; + break; + default: + printf("\n ZFP is not available for this typesize \n"); + return 0; + } + double rate = ratio * typesize * 8; // convert from output size / input size to output bits per input value + uint cellsize = 1u << (2 * ndim); + double min_rate; + switch (type) { + case zfp_type_float: + min_rate = (double) (1 + 8u) / cellsize; + if (rate < min_rate) { + BLOSC_TRACE_ERROR("\n ZFP minimum rate for this item type is %f. Compression will be done using this rate \n", min_rate); + } + break; + case zfp_type_double: + min_rate = (double) (1 + 11u) / cellsize; + if (rate < min_rate) { + BLOSC_TRACE_ERROR("\n ZFP minimum rate for this item type is %f. Compression will be done using this rate \n", min_rate); + } + break; + default: + free(shape); + free(chunkshape); + free(blockshape); + return 0; + } + zfp = zfp_stream_open(NULL); + stream = stream_open(output, output_len); + zfp_stream_set_bit_stream(zfp, stream); + zfp_stream_rewind(zfp); + + switch (ndim) { + case 1: + field = zfp_field_1d((void *) input, type, blockshape[0]); + break; + case 2: + field = zfp_field_2d((void *) input, type, blockshape[1], blockshape[0]); + break; + case 3: + field = zfp_field_3d((void *) input, type, blockshape[2], blockshape[1], blockshape[0]); + break; + case 4: + field = zfp_field_4d((void *) input, type, blockshape[3], blockshape[2], blockshape[1], blockshape[0]); + break; + default: + printf("\n ZFP is not available for this number of dims \n"); + free(shape); + free(chunkshape); + free(blockshape); + return 0; + } + + int zfp_maxout = (int) zfp_stream_maximum_size(zfp, field); + zfp_stream_close(zfp); + stream_close(stream); + uint8_t *aux_out = malloc(zfp_maxout); + zfp_aux = zfp_stream_open(NULL); + stream_aux = stream_open(aux_out, zfp_maxout); + zfp_stream_set_bit_stream(zfp_aux, stream_aux); + zfp_stream_rewind(zfp_aux); + zfp_stream_set_rate(zfp_aux, rate, type, ndim, zfp_false); + + zfpsize = zfp_compress(zfp_aux, field); + + /* clean up */ + zfp_field_free(field); + zfp_stream_close(zfp_aux); + stream_close(stream_aux); + free(shape); + free(chunkshape); + free(blockshape); + + if (zfpsize == 0) { + BLOSC_TRACE_ERROR("\n ZFP: Compression failed\n"); + free(aux_out); + return (int) zfpsize; + } + if ((int32_t) zfpsize >= input_len) { + BLOSC_TRACE_ERROR("\n ZFP: Compressed data is bigger than input! \n"); + free(aux_out); + return 0; + } + + memcpy(output, aux_out, zfpsize); + free(aux_out); + + return (int) zfpsize; +} + +int zfp_rate_decompress(const uint8_t *input, int32_t input_len, uint8_t *output, + int32_t output_len, uint8_t meta, blosc2_dparams *dparams, const void* chunk) { + ZFP_ERROR_NULL(input); + ZFP_ERROR_NULL(output); + ZFP_ERROR_NULL(dparams); + + size_t typesize; + int flags; + blosc1_cbuffer_metainfo(chunk, &typesize, &flags); + + double ratio = (double) meta / 100.0; + int8_t ndim; + int64_t *shape = malloc(8 * sizeof(int64_t)); + int32_t *chunkshape = malloc(8 * sizeof(int32_t)); + int32_t *blockshape = malloc(8 * sizeof(int32_t)); + uint8_t *smeta; + int32_t smeta_len; + if (blosc2_meta_get(dparams->schunk, "caterva", &smeta, &smeta_len) < 0) { + printf("Blosc error"); + free(shape); + free(chunkshape); + free(blockshape); + return -1; + } + deserialize_meta(smeta, smeta_len, &ndim, shape, chunkshape, blockshape); + free(smeta); + + zfp_type type; /* array scalar type */ + zfp_field *field; /* array meta data */ + zfp_stream *zfp; /* compressed stream */ + bitstream *stream; /* bit stream to write to or read from */ + size_t zfpsize; /* byte size of compressed stream */ + + switch (typesize) { + case sizeof(float): + type = zfp_type_float; + break; + case sizeof(double): + type = zfp_type_double; + break; + default: + printf("\n ZFP is not available for this typesize \n"); + free(shape); + free(chunkshape); + free(blockshape); + return 0; + } + double rate = ratio * (double) typesize * 8; // convert from output size / input size to output bits per input value + zfp = zfp_stream_open(NULL); + zfp_stream_set_rate(zfp, rate, type, ndim, zfp_false); + + stream = stream_open((void*) input, input_len); + zfp_stream_set_bit_stream(zfp, stream); + zfp_stream_rewind(zfp); + + switch (ndim) { + case 1: + field = zfp_field_1d((void *) output, type, blockshape[0]); + break; + case 2: + field = zfp_field_2d((void *) output, type, blockshape[1], blockshape[0]); + break; + case 3: + field = zfp_field_3d((void *) output, type, blockshape[2], blockshape[1], blockshape[0]); + break; + case 4: + field = zfp_field_4d((void *) output, type, blockshape[3], blockshape[2], blockshape[1], blockshape[0]); + break; + default: + printf("\n ZFP is not available for this number of dims \n"); + free(shape); + free(chunkshape); + free(blockshape); + return 0; + } + + zfpsize = zfp_decompress(zfp, field); + + /* clean up */ + zfp_field_free(field); + zfp_stream_close(zfp); + stream_close(stream); + free(shape); + free(chunkshape); + free(blockshape); + + if (zfpsize == 0) { + BLOSC_TRACE_ERROR("\n ZFP: Decompression failed\n"); + return (int) zfpsize; + } + + return (int) output_len; +} + +int zfp_getcell(void *thread_context, const uint8_t *block, int32_t cbytes, uint8_t *dest, int32_t destsize) { + struct thread_context *thread_ctx = thread_context; + blosc2_context *context = thread_ctx->parent_context; + bool meta = false; + int8_t ndim = ZFP_MAX_DIM + 1; + int32_t blockmeta[ZFP_MAX_DIM]; + if (context->schunk->blockshape == NULL) { + // blockshape is not filled yet. Use the Caterva layer to populate it. + for (int nmetalayer = 0; nmetalayer < context->schunk->nmetalayers; nmetalayer++) { + if (strcmp("caterva", context->schunk->metalayers[nmetalayer]->name) == 0) { + meta = true; + uint8_t *pmeta = context->schunk->metalayers[nmetalayer]->content; + ndim = (int8_t) pmeta[2]; + assert(ndim <= ZFP_MAX_DIM); + pmeta += (6 + ndim * 9 + ndim * 5); + for (int8_t i = 0; (uint8_t) i < ndim; i++) { + pmeta += 1; + swap_store(blockmeta + i, pmeta, sizeof(int32_t)); + pmeta += sizeof(int32_t); + } + } + } + if (!meta) { + return -1; + } + context->schunk->ndim = ndim; + context->schunk->blockshape = malloc(sizeof(int64_t) * ndim); + for (int i = 0; i < ndim; ++i) { + context->schunk->blockshape[i] = (int64_t) blockmeta[i]; + } + } + ndim = context->schunk->ndim; + int64_t *blockshape = context->schunk->blockshape; + + // Compute the coordinates of the cell + int64_t cell_start_ndim[ZFP_MAX_DIM]; + int64_t cell_ind_ndim[ZFP_MAX_DIM]; + int64_t ncell_ndim[ZFP_MAX_DIM]; + int64_t ind_strides[ZFP_MAX_DIM]; + int64_t cell_strides[ZFP_MAX_DIM]; + int64_t cell_ind, ncell; + blosc2_unidim_to_multidim(ndim, blockshape, thread_ctx->zfp_cell_start, cell_start_ndim); + for (int i = 0; i < ndim; ++i) { + cell_ind_ndim[i] = cell_start_ndim[i] % ZFP_MAX_DIM; + ncell_ndim[i] = cell_start_ndim[i] / ZFP_MAX_DIM; + } + ind_strides[ndim - 1] = cell_strides[ndim - 1] = 1; + for (int i = ndim - 2; i >= 0; --i) { + ind_strides[i] = ZFP_MAX_DIM * ind_strides[i + 1]; + cell_strides[i] = ((blockshape[i + 1] - 1) / ZFP_MAX_DIM + 1) * cell_strides[i + 1]; + } + blosc2_multidim_to_unidim(cell_ind_ndim, (int8_t) ndim, ind_strides, &cell_ind); + blosc2_multidim_to_unidim(ncell_ndim, (int8_t) ndim, cell_strides, &ncell); + int cell_nitems = (int) (1u << (2 * ndim)); + if ((thread_ctx->zfp_cell_nitems > cell_nitems) || + ((cell_ind + thread_ctx->zfp_cell_nitems) > cell_nitems)) { + return 0; + } + + // Get the ZFP stream + zfp_type type; /* array scalar type */ + zfp_stream *zfp; /* compressed stream */ + bitstream *stream; /* bit stream to write to or read from */ + int32_t typesize = context->typesize; + zfp = zfp_stream_open(NULL); + + switch (typesize) { + case sizeof(float): + type = zfp_type_float; + break; + case sizeof(double): + type = zfp_type_double; + break; + default: + printf("\n ZFP is not available for this typesize \n"); + return 0; + } + uint8_t compmeta = context->compcode_meta; // access to compressed chunk header + double rate = (double) (compmeta * typesize * 8) / + 100.0; // convert from output size / input size to output bits per input value + zfp_stream_set_rate(zfp, rate, type, ndim, zfp_false); + + stream = stream_open((void *) block, cbytes); + zfp_stream_set_bit_stream(zfp, stream); + zfp_stream_rewind(zfp); + + // Check that ncell is a valid index + int ncells = (int) ((cbytes * 8) / zfp->maxbits); + if (ncell >= ncells) { + BLOSC_TRACE_ERROR("Invalid cell index"); + return -1; + } + + // Position the stream at the ncell bit offset for reading + stream_rseek(zfp->stream, (size_t) (ncell * zfp->maxbits)); + + // Get the cell + size_t zfpsize; + uint8_t *cell = malloc(cell_nitems * typesize); + switch (ndim) { + case 1: + switch (type) { + case zfp_type_float: + zfpsize = zfp_decode_block_float_1(zfp, (float *) cell); + break; + case zfp_type_double: + zfpsize = zfp_decode_block_double_1(zfp, (double *) cell); + break; + default: + printf("\n ZFP is not available for this typesize \n"); + zfp_stream_close(zfp); + stream_close(stream); + return 0; + } + break; + case 2: + switch (type) { + case zfp_type_float: + zfpsize = zfp_decode_block_float_2(zfp, (float *) cell); + break; + case zfp_type_double: + zfpsize = zfp_decode_block_double_2(zfp, (double *) cell); + break; + default: + printf("\n ZFP is not available for this typesize \n"); + zfp_stream_close(zfp); + stream_close(stream); + return 0; + } + break; + case 3: + switch (type) { + case zfp_type_float: + zfpsize = zfp_decode_block_float_3(zfp, (float *) cell); + break; + case zfp_type_double: + zfpsize = zfp_decode_block_double_3(zfp, (double *) cell); + break; + default: + printf("\n ZFP is not available for this typesize \n"); + zfp_stream_close(zfp); + stream_close(stream); + return 0; + } + break; + case 4: + switch (type) { + case zfp_type_float: + zfpsize = zfp_decode_block_float_4(zfp, (float *) cell); + break; + case zfp_type_double: + zfpsize = zfp_decode_block_double_4(zfp, (double *) cell); + break; + default: + printf("\n ZFP is not available for this typesize \n"); + zfp_stream_close(zfp); + stream_close(stream); + return 0; + } + break; + default: + printf("\n ZFP is not available for this number of dims \n"); + return 0; + } + memcpy(dest, &cell[cell_ind * typesize], thread_ctx->zfp_cell_nitems * typesize); + zfp_stream_close(zfp); + stream_close(stream); + free(cell); + + if ((zfpsize == 0) || ((int32_t)zfpsize > (destsize * 8)) || + ((int32_t)zfpsize > (cell_nitems * typesize * 8)) || + ((thread_ctx->zfp_cell_nitems * typesize * 8) > (int32_t)zfpsize)) { + BLOSC_TRACE_ERROR("ZFP error or small destsize"); + return -1; + } + + return (int) (thread_ctx->zfp_cell_nitems * typesize); +} diff --git a/src/blosc2/plugins/codecs/zfp/blosc2-zfp.h b/src/blosc2/plugins/codecs/zfp/blosc2-zfp.h new file mode 100644 index 0000000..87e8f9f --- /dev/null +++ b/src/blosc2/plugins/codecs/zfp/blosc2-zfp.h @@ -0,0 +1,48 @@ +/********************************************************************* + Blosc - Blocked Shuffling and Compression Library + + Copyright (C) 2021 The Blosc Developers + https://blosc.org + License: BSD 3-Clause (see LICENSE.txt) + + See LICENSE.txt for details about copyright and rights to use. +**********************************************************************/ + + + +#ifndef BLOSC2_ZFP_H +#define BLOSC2_ZFP_H +#include "zfp-private.h" +#include "../plugins/plugin_utils.h" + +#if defined (__cplusplus) +extern "C" { +#endif + + +int zfp_acc_compress(const uint8_t *input, int32_t input_len, uint8_t *output, int32_t output_len, + uint8_t meta, blosc2_cparams *cparams, const void* chunk); + +int zfp_acc_decompress(const uint8_t *input, int32_t input_len, uint8_t *output, int32_t output_len, + uint8_t meta, blosc2_dparams *dparams, const void* chunk); + +int zfp_prec_compress(const uint8_t *input, int32_t input_len, uint8_t *output, int32_t output_len, + uint8_t meta, blosc2_cparams *cparams, const void* chunk); + +int zfp_prec_decompress(const uint8_t *input, int32_t input_len, uint8_t *output, int32_t output_len, + uint8_t meta, blosc2_dparams *dparams, const void* chunk); + +int zfp_rate_compress(const uint8_t *input, int32_t input_len, uint8_t *output, int32_t output_len, + uint8_t meta, blosc2_cparams *cparams, const void* chunk); + +int zfp_rate_decompress(const uint8_t *input, int32_t input_len, uint8_t *output, int32_t output_len, + uint8_t meta, blosc2_dparams *dparams, const void* chunk); + +int zfp_getcell(void *thread_context, const uint8_t *block, int32_t cbytes, uint8_t *dest, int32_t destsize); + + +#if defined (__cplusplus) +} +#endif + +#endif /* BLOSC2_ZFP_H */ diff --git a/src/blosc2/plugins/codecs/zfp/include/bitstream.h b/src/blosc2/plugins/codecs/zfp/include/bitstream.h new file mode 100644 index 0000000..c7b55d2 --- /dev/null +++ b/src/blosc2/plugins/codecs/zfp/include/bitstream.h @@ -0,0 +1,97 @@ +#ifndef ZFP_BITSTREAM_H +#define ZFP_BITSTREAM_H + +#include +#include "zfp/types.h" +#include "zfp/system.h" + +/* forward declaration of opaque type */ +typedef struct bitstream bitstream; + +extern_ const size_t stream_word_bits; /* bit stream granularity */ + +#ifndef inline_ +#ifdef __cplusplus +extern "C" { +#endif + +/* allocate and initialize bit stream */ +bitstream* stream_open(void* buffer, size_t bytes); + +/* close and deallocate bit stream */ +void stream_close(bitstream* stream); + +/* make a copy of bit stream to shared memory buffer */ +bitstream* stream_clone(const bitstream* stream); + +/* word size in bits (equal to stream_word_bits) */ +size_t stream_alignment(void); + +/* pointer to beginning of stream */ +void* stream_data(const bitstream* stream); + +/* current byte size of stream (if flushed) */ +size_t stream_size(const bitstream* stream); + +/* byte capacity of stream */ +size_t stream_capacity(const bitstream* stream); + +/* number of words per block */ +size_t stream_stride_block(const bitstream* stream); + +/* number of blocks between consecutive blocks */ +ptrdiff_t stream_stride_delta(const bitstream* stream); + +/* read single bit (0 or 1) */ +uint stream_read_bit(bitstream* stream); + +/* write single bit */ +uint stream_write_bit(bitstream* stream, uint bit); + +/* read 0 <= n <= 64 bits */ +uint64 stream_read_bits(bitstream* stream, uint n); + +/* write 0 <= n <= 64 low bits of value and return remaining bits */ +uint64 stream_write_bits(bitstream* stream, uint64 value, uint n); + +/* return bit offset to next bit to be read */ +size_t stream_rtell(const bitstream* stream); + +/* return bit offset to next bit to be written */ +size_t stream_wtell(const bitstream* stream); + +/* rewind stream to beginning */ +void stream_rewind(bitstream* stream); + +/* position stream for reading at given bit offset */ +void stream_rseek(bitstream* stream, size_t offset); + +/* position stream for writing at given bit offset */ +void stream_wseek(bitstream* stream, size_t offset); + +/* skip over the next n bits */ +void stream_skip(bitstream* stream, uint n); + +/* append n zero-bits to stream */ +void stream_pad(bitstream* stream, uint n); + +/* align stream on next word boundary */ +size_t stream_align(bitstream* stream); + +/* flush out any remaining buffered bits */ +size_t stream_flush(bitstream* stream); + +/* copy n bits from one bit stream to another */ +void stream_copy(bitstream* dst, bitstream* src, size_t n); + +#ifdef BIT_STREAM_STRIDED +/* set block size in number of words and spacing in number of blocks */ +int stream_set_stride(bitstream* stream, size_t block, ptrdiff_t delta); +#endif + +#ifdef __cplusplus +} +#endif +#endif /* !inline_ */ + +#endif diff --git a/src/blosc2/plugins/codecs/zfp/include/zfp.h b/src/blosc2/plugins/codecs/zfp/include/zfp.h new file mode 100644 index 0000000..936008b --- /dev/null +++ b/src/blosc2/plugins/codecs/zfp/include/zfp.h @@ -0,0 +1,794 @@ +/* +** Copyright (c) 2014-2019, Lawrence Livermore National Security, LLC and +** other zfp project contributors. See the top-level LICENSE file for details. +** SPDX-License-Identifier: BSD-3-Clause +*/ + +#ifndef ZFP_H +#define ZFP_H + +#include "zfp/types.h" +#include "zfp/system.h" +#include "zfp/version.h" +#include "bitstream.h" + +/* macros ------------------------------------------------------------------ */ + +/* default compression parameters */ +#define ZFP_MIN_BITS 1 /* minimum number of bits per block */ +#define ZFP_MAX_BITS 16658 /* maximum number of bits per block */ +#define ZFP_MAX_PREC 64 /* maximum precision supported */ +#define ZFP_MIN_EXP (-1074) /* minimum floating-point base-2 exponent */ + +/* header masks (enable via bitwise or; reader must use same mask) */ +#define ZFP_HEADER_NONE 0x0u /* no header */ +#define ZFP_HEADER_MAGIC 0x1u /* embed 64-bit magic */ +#define ZFP_HEADER_META 0x2u /* embed 52-bit field metadata */ +#define ZFP_HEADER_MODE 0x4u /* embed 12- or 64-bit compression mode */ +#define ZFP_HEADER_FULL 0x7u /* embed all of the above */ + +/* bit masks for specifying storage class */ +#define ZFP_DATA_UNUSED 0x01u /* allocated but unused storage */ +#define ZFP_DATA_PADDING 0x02u /* padding for alignment purposes */ +#define ZFP_DATA_META 0x04u /* class members and other fixed-size storage */ +#define ZFP_DATA_MISC 0x08u /* miscellaneous uncategorized storage */ +#define ZFP_DATA_PAYLOAD 0x10u /* compressed data */ +#define ZFP_DATA_INDEX 0x20u /* variable-rate block index information */ +#define ZFP_DATA_CACHE 0x40u /* uncompressed cached data */ +#define ZFP_DATA_HEADER 0x80u /* header information */ +#define ZFP_DATA_ALL 0xffu /* all storage */ + +/* field metadata indeterminate state and error code */ +#define ZFP_META_NULL (UINT64C(-1)) + +/* number of bits per header entry */ +#define ZFP_MAGIC_BITS 32 /* number of magic word bits */ +#define ZFP_META_BITS 52 /* number of field metadata bits */ +#define ZFP_MODE_SHORT_BITS 12 /* number of mode bits in short format */ +#define ZFP_MODE_LONG_BITS 64 /* number of mode bits in long format */ +#define ZFP_HEADER_MAX_BITS 148 /* max number of header bits */ +#define ZFP_MODE_SHORT_MAX ((1u << ZFP_MODE_SHORT_BITS) - 2) + +/* rounding mode for reducing bias; see build option ZFP_ROUNDING_MODE */ +#define ZFP_ROUND_FIRST (-1) /* round during compression */ +#define ZFP_ROUND_NEVER 0 /* never round */ +#define ZFP_ROUND_LAST 1 /* round during decompression */ + +/* types ------------------------------------------------------------------- */ + +/* Boolean constants */ +enum { + zfp_false = 0, /* false */ + zfp_true = !zfp_false /* true */ +}; + +typedef int zfp_bool; /* Boolean type */ + +/* execution policy */ +typedef enum { + zfp_exec_serial = 0, /* serial execution (default) */ + zfp_exec_omp = 1, /* OpenMP multi-threaded execution */ + zfp_exec_cuda = 2 /* CUDA parallel execution */ +} zfp_exec_policy; + +/* OpenMP execution parameters */ +typedef struct { + uint threads; /* number of requested threads */ + uint chunk_size; /* number of blocks per chunk (1D only) */ +} zfp_exec_params_omp; + +/* execution parameters */ +typedef union { + zfp_exec_params_omp omp; /* OpenMP parameters */ +} zfp_exec_params; + +typedef struct { + zfp_exec_policy policy; /* execution policy (serial, omp, ...) */ + zfp_exec_params params; /* execution parameters */ +} zfp_execution; + +/* compressed stream; use accessors to get/set members */ +typedef struct { + uint minbits; /* minimum number of bits to store per block */ + uint maxbits; /* maximum number of bits to store per block */ + uint maxprec; /* maximum number of bit planes to store */ + int minexp; /* minimum floating point bit plane number to store */ + bitstream* stream; /* compressed bit stream */ + zfp_execution exec; /* execution policy and parameters */ +} zfp_stream; + +/* compression mode */ +typedef enum { + zfp_mode_null = 0, /* an invalid configuration of the 4 params */ + zfp_mode_expert = 1, /* expert mode (4 params set manually) */ + zfp_mode_fixed_rate = 2, /* fixed rate mode */ + zfp_mode_fixed_precision = 3, /* fixed precision mode */ + zfp_mode_fixed_accuracy = 4, /* fixed accuracy mode */ + zfp_mode_reversible = 5 /* reversible (lossless) mode */ +} zfp_mode; + +/* compression mode and parameter settings */ +typedef struct { + zfp_mode mode; /* compression mode */ + union { + double rate; /* compressed bits/value (negative for word alignment) */ + uint precision; /* uncompressed bits/value */ + double tolerance; /* absolute error tolerance */ + struct { + uint minbits; /* min number of compressed bits/block */ + uint maxbits; /* max number of compressed bits/block */ + uint maxprec; /* max number of uncompressed bits/value */ + int minexp; /* min floating point bit plane number to store */ + } expert; /* expert mode arguments */ + } arg; /* arguments corresponding to compression mode */ +} zfp_config; + +/* scalar type */ +typedef enum { + zfp_type_none = 0, /* unspecified type */ + zfp_type_int32 = 1, /* 32-bit signed integer */ + zfp_type_int64 = 2, /* 64-bit signed integer */ + zfp_type_float = 3, /* single precision floating point */ + zfp_type_double = 4 /* double precision floating point */ +} zfp_type; + +/* uncompressed array; use accessors to get/set members */ +typedef struct { + zfp_type type; /* scalar type (e.g. int32, double) */ + size_t nx, ny, nz, nw; /* sizes (zero for unused dimensions) */ + ptrdiff_t sx, sy, sz, sw; /* strides (zero for contiguous array a[nw][nz][ny][nx]) */ + void* data; /* pointer to array data */ +} zfp_field; + +#ifdef __cplusplus +extern "C" { +#endif + +/* public data ------------------------------------------------------------- */ + +extern_ const uint zfp_codec_version; /* codec version ZFP_CODEC */ +extern_ const uint zfp_library_version; /* library version ZFP_VERSION */ +extern_ const char* const zfp_version_string; /* verbose version string */ + +/* high-level API: utility functions --------------------------------------- */ + +size_t /* byte size of scalar type */ +zfp_type_size( + zfp_type type /* scalar type */ +); + +/* high-level API: compressed stream construction/destruction -------------- */ + +/* open compressed stream and associate with bit stream */ +zfp_stream* /* allocated compressed stream */ +zfp_stream_open( + bitstream* stream /* bit stream to read from and write to (may be NULL) */ +); + +/* close and deallocate compressed stream (does not affect bit stream) */ +void +zfp_stream_close( + zfp_stream* stream /* compressed stream */ +); + +/* high-level API: compressed stream inspectors ---------------------------- */ + +/* bit stream associated with compressed stream */ +bitstream* /* bit stream associated with compressed stream */ +zfp_stream_bit_stream( + const zfp_stream* stream /* compressed stream */ +); + +/* enumerated compression mode */ +zfp_mode /* compression mode or zfp_mode_null if not set */ +zfp_stream_compression_mode( + const zfp_stream* stream /* compressed stream */ +); + +/* rate in compressed bits/scalar (when in fixed-rate mode) */ +double /* rate or zero upon failure */ +zfp_stream_rate( + const zfp_stream* stream, /* compressed stream */ + uint dims /* array dimensionality (1, 2, 3, or 4) */ +); + +/* precision in uncompressed bits/scalar (when in fixed-precision mode) */ +uint /* precision or zero upon failure */ +zfp_stream_precision( + const zfp_stream* stream /* compressed stream */ +); + +/* accuracy as absolute error tolerance (when in fixed-accuracy mode) */ +double /* tolerance or zero upon failure */ +zfp_stream_accuracy( + const zfp_stream* stream /* compressed stream */ +); + +/* get all compression parameters in a compact representation */ +uint64 /* 12- or 64-bit encoding of parameters */ +zfp_stream_mode( + const zfp_stream* stream /* compressed stream */ +); + +/* get all compression parameters (pointers may be NULL) */ +void +zfp_stream_params( + const zfp_stream* stream, /* compressed stream */ + uint* minbits, /* minimum number of bits per 4^d block */ + uint* maxbits, /* maximum number of bits per 4^d block */ + uint* maxprec, /* maximum precision (# bit planes coded) */ + int* minexp /* minimum base-2 exponent; error <= 2^minexp */ +); + +/* byte size of sequentially compressed stream (call after compression) */ +size_t /* actual number of bytes of compressed storage */ +zfp_stream_compressed_size( + const zfp_stream* stream /* compressed stream */ +); + +/* conservative estimate of compressed size in bytes */ +size_t /* maximum number of bytes of compressed storage */ +zfp_stream_maximum_size( + const zfp_stream* stream, /* compressed stream */ + const zfp_field* field /* array to compress */ +); + +/* high-level API: initialization of compressed stream parameters ---------- */ + +/* rewind bit stream to beginning for compression or decompression */ +void +zfp_stream_rewind( + zfp_stream* stream /* compressed bit stream */ +); + +/* associate bit stream with compressed stream */ +void +zfp_stream_set_bit_stream( + zfp_stream* stream, /* compressed stream */ + bitstream* bs /* bit stream to read from and write to */ +); + +/* enable reversible (lossless) compression */ +void +zfp_stream_set_reversible( + zfp_stream* stream /* compressed stream */ +); + +/* set size in compressed bits/scalar (fixed-rate mode) */ +double /* actual rate in compressed bits/scalar */ +zfp_stream_set_rate( + zfp_stream* stream, /* compressed stream */ + double rate, /* desired rate in compressed bits/scalar */ + zfp_type type, /* scalar type to compress */ + uint dims, /* array dimensionality (1, 2, 3, or 4) */ + zfp_bool align /* word-aligned blocks, e.g., for write random access */ +); + +/* set precision in uncompressed bits/scalar (fixed-precision mode) */ +uint /* actual precision */ +zfp_stream_set_precision( + zfp_stream* stream, /* compressed stream */ + uint precision /* desired precision in uncompressed bits/scalar */ +); + +/* set accuracy as absolute error tolerance (fixed-accuracy mode) */ +double /* actual error tolerance */ +zfp_stream_set_accuracy( + zfp_stream* stream, /* compressed stream */ + double tolerance /* desired error tolerance */ +); + +/* set parameters from compact encoding; leaves stream intact on failure */ +zfp_mode /* compression mode or zfp_mode_null upon failure */ +zfp_stream_set_mode( + zfp_stream* stream, /* compressed stream */ + uint64 mode /* 12- or 64-bit encoding of parameters */ +); + +/* set all parameters (expert mode); leaves stream intact on failure */ +zfp_bool /* true upon success */ +zfp_stream_set_params( + zfp_stream* stream, /* compressed stream */ + uint minbits, /* minimum number of bits per 4^d block */ + uint maxbits, /* maximum number of bits per 4^d block */ + uint maxprec, /* maximum precision (# bit planes coded) */ + int minexp /* minimum base-2 exponent; error <= 2^minexp */ +); + +/* high-level API: execution policy ---------------------------------------- */ + +/* current execution policy */ +zfp_exec_policy +zfp_stream_execution( + const zfp_stream* stream /* compressed stream */ +); + +/* number of OpenMP threads to use */ +uint /* number of threads (0 for default) */ +zfp_stream_omp_threads( + const zfp_stream* stream /* compressed stream */ +); + +/* number of blocks per OpenMP chunk (1D only) */ +uint /* number of blocks per chunk (0 for default) */ +zfp_stream_omp_chunk_size( + const zfp_stream* stream /* compressed stream */ +); + +/* set execution policy */ +zfp_bool /* true upon success */ +zfp_stream_set_execution( + zfp_stream* stream, /* compressed stream */ + zfp_exec_policy policy /* execution policy */ +); + +/* set OpenMP execution policy and number of threads */ +zfp_bool /* true upon success */ +zfp_stream_set_omp_threads( + zfp_stream* stream, /* compressed stream */ + uint threads /* number of OpenMP threads to use (0 for default) */ +); + +/* set OpenMP execution policy and number of blocks per chunk (1D only) */ +zfp_bool /* true upon success */ +zfp_stream_set_omp_chunk_size( + zfp_stream* stream, /* compressed stream */ + uint chunk_size /* number of blocks per chunk (0 for default) */ +); + +/* high-level API: compression mode and parameter settings ----------------- */ + +/* unspecified configuration */ +zfp_config /* compression mode and parameter settings */ +zfp_config_none(void); + +/* fixed-rate configuration */ +zfp_config /* compression mode and parameter settings */ +zfp_config_rate( + double rate, /* desired rate in compressed bits/scalar */ + zfp_bool align /* word-aligned blocks, e.g., for write random access */ +); + +/* fixed-precision configuration */ +zfp_config /* compression mode and parameter settings */ +zfp_config_precision( + uint precision /* desired precision in uncompressed bits/scalar */ +); + +/* fixed-accuracy configuration */ +zfp_config /* compression mode and parameter settings */ +zfp_config_accuracy( + double tolerance /* desired error tolerance */ +); + +/* reversible (lossless) configuration */ +zfp_config /* compression mode and parameter settings */ +zfp_config_reversible(void); + +/* expert configuration */ +zfp_config /* compression mode and parameter settings */ +zfp_config_expert( + uint minbits, /* minimum number of bits per 4^d block */ + uint maxbits, /* maximum number of bits per 4^d block */ + uint maxprec, /* maximum precision (# bit planes coded) */ + int minexp /* minimum base-2 exponent; error <= 2^minexp */ +); + +/* high-level API: uncompressed array construction/destruction ------------- */ + +/* allocate field struct */ +zfp_field* /* pointer to default initialized field */ +zfp_field_alloc(void); + +/* allocate metadata for 1D field f[nx] */ +zfp_field* /* allocated field metadata */ +zfp_field_1d( + void* pointer, /* pointer to uncompressed scalars (may be NULL) */ + zfp_type type, /* scalar type */ + size_t nx /* number of scalars */ +); + +/* allocate metadata for 2D field f[ny][nx] */ +zfp_field* /* allocated field metadata */ +zfp_field_2d( + void* pointer, /* pointer to uncompressed scalars (may be NULL) */ + zfp_type type, /* scalar type */ + size_t nx, /* number of scalars in x dimension */ + size_t ny /* number of scalars in y dimension */ +); + +/* allocate metadata for 3D field f[nz][ny][nx] */ +zfp_field* /* allocated field metadata */ +zfp_field_3d( + void* pointer, /* pointer to uncompressed scalars (may be NULL) */ + zfp_type type, /* scalar type */ + size_t nx, /* number of scalars in x dimension */ + size_t ny, /* number of scalars in y dimension */ + size_t nz /* number of scalars in z dimension */ +); + +/* allocate metadata for 4D field f[nw][nz][ny][nx] */ +zfp_field* /* allocated field metadata */ +zfp_field_4d( + void* pointer, /* pointer to uncompressed scalars (may be NULL) */ + zfp_type type, /* scalar type */ + size_t nx, /* number of scalars in x dimension */ + size_t ny, /* number of scalars in y dimension */ + size_t nz, /* number of scalars in z dimension */ + size_t nw /* number of scalars in w dimension */ +); + +/* deallocate field metadata */ +void +zfp_field_free( + zfp_field* field /* field metadata */ +); + +/* high-level API: uncompressed array inspectors --------------------------- */ + +/* pointer to first scalar in field */ +void* /* array pointer */ +zfp_field_pointer( + const zfp_field* field /* field metadata */ +); + +/* pointer to lowest memory address spanned by field */ +void* +zfp_field_begin( + const zfp_field* field /* field metadata */ +); + +/* field scalar type */ +zfp_type /* scalar type */ +zfp_field_type( + const zfp_field* field /* field metadata */ +); + +/* precision of field scalar type */ +uint /* scalar type precision in number of bits */ +zfp_field_precision( + const zfp_field* field /* field metadata */ +); + +/* field dimensionality (1, 2, 3, or 4) */ +uint /* number of dimensions */ +zfp_field_dimensionality( + const zfp_field* field /* field metadata */ +); + +/* field size in number of scalars */ +size_t /* total number of scalars */ +zfp_field_size( + const zfp_field* field, /* field metadata */ + size_t* size /* number of scalars per dimension (may be NULL) */ +); + +/* number of bytes spanned by field data including gaps (if any) */ +size_t +zfp_field_size_bytes( + const zfp_field* field /* field metadata */ +); + +/* field strides per dimension */ +zfp_bool /* true if array is not contiguous */ +zfp_field_stride( + const zfp_field* field, /* field metadata */ + ptrdiff_t* stride /* stride in scalars per dimension (may be NULL) */ +); + +/* field contiguity test */ +zfp_bool /* true if field layout is contiguous */ +zfp_field_is_contiguous( + const zfp_field* field /* field metadata */ +); + +/* field scalar type and dimensions */ +uint64 /* compact 52-bit encoding of metadata */ +zfp_field_metadata( + const zfp_field* field /* field metadata */ +); + +/* high-level API: uncompressed array specification ------------------------ */ + +/* set pointer to first scalar in field */ +void +zfp_field_set_pointer( + zfp_field* field, /* field metadata */ + void* pointer /* pointer to first scalar */ +); + +/* set field scalar type */ +zfp_type /* actual scalar type */ +zfp_field_set_type( + zfp_field* field, /* field metadata */ + zfp_type type /* desired scalar type */ +); + +/* set 1D field size */ +void +zfp_field_set_size_1d( + zfp_field* field, /* field metadata */ + size_t nx /* number of scalars */ +); + +/* set 2D field size */ +void +zfp_field_set_size_2d( + zfp_field* field, /* field metadata */ + size_t nx, /* number of scalars in x dimension */ + size_t ny /* number of scalars in y dimension */ +); + +/* set 3D field size */ +void +zfp_field_set_size_3d( + zfp_field* field, /* field metadata */ + size_t nx, /* number of scalars in x dimension */ + size_t ny, /* number of scalars in y dimension */ + size_t nz /* number of scalars in z dimension */ +); + +/* set 4D field size */ +void +zfp_field_set_size_4d( + zfp_field* field, /* field metadata */ + size_t nx, /* number of scalars in x dimension */ + size_t ny, /* number of scalars in y dimension */ + size_t nz, /* number of scalars in z dimension */ + size_t nw /* number of scalars in w dimension */ +); + +/* set 1D field stride in number of scalars */ +void +zfp_field_set_stride_1d( + zfp_field* field, /* field metadata */ + ptrdiff_t sx /* stride in number of scalars: &f[1] - &f[0] */ +); + +/* set 2D field strides in number of scalars */ +void +zfp_field_set_stride_2d( + zfp_field* field, /* field metadata */ + ptrdiff_t sx, /* stride in x dimension: &f[0][1] - &f[0][0] */ + ptrdiff_t sy /* stride in y dimension: &f[1][0] - &f[0][0] */ +); + +/* set 3D field strides in number of scalars */ +void +zfp_field_set_stride_3d( + zfp_field* field, /* field metadata */ + ptrdiff_t sx, /* stride in x dimension: &f[0][0][1] - &f[0][0][0] */ + ptrdiff_t sy, /* stride in y dimension: &f[0][1][0] - &f[0][0][0] */ + ptrdiff_t sz /* stride in z dimension: &f[1][0][0] - &f[0][0][0] */ +); + +/* set 4D field strides in number of scalars */ +void +zfp_field_set_stride_4d( + zfp_field* field, /* field metadata */ + ptrdiff_t sx, /* stride in x dimension: &f[0][0][0][1] - &f[0][0][0][0] */ + ptrdiff_t sy, /* stride in y dimension: &f[0][0][1][0] - &f[0][0][0][0] */ + ptrdiff_t sz, /* stride in z dimension: &f[0][1][0][0] - &f[0][0][0][0] */ + ptrdiff_t sw /* stride in w dimension: &f[1][0][0][0] - &f[0][0][0][0] */ +); + +/* set field scalar type and dimensions */ +zfp_bool /* true upon success */ +zfp_field_set_metadata( + zfp_field* field, /* field metadata */ + uint64 meta /* compact 52-bit encoding of metadata */ +); + +/* high-level API: compression and decompression --------------------------- */ + +/* compress entire field (nonzero return value upon success) */ +size_t /* cumulative number of bytes of compressed storage */ +zfp_compress( + zfp_stream* stream, /* compressed stream */ + const zfp_field* field /* field metadata */ +); + +/* decompress entire field (nonzero return value upon success) */ +size_t /* cumulative number of bytes of compressed storage */ +zfp_decompress( + zfp_stream* stream, /* compressed stream */ + zfp_field* field /* field metadata */ +); + +/* write compression parameters and field metadata (optional) */ +size_t /* number of bits written or zero upon failure */ +zfp_write_header( + zfp_stream* stream, /* compressed stream */ + const zfp_field* field, /* field metadata */ + uint mask /* information to write */ +); + +/* read compression parameters and field metadata when previously written */ +size_t /* number of bits read or zero upon failure */ +zfp_read_header( + zfp_stream* stream, /* compressed stream */ + zfp_field* field, /* field metadata */ + uint mask /* information to read */ +); + +/* low-level API: stream manipulation -------------------------------------- */ + +/* flush bit stream--must be called after last encode call or between seeks */ +size_t +zfp_stream_flush( + zfp_stream* stream /* compressed bit stream */ +); + +/* align bit stream on next word boundary (decoding analogy to flush) */ +size_t +zfp_stream_align( + zfp_stream* stream /* compressed bit stream */ +); + +/* low-level API: encoder -------------------------------------------------- */ + +/* +The functions below all compress either a complete contiguous d-dimensional +block of 4^d scalars or a complete or partial block assembled from a strided +array. In the latter case, p points to the first scalar; (nx, ny, nz) specify +the size of the block, with 1 <= nx, ny, nz <= 4; and (sx, sy, sz) specify the +strides, i.e. the number of scalars to advance to get to the next scalar along +each dimension. The functions return the number of bits of compressed storage +needed for the compressed block. +*/ + +/* encode 1D contiguous block of 4 values */ +size_t zfp_encode_block_int32_1(zfp_stream* stream, const int32* block); +size_t zfp_encode_block_int64_1(zfp_stream* stream, const int64* block); +size_t zfp_encode_block_float_1(zfp_stream* stream, const float* block); +size_t zfp_encode_block_double_1(zfp_stream* stream, const double* block); + +/* encode 1D complete or partial block from strided array */ +size_t zfp_encode_block_strided_int32_1(zfp_stream* stream, const int32* p, ptrdiff_t sx); +size_t zfp_encode_block_strided_int64_1(zfp_stream* stream, const int64* p, ptrdiff_t sx); +size_t zfp_encode_block_strided_float_1(zfp_stream* stream, const float* p, ptrdiff_t sx); +size_t zfp_encode_block_strided_double_1(zfp_stream* stream, const double* p, ptrdiff_t sx); +size_t zfp_encode_partial_block_strided_int32_1(zfp_stream* stream, const int32* p, size_t nx, ptrdiff_t sx); +size_t zfp_encode_partial_block_strided_int64_1(zfp_stream* stream, const int64* p, size_t nx, ptrdiff_t sx); +size_t zfp_encode_partial_block_strided_float_1(zfp_stream* stream, const float* p, size_t nx, ptrdiff_t sx); +size_t zfp_encode_partial_block_strided_double_1(zfp_stream* stream, const double* p, size_t nx, ptrdiff_t sx); + +/* encode 2D contiguous block of 4x4 values */ +size_t zfp_encode_block_int32_2(zfp_stream* stream, const int32* block); +size_t zfp_encode_block_int64_2(zfp_stream* stream, const int64* block); +size_t zfp_encode_block_float_2(zfp_stream* stream, const float* block); +size_t zfp_encode_block_double_2(zfp_stream* stream, const double* block); + +/* encode 2D complete or partial block from strided array */ +size_t zfp_encode_partial_block_strided_int32_2(zfp_stream* stream, const int32* p, size_t nx, size_t ny, ptrdiff_t sx, ptrdiff_t sy); +size_t zfp_encode_partial_block_strided_int64_2(zfp_stream* stream, const int64* p, size_t nx, size_t ny, ptrdiff_t sx, ptrdiff_t sy); +size_t zfp_encode_partial_block_strided_float_2(zfp_stream* stream, const float* p, size_t nx, size_t ny, ptrdiff_t sx, ptrdiff_t sy); +size_t zfp_encode_partial_block_strided_double_2(zfp_stream* stream, const double* p, size_t nx, size_t ny, ptrdiff_t sx, ptrdiff_t sy); +size_t zfp_encode_block_strided_int32_2(zfp_stream* stream, const int32* p, ptrdiff_t sx, ptrdiff_t sy); +size_t zfp_encode_block_strided_int64_2(zfp_stream* stream, const int64* p, ptrdiff_t sx, ptrdiff_t sy); +size_t zfp_encode_block_strided_float_2(zfp_stream* stream, const float* p, ptrdiff_t sx, ptrdiff_t sy); +size_t zfp_encode_block_strided_double_2(zfp_stream* stream, const double* p, ptrdiff_t sx, ptrdiff_t sy); + +/* encode 3D contiguous block of 4x4x4 values */ +size_t zfp_encode_block_int32_3(zfp_stream* stream, const int32* block); +size_t zfp_encode_block_int64_3(zfp_stream* stream, const int64* block); +size_t zfp_encode_block_float_3(zfp_stream* stream, const float* block); +size_t zfp_encode_block_double_3(zfp_stream* stream, const double* block); + +/* encode 3D complete or partial block from strided array */ +size_t zfp_encode_block_strided_int32_3(zfp_stream* stream, const int32* p, ptrdiff_t sx, ptrdiff_t sy, ptrdiff_t sz); +size_t zfp_encode_block_strided_int64_3(zfp_stream* stream, const int64* p, ptrdiff_t sx, ptrdiff_t sy, ptrdiff_t sz); +size_t zfp_encode_block_strided_float_3(zfp_stream* stream, const float* p, ptrdiff_t sx, ptrdiff_t sy, ptrdiff_t sz); +size_t zfp_encode_block_strided_double_3(zfp_stream* stream, const double* p, ptrdiff_t sx, ptrdiff_t sy, ptrdiff_t sz); +size_t zfp_encode_partial_block_strided_int32_3(zfp_stream* stream, const int32* p, size_t nx, size_t ny, size_t nz, ptrdiff_t sx, ptrdiff_t sy, ptrdiff_t sz); +size_t zfp_encode_partial_block_strided_int64_3(zfp_stream* stream, const int64* p, size_t nx, size_t ny, size_t nz, ptrdiff_t sx, ptrdiff_t sy, ptrdiff_t sz); +size_t zfp_encode_partial_block_strided_float_3(zfp_stream* stream, const float* p, size_t nx, size_t ny, size_t nz, ptrdiff_t sx, ptrdiff_t sy, ptrdiff_t sz); +size_t zfp_encode_partial_block_strided_double_3(zfp_stream* stream, const double* p, size_t nx, size_t ny, size_t nz, ptrdiff_t sx, ptrdiff_t sy, ptrdiff_t sz); + +/* encode 4D contiguous block of 4x4x4x4 values */ +size_t zfp_encode_block_int32_4(zfp_stream* stream, const int32* block); +size_t zfp_encode_block_int64_4(zfp_stream* stream, const int64* block); +size_t zfp_encode_block_float_4(zfp_stream* stream, const float* block); +size_t zfp_encode_block_double_4(zfp_stream* stream, const double* block); + +/* encode 4D complete or partial block from strided array */ +size_t zfp_encode_block_strided_int32_4(zfp_stream* stream, const int32* p, ptrdiff_t sx, ptrdiff_t sy, ptrdiff_t sz, ptrdiff_t sw); +size_t zfp_encode_block_strided_int64_4(zfp_stream* stream, const int64* p, ptrdiff_t sx, ptrdiff_t sy, ptrdiff_t sz, ptrdiff_t sw); +size_t zfp_encode_block_strided_float_4(zfp_stream* stream, const float* p, ptrdiff_t sx, ptrdiff_t sy, ptrdiff_t sz, ptrdiff_t sw); +size_t zfp_encode_block_strided_double_4(zfp_stream* stream, const double* p, ptrdiff_t sx, ptrdiff_t sy, ptrdiff_t sz, ptrdiff_t sw); +size_t zfp_encode_partial_block_strided_int32_4(zfp_stream* stream, const int32* p, size_t nx, size_t ny, size_t nz, size_t nw, ptrdiff_t sx, ptrdiff_t sy, ptrdiff_t sz, ptrdiff_t sw); +size_t zfp_encode_partial_block_strided_int64_4(zfp_stream* stream, const int64* p, size_t nx, size_t ny, size_t nz, size_t nw, ptrdiff_t sx, ptrdiff_t sy, ptrdiff_t sz, ptrdiff_t sw); +size_t zfp_encode_partial_block_strided_float_4(zfp_stream* stream, const float* p, size_t nx, size_t ny, size_t nz, size_t nw, ptrdiff_t sx, ptrdiff_t sy, ptrdiff_t sz, ptrdiff_t sw); +size_t zfp_encode_partial_block_strided_double_4(zfp_stream* stream, const double* p, size_t nx, size_t ny, size_t nz, size_t nw, ptrdiff_t sx, ptrdiff_t sy, ptrdiff_t sz, ptrdiff_t sw); + +/* low-level API: decoder -------------------------------------------------- */ + +/* +Each function below decompresses a single block and returns the number of bits +of compressed storage consumed. See corresponding encoder functions above for +further details. +*/ + +/* decode 1D contiguous block of 4 values */ +size_t zfp_decode_block_int32_1(zfp_stream* stream, int32* block); +size_t zfp_decode_block_int64_1(zfp_stream* stream, int64* block); +size_t zfp_decode_block_float_1(zfp_stream* stream, float* block); +size_t zfp_decode_block_double_1(zfp_stream* stream, double* block); + +/* decode 1D complete or partial block from strided array */ +size_t zfp_decode_block_strided_int32_1(zfp_stream* stream, int32* p, ptrdiff_t sx); +size_t zfp_decode_block_strided_int64_1(zfp_stream* stream, int64* p, ptrdiff_t sx); +size_t zfp_decode_block_strided_float_1(zfp_stream* stream, float* p, ptrdiff_t sx); +size_t zfp_decode_block_strided_double_1(zfp_stream* stream, double* p, ptrdiff_t sx); +size_t zfp_decode_partial_block_strided_int32_1(zfp_stream* stream, int32* p, size_t nx, ptrdiff_t sx); +size_t zfp_decode_partial_block_strided_int64_1(zfp_stream* stream, int64* p, size_t nx, ptrdiff_t sx); +size_t zfp_decode_partial_block_strided_float_1(zfp_stream* stream, float* p, size_t nx, ptrdiff_t sx); +size_t zfp_decode_partial_block_strided_double_1(zfp_stream* stream, double* p, size_t nx, ptrdiff_t sx); + +/* decode 2D contiguous block of 4x4 values */ +size_t zfp_decode_block_int32_2(zfp_stream* stream, int32* block); +size_t zfp_decode_block_int64_2(zfp_stream* stream, int64* block); +size_t zfp_decode_block_float_2(zfp_stream* stream, float* block); +size_t zfp_decode_block_double_2(zfp_stream* stream, double* block); + +/* decode 2D complete or partial block from strided array */ +size_t zfp_decode_block_strided_int32_2(zfp_stream* stream, int32* p, ptrdiff_t sx, ptrdiff_t sy); +size_t zfp_decode_block_strided_int64_2(zfp_stream* stream, int64* p, ptrdiff_t sx, ptrdiff_t sy); +size_t zfp_decode_block_strided_float_2(zfp_stream* stream, float* p, ptrdiff_t sx, ptrdiff_t sy); +size_t zfp_decode_block_strided_double_2(zfp_stream* stream, double* p, ptrdiff_t sx, ptrdiff_t sy); +size_t zfp_decode_partial_block_strided_int32_2(zfp_stream* stream, int32* p, size_t nx, size_t ny, ptrdiff_t sx, ptrdiff_t sy); +size_t zfp_decode_partial_block_strided_int64_2(zfp_stream* stream, int64* p, size_t nx, size_t ny, ptrdiff_t sx, ptrdiff_t sy); +size_t zfp_decode_partial_block_strided_float_2(zfp_stream* stream, float* p, size_t nx, size_t ny, ptrdiff_t sx, ptrdiff_t sy); +size_t zfp_decode_partial_block_strided_double_2(zfp_stream* stream, double* p, size_t nx, size_t ny, ptrdiff_t sx, ptrdiff_t sy); + +/* decode 3D contiguous block of 4x4x4 values */ +size_t zfp_decode_block_int32_3(zfp_stream* stream, int32* block); +size_t zfp_decode_block_int64_3(zfp_stream* stream, int64* block); +size_t zfp_decode_block_float_3(zfp_stream* stream, float* block); +size_t zfp_decode_block_double_3(zfp_stream* stream, double* block); + +/* decode 3D complete or partial block from strided array */ +size_t zfp_decode_block_strided_int32_3(zfp_stream* stream, int32* p, ptrdiff_t sx, ptrdiff_t sy, ptrdiff_t sz); +size_t zfp_decode_block_strided_int64_3(zfp_stream* stream, int64* p, ptrdiff_t sx, ptrdiff_t sy, ptrdiff_t sz); +size_t zfp_decode_block_strided_float_3(zfp_stream* stream, float* p, ptrdiff_t sx, ptrdiff_t sy, ptrdiff_t sz); +size_t zfp_decode_block_strided_double_3(zfp_stream* stream, double* p, ptrdiff_t sx, ptrdiff_t sy, ptrdiff_t sz); +size_t zfp_decode_partial_block_strided_int32_3(zfp_stream* stream, int32* p, size_t nx, size_t ny, size_t nz, ptrdiff_t sx, ptrdiff_t sy, ptrdiff_t sz); +size_t zfp_decode_partial_block_strided_int64_3(zfp_stream* stream, int64* p, size_t nx, size_t ny, size_t nz, ptrdiff_t sx, ptrdiff_t sy, ptrdiff_t sz); +size_t zfp_decode_partial_block_strided_float_3(zfp_stream* stream, float* p, size_t nx, size_t ny, size_t nz, ptrdiff_t sx, ptrdiff_t sy, ptrdiff_t sz); +size_t zfp_decode_partial_block_strided_double_3(zfp_stream* stream, double* p, size_t nx, size_t ny, size_t nz, ptrdiff_t sx, ptrdiff_t sy, ptrdiff_t sz); + +/* decode 4D contiguous block of 4x4x4x4 values */ +size_t zfp_decode_block_int32_4(zfp_stream* stream, int32* block); +size_t zfp_decode_block_int64_4(zfp_stream* stream, int64* block); +size_t zfp_decode_block_float_4(zfp_stream* stream, float* block); +size_t zfp_decode_block_double_4(zfp_stream* stream, double* block); + +/* decode 4D complete or partial block from strided array */ +size_t zfp_decode_block_strided_int32_4(zfp_stream* stream, int32* p, ptrdiff_t sx, ptrdiff_t sy, ptrdiff_t sz, ptrdiff_t sw); +size_t zfp_decode_block_strided_int64_4(zfp_stream* stream, int64* p, ptrdiff_t sx, ptrdiff_t sy, ptrdiff_t sz, ptrdiff_t sw); +size_t zfp_decode_block_strided_float_4(zfp_stream* stream, float* p, ptrdiff_t sx, ptrdiff_t sy, ptrdiff_t sz, ptrdiff_t sw); +size_t zfp_decode_block_strided_double_4(zfp_stream* stream, double* p, ptrdiff_t sx, ptrdiff_t sy, ptrdiff_t sz, ptrdiff_t sw); +size_t zfp_decode_partial_block_strided_int32_4(zfp_stream* stream, int32* p, size_t nx, size_t ny, size_t nz, size_t nw, ptrdiff_t sx, ptrdiff_t sy, ptrdiff_t sz, ptrdiff_t sw); +size_t zfp_decode_partial_block_strided_int64_4(zfp_stream* stream, int64* p, size_t nx, size_t ny, size_t nz, size_t nw, ptrdiff_t sx, ptrdiff_t sy, ptrdiff_t sz, ptrdiff_t sw); +size_t zfp_decode_partial_block_strided_float_4(zfp_stream* stream, float* p, size_t nx, size_t ny, size_t nz, size_t nw, ptrdiff_t sx, ptrdiff_t sy, ptrdiff_t sz, ptrdiff_t sw); +size_t zfp_decode_partial_block_strided_double_4(zfp_stream* stream, double* p, size_t nx, size_t ny, size_t nz, size_t nw, ptrdiff_t sx, ptrdiff_t sy, ptrdiff_t sz, ptrdiff_t sw); + +/* low-level API: utility functions ---------------------------------------- */ + +/* convert dims-dimensional contiguous block to 32-bit integer type */ +void zfp_promote_int8_to_int32(int32* oblock, const int8* iblock, uint dims); +void zfp_promote_uint8_to_int32(int32* oblock, const uint8* iblock, uint dims); +void zfp_promote_int16_to_int32(int32* oblock, const int16* iblock, uint dims); +void zfp_promote_uint16_to_int32(int32* oblock, const uint16* iblock, uint dims); + +/* convert dims-dimensional contiguous block from 32-bit integer type */ +void zfp_demote_int32_to_int8(int8* oblock, const int32* iblock, uint dims); +void zfp_demote_int32_to_uint8(uint8* oblock, const int32* iblock, uint dims); +void zfp_demote_int32_to_int16(int16* oblock, const int32* iblock, uint dims); +void zfp_demote_int32_to_uint16(uint16* oblock, const int32* iblock, uint dims); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/src/blosc2/plugins/codecs/zfp/include/zfp/macros.h b/src/blosc2/plugins/codecs/zfp/include/zfp/macros.h new file mode 100644 index 0000000..be3655c --- /dev/null +++ b/src/blosc2/plugins/codecs/zfp/include/zfp/macros.h @@ -0,0 +1,7 @@ +#ifndef ZFP_MACROS_H +#define ZFP_MACROS_H + +#define MIN(x, y) ((x) < (y) ? (x) : (y)) +#define MAX(x, y) ((x) > (y) ? (x) : (y)) + +#endif diff --git a/src/blosc2/plugins/codecs/zfp/include/zfp/system.h b/src/blosc2/plugins/codecs/zfp/include/zfp/system.h new file mode 100644 index 0000000..23c4936 --- /dev/null +++ b/src/blosc2/plugins/codecs/zfp/include/zfp/system.h @@ -0,0 +1,47 @@ +#ifndef ZFP_SYSTEM_H +#define ZFP_SYSTEM_H + +#if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L + /* C99: use restrict */ + #define restrict_ restrict +#else + /* C89: no restrict keyword */ + #define restrict_ +#endif + +/* macros for exporting and importing symbols */ +#if defined(_MSC_VER) && defined(ZFP_SHARED_LIBS) + /* export (import) symbols when ZFP_SOURCE is (is not) defined */ + #ifdef ZFP_SOURCE + #ifdef __cplusplus + #define extern_ extern "C" __declspec(dllexport) + #else + #define extern_ extern __declspec(dllexport) + #endif + #else + #ifdef __cplusplus + #define extern_ extern "C" __declspec(dllimport) + #else + #define extern_ extern __declspec(dllimport) + #endif + #endif +#else /* !(_MSC_VER && ZFP_SHARED_LIBS) */ + #ifdef __cplusplus + #define extern_ extern "C" + #else + #define extern_ extern + #endif +#endif + +/* L1 cache line size for alignment purposes */ +#ifndef ZFP_CACHE_LINE_SIZE + #define ZFP_CACHE_LINE_SIZE 0x100 +#endif +/* ZFP_CACHE_LINE_SIZE=0 disables alignment */ +#if defined(__GNUC__) && ZFP_CACHE_LINE_SIZE + #define cache_align_(x) x __attribute__((aligned(ZFP_CACHE_LINE_SIZE))) +#else + #define cache_align_(x) x +#endif + +#endif diff --git a/src/blosc2/plugins/codecs/zfp/include/zfp/types.h b/src/blosc2/plugins/codecs/zfp/include/zfp/types.h new file mode 100644 index 0000000..b209f37 --- /dev/null +++ b/src/blosc2/plugins/codecs/zfp/include/zfp/types.h @@ -0,0 +1,132 @@ +#ifndef ZFP_TYPES_H +#define ZFP_TYPES_H + +typedef unsigned char uchar; +typedef unsigned short ushort; +typedef unsigned int uint; +typedef unsigned long ulong; + +#if defined(__cplusplus) && __cplusplus >= 201103L + /* C++11: use standard integer types */ + #include + #include + #define INT64C(x) INT64_C(x) + #define UINT64C(x) UINT64_C(x) + #define INT64PRId PRId64 + #define INT64PRIi PRIi64 + #define UINT64PRIo PRIo64 + #define UINT64PRIu PRIu64 + #define UINT64PRIx PRIx64 + #define INT64SCNd SCNd64 + #define INT64SCNi SCNi64 + #define UINT64SCNo SCNo64 + #define UINT64SCNu SCNu64 + #define UINT64SCNx SCNx64 + typedef std::int8_t int8; + typedef std::uint8_t uint8; + typedef std::int16_t int16; + typedef std::uint16_t uint16; + typedef std::int32_t int32; + typedef std::uint32_t uint32; + typedef std::int64_t int64; + typedef std::uint64_t uint64; +#elif defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L + /* C99: use standard integer types */ + #include + #include + #define INT64C(x) INT64_C(x) + #define UINT64C(x) UINT64_C(x) + #define INT64PRId PRId64 + #define INT64PRIi PRIi64 + #define UINT64PRIo PRIo64 + #define UINT64PRIu PRIu64 + #define UINT64PRIx PRIx64 + #define INT64SCNd SCNd64 + #define INT64SCNi SCNi64 + #define UINT64SCNo SCNo64 + #define UINT64SCNu SCNu64 + #define UINT64SCNx SCNx64 + typedef int8_t int8; + typedef uint8_t uint8; + typedef int16_t int16; + typedef uint16_t uint16; + typedef int32_t int32; + typedef uint32_t uint32; + typedef int64_t int64; + typedef uint64_t uint64; +#else + /* C89: assume common integer types */ + typedef signed char int8; + typedef unsigned char uint8; + typedef signed short int16; + typedef unsigned short uint16; + + /* assume 32-bit integers (LP64, LLP64) */ + typedef signed int int32; + typedef unsigned int uint32; + + /* determine 64-bit data model */ + #if defined(_WIN32) || defined(_WIN64) + /* assume ILP32 or LLP64 (MSVC, MinGW) */ + #define ZFP_LLP64 1 + #else + /* assume LP64 (Linux, macOS, ...) */ + #define ZFP_LP64 1 + #endif + + /* concatenation for literal suffixes */ + #define _zfp_cat_(x, y) x ## y + #define _zfp_cat(x, y) _zfp_cat_(x, y) + + /* signed 64-bit integers */ + #if defined(ZFP_INT64) && defined(ZFP_INT64_SUFFIX) + #define INT64C(x) _zfp_cat(x, ZFP_INT64_SUFFIX) + #define INT64PRId #ZFP_INT64_SUFFIX "d" + #define INT64PRIi #ZFP_INT64_SUFFIX "i" + typedef ZFP_INT64 int64; + #elif ZFP_LP64 + #define INT64C(x) x ## l + #define INT64PRId "ld" + #define INT64PRIi "li" + typedef signed long int64; + #elif ZFP_LLP64 + #define INT64C(x) x ## ll + #define INT64PRId "lld" + #define INT64PRIi "lli" + typedef signed long long int64; + #else + #error "unknown 64-bit signed integer type" + #endif + #define INT64SCNd INT64PRId + #define INT64SCNi INT64PRIi + + /* unsigned 64-bit integers */ + #if defined(ZFP_UINT64) && defined(ZFP_UINT64_SUFFIX) + #define UINT64C(x) _zfp_cat(x, ZFP_UINT64_SUFFIX) + #ifdef ZFP_INT64_SUFFIX + #define UINT64PRIo #ZFP_INT64_SUFFIX "o" + #define UINT64PRIu #ZFP_INT64_SUFFIX "u" + #define UINT64PRIx #ZFP_INT64_SUFFIX "x" + #endif + typedef ZFP_UINT64 uint64; + #elif ZFP_LP64 + #define UINT64C(x) x ## ul + #define UINT64PRIo "lo" + #define UINT64PRIu "lu" + #define UINT64PRIx "lx" + typedef unsigned long uint64; + #elif ZFP_LLP64 + #define UINT64C(x) x ## ull + #define UINT64PRIo "llo" + #define UINT64PRIu "llu" + #define UINT64PRIx "llx" + typedef unsigned long long uint64; + #else + #error "unknown 64-bit unsigned integer type" + #endif + #define UINT64SCNo UINT64PRIo + #define UINT64SCNu UINT64PRIu + #define UINT64SCNx UINT64PRIx +#endif + +#endif diff --git a/src/blosc2/plugins/codecs/zfp/include/zfp/version.h b/src/blosc2/plugins/codecs/zfp/include/zfp/version.h new file mode 100644 index 0000000..87a1d22 --- /dev/null +++ b/src/blosc2/plugins/codecs/zfp/include/zfp/version.h @@ -0,0 +1,29 @@ +#ifndef ZFP_VERSION_H +#define ZFP_VERSION_H + +/* stringification */ +#define _zfp_str_(x) # x +#define _zfp_str(x) _zfp_str_(x) + +/* library version information */ +#define ZFP_VERSION_MAJOR 0 /* library major version number */ +#define ZFP_VERSION_MINOR 5 /* library minor version number */ +#define ZFP_VERSION_PATCH 5 /* library patch version number */ +#define ZFP_VERSION_RELEASE ZFP_VERSION_PATCH + +/* codec version number (see also zfp_codec_version) */ +#define ZFP_CODEC 5 + +/* library version number (see also zfp_library_version) */ +#define ZFP_VERSION \ + ((ZFP_VERSION_MAJOR << 8) + \ + (ZFP_VERSION_MINOR << 4) + \ + (ZFP_VERSION_PATCH << 0)) + +/* library version string (see also zfp_version_string) */ +#define ZFP_VERSION_STRING \ + _zfp_str(ZFP_VERSION_MAJOR) "." \ + _zfp_str(ZFP_VERSION_MINOR) "." \ + _zfp_str(ZFP_VERSION_PATCH) + +#endif diff --git a/src/blosc2/plugins/codecs/zfp/src/bitstream.c b/src/blosc2/plugins/codecs/zfp/src/bitstream.c new file mode 100644 index 0000000..3a74766 --- /dev/null +++ b/src/blosc2/plugins/codecs/zfp/src/bitstream.c @@ -0,0 +1,4 @@ +#include "bitstream.h" +#include "inline/bitstream.c" + +const size_t stream_word_bits = wsize; diff --git a/src/blosc2/plugins/codecs/zfp/src/block1.h b/src/blosc2/plugins/codecs/zfp/src/block1.h new file mode 100644 index 0000000..035d9c9 --- /dev/null +++ b/src/blosc2/plugins/codecs/zfp/src/block1.h @@ -0,0 +1 @@ +#define DIMS 1 diff --git a/src/blosc2/plugins/codecs/zfp/src/block2.h b/src/blosc2/plugins/codecs/zfp/src/block2.h new file mode 100644 index 0000000..e87ab62 --- /dev/null +++ b/src/blosc2/plugins/codecs/zfp/src/block2.h @@ -0,0 +1 @@ +#define DIMS 2 diff --git a/src/blosc2/plugins/codecs/zfp/src/block3.h b/src/blosc2/plugins/codecs/zfp/src/block3.h new file mode 100644 index 0000000..a683568 --- /dev/null +++ b/src/blosc2/plugins/codecs/zfp/src/block3.h @@ -0,0 +1 @@ +#define DIMS 3 diff --git a/src/blosc2/plugins/codecs/zfp/src/block4.h b/src/blosc2/plugins/codecs/zfp/src/block4.h new file mode 100644 index 0000000..6737fb2 --- /dev/null +++ b/src/blosc2/plugins/codecs/zfp/src/block4.h @@ -0,0 +1 @@ +#define DIMS 4 diff --git a/src/blosc2/plugins/codecs/zfp/src/decode1d.c b/src/blosc2/plugins/codecs/zfp/src/decode1d.c new file mode 100644 index 0000000..b5bbd66 --- /dev/null +++ b/src/blosc2/plugins/codecs/zfp/src/decode1d.c @@ -0,0 +1,18 @@ +#include "inline/inline.h" +#include "zfp.h" +#include "zfp/macros.h" +#include "block1.h" +#include "traitsd.h" +#include "template/template.h" +#include "template/codec.h" +#include "inline/bitstream.c" +#include "template/codec.c" +#include "template/codecf.c" +#include "template/codec1.c" +#include "template/decode.c" +#include "template/decodef.c" +#include "template/decode1.c" +#include "template/revcodecf.c" +#include "template/revdecode.c" +#include "template/revdecodef.c" +#include "template/revdecode1.c" diff --git a/src/blosc2/plugins/codecs/zfp/src/decode1f.c b/src/blosc2/plugins/codecs/zfp/src/decode1f.c new file mode 100644 index 0000000..83085a8 --- /dev/null +++ b/src/blosc2/plugins/codecs/zfp/src/decode1f.c @@ -0,0 +1,18 @@ +#include "inline/inline.h" +#include "zfp.h" +#include "zfp/macros.h" +#include "block1.h" +#include "traitsf.h" +#include "template/template.h" +#include "template/codec.h" +#include "inline/bitstream.c" +#include "template/codec.c" +#include "template/codecf.c" +#include "template/codec1.c" +#include "template/decode.c" +#include "template/decodef.c" +#include "template/decode1.c" +#include "template/revcodecf.c" +#include "template/revdecode.c" +#include "template/revdecodef.c" +#include "template/revdecode1.c" diff --git a/src/blosc2/plugins/codecs/zfp/src/decode1i.c b/src/blosc2/plugins/codecs/zfp/src/decode1i.c new file mode 100644 index 0000000..b0e61ab --- /dev/null +++ b/src/blosc2/plugins/codecs/zfp/src/decode1i.c @@ -0,0 +1,15 @@ +#include "inline/inline.h" +#include "zfp.h" +#include "zfp/macros.h" +#include "block1.h" +#include "traitsi.h" +#include "template/template.h" +#include "template/codec.h" +#include "inline/bitstream.c" +#include "template/codec.c" +#include "template/codec1.c" +#include "template/decode.c" +#include "template/decodei.c" +#include "template/decode1.c" +#include "template/revdecode.c" +#include "template/revdecode1.c" diff --git a/src/blosc2/plugins/codecs/zfp/src/decode1l.c b/src/blosc2/plugins/codecs/zfp/src/decode1l.c new file mode 100644 index 0000000..a063a1f --- /dev/null +++ b/src/blosc2/plugins/codecs/zfp/src/decode1l.c @@ -0,0 +1,15 @@ +#include "inline/inline.h" +#include "zfp.h" +#include "zfp/macros.h" +#include "block1.h" +#include "traitsl.h" +#include "template/template.h" +#include "template/codec.h" +#include "inline/bitstream.c" +#include "template/codec.c" +#include "template/codec1.c" +#include "template/decode.c" +#include "template/decodei.c" +#include "template/decode1.c" +#include "template/revdecode.c" +#include "template/revdecode1.c" diff --git a/src/blosc2/plugins/codecs/zfp/src/decode2d.c b/src/blosc2/plugins/codecs/zfp/src/decode2d.c new file mode 100644 index 0000000..ced3d5d --- /dev/null +++ b/src/blosc2/plugins/codecs/zfp/src/decode2d.c @@ -0,0 +1,18 @@ +#include "inline/inline.h" +#include "zfp.h" +#include "zfp/macros.h" +#include "block2.h" +#include "traitsd.h" +#include "template/template.h" +#include "template/codec.h" +#include "inline/bitstream.c" +#include "template/codec.c" +#include "template/codecf.c" +#include "template/codec2.c" +#include "template/decode.c" +#include "template/decodef.c" +#include "template/decode2.c" +#include "template/revcodecf.c" +#include "template/revdecode.c" +#include "template/revdecodef.c" +#include "template/revdecode2.c" diff --git a/src/blosc2/plugins/codecs/zfp/src/decode2f.c b/src/blosc2/plugins/codecs/zfp/src/decode2f.c new file mode 100644 index 0000000..f6d823a --- /dev/null +++ b/src/blosc2/plugins/codecs/zfp/src/decode2f.c @@ -0,0 +1,18 @@ +#include "inline/inline.h" +#include "zfp.h" +#include "zfp/macros.h" +#include "block2.h" +#include "traitsf.h" +#include "template/template.h" +#include "template/codec.h" +#include "inline/bitstream.c" +#include "template/codec.c" +#include "template/codecf.c" +#include "template/codec2.c" +#include "template/decode.c" +#include "template/decodef.c" +#include "template/decode2.c" +#include "template/revcodecf.c" +#include "template/revdecode.c" +#include "template/revdecodef.c" +#include "template/revdecode2.c" diff --git a/src/blosc2/plugins/codecs/zfp/src/decode2i.c b/src/blosc2/plugins/codecs/zfp/src/decode2i.c new file mode 100644 index 0000000..e4b5235 --- /dev/null +++ b/src/blosc2/plugins/codecs/zfp/src/decode2i.c @@ -0,0 +1,15 @@ +#include "inline/inline.h" +#include "zfp.h" +#include "zfp/macros.h" +#include "block2.h" +#include "traitsi.h" +#include "template/template.h" +#include "template/codec.h" +#include "inline/bitstream.c" +#include "template/codec.c" +#include "template/codec2.c" +#include "template/decode.c" +#include "template/decodei.c" +#include "template/decode2.c" +#include "template/revdecode.c" +#include "template/revdecode2.c" diff --git a/src/blosc2/plugins/codecs/zfp/src/decode2l.c b/src/blosc2/plugins/codecs/zfp/src/decode2l.c new file mode 100644 index 0000000..80031ea --- /dev/null +++ b/src/blosc2/plugins/codecs/zfp/src/decode2l.c @@ -0,0 +1,15 @@ +#include "inline/inline.h" +#include "zfp.h" +#include "zfp/macros.h" +#include "block2.h" +#include "traitsl.h" +#include "template/template.h" +#include "template/codec.h" +#include "inline/bitstream.c" +#include "template/codec.c" +#include "template/codec2.c" +#include "template/decode.c" +#include "template/decodei.c" +#include "template/decode2.c" +#include "template/revdecode.c" +#include "template/revdecode2.c" diff --git a/src/blosc2/plugins/codecs/zfp/src/decode3d.c b/src/blosc2/plugins/codecs/zfp/src/decode3d.c new file mode 100644 index 0000000..823c64f --- /dev/null +++ b/src/blosc2/plugins/codecs/zfp/src/decode3d.c @@ -0,0 +1,18 @@ +#include "inline/inline.h" +#include "zfp.h" +#include "zfp/macros.h" +#include "block3.h" +#include "traitsd.h" +#include "template/template.h" +#include "template/codec.h" +#include "inline/bitstream.c" +#include "template/codec.c" +#include "template/codecf.c" +#include "template/codec3.c" +#include "template/decode.c" +#include "template/decodef.c" +#include "template/decode3.c" +#include "template/revcodecf.c" +#include "template/revdecode.c" +#include "template/revdecodef.c" +#include "template/revdecode3.c" diff --git a/src/blosc2/plugins/codecs/zfp/src/decode3f.c b/src/blosc2/plugins/codecs/zfp/src/decode3f.c new file mode 100644 index 0000000..2e2724f --- /dev/null +++ b/src/blosc2/plugins/codecs/zfp/src/decode3f.c @@ -0,0 +1,18 @@ +#include "inline/inline.h" +#include "zfp.h" +#include "zfp/macros.h" +#include "block3.h" +#include "traitsf.h" +#include "template/template.h" +#include "template/codec.h" +#include "inline/bitstream.c" +#include "template/codec.c" +#include "template/codecf.c" +#include "template/codec3.c" +#include "template/decode.c" +#include "template/decodef.c" +#include "template/decode3.c" +#include "template/revcodecf.c" +#include "template/revdecode.c" +#include "template/revdecodef.c" +#include "template/revdecode3.c" diff --git a/src/blosc2/plugins/codecs/zfp/src/decode3i.c b/src/blosc2/plugins/codecs/zfp/src/decode3i.c new file mode 100644 index 0000000..6888f24 --- /dev/null +++ b/src/blosc2/plugins/codecs/zfp/src/decode3i.c @@ -0,0 +1,15 @@ +#include "inline/inline.h" +#include "zfp.h" +#include "zfp/macros.h" +#include "block3.h" +#include "traitsi.h" +#include "template/template.h" +#include "template/codec.h" +#include "inline/bitstream.c" +#include "template/codec.c" +#include "template/codec3.c" +#include "template/decode.c" +#include "template/decodei.c" +#include "template/decode3.c" +#include "template/revdecode.c" +#include "template/revdecode3.c" diff --git a/src/blosc2/plugins/codecs/zfp/src/decode3l.c b/src/blosc2/plugins/codecs/zfp/src/decode3l.c new file mode 100644 index 0000000..225b6bc --- /dev/null +++ b/src/blosc2/plugins/codecs/zfp/src/decode3l.c @@ -0,0 +1,15 @@ +#include "inline/inline.h" +#include "zfp.h" +#include "zfp/macros.h" +#include "block3.h" +#include "traitsl.h" +#include "template/template.h" +#include "template/codec.h" +#include "inline/bitstream.c" +#include "template/codec.c" +#include "template/codec3.c" +#include "template/decode.c" +#include "template/decodei.c" +#include "template/decode3.c" +#include "template/revdecode.c" +#include "template/revdecode3.c" diff --git a/src/blosc2/plugins/codecs/zfp/src/decode4d.c b/src/blosc2/plugins/codecs/zfp/src/decode4d.c new file mode 100644 index 0000000..8a047b6 --- /dev/null +++ b/src/blosc2/plugins/codecs/zfp/src/decode4d.c @@ -0,0 +1,18 @@ +#include "inline/inline.h" +#include "zfp.h" +#include "zfp/macros.h" +#include "block4.h" +#include "traitsd.h" +#include "template/template.h" +#include "template/codec.h" +#include "inline/bitstream.c" +#include "template/codec.c" +#include "template/codecf.c" +#include "template/codec4.c" +#include "template/decode.c" +#include "template/decodef.c" +#include "template/decode4.c" +#include "template/revcodecf.c" +#include "template/revdecode.c" +#include "template/revdecodef.c" +#include "template/revdecode4.c" diff --git a/src/blosc2/plugins/codecs/zfp/src/decode4f.c b/src/blosc2/plugins/codecs/zfp/src/decode4f.c new file mode 100644 index 0000000..6ec6bf0 --- /dev/null +++ b/src/blosc2/plugins/codecs/zfp/src/decode4f.c @@ -0,0 +1,18 @@ +#include "inline/inline.h" +#include "zfp.h" +#include "zfp/macros.h" +#include "block4.h" +#include "traitsf.h" +#include "template/template.h" +#include "template/codec.h" +#include "inline/bitstream.c" +#include "template/codec.c" +#include "template/codecf.c" +#include "template/codec4.c" +#include "template/decode.c" +#include "template/decodef.c" +#include "template/decode4.c" +#include "template/revcodecf.c" +#include "template/revdecode.c" +#include "template/revdecodef.c" +#include "template/revdecode4.c" diff --git a/src/blosc2/plugins/codecs/zfp/src/decode4i.c b/src/blosc2/plugins/codecs/zfp/src/decode4i.c new file mode 100644 index 0000000..309eee2 --- /dev/null +++ b/src/blosc2/plugins/codecs/zfp/src/decode4i.c @@ -0,0 +1,15 @@ +#include "inline/inline.h" +#include "zfp.h" +#include "zfp/macros.h" +#include "block4.h" +#include "traitsi.h" +#include "template/template.h" +#include "template/codec.h" +#include "inline/bitstream.c" +#include "template/codec.c" +#include "template/codec4.c" +#include "template/decode.c" +#include "template/decodei.c" +#include "template/decode4.c" +#include "template/revdecode.c" +#include "template/revdecode4.c" diff --git a/src/blosc2/plugins/codecs/zfp/src/decode4l.c b/src/blosc2/plugins/codecs/zfp/src/decode4l.c new file mode 100644 index 0000000..c07c85c --- /dev/null +++ b/src/blosc2/plugins/codecs/zfp/src/decode4l.c @@ -0,0 +1,15 @@ +#include "inline/inline.h" +#include "zfp.h" +#include "zfp/macros.h" +#include "block4.h" +#include "traitsl.h" +#include "template/template.h" +#include "template/codec.h" +#include "inline/bitstream.c" +#include "template/codec.c" +#include "template/codec4.c" +#include "template/decode.c" +#include "template/decodei.c" +#include "template/decode4.c" +#include "template/revdecode.c" +#include "template/revdecode4.c" diff --git a/src/blosc2/plugins/codecs/zfp/src/encode1d.c b/src/blosc2/plugins/codecs/zfp/src/encode1d.c new file mode 100644 index 0000000..7f8f3ca --- /dev/null +++ b/src/blosc2/plugins/codecs/zfp/src/encode1d.c @@ -0,0 +1,18 @@ +#include "inline/inline.h" +#include "zfp.h" +#include "zfp/macros.h" +#include "block1.h" +#include "traitsd.h" +#include "template/template.h" +#include "template/codec.h" +#include "inline/bitstream.c" +#include "template/codec.c" +#include "template/codecf.c" +#include "template/codec1.c" +#include "template/encode.c" +#include "template/encodef.c" +#include "template/encode1.c" +#include "template/revcodecf.c" +#include "template/revencode.c" +#include "template/revencodef.c" +#include "template/revencode1.c" diff --git a/src/blosc2/plugins/codecs/zfp/src/encode1f.c b/src/blosc2/plugins/codecs/zfp/src/encode1f.c new file mode 100644 index 0000000..5fe4812 --- /dev/null +++ b/src/blosc2/plugins/codecs/zfp/src/encode1f.c @@ -0,0 +1,18 @@ +#include "inline/inline.h" +#include "zfp.h" +#include "zfp/macros.h" +#include "block1.h" +#include "traitsf.h" +#include "template/template.h" +#include "template/codec.h" +#include "inline/bitstream.c" +#include "template/codec.c" +#include "template/codecf.c" +#include "template/codec1.c" +#include "template/encode.c" +#include "template/encodef.c" +#include "template/encode1.c" +#include "template/revcodecf.c" +#include "template/revencode.c" +#include "template/revencodef.c" +#include "template/revencode1.c" diff --git a/src/blosc2/plugins/codecs/zfp/src/encode1i.c b/src/blosc2/plugins/codecs/zfp/src/encode1i.c new file mode 100644 index 0000000..f3069de --- /dev/null +++ b/src/blosc2/plugins/codecs/zfp/src/encode1i.c @@ -0,0 +1,15 @@ +#include "inline/inline.h" +#include "zfp.h" +#include "zfp/macros.h" +#include "block1.h" +#include "traitsi.h" +#include "template/template.h" +#include "template/codec.h" +#include "inline/bitstream.c" +#include "template/codec.c" +#include "template/codec1.c" +#include "template/encode.c" +#include "template/encodei.c" +#include "template/encode1.c" +#include "template/revencode.c" +#include "template/revencode1.c" diff --git a/src/blosc2/plugins/codecs/zfp/src/encode1l.c b/src/blosc2/plugins/codecs/zfp/src/encode1l.c new file mode 100644 index 0000000..62a1814 --- /dev/null +++ b/src/blosc2/plugins/codecs/zfp/src/encode1l.c @@ -0,0 +1,15 @@ +#include "inline/inline.h" +#include "zfp.h" +#include "zfp/macros.h" +#include "block1.h" +#include "traitsl.h" +#include "template/template.h" +#include "template/codec.h" +#include "inline/bitstream.c" +#include "template/codec.c" +#include "template/codec1.c" +#include "template/encode.c" +#include "template/encodei.c" +#include "template/encode1.c" +#include "template/revencode.c" +#include "template/revencode1.c" diff --git a/src/blosc2/plugins/codecs/zfp/src/encode2d.c b/src/blosc2/plugins/codecs/zfp/src/encode2d.c new file mode 100644 index 0000000..4dba892 --- /dev/null +++ b/src/blosc2/plugins/codecs/zfp/src/encode2d.c @@ -0,0 +1,18 @@ +#include "inline/inline.h" +#include "zfp.h" +#include "zfp/macros.h" +#include "block2.h" +#include "traitsd.h" +#include "template/template.h" +#include "template/codec.h" +#include "inline/bitstream.c" +#include "template/codec.c" +#include "template/codecf.c" +#include "template/codec2.c" +#include "template/encode.c" +#include "template/encodef.c" +#include "template/encode2.c" +#include "template/revcodecf.c" +#include "template/revencode.c" +#include "template/revencodef.c" +#include "template/revencode2.c" diff --git a/src/blosc2/plugins/codecs/zfp/src/encode2f.c b/src/blosc2/plugins/codecs/zfp/src/encode2f.c new file mode 100644 index 0000000..d667ff7 --- /dev/null +++ b/src/blosc2/plugins/codecs/zfp/src/encode2f.c @@ -0,0 +1,18 @@ +#include "inline/inline.h" +#include "zfp.h" +#include "zfp/macros.h" +#include "block2.h" +#include "traitsf.h" +#include "template/template.h" +#include "template/codec.h" +#include "inline/bitstream.c" +#include "template/codec.c" +#include "template/codecf.c" +#include "template/codec2.c" +#include "template/encode.c" +#include "template/encodef.c" +#include "template/encode2.c" +#include "template/revcodecf.c" +#include "template/revencode.c" +#include "template/revencodef.c" +#include "template/revencode2.c" diff --git a/src/blosc2/plugins/codecs/zfp/src/encode2i.c b/src/blosc2/plugins/codecs/zfp/src/encode2i.c new file mode 100644 index 0000000..264c449 --- /dev/null +++ b/src/blosc2/plugins/codecs/zfp/src/encode2i.c @@ -0,0 +1,15 @@ +#include "inline/inline.h" +#include "zfp.h" +#include "zfp/macros.h" +#include "block2.h" +#include "traitsi.h" +#include "template/template.h" +#include "template/codec.h" +#include "inline/bitstream.c" +#include "template/codec.c" +#include "template/codec2.c" +#include "template/encode.c" +#include "template/encodei.c" +#include "template/encode2.c" +#include "template/revencode.c" +#include "template/revencode2.c" diff --git a/src/blosc2/plugins/codecs/zfp/src/encode2l.c b/src/blosc2/plugins/codecs/zfp/src/encode2l.c new file mode 100644 index 0000000..fdea8a0 --- /dev/null +++ b/src/blosc2/plugins/codecs/zfp/src/encode2l.c @@ -0,0 +1,15 @@ +#include "inline/inline.h" +#include "zfp.h" +#include "zfp/macros.h" +#include "block2.h" +#include "traitsl.h" +#include "template/template.h" +#include "template/codec.h" +#include "inline/bitstream.c" +#include "template/codec.c" +#include "template/codec2.c" +#include "template/encode.c" +#include "template/encodei.c" +#include "template/encode2.c" +#include "template/revencode.c" +#include "template/revencode2.c" diff --git a/src/blosc2/plugins/codecs/zfp/src/encode3d.c b/src/blosc2/plugins/codecs/zfp/src/encode3d.c new file mode 100644 index 0000000..5485421 --- /dev/null +++ b/src/blosc2/plugins/codecs/zfp/src/encode3d.c @@ -0,0 +1,18 @@ +#include "inline/inline.h" +#include "zfp.h" +#include "zfp/macros.h" +#include "block3.h" +#include "traitsd.h" +#include "template/template.h" +#include "template/codec.h" +#include "inline/bitstream.c" +#include "template/codec.c" +#include "template/codecf.c" +#include "template/codec3.c" +#include "template/encode.c" +#include "template/encodef.c" +#include "template/encode3.c" +#include "template/revcodecf.c" +#include "template/revencode.c" +#include "template/revencodef.c" +#include "template/revencode3.c" diff --git a/src/blosc2/plugins/codecs/zfp/src/encode3f.c b/src/blosc2/plugins/codecs/zfp/src/encode3f.c new file mode 100644 index 0000000..2e9db3f --- /dev/null +++ b/src/blosc2/plugins/codecs/zfp/src/encode3f.c @@ -0,0 +1,18 @@ +#include "inline/inline.h" +#include "zfp.h" +#include "zfp/macros.h" +#include "block3.h" +#include "traitsf.h" +#include "template/template.h" +#include "template/codec.h" +#include "inline/bitstream.c" +#include "template/codec.c" +#include "template/codecf.c" +#include "template/codec3.c" +#include "template/encode.c" +#include "template/encodef.c" +#include "template/encode3.c" +#include "template/revcodecf.c" +#include "template/revencode.c" +#include "template/revencodef.c" +#include "template/revencode3.c" diff --git a/src/blosc2/plugins/codecs/zfp/src/encode3i.c b/src/blosc2/plugins/codecs/zfp/src/encode3i.c new file mode 100644 index 0000000..6dcfbe5 --- /dev/null +++ b/src/blosc2/plugins/codecs/zfp/src/encode3i.c @@ -0,0 +1,15 @@ +#include "inline/inline.h" +#include "zfp.h" +#include "zfp/macros.h" +#include "block3.h" +#include "traitsi.h" +#include "template/template.h" +#include "template/codec.h" +#include "inline/bitstream.c" +#include "template/codec.c" +#include "template/codec3.c" +#include "template/encode.c" +#include "template/encodei.c" +#include "template/encode3.c" +#include "template/revencode.c" +#include "template/revencode3.c" diff --git a/src/blosc2/plugins/codecs/zfp/src/encode3l.c b/src/blosc2/plugins/codecs/zfp/src/encode3l.c new file mode 100644 index 0000000..26c9077 --- /dev/null +++ b/src/blosc2/plugins/codecs/zfp/src/encode3l.c @@ -0,0 +1,15 @@ +#include "inline/inline.h" +#include "zfp.h" +#include "zfp/macros.h" +#include "block3.h" +#include "traitsl.h" +#include "template/template.h" +#include "template/codec.h" +#include "inline/bitstream.c" +#include "template/codec.c" +#include "template/codec3.c" +#include "template/encode.c" +#include "template/encodei.c" +#include "template/encode3.c" +#include "template/revencode.c" +#include "template/revencode3.c" diff --git a/src/blosc2/plugins/codecs/zfp/src/encode4d.c b/src/blosc2/plugins/codecs/zfp/src/encode4d.c new file mode 100644 index 0000000..c96aae0 --- /dev/null +++ b/src/blosc2/plugins/codecs/zfp/src/encode4d.c @@ -0,0 +1,18 @@ +#include "inline/inline.h" +#include "zfp.h" +#include "zfp/macros.h" +#include "block4.h" +#include "traitsd.h" +#include "template/template.h" +#include "template/codec.h" +#include "inline/bitstream.c" +#include "template/codec.c" +#include "template/codecf.c" +#include "template/codec4.c" +#include "template/encode.c" +#include "template/encodef.c" +#include "template/encode4.c" +#include "template/revcodecf.c" +#include "template/revencode.c" +#include "template/revencodef.c" +#include "template/revencode4.c" diff --git a/src/blosc2/plugins/codecs/zfp/src/encode4f.c b/src/blosc2/plugins/codecs/zfp/src/encode4f.c new file mode 100644 index 0000000..36cc283 --- /dev/null +++ b/src/blosc2/plugins/codecs/zfp/src/encode4f.c @@ -0,0 +1,18 @@ +#include "inline/inline.h" +#include "zfp.h" +#include "zfp/macros.h" +#include "block4.h" +#include "traitsf.h" +#include "template/template.h" +#include "template/codec.h" +#include "inline/bitstream.c" +#include "template/codec.c" +#include "template/codecf.c" +#include "template/codec4.c" +#include "template/encode.c" +#include "template/encodef.c" +#include "template/encode4.c" +#include "template/revcodecf.c" +#include "template/revencode.c" +#include "template/revencodef.c" +#include "template/revencode4.c" diff --git a/src/blosc2/plugins/codecs/zfp/src/encode4i.c b/src/blosc2/plugins/codecs/zfp/src/encode4i.c new file mode 100644 index 0000000..86fc5ac --- /dev/null +++ b/src/blosc2/plugins/codecs/zfp/src/encode4i.c @@ -0,0 +1,15 @@ +#include "inline/inline.h" +#include "zfp.h" +#include "zfp/macros.h" +#include "block4.h" +#include "traitsi.h" +#include "template/template.h" +#include "template/codec.h" +#include "inline/bitstream.c" +#include "template/codec.c" +#include "template/codec4.c" +#include "template/encode.c" +#include "template/encodei.c" +#include "template/encode4.c" +#include "template/revencode.c" +#include "template/revencode4.c" diff --git a/src/blosc2/plugins/codecs/zfp/src/encode4l.c b/src/blosc2/plugins/codecs/zfp/src/encode4l.c new file mode 100644 index 0000000..e8a3828 --- /dev/null +++ b/src/blosc2/plugins/codecs/zfp/src/encode4l.c @@ -0,0 +1,15 @@ +#include "inline/inline.h" +#include "zfp.h" +#include "zfp/macros.h" +#include "block4.h" +#include "traitsl.h" +#include "template/template.h" +#include "template/codec.h" +#include "inline/bitstream.c" +#include "template/codec.c" +#include "template/codec4.c" +#include "template/encode.c" +#include "template/encodei.c" +#include "template/encode4.c" +#include "template/revencode.c" +#include "template/revencode4.c" diff --git a/src/blosc2/plugins/codecs/zfp/src/inline/bitstream.c b/src/blosc2/plugins/codecs/zfp/src/inline/bitstream.c new file mode 100644 index 0000000..3007bf4 --- /dev/null +++ b/src/blosc2/plugins/codecs/zfp/src/inline/bitstream.c @@ -0,0 +1,464 @@ +/* +High-speed in-memory bit stream I/O that supports reading and writing between +0 and 64 bits at a time. The implementation, which relies heavily on bit +shifts, has been carefully written to ensure that all shifts are between +zero and one less the width of the type being shifted to avoid undefined +behavior. This occasionally causes somewhat convoluted code. + +The following assumptions and restrictions apply: + +1. The user must allocate a memory buffer large enough to hold the bit stream, + whether for reading, writing, or both. This buffer is associated with the + bit stream via stream_open(buffer, bytes), which allocates and returns a + pointer to an opaque bit stream struct. Call stream_close(stream) to + deallocate this struct. + +2. The stream is either in a read or write state (or, initially, in both + states). When done writing, call stream_flush(stream) before entering + read mode to ensure any buffered bits are output. To enter read mode, + call stream_rewind(stream) or stream_rseek(stream, offset) to position + the stream at the beginning or at a particular bit offset. Conversely, + stream_rewind(stream) or stream_wseek(stream, offset) positions the + stream for writing. In read mode, the following functions may be called: + + size_t stream_size(stream); + size_t stream_rtell(stream); + void stream_rewind(stream); + void stream_rseek(stream, offset); + void stream_skip(stream, uint n); + size_t stream_align(stream); + uint stream_read_bit(stream); + uint64 stream_read_bits(stream, n); + + Each of the above read calls has a corresponding write call: + + size_t stream_size(stream); + size_t stream_wtell(stream); + void stream_rewind(stream); + void stream_wseek(stream, offset); + void stream_pad(stream, n); + size_t stream_flush(stream); + uint stream_write_bit(stream, bit); + uint64 stream_write_bits(stream, value, n); + +3. The stream buffer is an unsigned integer of a user-specified type given + by the BIT_STREAM_WORD_TYPE macro. Bits are read and written in units of + this integer word type. Supported types are 8, 16, 32, or 64 bits wide. + The bit width of the buffer is denoted by 'wsize' and can be accessed via + the global constant stream_word_bits. A small wsize allows for fine + granularity reads and writes, and may be preferable when working with many + small blocks of data that require non-sequential access. The default + maximum size of 64 bits ensures maximum speed. Note that even when + wsize < 64, it is still possible to read and write up to 64 bits at a time + using stream_read_bits() and stream_write_bits(). + +4. If BIT_STREAM_STRIDED is defined, words read from or written to the stream + may be accessed noncontiguously by setting a power-of-two block size (which + by default is one word) and a block stride (defaults to zero blocks). The + word pointer is always incremented by one word each time a word is accessed. + Once advanced past a block boundary, the word pointer is also advanced by + the stride to the next block. This feature may be used to store blocks of + data interleaved, e.g. for progressive coding or for noncontiguous parallel + access to the bit stream Note that the block size is measured in words, + while the stride is measured in multiples of the block size. Strided access + can have a significant performance penalty. + +5. Multiple bits are read and written in order of least to most significant + bit. Thus, the statement + + value = stream_write_bits(stream, value, n); + + is essentially equivalent to (but faster than) + + for (i = 0; i < n; i++, value >>= 1) + stream_write_bit(value & 1); + + when 0 <= n <= 64. The same holds for read calls, and thus + + value = stream_read_bits(stream, n); + + is essentially equivalent to + + for (i = 0, value = 0; i < n; i++) + value += (uint64)stream_read_bit() << i; + + Note that it is possible to write fewer bits than the argument 'value' + holds (possibly even no bits), in which case any unwritten bits are + returned. + +6. Although the stream_wseek(stream, offset) call allows positioning the + stream for writing at any bit offset without any data loss (i.e. all + previously written bits preceding the offset remain valid), for efficiency + the stream_flush(stream) operation will zero all bits up to the next + multiple of wsize bits, thus overwriting bits that were previously stored + at that location. Consequently, random write access is effectively + supported only at wsize granularity. For sequential access, the largest + possible wsize is preferred due to higher speed. + +7. It is up to the user to adhere to these rules. For performance reasons, + no error checking is done, and in particular buffer overruns are not + caught. +*/ + +#include +#include + +#ifndef inline_ + #define inline_ +#endif + +/* satisfy compiler when args unused */ +#define unused_(x) ((void)(x)) + +/* bit stream word/buffer type; granularity of stream I/O operations */ +#ifdef BIT_STREAM_WORD_TYPE + /* may be 8-, 16-, 32-, or 64-bit unsigned integer type */ + typedef BIT_STREAM_WORD_TYPE word; +#else + /* use maximum word size by default for highest speed */ + typedef uint64 word; +#endif + +/* number of bits in a buffered word */ +#define wsize ((uint)(CHAR_BIT * sizeof(word))) + +/* bit stream structure (opaque to caller) */ +struct bitstream { + uint bits; /* number of buffered bits (0 <= bits < wsize) */ + word buffer; /* buffer for incoming/outgoing bits (buffer < 2^bits) */ + word* ptr; /* pointer to next word to be read/written */ + word* begin; /* beginning of stream */ + word* end; /* end of stream (currently unused) */ +#ifdef BIT_STREAM_STRIDED + size_t mask; /* one less the block size in number of words */ + ptrdiff_t delta; /* number of words between consecutive blocks */ +#endif +}; + +/* private functions ------------------------------------------------------- */ + +/* read a single word from memory */ +static word +stream_read_word(bitstream* s) +{ + word w = *s->ptr++; +#ifdef BIT_STREAM_STRIDED + if (!((s->ptr - s->begin) & s->mask)) + s->ptr += s->delta; +#endif + return w; +} + +/* write a single word to memory */ +static void +stream_write_word(bitstream* s, word value) +{ + *s->ptr++ = value; +#ifdef BIT_STREAM_STRIDED + if (!((s->ptr - s->begin) & s->mask)) + s->ptr += s->delta; +#endif +} + +/* public functions -------------------------------------------------------- */ + +/* word size in bits (equals stream_word_bits) */ +inline_ size_t +stream_alignment(void) +{ + return wsize; +} + +/* pointer to beginning of stream */ +inline_ void* +stream_data(const bitstream* s) +{ + return s->begin; +} + +/* current byte size of stream (if flushed) */ +inline_ size_t +stream_size(const bitstream* s) +{ + return sizeof(word) * (size_t)(s->ptr - s->begin); +} + +/* byte capacity of stream */ +inline_ size_t +stream_capacity(const bitstream* s) +{ + return sizeof(word) * (size_t)(s->end - s->begin); +} + +/* number of words per block */ +inline_ size_t +stream_stride_block(const bitstream* s) +{ +#ifdef BIT_STREAM_STRIDED + return s->mask + 1; +#else + unused_(s); + return 1; +#endif +} + +/* number of blocks between consecutive stream blocks */ +inline_ ptrdiff_t +stream_stride_delta(const bitstream* s) +{ +#ifdef BIT_STREAM_STRIDED + return s->delta / (s->mask + 1); +#else + unused_(s); + return 0; +#endif +} + +/* read single bit (0 or 1) */ +inline_ uint +stream_read_bit(bitstream* s) +{ + uint bit; + if (!s->bits) { + s->buffer = stream_read_word(s); + s->bits = wsize; + } + s->bits--; + bit = (uint)s->buffer & 1u; + s->buffer >>= 1; + return bit; +} + +/* write single bit (must be 0 or 1) */ +inline_ uint +stream_write_bit(bitstream* s, uint bit) +{ + s->buffer += (word)bit << s->bits; + if (++s->bits == wsize) { + stream_write_word(s, s->buffer); + s->buffer = 0; + s->bits = 0; + } + return bit; +} + +/* read 0 <= n <= 64 bits */ +inline_ uint64 +stream_read_bits(bitstream* s, uint n) +{ + uint64 value = s->buffer; + if (s->bits < n) { + /* keep fetching wsize bits until enough bits are buffered */ + do { + /* assert: 0 <= s->bits < n <= 64 */ + s->buffer = stream_read_word(s); + value += (uint64)s->buffer << s->bits; + s->bits += wsize; + } while (sizeof(s->buffer) < sizeof(value) && s->bits < n); + /* assert: 1 <= n <= s->bits < n + wsize */ + s->bits -= n; + if (!s->bits) { + /* value holds exactly n bits; no need for masking */ + s->buffer = 0; + } + else { + /* assert: 1 <= s->bits < wsize */ + s->buffer >>= wsize - s->bits; + /* assert: 1 <= n <= 64 */ + value &= ((uint64)2 << (n - 1)) - 1; + } + } + else { + /* assert: 0 <= n <= s->bits < wsize <= 64 */ + s->bits -= n; + s->buffer >>= n; + value &= ((uint64)1 << n) - 1; + } + return value; +} + +/* write 0 <= n <= 64 low bits of value and return remaining bits */ +inline_ uint64 +stream_write_bits(bitstream* s, uint64 value, uint n) +{ + /* append bit string to buffer */ + s->buffer += (word)(value << s->bits); + s->bits += n; + /* is buffer full? */ + if (s->bits >= wsize) { + /* 1 <= n <= 64; decrement n to ensure valid right shifts below */ + value >>= 1; + n--; + /* assert: 0 <= n < 64; wsize <= s->bits <= wsize + n */ + do { + /* output wsize bits while buffer is full */ + s->bits -= wsize; + /* assert: 0 <= s->bits <= n */ + stream_write_word(s, s->buffer); + /* assert: 0 <= n - s->bits < 64 */ + s->buffer = (word)(value >> (n - s->bits)); + } while (sizeof(s->buffer) < sizeof(value) && s->bits >= wsize); + } + /* assert: 0 <= s->bits < wsize */ + s->buffer &= ((word)1 << s->bits) - 1; + /* assert: 0 <= n < 64 */ + return value >> n; +} + +/* return bit offset to next bit to be read */ +inline_ size_t +stream_rtell(const bitstream* s) +{ + return wsize * (size_t)(s->ptr - s->begin) - s->bits; +} + +/* return bit offset to next bit to be written */ +inline_ size_t +stream_wtell(const bitstream* s) +{ + return wsize * (size_t)(s->ptr - s->begin) + s->bits; +} + +/* position stream for reading or writing at beginning */ +inline_ void +stream_rewind(bitstream* s) +{ + s->ptr = s->begin; + s->buffer = 0; + s->bits = 0; +} + +/* position stream for reading at given bit offset */ +inline_ void +stream_rseek(bitstream* s, size_t offset) +{ + uint n = offset % wsize; + s->ptr = s->begin + offset / wsize; + if (n) { + s->buffer = stream_read_word(s) >> n; + s->bits = wsize - n; + } + else { + s->buffer = 0; + s->bits = 0; + } +} + +/* position stream for writing at given bit offset */ +inline_ void +stream_wseek(bitstream* s, size_t offset) +{ + uint n = offset % wsize; + s->ptr = s->begin + offset / wsize; + if (n) { + word buffer = *s->ptr; + buffer &= ((word)1 << n) - 1; + s->buffer = buffer; + s->bits = n; + } + else { + s->buffer = 0; + s->bits = 0; + } +} + +/* skip over the next n bits (n >= 0) */ +inline_ void +stream_skip(bitstream* s, uint n) +{ + stream_rseek(s, stream_rtell(s) + n); +} + +/* append n zero-bits to stream (n >= 0) */ +inline_ void +stream_pad(bitstream* s, uint n) +{ + for (s->bits += n; s->bits >= wsize; s->bits -= wsize) { + stream_write_word(s, s->buffer); + s->buffer = 0; + } +} + +/* align stream on next word boundary */ +inline_ size_t +stream_align(bitstream* s) +{ + uint bits = s->bits; + if (bits) + stream_skip(s, bits); + return bits; +} + +/* write any remaining buffered bits and align stream on next word boundary */ +inline_ size_t +stream_flush(bitstream* s) +{ + uint bits = (wsize - s->bits) % wsize; + if (bits) + stream_pad(s, bits); + return bits; +} + +/* copy n bits from one bit stream to another */ +inline_ void +stream_copy(bitstream* dst, bitstream* src, size_t n) +{ + while (n > wsize) { + word w = (word)stream_read_bits(src, wsize); + stream_write_bits(dst, w, wsize); + n -= wsize; + } + if (n) { + word w = (word)stream_read_bits(src, (uint)n); + stream_write_bits(dst, w, (uint)n); + } +} + +#ifdef BIT_STREAM_STRIDED +/* set block size in number of words and spacing in number of blocks */ +inline_ int +stream_set_stride(bitstream* s, size_t block, ptrdiff_t delta) +{ + /* ensure block size is a power of two */ + if (block & (block - 1)) + return 0; + s->mask = block - 1; + s->delta = delta * block; + return 1; +} +#endif + +/* allocate and initialize bit stream to user-allocated buffer */ +inline_ bitstream* +stream_open(void* buffer, size_t bytes) +{ + bitstream* s = (bitstream*)malloc(sizeof(bitstream)); + if (s) { + s->begin = (word*)buffer; + s->end = s->begin + bytes / sizeof(word); +#ifdef BIT_STREAM_STRIDED + stream_set_stride(s, 0, 0); +#endif + stream_rewind(s); + } + return s; +} + +/* close and deallocate bit stream */ +inline_ void +stream_close(bitstream* s) +{ + free(s); +} + +/* make a copy of bit stream to shared memory buffer */ +inline_ bitstream* +stream_clone(const bitstream* s) +{ + bitstream* c = (bitstream*)malloc(sizeof(bitstream)); + if (c) + *c = *s; + return c; +} + +#undef unused_ diff --git a/src/blosc2/plugins/codecs/zfp/src/inline/inline.h b/src/blosc2/plugins/codecs/zfp/src/inline/inline.h new file mode 100644 index 0000000..e9ade3f --- /dev/null +++ b/src/blosc2/plugins/codecs/zfp/src/inline/inline.h @@ -0,0 +1,12 @@ +#ifndef INLINE_H +#define INLINE_H + +#ifndef inline_ + #if __STDC_VERSION__ >= 199901L + #define inline_ static inline + #else + #define inline_ static + #endif +#endif + +#endif diff --git a/src/blosc2/plugins/codecs/zfp/src/share/omp.c b/src/blosc2/plugins/codecs/zfp/src/share/omp.c new file mode 100644 index 0000000..e0300ca --- /dev/null +++ b/src/blosc2/plugins/codecs/zfp/src/share/omp.c @@ -0,0 +1,30 @@ +#ifdef _OPENMP +#include +#include + +/* number of omp threads to use */ +static uint +thread_count_omp(const zfp_stream* stream) +{ + uint count = stream->exec.params.omp.threads; + /* if no thread count is specified, use default number of threads */ + if (!count) + count = omp_get_max_threads(); + return count; +} + +/* number of chunks to partition array into */ +static size_t +chunk_count_omp(const zfp_stream* stream, size_t blocks, uint threads) +{ + size_t chunk_size = stream->exec.params.omp.chunk_size; + /* if no chunk size is specified, assign one chunk per thread */ + size_t chunks = chunk_size ? (blocks + chunk_size - 1) / chunk_size : threads; + /* each chunk must contain at least one block */ + chunks = MIN(chunks, blocks); + /* OpenMP 2.0 loop counters must be ints */ + chunks = MIN(chunks, INT_MAX); + return chunks; +} + +#endif diff --git a/src/blosc2/plugins/codecs/zfp/src/share/parallel.c b/src/blosc2/plugins/codecs/zfp/src/share/parallel.c new file mode 100644 index 0000000..0cbdc02 --- /dev/null +++ b/src/blosc2/plugins/codecs/zfp/src/share/parallel.c @@ -0,0 +1,104 @@ +#ifdef _OPENMP + +/* block index at which chunk begins */ +static size_t +chunk_offset(size_t blocks, size_t chunks, size_t chunk) +{ + return (size_t)(((uint64)blocks * (uint64)chunk) / chunks); +} + +/* initialize per-thread bit streams for parallel compression */ +static bitstream** +compress_init_par(zfp_stream* stream, const zfp_field* field, size_t chunks, size_t blocks) +{ + bitstream** bs; + zfp_bool copy; + size_t n = 4 * (blocks + chunks - 1) / chunks; + size_t size; + size_t chunk; + + /* determine maximum size buffer needed per thread */ + zfp_field f = *field; + switch (zfp_field_dimensionality(field)) { + case 1: + f.nx = n; + break; + case 2: + f.nx = 4; + f.ny = n; + break; + case 3: + f.nx = 4; + f.ny = 4; + f.nz = n; + break; + case 4: + f.nx = 4; + f.ny = 4; + f.nz = 4; + f.nw = n; + break; + default: + return NULL; + } + size = zfp_stream_maximum_size(stream, &f); + + /* avoid copies in fixed-rate mode when each bitstream is word aligned */ + copy = (stream->minbits != stream->maxbits) || + (stream->maxbits % stream_word_bits != 0) || + (stream_wtell(stream->stream) % stream_word_bits != 0); + + /* set up buffer for each thread to compress to */ + bs = (bitstream**)malloc(chunks * sizeof(bitstream*)); + if (!bs) + return NULL; + for (chunk = 0; chunk < chunks; chunk++) { + size_t block = chunk_offset(blocks, chunks, chunk); + void* buffer = copy ? malloc(size) : (uchar*)stream_data(stream->stream) + stream_size(stream->stream) + block * (stream->maxbits / CHAR_BIT); + if (!buffer) + break; + bs[chunk] = stream_open(buffer, size); + } + + /* handle memory allocation failure */ + if (copy && chunk < chunks) { + while (chunk--) { + free(stream_data(bs[chunk])); + stream_close(bs[chunk]); + } + free(bs); + bs = NULL; + } + + return bs; +} + +/* flush and concatenate bit streams if needed */ +static void +compress_finish_par(zfp_stream* stream, bitstream** src, size_t chunks) +{ + bitstream* dst = zfp_stream_bit_stream(stream); + zfp_bool copy = (stream_data(dst) != stream_data(*src)); + size_t offset = stream_wtell(dst); + size_t chunk; + + /* flush each stream and concatenate if necessary */ + for (chunk = 0; chunk < chunks; chunk++) { + size_t bits = stream_wtell(src[chunk]); + offset += bits; + stream_flush(src[chunk]); + /* concatenate streams if they are not already contiguous */ + if (copy) { + stream_rewind(src[chunk]); + stream_copy(dst, src[chunk], bits); + free(stream_data(src[chunk])); + } + stream_close(src[chunk]); + } + + free(src); + if (!copy) + stream_wseek(dst, offset); +} + +#endif diff --git a/src/blosc2/plugins/codecs/zfp/src/template/codec.c b/src/blosc2/plugins/codecs/zfp/src/template/codec.c new file mode 100644 index 0000000..539bca9 --- /dev/null +++ b/src/blosc2/plugins/codecs/zfp/src/template/codec.c @@ -0,0 +1,6 @@ +/* true if max compressed size exceeds maxbits */ +static int +with_maxbits(uint maxbits, uint maxprec, uint size) +{ + return (maxprec + 1) * size - 1 > maxbits; +} diff --git a/src/blosc2/plugins/codecs/zfp/src/template/codec.h b/src/blosc2/plugins/codecs/zfp/src/template/codec.h new file mode 100644 index 0000000..e4b4acf --- /dev/null +++ b/src/blosc2/plugins/codecs/zfp/src/template/codec.h @@ -0,0 +1,4 @@ +#define PERM _t1(perm, DIMS) /* coefficient order */ +#define BLOCK_SIZE (1 << (2 * DIMS)) /* values per block */ +#define EBIAS ((1 << (EBITS - 1)) - 1) /* exponent bias */ +#define REVERSIBLE(zfp) ((zfp)->minexp < ZFP_MIN_EXP) /* reversible mode? */ diff --git a/src/blosc2/plugins/codecs/zfp/src/template/codec1.c b/src/blosc2/plugins/codecs/zfp/src/template/codec1.c new file mode 100644 index 0000000..5a47864 --- /dev/null +++ b/src/blosc2/plugins/codecs/zfp/src/template/codec1.c @@ -0,0 +1,4 @@ +/* order coefficients by polynomial degree/frequency */ +cache_align_(static const uchar perm_1[4]) = { + 0, 1, 2, 3 +}; diff --git a/src/blosc2/plugins/codecs/zfp/src/template/codec2.c b/src/blosc2/plugins/codecs/zfp/src/template/codec2.c new file mode 100644 index 0000000..a0f9770 --- /dev/null +++ b/src/blosc2/plugins/codecs/zfp/src/template/codec2.c @@ -0,0 +1,32 @@ +#define index(i, j) ((i) + 4 * (j)) + +/* order coefficients (i, j) by i + j, then i^2 + j^2 */ +cache_align_(static const uchar perm_2[16]) = { + index(0, 0), /* 0 : 0 */ + + index(1, 0), /* 1 : 1 */ + index(0, 1), /* 2 : 1 */ + + index(1, 1), /* 3 : 2 */ + + index(2, 0), /* 4 : 2 */ + index(0, 2), /* 5 : 2 */ + + index(2, 1), /* 6 : 3 */ + index(1, 2), /* 7 : 3 */ + + index(3, 0), /* 8 : 3 */ + index(0, 3), /* 9 : 3 */ + + index(2, 2), /* 10 : 4 */ + + index(3, 1), /* 11 : 4 */ + index(1, 3), /* 12 : 4 */ + + index(3, 2), /* 13 : 5 */ + index(2, 3), /* 14 : 5 */ + + index(3, 3), /* 15 : 6 */ +}; + +#undef index diff --git a/src/blosc2/plugins/codecs/zfp/src/template/codec3.c b/src/blosc2/plugins/codecs/zfp/src/template/codec3.c new file mode 100644 index 0000000..b95f302 --- /dev/null +++ b/src/blosc2/plugins/codecs/zfp/src/template/codec3.c @@ -0,0 +1,90 @@ +#define index(i, j, k) ((i) + 4 * ((j) + 4 * (k))) + +/* order coefficients (i, j, k) by i + j + k, then i^2 + j^2 + k^2 */ +cache_align_(static const uchar perm_3[64]) = { + index(0, 0, 0), /* 0 : 0 */ + + index(1, 0, 0), /* 1 : 1 */ + index(0, 1, 0), /* 2 : 1 */ + index(0, 0, 1), /* 3 : 1 */ + + index(0, 1, 1), /* 4 : 2 */ + index(1, 0, 1), /* 5 : 2 */ + index(1, 1, 0), /* 6 : 2 */ + + index(2, 0, 0), /* 7 : 2 */ + index(0, 2, 0), /* 8 : 2 */ + index(0, 0, 2), /* 9 : 2 */ + + index(1, 1, 1), /* 10 : 3 */ + + index(2, 1, 0), /* 11 : 3 */ + index(2, 0, 1), /* 12 : 3 */ + index(0, 2, 1), /* 13 : 3 */ + index(1, 2, 0), /* 14 : 3 */ + index(1, 0, 2), /* 15 : 3 */ + index(0, 1, 2), /* 16 : 3 */ + + index(3, 0, 0), /* 17 : 3 */ + index(0, 3, 0), /* 18 : 3 */ + index(0, 0, 3), /* 19 : 3 */ + + index(2, 1, 1), /* 20 : 4 */ + index(1, 2, 1), /* 21 : 4 */ + index(1, 1, 2), /* 22 : 4 */ + + index(0, 2, 2), /* 23 : 4 */ + index(2, 0, 2), /* 24 : 4 */ + index(2, 2, 0), /* 25 : 4 */ + + index(3, 1, 0), /* 26 : 4 */ + index(3, 0, 1), /* 27 : 4 */ + index(0, 3, 1), /* 28 : 4 */ + index(1, 3, 0), /* 29 : 4 */ + index(1, 0, 3), /* 30 : 4 */ + index(0, 1, 3), /* 31 : 4 */ + + index(1, 2, 2), /* 32 : 5 */ + index(2, 1, 2), /* 33 : 5 */ + index(2, 2, 1), /* 34 : 5 */ + + index(3, 1, 1), /* 35 : 5 */ + index(1, 3, 1), /* 36 : 5 */ + index(1, 1, 3), /* 37 : 5 */ + + index(3, 2, 0), /* 38 : 5 */ + index(3, 0, 2), /* 39 : 5 */ + index(0, 3, 2), /* 40 : 5 */ + index(2, 3, 0), /* 41 : 5 */ + index(2, 0, 3), /* 42 : 5 */ + index(0, 2, 3), /* 43 : 5 */ + + index(2, 2, 2), /* 44 : 6 */ + + index(3, 2, 1), /* 45 : 6 */ + index(3, 1, 2), /* 46 : 6 */ + index(1, 3, 2), /* 47 : 6 */ + index(2, 3, 1), /* 48 : 6 */ + index(2, 1, 3), /* 49 : 6 */ + index(1, 2, 3), /* 50 : 6 */ + + index(0, 3, 3), /* 51 : 6 */ + index(3, 0, 3), /* 52 : 6 */ + index(3, 3, 0), /* 53 : 6 */ + + index(3, 2, 2), /* 54 : 7 */ + index(2, 3, 2), /* 55 : 7 */ + index(2, 2, 3), /* 56 : 7 */ + + index(1, 3, 3), /* 57 : 7 */ + index(3, 1, 3), /* 58 : 7 */ + index(3, 3, 1), /* 59 : 7 */ + + index(2, 3, 3), /* 60 : 8 */ + index(3, 2, 3), /* 61 : 8 */ + index(3, 3, 2), /* 62 : 8 */ + + index(3, 3, 3), /* 63 : 9 */ +}; + +#undef index diff --git a/src/blosc2/plugins/codecs/zfp/src/template/codec4.c b/src/blosc2/plugins/codecs/zfp/src/template/codec4.c new file mode 100644 index 0000000..b831452 --- /dev/null +++ b/src/blosc2/plugins/codecs/zfp/src/template/codec4.c @@ -0,0 +1,297 @@ +#define index(i, j, k, l) ((i) + 4 * ((j) + 4 * ((k) + 4 * (l)))) + +/* order coefficients (i, j, k, l) by i + j + k + l, then i^2 + j^2 + k^2 + l^2 */ +cache_align_(static const uchar perm_4[256]) = { + index(0, 0, 0, 0), /* 0 : 0 */ + + index(1, 0, 0, 0), /* 1 : 1 */ + index(0, 1, 0, 0), /* 2 : 1 */ + index(0, 0, 1, 0), /* 3 : 1 */ + index(0, 0, 0, 1), /* 4 : 1 */ + + index(1, 1, 0, 0), /* 5 : 2 */ + index(0, 0, 1, 1), /* 6 : 2 */ + index(1, 0, 1, 0), /* 7 : 2 */ + index(0, 1, 0, 1), /* 8 : 2 */ + index(1, 0, 0, 1), /* 9 : 2 */ + index(0, 1, 1, 0), /* 10 : 2 */ + + index(2, 0, 0, 0), /* 11 : 2 */ + index(0, 2, 0, 0), /* 12 : 2 */ + index(0, 0, 2, 0), /* 13 : 2 */ + index(0, 0, 0, 2), /* 14 : 2 */ + + index(0, 1, 1, 1), /* 15 : 3 */ + index(1, 0, 1, 1), /* 16 : 3 */ + index(1, 1, 0, 1), /* 17 : 3 */ + index(1, 1, 1, 0), /* 18 : 3 */ + + index(2, 1, 0, 0), /* 19 : 3 */ + index(2, 0, 1, 0), /* 20 : 3 */ + index(2, 0, 0, 1), /* 21 : 3 */ + index(0, 2, 1, 0), /* 22 : 3 */ + index(0, 2, 0, 1), /* 23 : 3 */ + index(1, 2, 0, 0), /* 24 : 3 */ + index(0, 0, 2, 1), /* 25 : 3 */ + index(1, 0, 2, 0), /* 26 : 3 */ + index(0, 1, 2, 0), /* 27 : 3 */ + index(1, 0, 0, 2), /* 28 : 3 */ + index(0, 1, 0, 2), /* 29 : 3 */ + index(0, 0, 1, 2), /* 30 : 3 */ + + index(3, 0, 0, 0), /* 31 : 3 */ + index(0, 3, 0, 0), /* 32 : 3 */ + index(0, 0, 3, 0), /* 33 : 3 */ + index(0, 0, 0, 3), /* 34 : 3 */ + + index(1, 1, 1, 1), /* 35 : 4 */ + + index(2, 0, 1, 1), /* 36 : 4 */ + index(2, 1, 0, 1), /* 37 : 4 */ + index(2, 1, 1, 0), /* 38 : 4 */ + index(1, 2, 0, 1), /* 39 : 4 */ + index(1, 2, 1, 0), /* 40 : 4 */ + index(0, 2, 1, 1), /* 41 : 4 */ + index(1, 1, 2, 0), /* 42 : 4 */ + index(0, 1, 2, 1), /* 43 : 4 */ + index(1, 0, 2, 1), /* 44 : 4 */ + index(0, 1, 1, 2), /* 45 : 4 */ + index(1, 0, 1, 2), /* 46 : 4 */ + index(1, 1, 0, 2), /* 47 : 4 */ + + index(2, 2, 0, 0), /* 48 : 4 */ + index(0, 0, 2, 2), /* 49 : 4 */ + index(2, 0, 2, 0), /* 50 : 4 */ + index(0, 2, 0, 2), /* 51 : 4 */ + index(2, 0, 0, 2), /* 52 : 4 */ + index(0, 2, 2, 0), /* 53 : 4 */ + + index(3, 1, 0, 0), /* 54 : 4 */ + index(3, 0, 1, 0), /* 55 : 4 */ + index(3, 0, 0, 1), /* 56 : 4 */ + index(0, 3, 1, 0), /* 57 : 4 */ + index(0, 3, 0, 1), /* 58 : 4 */ + index(1, 3, 0, 0), /* 59 : 4 */ + index(0, 0, 3, 1), /* 60 : 4 */ + index(1, 0, 3, 0), /* 61 : 4 */ + index(0, 1, 3, 0), /* 62 : 4 */ + index(1, 0, 0, 3), /* 63 : 4 */ + index(0, 1, 0, 3), /* 64 : 4 */ + index(0, 0, 1, 3), /* 65 : 4 */ + + index(2, 1, 1, 1), /* 66 : 5 */ + index(1, 2, 1, 1), /* 67 : 5 */ + index(1, 1, 2, 1), /* 68 : 5 */ + index(1, 1, 1, 2), /* 69 : 5 */ + + index(1, 0, 2, 2), /* 70 : 5 */ + index(1, 2, 0, 2), /* 71 : 5 */ + index(1, 2, 2, 0), /* 72 : 5 */ + index(2, 1, 0, 2), /* 73 : 5 */ + index(2, 1, 2, 0), /* 74 : 5 */ + index(0, 1, 2, 2), /* 75 : 5 */ + index(2, 2, 1, 0), /* 76 : 5 */ + index(0, 2, 1, 2), /* 77 : 5 */ + index(2, 0, 1, 2), /* 78 : 5 */ + index(0, 2, 2, 1), /* 79 : 5 */ + index(2, 0, 2, 1), /* 80 : 5 */ + index(2, 2, 0, 1), /* 81 : 5 */ + + index(3, 0, 1, 1), /* 82 : 5 */ + index(3, 1, 0, 1), /* 83 : 5 */ + index(3, 1, 1, 0), /* 84 : 5 */ + index(1, 3, 0, 1), /* 85 : 5 */ + index(1, 3, 1, 0), /* 86 : 5 */ + index(0, 3, 1, 1), /* 87 : 5 */ + index(1, 1, 3, 0), /* 88 : 5 */ + index(0, 1, 3, 1), /* 89 : 5 */ + index(1, 0, 3, 1), /* 90 : 5 */ + index(0, 1, 1, 3), /* 91 : 5 */ + index(1, 0, 1, 3), /* 92 : 5 */ + index(1, 1, 0, 3), /* 93 : 5 */ + + index(3, 2, 0, 0), /* 94 : 5 */ + index(3, 0, 2, 0), /* 95 : 5 */ + index(3, 0, 0, 2), /* 96 : 5 */ + index(0, 3, 2, 0), /* 97 : 5 */ + index(0, 3, 0, 2), /* 98 : 5 */ + index(2, 3, 0, 0), /* 99 : 5 */ + index(0, 0, 3, 2), /* 100 : 5 */ + index(2, 0, 3, 0), /* 101 : 5 */ + index(0, 2, 3, 0), /* 102 : 5 */ + index(2, 0, 0, 3), /* 103 : 5 */ + index(0, 2, 0, 3), /* 104 : 5 */ + index(0, 0, 2, 3), /* 105 : 5 */ + + index(2, 2, 1, 1), /* 106 : 6 */ + index(1, 1, 2, 2), /* 107 : 6 */ + index(2, 1, 2, 1), /* 108 : 6 */ + index(1, 2, 1, 2), /* 109 : 6 */ + index(2, 1, 1, 2), /* 110 : 6 */ + index(1, 2, 2, 1), /* 111 : 6 */ + + index(0, 2, 2, 2), /* 112 : 6 */ + index(2, 0, 2, 2), /* 113 : 6 */ + index(2, 2, 0, 2), /* 114 : 6 */ + index(2, 2, 2, 0), /* 115 : 6 */ + + index(3, 1, 1, 1), /* 116 : 6 */ + index(1, 3, 1, 1), /* 117 : 6 */ + index(1, 1, 3, 1), /* 118 : 6 */ + index(1, 1, 1, 3), /* 119 : 6 */ + + index(3, 2, 1, 0), /* 120 : 6 */ + index(3, 2, 0, 1), /* 121 : 6 */ + index(3, 0, 2, 1), /* 122 : 6 */ + index(3, 1, 2, 0), /* 123 : 6 */ + index(3, 1, 0, 2), /* 124 : 6 */ + index(3, 0, 1, 2), /* 125 : 6 */ + index(0, 3, 2, 1), /* 126 : 6 */ + index(1, 3, 2, 0), /* 127 : 6 */ + index(1, 3, 0, 2), /* 128 : 6 */ + index(0, 3, 1, 2), /* 129 : 6 */ + index(2, 3, 1, 0), /* 130 : 6 */ + index(2, 3, 0, 1), /* 131 : 6 */ + index(1, 0, 3, 2), /* 132 : 6 */ + index(0, 1, 3, 2), /* 133 : 6 */ + index(2, 1, 3, 0), /* 134 : 6 */ + index(2, 0, 3, 1), /* 135 : 6 */ + index(0, 2, 3, 1), /* 136 : 6 */ + index(1, 2, 3, 0), /* 137 : 6 */ + index(2, 1, 0, 3), /* 138 : 6 */ + index(2, 0, 1, 3), /* 139 : 6 */ + index(0, 2, 1, 3), /* 140 : 6 */ + index(1, 2, 0, 3), /* 141 : 6 */ + index(1, 0, 2, 3), /* 142 : 6 */ + index(0, 1, 2, 3), /* 143 : 6 */ + + index(3, 3, 0, 0), /* 144 : 6 */ + index(0, 0, 3, 3), /* 145 : 6 */ + index(3, 0, 3, 0), /* 146 : 6 */ + index(0, 3, 0, 3), /* 147 : 6 */ + index(3, 0, 0, 3), /* 148 : 6 */ + index(0, 3, 3, 0), /* 149 : 6 */ + + index(1, 2, 2, 2), /* 150 : 7 */ + index(2, 1, 2, 2), /* 151 : 7 */ + index(2, 2, 1, 2), /* 152 : 7 */ + index(2, 2, 2, 1), /* 153 : 7 */ + + index(3, 2, 1, 1), /* 154 : 7 */ + index(3, 1, 2, 1), /* 155 : 7 */ + index(3, 1, 1, 2), /* 156 : 7 */ + index(1, 3, 2, 1), /* 157 : 7 */ + index(1, 3, 1, 2), /* 158 : 7 */ + index(2, 3, 1, 1), /* 159 : 7 */ + index(1, 1, 3, 2), /* 160 : 7 */ + index(2, 1, 3, 1), /* 161 : 7 */ + index(1, 2, 3, 1), /* 162 : 7 */ + index(2, 1, 1, 3), /* 163 : 7 */ + index(1, 2, 1, 3), /* 164 : 7 */ + index(1, 1, 2, 3), /* 165 : 7 */ + + index(3, 0, 2, 2), /* 166 : 7 */ + index(3, 2, 0, 2), /* 167 : 7 */ + index(3, 2, 2, 0), /* 168 : 7 */ + index(2, 3, 0, 2), /* 169 : 7 */ + index(2, 3, 2, 0), /* 170 : 7 */ + index(0, 3, 2, 2), /* 171 : 7 */ + index(2, 2, 3, 0), /* 172 : 7 */ + index(0, 2, 3, 2), /* 173 : 7 */ + index(2, 0, 3, 2), /* 174 : 7 */ + index(0, 2, 2, 3), /* 175 : 7 */ + index(2, 0, 2, 3), /* 176 : 7 */ + index(2, 2, 0, 3), /* 177 : 7 */ + + index(1, 0, 3, 3), /* 178 : 7 */ + index(1, 3, 0, 3), /* 179 : 7 */ + index(1, 3, 3, 0), /* 180 : 7 */ + index(3, 1, 0, 3), /* 181 : 7 */ + index(3, 1, 3, 0), /* 182 : 7 */ + index(0, 1, 3, 3), /* 183 : 7 */ + index(3, 3, 1, 0), /* 184 : 7 */ + index(0, 3, 1, 3), /* 185 : 7 */ + index(3, 0, 1, 3), /* 186 : 7 */ + index(0, 3, 3, 1), /* 187 : 7 */ + index(3, 0, 3, 1), /* 188 : 7 */ + index(3, 3, 0, 1), /* 189 : 7 */ + + index(2, 2, 2, 2), /* 190 : 8 */ + + index(3, 1, 2, 2), /* 191 : 8 */ + index(3, 2, 1, 2), /* 192 : 8 */ + index(3, 2, 2, 1), /* 193 : 8 */ + index(2, 3, 1, 2), /* 194 : 8 */ + index(2, 3, 2, 1), /* 195 : 8 */ + index(1, 3, 2, 2), /* 196 : 8 */ + index(2, 2, 3, 1), /* 197 : 8 */ + index(1, 2, 3, 2), /* 198 : 8 */ + index(2, 1, 3, 2), /* 199 : 8 */ + index(1, 2, 2, 3), /* 200 : 8 */ + index(2, 1, 2, 3), /* 201 : 8 */ + index(2, 2, 1, 3), /* 202 : 8 */ + + index(3, 3, 1, 1), /* 203 : 8 */ + index(1, 1, 3, 3), /* 204 : 8 */ + index(3, 1, 3, 1), /* 205 : 8 */ + index(1, 3, 1, 3), /* 206 : 8 */ + index(3, 1, 1, 3), /* 207 : 8 */ + index(1, 3, 3, 1), /* 208 : 8 */ + + index(2, 0, 3, 3), /* 209 : 8 */ + index(2, 3, 0, 3), /* 210 : 8 */ + index(2, 3, 3, 0), /* 211 : 8 */ + index(3, 2, 0, 3), /* 212 : 8 */ + index(3, 2, 3, 0), /* 213 : 8 */ + index(0, 2, 3, 3), /* 214 : 8 */ + index(3, 3, 2, 0), /* 215 : 8 */ + index(0, 3, 2, 3), /* 216 : 8 */ + index(3, 0, 2, 3), /* 217 : 8 */ + index(0, 3, 3, 2), /* 218 : 8 */ + index(3, 0, 3, 2), /* 219 : 8 */ + index(3, 3, 0, 2), /* 220 : 8 */ + + index(3, 2, 2, 2), /* 221 : 9 */ + index(2, 3, 2, 2), /* 222 : 9 */ + index(2, 2, 3, 2), /* 223 : 9 */ + index(2, 2, 2, 3), /* 224 : 9 */ + + index(2, 1, 3, 3), /* 225 : 9 */ + index(2, 3, 1, 3), /* 226 : 9 */ + index(2, 3, 3, 1), /* 227 : 9 */ + index(3, 2, 1, 3), /* 228 : 9 */ + index(3, 2, 3, 1), /* 229 : 9 */ + index(1, 2, 3, 3), /* 230 : 9 */ + index(3, 3, 2, 1), /* 231 : 9 */ + index(1, 3, 2, 3), /* 232 : 9 */ + index(3, 1, 2, 3), /* 233 : 9 */ + index(1, 3, 3, 2), /* 234 : 9 */ + index(3, 1, 3, 2), /* 235 : 9 */ + index(3, 3, 1, 2), /* 236 : 9 */ + + index(0, 3, 3, 3), /* 237 : 9 */ + index(3, 0, 3, 3), /* 238 : 9 */ + index(3, 3, 0, 3), /* 239 : 9 */ + index(3, 3, 3, 0), /* 240 : 9 */ + + index(3, 3, 2, 2), /* 241 : 10 */ + index(2, 2, 3, 3), /* 242 : 10 */ + index(3, 2, 3, 2), /* 243 : 10 */ + index(2, 3, 2, 3), /* 244 : 10 */ + index(3, 2, 2, 3), /* 245 : 10 */ + index(2, 3, 3, 2), /* 246 : 10 */ + + index(1, 3, 3, 3), /* 247 : 10 */ + index(3, 1, 3, 3), /* 248 : 10 */ + index(3, 3, 1, 3), /* 249 : 10 */ + index(3, 3, 3, 1), /* 250 : 10 */ + + index(2, 3, 3, 3), /* 251 : 11 */ + index(3, 2, 3, 3), /* 252 : 11 */ + index(3, 3, 2, 3), /* 253 : 11 */ + index(3, 3, 3, 2), /* 254 : 11 */ + + index(3, 3, 3, 3), /* 255 : 12 */ +}; + +#undef index diff --git a/src/blosc2/plugins/codecs/zfp/src/template/codecf.c b/src/blosc2/plugins/codecs/zfp/src/template/codecf.c new file mode 100644 index 0000000..50929fa --- /dev/null +++ b/src/blosc2/plugins/codecs/zfp/src/template/codecf.c @@ -0,0 +1,32 @@ +#include +#include + +/* maximum number of bit planes to encode */ +static uint +precision(int maxexp, uint maxprec, int minexp, int dims) +{ +#if (ZFP_ROUNDING_MODE != ZFP_ROUND_NEVER) && defined(ZFP_WITH_TIGHT_ERROR) + return MIN(maxprec, (uint)MAX(0, maxexp - minexp + 2 * dims + 1)); +#else + return MIN(maxprec, (uint)MAX(0, maxexp - minexp + 2 * dims + 2)); +#endif +} + +/* map integer x relative to exponent e to floating-point number */ +static Scalar +_t1(dequantize, Scalar)(Int x, int e) +{ + return LDEXP((Scalar)x, e - ((int)(CHAR_BIT * sizeof(Scalar)) - 2)); +} + +/* inverse block-floating-point transform from signed integers */ +static void +_t1(inv_cast, Scalar)(const Int* iblock, Scalar* fblock, uint n, int emax) +{ + /* compute power-of-two scale factor s */ + Scalar s = _t1(dequantize, Scalar)(1, emax); + /* compute p-bit float x = s*y where |y| <= 2^(p-2) - 1 */ + do + *fblock++ = (Scalar)(s * *iblock++); + while (--n); +} diff --git a/src/blosc2/plugins/codecs/zfp/src/template/compress.c b/src/blosc2/plugins/codecs/zfp/src/template/compress.c new file mode 100644 index 0000000..74983c5 --- /dev/null +++ b/src/blosc2/plugins/codecs/zfp/src/template/compress.c @@ -0,0 +1,109 @@ +/* compress 1d contiguous array */ +static void +_t2(compress, Scalar, 1)(zfp_stream* stream, const zfp_field* field) +{ + const Scalar* data = (const Scalar*)field->data; + size_t nx = field->nx; + size_t mx = nx & ~3u; + size_t x; + + /* compress array one block of 4 values at a time */ + for (x = 0; x < mx; x += 4, data += 4) + _t2(zfp_encode_block, Scalar, 1)(stream, data); + if (x < nx) + _t2(zfp_encode_partial_block_strided, Scalar, 1)(stream, data, nx - x, 1); +} + +/* compress 1d strided array */ +static void +_t2(compress_strided, Scalar, 1)(zfp_stream* stream, const zfp_field* field) +{ + const Scalar* data = field->data; + size_t nx = field->nx; + ptrdiff_t sx = field->sx ? field->sx : 1; + size_t x; + + /* compress array one block of 4 values at a time */ + for (x = 0; x < nx; x += 4) { + const Scalar* p = data + sx * (ptrdiff_t)x; + if (nx - x < 4) + _t2(zfp_encode_partial_block_strided, Scalar, 1)(stream, p, nx - x, sx); + else + _t2(zfp_encode_block_strided, Scalar, 1)(stream, p, sx); + } +} + +/* compress 2d strided array */ +static void +_t2(compress_strided, Scalar, 2)(zfp_stream* stream, const zfp_field* field) +{ + const Scalar* data = (const Scalar*)field->data; + size_t nx = field->nx; + size_t ny = field->ny; + ptrdiff_t sx = field->sx ? field->sx : 1; + ptrdiff_t sy = field->sy ? field->sy : (ptrdiff_t)nx; + size_t x, y; + + /* compress array one block of 4x4 values at a time */ + for (y = 0; y < ny; y += 4) + for (x = 0; x < nx; x += 4) { + const Scalar* p = data + sx * (ptrdiff_t)x + sy * (ptrdiff_t)y; + if (nx - x < 4 || ny - y < 4) + _t2(zfp_encode_partial_block_strided, Scalar, 2)(stream, p, MIN(nx - x, 4u), MIN(ny - y, 4u), sx, sy); + else + _t2(zfp_encode_block_strided, Scalar, 2)(stream, p, sx, sy); + } +} + +/* compress 3d strided array */ +static void +_t2(compress_strided, Scalar, 3)(zfp_stream* stream, const zfp_field* field) +{ + const Scalar* data = (const Scalar*)field->data; + size_t nx = field->nx; + size_t ny = field->ny; + size_t nz = field->nz; + ptrdiff_t sx = field->sx ? field->sx : 1; + ptrdiff_t sy = field->sy ? field->sy : (ptrdiff_t)nx; + ptrdiff_t sz = field->sz ? field->sz : (ptrdiff_t)(nx * ny); + size_t x, y, z; + + /* compress array one block of 4x4x4 values at a time */ + for (z = 0; z < nz; z += 4) + for (y = 0; y < ny; y += 4) + for (x = 0; x < nx; x += 4) { + const Scalar* p = data + sx * (ptrdiff_t)x + sy * (ptrdiff_t)y + sz * (ptrdiff_t)z; + if (nx - x < 4 || ny - y < 4 || nz - z < 4) + _t2(zfp_encode_partial_block_strided, Scalar, 3)(stream, p, MIN(nx - x, 4u), MIN(ny - y, 4u), MIN(nz - z, 4u), sx, sy, sz); + else + _t2(zfp_encode_block_strided, Scalar, 3)(stream, p, sx, sy, sz); + } +} + +/* compress 4d strided array */ +static void +_t2(compress_strided, Scalar, 4)(zfp_stream* stream, const zfp_field* field) +{ + const Scalar* data = field->data; + size_t nx = field->nx; + size_t ny = field->ny; + size_t nz = field->nz; + size_t nw = field->nw; + ptrdiff_t sx = field->sx ? field->sx : 1; + ptrdiff_t sy = field->sy ? field->sy : (ptrdiff_t)nx; + ptrdiff_t sz = field->sz ? field->sz : (ptrdiff_t)(nx * ny); + ptrdiff_t sw = field->sw ? field->sw : (ptrdiff_t)(nx * ny * nz); + size_t x, y, z, w; + + /* compress array one block of 4x4x4x4 values at a time */ + for (w = 0; w < nw; w += 4) + for (z = 0; z < nz; z += 4) + for (y = 0; y < ny; y += 4) + for (x = 0; x < nx; x += 4) { + const Scalar* p = data + sx * (ptrdiff_t)x + sy * (ptrdiff_t)y + sz * (ptrdiff_t)z + sw * (ptrdiff_t)w; + if (nx - x < 4 || ny - y < 4 || nz - z < 4 || nw - w < 4) + _t2(zfp_encode_partial_block_strided, Scalar, 4)(stream, p, MIN(nx - x, 4u), MIN(ny - y, 4u), MIN(nz - z, 4u), MIN(nw - w, 4u), sx, sy, sz, sw); + else + _t2(zfp_encode_block_strided, Scalar, 4)(stream, p, sx, sy, sz, sw); + } +} diff --git a/src/blosc2/plugins/codecs/zfp/src/template/decode.c b/src/blosc2/plugins/codecs/zfp/src/template/decode.c new file mode 100644 index 0000000..990f838 --- /dev/null +++ b/src/blosc2/plugins/codecs/zfp/src/template/decode.c @@ -0,0 +1,287 @@ +#include + +static void _t2(inv_xform, Int, DIMS)(Int* p); + +/* private functions ------------------------------------------------------- */ + +/* inverse lifting transform of 4-vector */ +static void +_t1(inv_lift, Int)(Int* p, ptrdiff_t s) +{ + Int x, y, z, w; + x = *p; p += s; + y = *p; p += s; + z = *p; p += s; + w = *p; p += s; + + /* + ** non-orthogonal transform + ** ( 4 6 -4 -1) (x) + ** 1/4 * ( 4 2 4 5) (y) + ** ( 4 -2 4 -5) (z) + ** ( 4 -6 -4 1) (w) + */ + y += w >> 1; w -= y >> 1; + y += w; w <<= 1; w -= y; + z += x; x <<= 1; x -= z; + y += z; z <<= 1; z -= y; + w += x; x <<= 1; x -= w; + + p -= s; *p = w; + p -= s; *p = z; + p -= s; *p = y; + p -= s; *p = x; +} + +#if ZFP_ROUNDING_MODE == ZFP_ROUND_LAST +/* bias values such that truncation is equivalent to round to nearest */ +static void +_t1(inv_round, UInt)(UInt* ublock, uint n, uint m, uint prec) +{ + /* add 1/6 ulp to unbias errors */ + if (prec < (uint)(CHAR_BIT * sizeof(UInt) - 1)) { + /* the first m values (0 <= m <= n) have one more bit of precision */ + n -= m; + while (m--) *ublock++ += ((NBMASK >> 2) >> prec); + while (n--) *ublock++ += ((NBMASK >> 1) >> prec); + } +} +#endif + +/* map two's complement signed integer to negabinary unsigned integer */ +static Int +_t1(uint2int, UInt)(UInt x) +{ + return (Int)((x ^ NBMASK) - NBMASK); +} + +/* reorder unsigned coefficients and convert to signed integer */ +static void +_t1(inv_order, Int)(const UInt* ublock, Int* iblock, const uchar* perm, uint n) +{ + do + iblock[*perm++] = _t1(uint2int, UInt)(*ublock++); + while (--n); +} + +/* decompress sequence of size <= 64 unsigned integers */ +static uint +_t1(decode_few_ints, UInt)(bitstream* restrict_ stream, uint maxbits, uint maxprec, UInt* restrict_ data, uint size) +{ + /* make a copy of bit stream to avoid aliasing */ + bitstream s = *stream; + uint intprec = (uint)(CHAR_BIT * sizeof(UInt)); + uint kmin = intprec > maxprec ? intprec - maxprec : 0; + uint bits = maxbits; + uint i, k, m, n; + uint64 x; + + /* initialize data array to all zeros */ + for (i = 0; i < size; i++) + data[i] = 0; + + /* decode one bit plane at a time from MSB to LSB */ + for (k = intprec, m = n = 0; bits && (m = 0, k-- > kmin);) { + /* step 1: decode first n bits of bit plane #k */ + m = MIN(n, bits); + bits -= m; + x = stream_read_bits(&s, m); + /* step 2: unary run-length decode remainder of bit plane */ + for (; bits && n < size; n++, m = n) { + bits--; + if (stream_read_bit(&s)) { + /* positive group test; scan for next one-bit */ + for (; bits && n < size - 1; n++) { + bits--; + if (stream_read_bit(&s)) + break; + } + /* set bit and continue decoding bit plane */ + x += (uint64)1 << n; + } + else { + /* negative group test; done with bit plane */ + m = size; + break; + } + } + /* step 3: deposit bit plane from x */ + for (i = 0; x; i++, x >>= 1) + data[i] += (UInt)(x & 1u) << k; + } + +#if ZFP_ROUNDING_MODE == ZFP_ROUND_LAST + /* bias values to achieve proper rounding */ + _t1(inv_round, UInt)(data, size, m, intprec - k); +#endif + + *stream = s; + return maxbits - bits; +} + +/* decompress sequence of size > 64 unsigned integers */ +static uint +_t1(decode_many_ints, UInt)(bitstream* restrict_ stream, uint maxbits, uint maxprec, UInt* restrict_ data, uint size) +{ + /* make a copy of bit stream to avoid aliasing */ + bitstream s = *stream; + uint intprec = (uint)(CHAR_BIT * sizeof(UInt)); + uint kmin = intprec > maxprec ? intprec - maxprec : 0; + uint bits = maxbits; + uint i, k, m, n; + + /* initialize data array to all zeros */ + for (i = 0; i < size; i++) + data[i] = 0; + + /* decode one bit plane at a time from MSB to LSB */ + for (k = intprec, m = n = 0; bits && (m = 0, k-- > kmin);) { + /* step 1: decode first n bits of bit plane #k */ + m = MIN(n, bits); + bits -= m; + for (i = 0; i < m; i++) + if (stream_read_bit(&s)) + data[i] += (UInt)1 << k; + /* step 2: unary run-length decode remainder of bit plane */ + for (; bits && n < size; n++, m = n) { + bits--; + if (stream_read_bit(&s)) { + /* positive group test; scan for next one-bit */ + for (; bits && n < size - 1; n++) { + bits--; + if (stream_read_bit(&s)) + break; + } + /* set bit and continue decoding bit plane */ + data[n] += (UInt)1 << k; + } + else { + /* negative group test; done with bit plane */ + m = size; + break; + } + } + } + +#if ZFP_ROUNDING_MODE == ZFP_ROUND_LAST + /* bias values to achieve proper rounding */ + _t1(inv_round, UInt)(data, size, m, intprec - k); +#endif + + *stream = s; + return maxbits - bits; +} + +/* decompress sequence of size <= 64 unsigned integers with no rate constraint */ +static uint +_t1(decode_few_ints_prec, UInt)(bitstream* restrict_ stream, uint maxprec, UInt* restrict_ data, uint size) +{ + /* make a copy of bit stream to avoid aliasing */ + bitstream s = *stream; + size_t offset = stream_rtell(&s); + uint intprec = (uint)(CHAR_BIT * sizeof(UInt)); + uint kmin = intprec > maxprec ? intprec - maxprec : 0; + uint i, k, n; + + /* initialize data array to all zeros */ + for (i = 0; i < size; i++) + data[i] = 0; + + /* decode one bit plane at a time from MSB to LSB */ + for (k = intprec, n = 0; k-- > kmin;) { + /* step 1: decode first n bits of bit plane #k */ + uint64 x = stream_read_bits(&s, n); + /* step 2: unary run-length decode remainder of bit plane */ + for (; n < size && stream_read_bit(&s); x += (uint64)1 << n, n++) + for (; n < size - 1 && !stream_read_bit(&s); n++) + ; + /* step 3: deposit bit plane from x */ + for (i = 0; x; i++, x >>= 1) + data[i] += (UInt)(x & 1u) << k; + } + +#if ZFP_ROUNDING_MODE == ZFP_ROUND_LAST + /* bias values to achieve proper rounding */ + _t1(inv_round, UInt)(data, size, 0, intprec - k); +#endif + + *stream = s; + return (uint)(stream_rtell(&s) - offset); +} + +/* decompress sequence of size > 64 unsigned integers with no rate constraint */ +static uint +_t1(decode_many_ints_prec, UInt)(bitstream* restrict_ stream, uint maxprec, UInt* restrict_ data, uint size) +{ + /* make a copy of bit stream to avoid aliasing */ + bitstream s = *stream; + size_t offset = stream_rtell(&s); + uint intprec = (uint)(CHAR_BIT * sizeof(UInt)); + uint kmin = intprec > maxprec ? intprec - maxprec : 0; + uint i, k, n; + + /* initialize data array to all zeros */ + for (i = 0; i < size; i++) + data[i] = 0; + + /* decode one bit plane at a time from MSB to LSB */ + for (k = intprec, n = 0; k-- > kmin;) { + /* step 1: decode first n bits of bit plane #k */ + for (i = 0; i < n; i++) + if (stream_read_bit(&s)) + data[i] += (UInt)1 << k; + /* step 2: unary run-length decode remainder of bit plane */ + for (; n < size && stream_read_bit(&s); data[n] += (UInt)1 << k, n++) + for (; n < size - 1 && !stream_read_bit(&s); n++) + ; + } + +#if ZFP_ROUNDING_MODE == ZFP_ROUND_LAST + /* bias values to achieve proper rounding */ + _t1(inv_round, UInt)(data, size, 0, intprec - k); +#endif + + *stream = s; + return (uint)(stream_rtell(&s) - offset); +} + +/* decompress sequence of size unsigned integers */ +static uint +_t1(decode_ints, UInt)(bitstream* restrict_ stream, uint maxbits, uint maxprec, UInt* restrict_ data, uint size) +{ + /* use fastest available decoder implementation */ + if (with_maxbits(maxbits, maxprec, size)) { + /* rate constrained path: decode partial bit planes */ + if (size <= 64) + return _t1(decode_few_ints, UInt)(stream, maxbits, maxprec, data, size); /* 1D, 2D, 3D blocks */ + else + return _t1(decode_many_ints, UInt)(stream, maxbits, maxprec, data, size); /* 4D blocks */ + } + else { + /* variable-rate path: decode whole bit planes */ + if (size <= 64) + return _t1(decode_few_ints_prec, UInt)(stream, maxprec, data, size); /* 1D, 2D, 3D blocks */ + else + return _t1(decode_many_ints_prec, UInt)(stream, maxprec, data, size); /* 4D blocks */ + } +} + +/* decode block of integers */ +static uint +_t2(decode_block, Int, DIMS)(bitstream* stream, int minbits, int maxbits, int maxprec, Int* iblock) +{ + int bits; + cache_align_(UInt ublock[BLOCK_SIZE]); + /* decode integer coefficients */ + bits = _t1(decode_ints, UInt)(stream, maxbits, maxprec, ublock, BLOCK_SIZE); + /* read at least minbits bits */ + if (bits < minbits) { + stream_skip(stream, minbits - bits); + bits = minbits; + } + /* reorder unsigned coefficients and convert to signed integer */ + _t1(inv_order, Int)(ublock, iblock, PERM, BLOCK_SIZE); + /* perform decorrelating transform */ + _t2(inv_xform, Int, DIMS)(iblock); + return bits; +} diff --git a/src/blosc2/plugins/codecs/zfp/src/template/decode1.c b/src/blosc2/plugins/codecs/zfp/src/template/decode1.c new file mode 100644 index 0000000..76444d8 --- /dev/null +++ b/src/blosc2/plugins/codecs/zfp/src/template/decode1.c @@ -0,0 +1,53 @@ +/* private functions ------------------------------------------------------- */ + +/* scatter 4-value block to strided array */ +static void +_t2(scatter, Scalar, 1)(const Scalar* q, Scalar* p, ptrdiff_t sx) +{ + uint x; + for (x = 0; x < 4; x++, p += sx) + *p = *q++; +} + +/* scatter nx-value block to strided array */ +static void +_t2(scatter_partial, Scalar, 1)(const Scalar* q, Scalar* p, size_t nx, ptrdiff_t sx) +{ + size_t x; + for (x = 0; x < nx; x++, p += sx) + *p = *q++; +} + +/* inverse decorrelating 1D transform */ +static void +_t2(inv_xform, Int, 1)(Int* p) +{ + /* transform along x */ + _t1(inv_lift, Int)(p, 1); +} + +/* public functions -------------------------------------------------------- */ + +/* decode 4-value block and store at p using stride sx */ +size_t +_t2(zfp_decode_block_strided, Scalar, 1)(zfp_stream* stream, Scalar* p, ptrdiff_t sx) +{ + /* decode contiguous block */ + cache_align_(Scalar block[4]); + size_t bits = _t2(zfp_decode_block, Scalar, 1)(stream, block); + /* scatter block to strided array */ + _t2(scatter, Scalar, 1)(block, p, sx); + return bits; +} + +/* decode nx-value block and store at p using stride sx */ +size_t +_t2(zfp_decode_partial_block_strided, Scalar, 1)(zfp_stream* stream, Scalar* p, size_t nx, ptrdiff_t sx) +{ + /* decode contiguous block */ + cache_align_(Scalar block[4]); + size_t bits = _t2(zfp_decode_block, Scalar, 1)(stream, block); + /* scatter block to strided array */ + _t2(scatter_partial, Scalar, 1)(block, p, nx, sx); + return bits; +} diff --git a/src/blosc2/plugins/codecs/zfp/src/template/decode2.c b/src/blosc2/plugins/codecs/zfp/src/template/decode2.c new file mode 100644 index 0000000..4d3d5bc --- /dev/null +++ b/src/blosc2/plugins/codecs/zfp/src/template/decode2.c @@ -0,0 +1,60 @@ +/* private functions ------------------------------------------------------- */ + +/* scatter 4*4 block to strided array */ +static void +_t2(scatter, Scalar, 2)(const Scalar* q, Scalar* p, ptrdiff_t sx, ptrdiff_t sy) +{ + uint x, y; + for (y = 0; y < 4; y++, p += sy - 4 * sx) + for (x = 0; x < 4; x++, p += sx) + *p = *q++; +} + +/* scatter nx*ny block to strided array */ +static void +_t2(scatter_partial, Scalar, 2)(const Scalar* q, Scalar* p, size_t nx, size_t ny, ptrdiff_t sx, ptrdiff_t sy) +{ + size_t x, y; + for (y = 0; y < ny; y++, p += sy - (ptrdiff_t)nx * sx, q += 4 - nx) + for (x = 0; x < nx; x++, p += sx, q++) + *p = *q; +} + +/* inverse decorrelating 2D transform */ +static void +_t2(inv_xform, Int, 2)(Int* p) +{ + uint x, y; + /* transform along y */ + for (x = 0; x < 4; x++) + _t1(inv_lift, Int)(p + 1 * x, 4); + /* transform along x */ + for (y = 0; y < 4; y++) + _t1(inv_lift, Int)(p + 4 * y, 1); +} + +/* public functions -------------------------------------------------------- */ + +/* decode 4*4 block and store at p using strides (sx, sy) */ +size_t +_t2(zfp_decode_block_strided, Scalar, 2)(zfp_stream* stream, Scalar* p, ptrdiff_t sx, ptrdiff_t sy) +{ + /* decode contiguous block */ + cache_align_(Scalar block[16]); + size_t bits = _t2(zfp_decode_block, Scalar, 2)(stream, block); + /* scatter block to strided array */ + _t2(scatter, Scalar, 2)(block, p, sx, sy); + return bits; +} + +/* decode nx*ny block and store at p using strides (sx, sy) */ +size_t +_t2(zfp_decode_partial_block_strided, Scalar, 2)(zfp_stream* stream, Scalar* p, size_t nx, size_t ny, ptrdiff_t sx, ptrdiff_t sy) +{ + /* decode contiguous block */ + cache_align_(Scalar block[16]); + size_t bits = _t2(zfp_decode_block, Scalar, 2)(stream, block); + /* scatter block to strided array */ + _t2(scatter_partial, Scalar, 2)(block, p, nx, ny, sx, sy); + return bits; +} diff --git a/src/blosc2/plugins/codecs/zfp/src/template/decode3.c b/src/blosc2/plugins/codecs/zfp/src/template/decode3.c new file mode 100644 index 0000000..c9232fc --- /dev/null +++ b/src/blosc2/plugins/codecs/zfp/src/template/decode3.c @@ -0,0 +1,68 @@ +/* private functions ------------------------------------------------------- */ + +/* scatter 4*4*4 block to strided array */ +static void +_t2(scatter, Scalar, 3)(const Scalar* q, Scalar* p, ptrdiff_t sx, ptrdiff_t sy, ptrdiff_t sz) +{ + uint x, y, z; + for (z = 0; z < 4; z++, p += sz - 4 * sy) + for (y = 0; y < 4; y++, p += sy - 4 * sx) + for (x = 0; x < 4; x++, p += sx) + *p = *q++; +} + +/* scatter nx*ny*nz block to strided array */ +static void +_t2(scatter_partial, Scalar, 3)(const Scalar* q, Scalar* p, size_t nx, size_t ny, size_t nz, ptrdiff_t sx, ptrdiff_t sy, ptrdiff_t sz) +{ + size_t x, y, z; + for (z = 0; z < nz; z++, p += sz - (ptrdiff_t)ny * sy, q += 4 * (4 - ny)) + for (y = 0; y < ny; y++, p += sy - (ptrdiff_t)nx * sx, q += 1 * (4 - nx)) + for (x = 0; x < nx; x++, p += sx, q++) + *p = *q; +} + +/* inverse decorrelating 3D transform */ +static void +_t2(inv_xform, Int, 3)(Int* p) +{ + uint x, y, z; + /* transform along z */ + for (y = 0; y < 4; y++) + for (x = 0; x < 4; x++) + _t1(inv_lift, Int)(p + 1 * x + 4 * y, 16); + /* transform along y */ + for (x = 0; x < 4; x++) + for (z = 0; z < 4; z++) + _t1(inv_lift, Int)(p + 16 * z + 1 * x, 4); + /* transform along x */ + for (z = 0; z < 4; z++) + for (y = 0; y < 4; y++) + _t1(inv_lift, Int)(p + 4 * y + 16 * z, 1); +} + +/* public functions -------------------------------------------------------- */ + +/* decode 4*4*4 block and store at p using strides (sx, sy, sz) */ +size_t +_t2(zfp_decode_block_strided, Scalar, 3)(zfp_stream* stream, Scalar* p, ptrdiff_t sx, ptrdiff_t sy, ptrdiff_t sz) +{ + /* decode contiguous block */ + cache_align_(Scalar block[64]); + size_t bits = _t2(zfp_decode_block, Scalar, 3)(stream, block); + /* scatter block to strided array */ + _t2(scatter, Scalar, 3)(block, p, sx, sy, sz); + return bits; +} + +/* decode nx*ny*nz block and store at p using strides (sx, sy, sz) */ +size_t +_t2(zfp_decode_partial_block_strided, Scalar, 3)(zfp_stream* stream, Scalar* p, size_t nx, size_t ny, size_t nz, ptrdiff_t sx, ptrdiff_t sy, ptrdiff_t sz) +{ + /* decode contiguous block */ + cache_align_(Scalar block[64]); + size_t bits = _t2(zfp_decode_block, Scalar, 3)(stream, block); + /* scatter block to strided array */ + _t2(scatter_partial, Scalar, 3)(block, p, nx, ny, nz, sx, sy, sz); + return bits; +} diff --git a/src/blosc2/plugins/codecs/zfp/src/template/decode4.c b/src/blosc2/plugins/codecs/zfp/src/template/decode4.c new file mode 100644 index 0000000..3274b42 --- /dev/null +++ b/src/blosc2/plugins/codecs/zfp/src/template/decode4.c @@ -0,0 +1,78 @@ +/* private functions ------------------------------------------------------- */ + +/* scatter 4*4*4*4 block to strided array */ +static void +_t2(scatter, Scalar, 4)(const Scalar* q, Scalar* p, ptrdiff_t sx, ptrdiff_t sy, ptrdiff_t sz, ptrdiff_t sw) +{ + uint x, y, z, w; + for (w = 0; w < 4; w++, p += sw - 4 * sz) + for (z = 0; z < 4; z++, p += sz - 4 * sy) + for (y = 0; y < 4; y++, p += sy - 4 * sx) + for (x = 0; x < 4; x++, p += sx) + *p = *q++; +} + +/* scatter nx*ny*nz*nw block to strided array */ +static void +_t2(scatter_partial, Scalar, 4)(const Scalar* q, Scalar* p, size_t nx, size_t ny, size_t nz, size_t nw, ptrdiff_t sx, ptrdiff_t sy, ptrdiff_t sz, ptrdiff_t sw) +{ + size_t x, y, z, w; + for (w = 0; w < nw; w++, p += sw - (ptrdiff_t)nz * sz, q += 16 * (4 - nz)) + for (z = 0; z < nz; z++, p += sz - (ptrdiff_t)ny * sy, q += 4 * (4 - ny)) + for (y = 0; y < ny; y++, p += sy - (ptrdiff_t)nx * sx, q += 1 * (4 - nx)) + for (x = 0; x < nx; x++, p += sx, q++) + *p = *q; +} + +/* inverse decorrelating 4D transform */ +static void +_t2(inv_xform, Int, 4)(Int* p) +{ + uint x, y, z, w; + /* transform along w */ + for (z = 0; z < 4; z++) + for (y = 0; y < 4; y++) + for (x = 0; x < 4; x++) + _t1(inv_lift, Int)(p + 1 * x + 4 * y + 16 * z, 64); + /* transform along z */ + for (y = 0; y < 4; y++) + for (x = 0; x < 4; x++) + for (w = 0; w < 4; w++) + _t1(inv_lift, Int)(p + 64 * w + 1 * x + 4 * y, 16); + /* transform along y */ + for (x = 0; x < 4; x++) + for (w = 0; w < 4; w++) + for (z = 0; z < 4; z++) + _t1(inv_lift, Int)(p + 16 * z + 64 * w + 1 * x, 4); + /* transform along x */ + for (w = 0; w < 4; w++) + for (z = 0; z < 4; z++) + for (y = 0; y < 4; y++) + _t1(inv_lift, Int)(p + 4 * y + 16 * z + 64 * w, 1); +} + +/* public functions -------------------------------------------------------- */ + +/* decode 4*4*4*4 block and store at p using strides (sx, sy, sz, sw) */ +size_t +_t2(zfp_decode_block_strided, Scalar, 4)(zfp_stream* stream, Scalar* p, ptrdiff_t sx, ptrdiff_t sy, ptrdiff_t sz, ptrdiff_t sw) +{ + /* decode contiguous block */ + cache_align_(Scalar block[256]); + size_t bits = _t2(zfp_decode_block, Scalar, 4)(stream, block); + /* scatter block to strided array */ + _t2(scatter, Scalar, 4)(block, p, sx, sy, sz, sw); + return bits; +} + +/* decode nx*ny*nz*nw block and store at p using strides (sx, sy, sz, sw) */ +size_t +_t2(zfp_decode_partial_block_strided, Scalar, 4)(zfp_stream* stream, Scalar* p, size_t nx, size_t ny, size_t nz, size_t nw, ptrdiff_t sx, ptrdiff_t sy, ptrdiff_t sz, ptrdiff_t sw) +{ + /* decode contiguous block */ + cache_align_(Scalar block[256]); + size_t bits = _t2(zfp_decode_block, Scalar, 4)(stream, block); + /* scatter block to strided array */ + _t2(scatter_partial, Scalar, 4)(block, p, nx, ny, nz, nw, sx, sy, sz, sw); + return bits; +} diff --git a/src/blosc2/plugins/codecs/zfp/src/template/decodef.c b/src/blosc2/plugins/codecs/zfp/src/template/decodef.c new file mode 100644 index 0000000..3191c94 --- /dev/null +++ b/src/blosc2/plugins/codecs/zfp/src/template/decodef.c @@ -0,0 +1,43 @@ +static uint _t2(rev_decode_block, Scalar, DIMS)(zfp_stream* zfp, Scalar* fblock); + +/* private functions ------------------------------------------------------- */ + +/* decode contiguous floating-point block using lossy algorithm */ +static uint +_t2(decode_block, Scalar, DIMS)(zfp_stream* zfp, Scalar* fblock) +{ + uint bits = 1; + /* test if block has nonzero values */ + if (stream_read_bit(zfp->stream)) { + cache_align_(Int iblock[BLOCK_SIZE]); + int emax, maxprec; + /* decode common exponent */ + bits += EBITS; + emax = (int)stream_read_bits(zfp->stream, EBITS) - EBIAS; + maxprec = precision(emax, zfp->maxprec, zfp->minexp, DIMS); + /* decode integer block */ + bits += _t2(decode_block, Int, DIMS)(zfp->stream, zfp->minbits - bits, zfp->maxbits - bits, maxprec, iblock); + /* perform inverse block-floating-point transform */ + _t1(inv_cast, Scalar)(iblock, fblock, BLOCK_SIZE, emax); + } + else { + /* set all values to zero */ + uint i; + for (i = 0; i < BLOCK_SIZE; i++) + *fblock++ = 0; + if (zfp->minbits > bits) { + stream_skip(zfp->stream, zfp->minbits - bits); + bits = zfp->minbits; + } + } + return bits; +} + +/* public functions -------------------------------------------------------- */ + +/* decode contiguous floating-point block */ +size_t +_t2(zfp_decode_block, Scalar, DIMS)(zfp_stream* zfp, Scalar* fblock) +{ + return REVERSIBLE(zfp) ? _t2(rev_decode_block, Scalar, DIMS)(zfp, fblock) : _t2(decode_block, Scalar, DIMS)(zfp, fblock); +} diff --git a/src/blosc2/plugins/codecs/zfp/src/template/decodei.c b/src/blosc2/plugins/codecs/zfp/src/template/decodei.c new file mode 100644 index 0000000..8a35a8d --- /dev/null +++ b/src/blosc2/plugins/codecs/zfp/src/template/decodei.c @@ -0,0 +1,10 @@ +static uint _t2(rev_decode_block, Int, DIMS)(bitstream* stream, int minbits, int maxbits, Int* iblock); + +/* public functions -------------------------------------------------------- */ + +/* decode contiguous integer block */ +size_t +_t2(zfp_decode_block, Int, DIMS)(zfp_stream* zfp, Int* iblock) +{ + return REVERSIBLE(zfp) ? _t2(rev_decode_block, Int, DIMS)(zfp->stream, zfp->minbits, zfp->maxbits, iblock) : _t2(decode_block, Int, DIMS)(zfp->stream, zfp->minbits, zfp->maxbits, zfp->maxprec, iblock); +} diff --git a/src/blosc2/plugins/codecs/zfp/src/template/decompress.c b/src/blosc2/plugins/codecs/zfp/src/template/decompress.c new file mode 100644 index 0000000..7261077 --- /dev/null +++ b/src/blosc2/plugins/codecs/zfp/src/template/decompress.c @@ -0,0 +1,109 @@ +/* decompress 1d contiguous array */ +static void +_t2(decompress, Scalar, 1)(zfp_stream* stream, zfp_field* field) +{ + Scalar* data = (Scalar*)field->data; + size_t nx = field->nx; + size_t mx = nx & ~3u; + size_t x; + + /* decompress array one block of 4 values at a time */ + for (x = 0; x < mx; x += 4, data += 4) + _t2(zfp_decode_block, Scalar, 1)(stream, data); + if (x < nx) + _t2(zfp_decode_partial_block_strided, Scalar, 1)(stream, data, nx - x, 1); +} + +/* decompress 1d strided array */ +static void +_t2(decompress_strided, Scalar, 1)(zfp_stream* stream, zfp_field* field) +{ + Scalar* data = field->data; + size_t nx = field->nx; + ptrdiff_t sx = field->sx ? field->sx : 1; + size_t x; + + /* decompress array one block of 4 values at a time */ + for (x = 0; x < nx; x += 4) { + Scalar* p = data + sx * (ptrdiff_t)x; + if (nx - x < 4) + _t2(zfp_decode_partial_block_strided, Scalar, 1)(stream, p, nx - x, sx); + else + _t2(zfp_decode_block_strided, Scalar, 1)(stream, p, sx); + } +} + +/* decompress 2d strided array */ +static void +_t2(decompress_strided, Scalar, 2)(zfp_stream* stream, zfp_field* field) +{ + Scalar* data = (Scalar*)field->data; + size_t nx = field->nx; + size_t ny = field->ny; + ptrdiff_t sx = field->sx ? field->sx : 1; + ptrdiff_t sy = field->sy ? field->sy : (ptrdiff_t)nx; + size_t x, y; + + /* decompress array one block of 4x4 values at a time */ + for (y = 0; y < ny; y += 4) + for (x = 0; x < nx; x += 4) { + Scalar* p = data + sx * (ptrdiff_t)x + sy * (ptrdiff_t)y; + if (nx - x < 4 || ny - y < 4) + _t2(zfp_decode_partial_block_strided, Scalar, 2)(stream, p, MIN(nx - x, 4u), MIN(ny - y, 4u), sx, sy); + else + _t2(zfp_decode_block_strided, Scalar, 2)(stream, p, sx, sy); + } +} + +/* decompress 3d strided array */ +static void +_t2(decompress_strided, Scalar, 3)(zfp_stream* stream, zfp_field* field) +{ + Scalar* data = (Scalar*)field->data; + size_t nx = field->nx; + size_t ny = field->ny; + size_t nz = field->nz; + ptrdiff_t sx = field->sx ? field->sx : 1; + ptrdiff_t sy = field->sy ? field->sy : (ptrdiff_t)nx; + ptrdiff_t sz = field->sz ? field->sz : (ptrdiff_t)(nx * ny); + size_t x, y, z; + + /* decompress array one block of 4x4x4 values at a time */ + for (z = 0; z < nz; z += 4) + for (y = 0; y < ny; y += 4) + for (x = 0; x < nx; x += 4) { + Scalar* p = data + sx * (ptrdiff_t)x + sy * (ptrdiff_t)y + sz * (ptrdiff_t)z; + if (nx - x < 4 || ny - y < 4 || nz - z < 4) + _t2(zfp_decode_partial_block_strided, Scalar, 3)(stream, p, MIN(nx - x, 4u), MIN(ny - y, 4u), MIN(nz - z, 4u), sx, sy, sz); + else + _t2(zfp_decode_block_strided, Scalar, 3)(stream, p, sx, sy, sz); + } +} + +/* decompress 4d strided array */ +static void +_t2(decompress_strided, Scalar, 4)(zfp_stream* stream, zfp_field* field) +{ + Scalar* data = field->data; + size_t nx = field->nx; + size_t ny = field->ny; + size_t nz = field->nz; + size_t nw = field->nw; + ptrdiff_t sx = field->sx ? field->sx : 1; + ptrdiff_t sy = field->sy ? field->sy : (ptrdiff_t)nx; + ptrdiff_t sz = field->sz ? field->sz : (ptrdiff_t)(nx * ny); + ptrdiff_t sw = field->sw ? field->sw : (ptrdiff_t)(nx * ny * nz); + size_t x, y, z, w; + + /* decompress array one block of 4x4x4x4 values at a time */ + for (w = 0; w < nw; w += 4) + for (z = 0; z < nz; z += 4) + for (y = 0; y < ny; y += 4) + for (x = 0; x < nx; x += 4) { + Scalar* p = data + sx * (ptrdiff_t)x + sy * (ptrdiff_t)y + sz * (ptrdiff_t)z + sw * (ptrdiff_t)w; + if (nx - x < 4 || ny - y < 4 || nz - z < 4 || nw - w < 4) + _t2(zfp_decode_partial_block_strided, Scalar, 4)(stream, p, MIN(nx - x, 4u), MIN(ny - y, 4u), MIN(nz - z, 4u), MIN(nw - w, 4u), sx, sy, sz, sw); + else + _t2(zfp_decode_block_strided, Scalar, 4)(stream, p, sx, sy, sz, sw); + } +} diff --git a/src/blosc2/plugins/codecs/zfp/src/template/encode.c b/src/blosc2/plugins/codecs/zfp/src/template/encode.c new file mode 100644 index 0000000..61f112a --- /dev/null +++ b/src/blosc2/plugins/codecs/zfp/src/template/encode.c @@ -0,0 +1,280 @@ +#include + +static void _t2(fwd_xform, Int, DIMS)(Int* p); + +/* private functions ------------------------------------------------------- */ + +/* pad partial block of width n <= 4 and stride s */ +static void +_t1(pad_block, Scalar)(Scalar* p, size_t n, ptrdiff_t s) +{ + switch (n) { + case 0: + p[0 * s] = 0; + /* FALLTHROUGH */ + case 1: + p[1 * s] = p[0 * s]; + /* FALLTHROUGH */ + case 2: + p[2 * s] = p[1 * s]; + /* FALLTHROUGH */ + case 3: + p[3 * s] = p[0 * s]; + /* FALLTHROUGH */ + default: + break; + } +} + +/* forward lifting transform of 4-vector */ +static void +_t1(fwd_lift, Int)(Int* p, ptrdiff_t s) +{ + Int x, y, z, w; + x = *p; p += s; + y = *p; p += s; + z = *p; p += s; + w = *p; p += s; + + /* + ** non-orthogonal transform + ** ( 4 4 4 4) (x) + ** 1/16 * ( 5 1 -1 -5) (y) + ** (-4 4 4 -4) (z) + ** (-2 6 -6 2) (w) + */ + x += w; x >>= 1; w -= x; + z += y; z >>= 1; y -= z; + x += z; x >>= 1; z -= x; + w += y; w >>= 1; y -= w; + w += y >> 1; y -= w >> 1; + + p -= s; *p = w; + p -= s; *p = z; + p -= s; *p = y; + p -= s; *p = x; +} + +#if ZFP_ROUNDING_MODE == ZFP_ROUND_FIRST +/* bias values such that truncation is equivalent to round to nearest */ +static void +_t1(fwd_round, Int)(Int* iblock, uint n, uint maxprec) +{ + /* add or subtract 1/6 ulp to unbias errors */ + if (maxprec < (uint)(CHAR_BIT * sizeof(Int))) { + Int bias = (NBMASK >> 2) >> maxprec; + if (maxprec & 1u) + do *iblock++ += bias; while (--n); + else + do *iblock++ -= bias; while (--n); + } +} +#endif + +/* map two's complement signed integer to negabinary unsigned integer */ +static UInt +_t1(int2uint, Int)(Int x) +{ + return ((UInt)x + NBMASK) ^ NBMASK; +} + +/* reorder signed coefficients and convert to unsigned integer */ +static void +_t1(fwd_order, Int)(UInt* ublock, const Int* iblock, const uchar* perm, uint n) +{ + do + *ublock++ = _t1(int2uint, Int)(iblock[*perm++]); + while (--n); +} + +/* compress sequence of size <= 64 unsigned integers */ +static uint +_t1(encode_few_ints, UInt)(bitstream* restrict_ stream, uint maxbits, uint maxprec, const UInt* restrict_ data, uint size) +{ + /* make a copy of bit stream to avoid aliasing */ + bitstream s = *stream; + uint intprec = (uint)(CHAR_BIT * sizeof(UInt)); + uint kmin = intprec > maxprec ? intprec - maxprec : 0; + uint bits = maxbits; + uint i, k, m, n; + uint64 x; + + /* encode one bit plane at a time from MSB to LSB */ + for (k = intprec, n = 0; bits && k-- > kmin;) { + /* step 1: extract bit plane #k to x */ + x = 0; + for (i = 0; i < size; i++) + x += (uint64)((data[i] >> k) & 1u) << i; + /* step 2: encode first n bits of bit plane */ + m = MIN(n, bits); + bits -= m; + x = stream_write_bits(&s, x, m); + /* step 3: unary run-length encode remainder of bit plane */ + for (; bits && n < size; x >>= 1, n++) { + bits--; + if (stream_write_bit(&s, !!x)) { + /* positive group test (x != 0); scan for one-bit */ + for (; bits && n < size - 1; x >>= 1, n++) { + bits--; + if (stream_write_bit(&s, x & 1u)) + break; + } + } + else { + /* negative group test (x == 0); done with bit plane */ + break; + } + } + } + + *stream = s; + return maxbits - bits; +} + +/* compress sequence of size > 64 unsigned integers */ +static uint +_t1(encode_many_ints, UInt)(bitstream* restrict_ stream, uint maxbits, uint maxprec, const UInt* restrict_ data, uint size) +{ + /* make a copy of bit stream to avoid aliasing */ + bitstream s = *stream; + uint intprec = (uint)(CHAR_BIT * sizeof(UInt)); + uint kmin = intprec > maxprec ? intprec - maxprec : 0; + uint bits = maxbits; + uint i, k, m, n, c; + + /* encode one bit plane at a time from MSB to LSB */ + for (k = intprec, n = 0; bits && k-- > kmin;) { + /* step 1: encode first n bits of bit plane #k */ + m = MIN(n, bits); + bits -= m; + for (i = 0; i < m; i++) + stream_write_bit(&s, (data[i] >> k) & 1u); + /* step 2: count remaining one-bits in bit plane */ + c = 0; + for (i = m; i < size; i++) + c += (data[i] >> k) & 1u; + /* step 3: unary run-length encode remainder of bit plane */ + for (; bits && n < size; n++) { + bits--; + if (stream_write_bit(&s, !!c)) { + /* positive group test (c > 0); scan for one-bit */ + for (c--; bits && n < size - 1; n++) { + bits--; + if (stream_write_bit(&s, (data[n] >> k) & 1u)) + break; + } + } + else { + /* negative group test (c == 0); done with bit plane */ + break; + } + } + } + + *stream = s; + return maxbits - bits; +} + +/* compress sequence of size <= 64 unsigned integers with no rate constraint */ +static uint +_t1(encode_few_ints_prec, UInt)(bitstream* restrict_ stream, uint maxprec, const UInt* restrict_ data, uint size) +{ + /* make a copy of bit stream to avoid aliasing */ + bitstream s = *stream; + size_t offset = stream_wtell(&s); + uint intprec = (uint)(CHAR_BIT * sizeof(UInt)); + uint kmin = intprec > maxprec ? intprec - maxprec : 0; + uint i, k, n; + + /* encode one bit plane at a time from MSB to LSB */ + for (k = intprec, n = 0; k-- > kmin;) { + /* step 1: extract bit plane #k to x */ + uint64 x = 0; + for (i = 0; i < size; i++) + x += (uint64)((data[i] >> k) & 1u) << i; + /* step 2: encode first n bits of bit plane */ + x = stream_write_bits(&s, x, n); + /* step 3: unary run-length encode remainder of bit plane */ + for (; n < size && stream_write_bit(&s, !!x); x >>= 1, n++) + for (; n < size - 1 && !stream_write_bit(&s, x & 1u); x >>= 1, n++) + ; + } + + *stream = s; + return (uint)(stream_wtell(&s) - offset); +} + +/* compress sequence of size > 64 unsigned integers with no rate constraint */ +static uint +_t1(encode_many_ints_prec, UInt)(bitstream* restrict_ stream, uint maxprec, const UInt* restrict_ data, uint size) +{ + /* make a copy of bit stream to avoid aliasing */ + bitstream s = *stream; + size_t offset = stream_wtell(&s); + uint intprec = (uint)(CHAR_BIT * sizeof(UInt)); + uint kmin = intprec > maxprec ? intprec - maxprec : 0; + uint i, k, n, c; + + /* encode one bit plane at a time from MSB to LSB */ + for (k = intprec, n = 0; k-- > kmin;) { + /* step 1: encode first n bits of bit plane #k */ + for (i = 0; i < n; i++) + stream_write_bit(&s, (data[i] >> k) & 1u); + /* step 2: count remaining one-bits in bit plane */ + c = 0; + for (i = n; i < size; i++) + c += (data[i] >> k) & 1u; + /* step 3: unary run-length encode remainder of bit plane */ + for (; n < size && stream_write_bit(&s, !!c); n++) + for (c--; n < size - 1 && !stream_write_bit(&s, (data[n] >> k) & 1u); n++) + ; + } + + *stream = s; + return (uint)(stream_wtell(&s) - offset); +} + +/* compress sequence of size unsigned integers */ +static uint +_t1(encode_ints, UInt)(bitstream* restrict_ stream, uint maxbits, uint maxprec, const UInt* restrict_ data, uint size) +{ + /* use fastest available encoder implementation */ + if (with_maxbits(maxbits, maxprec, size)) { + /* rate constrained path: encode partial bit planes */ + if (size <= 64) + return _t1(encode_few_ints, UInt)(stream, maxbits, maxprec, data, size); /* 1D, 2D, 3D blocks */ + else + return _t1(encode_many_ints, UInt)(stream, maxbits, maxprec, data, size); /* 4D blocks */ + } + else { + /* variable-rate path: encode whole bit planes */ + if (size <= 64) + return _t1(encode_few_ints_prec, UInt)(stream, maxprec, data, size); /* 1D, 2D, 3D blocks */ + else + return _t1(encode_many_ints_prec, UInt)(stream, maxprec, data, size); /* 4D blocks */ + } +} + +/* encode block of integers */ +static uint +_t2(encode_block, Int, DIMS)(bitstream* stream, int minbits, int maxbits, int maxprec, Int* iblock) +{ + int bits; + cache_align_(UInt ublock[BLOCK_SIZE]); + /* perform decorrelating transform */ + _t2(fwd_xform, Int, DIMS)(iblock); +#if ZFP_ROUNDING_MODE == ZFP_ROUND_FIRST + /* bias values to achieve proper rounding */ + _t1(fwd_round, Int)(iblock, BLOCK_SIZE, maxprec); +#endif + /* reorder signed coefficients and convert to unsigned integer */ + _t1(fwd_order, Int)(ublock, iblock, PERM, BLOCK_SIZE); + /* encode integer coefficients */ + bits = _t1(encode_ints, UInt)(stream, maxbits, maxprec, ublock, BLOCK_SIZE); + /* write at least minbits bits by padding with zeros */ + if (bits < minbits) { + stream_pad(stream, minbits - bits); + bits = minbits; + } + return bits; +} diff --git a/src/blosc2/plugins/codecs/zfp/src/template/encode1.c b/src/blosc2/plugins/codecs/zfp/src/template/encode1.c new file mode 100644 index 0000000..ff9d5c0 --- /dev/null +++ b/src/blosc2/plugins/codecs/zfp/src/template/encode1.c @@ -0,0 +1,52 @@ +/* private functions ------------------------------------------------------- */ + +/* gather 4-value block from strided array */ +static void +_t2(gather, Scalar, 1)(Scalar* q, const Scalar* p, ptrdiff_t sx) +{ + uint x; + for (x = 0; x < 4; x++, p += sx) + *q++ = *p; +} + +/* gather nx-value block from strided array */ +static void +_t2(gather_partial, Scalar, 1)(Scalar* q, const Scalar* p, size_t nx, ptrdiff_t sx) +{ + size_t x; + for (x = 0; x < nx; x++, p += sx) + q[x] = *p; + _t1(pad_block, Scalar)(q, nx, 1); +} + +/* forward decorrelating 1D transform */ +static void +_t2(fwd_xform, Int, 1)(Int* p) +{ + /* transform along x */ + _t1(fwd_lift, Int)(p, 1); +} + +/* public functions -------------------------------------------------------- */ + +/* encode 4-value block stored at p using stride sx */ +size_t +_t2(zfp_encode_block_strided, Scalar, 1)(zfp_stream* stream, const Scalar* p, ptrdiff_t sx) +{ + /* gather block from strided array */ + cache_align_(Scalar block[4]); + _t2(gather, Scalar, 1)(block, p, sx); + /* encode block */ + return _t2(zfp_encode_block, Scalar, 1)(stream, block); +} + +/* encode nx-value block stored at p using stride sx */ +size_t +_t2(zfp_encode_partial_block_strided, Scalar, 1)(zfp_stream* stream, const Scalar* p, size_t nx, ptrdiff_t sx) +{ + /* gather block from strided array */ + cache_align_(Scalar block[4]); + _t2(gather_partial, Scalar, 1)(block, p, nx, sx); + /* encode block */ + return _t2(zfp_encode_block, Scalar, 1)(stream, block); +} diff --git a/src/blosc2/plugins/codecs/zfp/src/template/encode2.c b/src/blosc2/plugins/codecs/zfp/src/template/encode2.c new file mode 100644 index 0000000..b77b439 --- /dev/null +++ b/src/blosc2/plugins/codecs/zfp/src/template/encode2.c @@ -0,0 +1,62 @@ +/* private functions ------------------------------------------------------- */ + +/* gather 4*4 block from strided array */ +static void +_t2(gather, Scalar, 2)(Scalar* q, const Scalar* p, ptrdiff_t sx, ptrdiff_t sy) +{ + uint x, y; + for (y = 0; y < 4; y++, p += sy - 4 * sx) + for (x = 0; x < 4; x++, p += sx) + *q++ = *p; +} + +/* gather nx*ny block from strided array */ +static void +_t2(gather_partial, Scalar, 2)(Scalar* q, const Scalar* p, size_t nx, size_t ny, ptrdiff_t sx, ptrdiff_t sy) +{ + size_t x, y; + for (y = 0; y < ny; y++, p += sy - (ptrdiff_t)nx * sx) { + for (x = 0; x < nx; x++, p += sx) + q[4 * y + x] = *p; + _t1(pad_block, Scalar)(q + 4 * y, nx, 1); + } + for (x = 0; x < 4; x++) + _t1(pad_block, Scalar)(q + x, ny, 4); +} + +/* forward decorrelating 2D transform */ +static void +_t2(fwd_xform, Int, 2)(Int* p) +{ + uint x, y; + /* transform along x */ + for (y = 0; y < 4; y++) + _t1(fwd_lift, Int)(p + 4 * y, 1); + /* transform along y */ + for (x = 0; x < 4; x++) + _t1(fwd_lift, Int)(p + 1 * x, 4); +} + +/* public functions -------------------------------------------------------- */ + +/* encode 4*4 block stored at p using strides (sx, sy) */ +size_t +_t2(zfp_encode_block_strided, Scalar, 2)(zfp_stream* stream, const Scalar* p, ptrdiff_t sx, ptrdiff_t sy) +{ + /* gather block from strided array */ + cache_align_(Scalar block[16]); + _t2(gather, Scalar, 2)(block, p, sx, sy); + /* encode block */ + return _t2(zfp_encode_block, Scalar, 2)(stream, block); +} + +/* encode nx*ny block stored at p using strides (sx, sy) */ +size_t +_t2(zfp_encode_partial_block_strided, Scalar, 2)(zfp_stream* stream, const Scalar* p, size_t nx, size_t ny, ptrdiff_t sx, ptrdiff_t sy) +{ + /* gather block from strided array */ + cache_align_(Scalar block[16]); + _t2(gather_partial, Scalar, 2)(block, p, nx, ny, sx, sy); + /* encode block */ + return _t2(zfp_encode_block, Scalar, 2)(stream, block); +} diff --git a/src/blosc2/plugins/codecs/zfp/src/template/encode3.c b/src/blosc2/plugins/codecs/zfp/src/template/encode3.c new file mode 100644 index 0000000..3206060 --- /dev/null +++ b/src/blosc2/plugins/codecs/zfp/src/template/encode3.c @@ -0,0 +1,74 @@ +/* private functions ------------------------------------------------------- */ + +/* gather 4*4*4 block from strided array */ +static void +_t2(gather, Scalar, 3)(Scalar* q, const Scalar* p, ptrdiff_t sx, ptrdiff_t sy, ptrdiff_t sz) +{ + uint x, y, z; + for (z = 0; z < 4; z++, p += sz - 4 * sy) + for (y = 0; y < 4; y++, p += sy - 4 * sx) + for (x = 0; x < 4; x++, p += sx) + *q++ = *p; +} + +/* gather nx*ny*nz block from strided array */ +static void +_t2(gather_partial, Scalar, 3)(Scalar* q, const Scalar* p, size_t nx, size_t ny, size_t nz, ptrdiff_t sx, ptrdiff_t sy, ptrdiff_t sz) +{ + size_t x, y, z; + for (z = 0; z < nz; z++, p += sz - (ptrdiff_t)ny * sy) { + for (y = 0; y < ny; y++, p += sy - (ptrdiff_t)nx * sx) { + for (x = 0; x < nx; x++, p += sx) + q[16 * z + 4 * y + x] = *p; + _t1(pad_block, Scalar)(q + 16 * z + 4 * y, nx, 1); + } + for (x = 0; x < 4; x++) + _t1(pad_block, Scalar)(q + 16 * z + x, ny, 4); + } + for (y = 0; y < 4; y++) + for (x = 0; x < 4; x++) + _t1(pad_block, Scalar)(q + 4 * y + x, nz, 16); +} + +/* forward decorrelating 3D transform */ +static void +_t2(fwd_xform, Int, 3)(Int* p) +{ + uint x, y, z; + /* transform along x */ + for (z = 0; z < 4; z++) + for (y = 0; y < 4; y++) + _t1(fwd_lift, Int)(p + 4 * y + 16 * z, 1); + /* transform along y */ + for (x = 0; x < 4; x++) + for (z = 0; z < 4; z++) + _t1(fwd_lift, Int)(p + 16 * z + 1 * x, 4); + /* transform along z */ + for (y = 0; y < 4; y++) + for (x = 0; x < 4; x++) + _t1(fwd_lift, Int)(p + 1 * x + 4 * y, 16); +} + +/* public functions -------------------------------------------------------- */ + +/* encode 4*4*4 block stored at p using strides (sx, sy, sz) */ +size_t +_t2(zfp_encode_block_strided, Scalar, 3)(zfp_stream* stream, const Scalar* p, ptrdiff_t sx, ptrdiff_t sy, ptrdiff_t sz) +{ + /* gather block from strided array */ + cache_align_(Scalar block[64]); + _t2(gather, Scalar, 3)(block, p, sx, sy, sz); + /* encode block */ + return _t2(zfp_encode_block, Scalar, 3)(stream, block); +} + +/* encode nx*ny*nz block stored at p using strides (sx, sy, sz) */ +size_t +_t2(zfp_encode_partial_block_strided, Scalar, 3)(zfp_stream* stream, const Scalar* p, size_t nx, size_t ny, size_t nz, ptrdiff_t sx, ptrdiff_t sy, ptrdiff_t sz) +{ + /* gather block from strided array */ + cache_align_(Scalar block[64]); + _t2(gather_partial, Scalar, 3)(block, p, nx, ny, nz, sx, sy, sz); + /* encode block */ + return _t2(zfp_encode_block, Scalar, 3)(stream, block); +} diff --git a/src/blosc2/plugins/codecs/zfp/src/template/encode4.c b/src/blosc2/plugins/codecs/zfp/src/template/encode4.c new file mode 100644 index 0000000..90ca40a --- /dev/null +++ b/src/blosc2/plugins/codecs/zfp/src/template/encode4.c @@ -0,0 +1,89 @@ +/* private functions ------------------------------------------------------- */ + +/* gather 4*4*4*4 block from strided array */ +static void +_t2(gather, Scalar, 4)(Scalar* q, const Scalar* p, ptrdiff_t sx, ptrdiff_t sy, ptrdiff_t sz, ptrdiff_t sw) +{ + uint x, y, z, w; + for (w = 0; w < 4; w++, p += sw - 4 * sz) + for (z = 0; z < 4; z++, p += sz - 4 * sy) + for (y = 0; y < 4; y++, p += sy - 4 * sx) + for (x = 0; x < 4; x++, p += sx) + *q++ = *p; +} + +/* gather nx*ny*nz*nw block from strided array */ +static void +_t2(gather_partial, Scalar, 4)(Scalar* q, const Scalar* p, size_t nx, size_t ny, size_t nz, size_t nw, ptrdiff_t sx, ptrdiff_t sy, ptrdiff_t sz, ptrdiff_t sw) +{ + size_t x, y, z, w; + for (w = 0; w < nw; w++, p += sw - (ptrdiff_t)nz * sz) { + for (z = 0; z < nz; z++, p += sz - (ptrdiff_t)ny * sy) { + for (y = 0; y < ny; y++, p += sy - (ptrdiff_t)nx * sx) { + for (x = 0; x < nx; x++, p += sx) + q[64 * w + 16 * z + 4 * y + x] = *p; + _t1(pad_block, Scalar)(q + 64 * w + 16 * z + 4 * y, nx, 1); + } + for (x = 0; x < 4; x++) + _t1(pad_block, Scalar)(q + 64 * w + 16 * z + x, ny, 4); + } + for (y = 0; y < 4; y++) + for (x = 0; x < 4; x++) + _t1(pad_block, Scalar)(q + 64 * w + 4 * y + x, nz, 16); + } + for (z = 0; z < 4; z++) + for (y = 0; y < 4; y++) + for (x = 0; x < 4; x++) + _t1(pad_block, Scalar)(q + 16 * z + 4 * y + x, nw, 64); +} + +/* forward decorrelating 4D transform */ +static void +_t2(fwd_xform, Int, 4)(Int* p) +{ + uint x, y, z, w; + /* transform along x */ + for (w = 0; w < 4; w++) + for (z = 0; z < 4; z++) + for (y = 0; y < 4; y++) + _t1(fwd_lift, Int)(p + 4 * y + 16 * z + 64 * w, 1); + /* transform along y */ + for (x = 0; x < 4; x++) + for (w = 0; w < 4; w++) + for (z = 0; z < 4; z++) + _t1(fwd_lift, Int)(p + 16 * z + 64 * w + 1 * x, 4); + /* transform along z */ + for (y = 0; y < 4; y++) + for (x = 0; x < 4; x++) + for (w = 0; w < 4; w++) + _t1(fwd_lift, Int)(p + 64 * w + 1 * x + 4 * y, 16); + /* transform along w */ + for (z = 0; z < 4; z++) + for (y = 0; y < 4; y++) + for (x = 0; x < 4; x++) + _t1(fwd_lift, Int)(p + 1 * x + 4 * y + 16 * z, 64); +} + +/* public functions -------------------------------------------------------- */ + +/* encode 4*4*4*4 block stored at p using strides (sx, sy, sz, sw) */ +size_t +_t2(zfp_encode_block_strided, Scalar, 4)(zfp_stream* stream, const Scalar* p, ptrdiff_t sx, ptrdiff_t sy, ptrdiff_t sz, ptrdiff_t sw) +{ + /* gather block from strided array */ + cache_align_(Scalar block[256]); + _t2(gather, Scalar, 4)(block, p, sx, sy, sz, sw); + /* encode block */ + return _t2(zfp_encode_block, Scalar, 4)(stream, block); +} + +/* encode nx*ny*nz*nw block stored at p using strides (sx, sy, sz, sw) */ +size_t +_t2(zfp_encode_partial_block_strided, Scalar, 4)(zfp_stream* stream, const Scalar* p, size_t nx, size_t ny, size_t nz, size_t nw, ptrdiff_t sx, ptrdiff_t sy, ptrdiff_t sz, ptrdiff_t sw) +{ + /* gather block from strided array */ + cache_align_(Scalar block[256]); + _t2(gather_partial, Scalar, 4)(block, p, nx, ny, nz, nw, sx, sy, sz, sw); + /* encode block */ + return _t2(zfp_encode_block, Scalar, 4)(stream, block); +} diff --git a/src/blosc2/plugins/codecs/zfp/src/template/encodef.c b/src/blosc2/plugins/codecs/zfp/src/template/encodef.c new file mode 100644 index 0000000..86da10c --- /dev/null +++ b/src/blosc2/plugins/codecs/zfp/src/template/encodef.c @@ -0,0 +1,99 @@ +#include +#include +#include + +static uint _t2(rev_encode_block, Scalar, DIMS)(zfp_stream* zfp, const Scalar* fblock); + +/* private functions ------------------------------------------------------- */ + +/* return normalized floating-point exponent for x >= 0 */ +static int +_t1(exponent, Scalar)(Scalar x) +{ + /* use e = -EBIAS when x = 0 */ + int e = -EBIAS; +#ifdef ZFP_WITH_DAZ + /* treat subnormals as zero; resolves issue #119 by avoiding overflow */ + if (x >= SCALAR_MIN) + FREXP(x, &e); +#else + if (x > 0) { + FREXP(x, &e); + /* clamp exponent in case x is subnormal; may still result in overflow */ + e = MAX(e, 1 - EBIAS); + } +#endif + return e; +} + +/* compute maximum floating-point exponent in block of n values */ +static int +_t1(exponent_block, Scalar)(const Scalar* p, uint n) +{ + Scalar max = 0; + do { + Scalar f = FABS(*p++); + if (max < f) + max = f; + } while (--n); + return _t1(exponent, Scalar)(max); +} + +/* map floating-point number x to integer relative to exponent e */ +static Scalar +_t1(quantize, Scalar)(Scalar x, int e) +{ + return LDEXP(x, ((int)(CHAR_BIT * sizeof(Scalar)) - 2) - e); +} + +/* forward block-floating-point transform to signed integers */ +static void +_t1(fwd_cast, Scalar)(Int* iblock, const Scalar* fblock, uint n, int emax) +{ + /* compute power-of-two scale factor s */ + Scalar s = _t1(quantize, Scalar)(1, emax); + /* compute p-bit int y = s*x where x is floating and |y| <= 2^(p-2) - 1 */ + do + *iblock++ = (Int)(s * *fblock++); + while (--n); +} + +/* encode contiguous floating-point block using lossy algorithm */ +static uint +_t2(encode_block, Scalar, DIMS)(zfp_stream* zfp, const Scalar* fblock) +{ + uint bits = 1; + /* compute maximum exponent */ + int emax = _t1(exponent_block, Scalar)(fblock, BLOCK_SIZE); + int maxprec = precision(emax, zfp->maxprec, zfp->minexp, DIMS); + uint e = maxprec ? emax + EBIAS : 0; + /* encode block only if biased exponent is nonzero */ + if (e) { + cache_align_(Int iblock[BLOCK_SIZE]); + /* encode common exponent; LSB indicates that exponent is nonzero */ + bits += EBITS; + stream_write_bits(zfp->stream, 2 * e + 1, bits); + /* perform forward block-floating-point transform */ + _t1(fwd_cast, Scalar)(iblock, fblock, BLOCK_SIZE, emax); + /* encode integer block */ + bits += _t2(encode_block, Int, DIMS)(zfp->stream, zfp->minbits - bits, zfp->maxbits - bits, maxprec, iblock); + } + else { + /* write single zero-bit to indicate that all values are zero */ + stream_write_bit(zfp->stream, 0); + if (zfp->minbits > bits) { + stream_pad(zfp->stream, zfp->minbits - bits); + bits = zfp->minbits; + } + } + return bits; +} + +/* public functions -------------------------------------------------------- */ + +/* encode contiguous floating-point block */ +size_t +_t2(zfp_encode_block, Scalar, DIMS)(zfp_stream* zfp, const Scalar* fblock) +{ + return REVERSIBLE(zfp) ? _t2(rev_encode_block, Scalar, DIMS)(zfp, fblock) : _t2(encode_block, Scalar, DIMS)(zfp, fblock); +} diff --git a/src/blosc2/plugins/codecs/zfp/src/template/encodei.c b/src/blosc2/plugins/codecs/zfp/src/template/encodei.c new file mode 100644 index 0000000..46b6e45 --- /dev/null +++ b/src/blosc2/plugins/codecs/zfp/src/template/encodei.c @@ -0,0 +1,15 @@ +static uint _t2(rev_encode_block, Int, DIMS)(bitstream* stream, int minbits, int maxbits, int maxprec, Int* iblock); + +/* public functions -------------------------------------------------------- */ + +/* encode contiguous integer block */ +size_t +_t2(zfp_encode_block, Int, DIMS)(zfp_stream* zfp, const Int* iblock) +{ + cache_align_(Int block[BLOCK_SIZE]); + uint i; + /* copy block */ + for (i = 0; i < BLOCK_SIZE; i++) + block[i] = iblock[i]; + return REVERSIBLE(zfp) ? _t2(rev_encode_block, Int, DIMS)(zfp->stream, zfp->minbits, zfp->maxbits, zfp->maxprec, block) : _t2(encode_block, Int, DIMS)(zfp->stream, zfp->minbits, zfp->maxbits, zfp->maxprec, block); +} diff --git a/src/blosc2/plugins/codecs/zfp/src/template/ompcompress.c b/src/blosc2/plugins/codecs/zfp/src/template/ompcompress.c new file mode 100644 index 0000000..4e4365c --- /dev/null +++ b/src/blosc2/plugins/codecs/zfp/src/template/ompcompress.c @@ -0,0 +1,275 @@ +#ifdef _OPENMP + +/* compress 1d contiguous array in parallel */ +static void +_t2(compress_omp, Scalar, 1)(zfp_stream* stream, const zfp_field* field) +{ + /* array metadata */ + const Scalar* data = (const Scalar*)field->data; + size_t nx = field->nx; + + /* number of omp threads, blocks, and chunks */ + uint threads = thread_count_omp(stream); + size_t blocks = (nx + 3) / 4; + size_t chunks = chunk_count_omp(stream, blocks, threads); + int chunk; /* OpenMP 2.0 requires int loop counter */ + + /* allocate per-thread streams */ + bitstream** bs = compress_init_par(stream, field, chunks, blocks); + if (!bs) + return; + + /* compress chunks of blocks in parallel */ + #pragma omp parallel for num_threads(threads) + for (chunk = 0; chunk < (int)chunks; chunk++) { + /* determine range of block indices assigned to this thread */ + size_t bmin = chunk_offset(blocks, chunks, chunk + 0); + size_t bmax = chunk_offset(blocks, chunks, chunk + 1); + size_t block; + /* set up thread-local bit stream */ + zfp_stream s = *stream; + zfp_stream_set_bit_stream(&s, bs[chunk]); + /* compress sequence of blocks */ + for (block = bmin; block < bmax; block++) { + /* determine block origin x within array */ + const Scalar* p = data; + size_t x = 4 * block; + p += x; + /* compress partial or full block */ + if (nx - x < 4u) + _t2(zfp_encode_partial_block_strided, Scalar, 1)(&s, p, nx - x, 1); + else + _t2(zfp_encode_block, Scalar, 1)(&s, p); + } + } + + /* concatenate per-thread streams */ + compress_finish_par(stream, bs, chunks); +} + +/* compress 1d strided array in parallel */ +static void +_t2(compress_strided_omp, Scalar, 1)(zfp_stream* stream, const zfp_field* field) +{ + /* array metadata */ + const Scalar* data = (const Scalar*)field->data; + size_t nx = field->nx; + ptrdiff_t sx = field->sx ? field->sx : 1; + + /* number of omp threads, blocks, and chunks */ + uint threads = thread_count_omp(stream); + size_t blocks = (nx + 3) / 4; + size_t chunks = chunk_count_omp(stream, blocks, threads); + int chunk; /* OpenMP 2.0 requires int loop counter */ + + /* allocate per-thread streams */ + bitstream** bs = compress_init_par(stream, field, chunks, blocks); + if (!bs) + return; + + /* compress chunks of blocks in parallel */ + #pragma omp parallel for num_threads(threads) + for (chunk = 0; chunk < (int)chunks; chunk++) { + /* determine range of block indices assigned to this thread */ + size_t bmin = chunk_offset(blocks, chunks, chunk + 0); + size_t bmax = chunk_offset(blocks, chunks, chunk + 1); + size_t block; + /* set up thread-local bit stream */ + zfp_stream s = *stream; + zfp_stream_set_bit_stream(&s, bs[chunk]); + /* compress sequence of blocks */ + for (block = bmin; block < bmax; block++) { + /* determine block origin x within array */ + const Scalar* p = data; + size_t x = 4 * block; + p += sx * (ptrdiff_t)x; + /* compress partial or full block */ + if (nx - x < 4u) + _t2(zfp_encode_partial_block_strided, Scalar, 1)(&s, p, nx - x, sx); + else + _t2(zfp_encode_block_strided, Scalar, 1)(&s, p, sx); + } + } + + /* concatenate per-thread streams */ + compress_finish_par(stream, bs, chunks); +} + +/* compress 2d strided array in parallel */ +static void +_t2(compress_strided_omp, Scalar, 2)(zfp_stream* stream, const zfp_field* field) +{ + /* array metadata */ + const Scalar* data = (const Scalar*)field->data; + size_t nx = field->nx; + size_t ny = field->ny; + ptrdiff_t sx = field->sx ? field->sx : 1; + ptrdiff_t sy = field->sy ? field->sy : (ptrdiff_t)nx; + + /* number of omp threads, blocks, and chunks */ + uint threads = thread_count_omp(stream); + size_t bx = (nx + 3) / 4; + size_t by = (ny + 3) / 4; + size_t blocks = bx * by; + size_t chunks = chunk_count_omp(stream, blocks, threads); + int chunk; /* OpenMP 2.0 requires int loop counter */ + + /* allocate per-thread streams */ + bitstream** bs = compress_init_par(stream, field, chunks, blocks); + if (!bs) + return; + + /* compress chunks of blocks in parallel */ + #pragma omp parallel for num_threads(threads) + for (chunk = 0; chunk < (int)chunks; chunk++) { + /* determine range of block indices assigned to this thread */ + size_t bmin = chunk_offset(blocks, chunks, chunk + 0); + size_t bmax = chunk_offset(blocks, chunks, chunk + 1); + size_t block; + /* set up thread-local bit stream */ + zfp_stream s = *stream; + zfp_stream_set_bit_stream(&s, bs[chunk]); + /* compress sequence of blocks */ + for (block = bmin; block < bmax; block++) { + /* determine block origin (x, y) within array */ + const Scalar* p = data; + size_t b = block; + size_t x, y; + x = 4 * (b % bx); b /= bx; + y = 4 * b; + p += sx * (ptrdiff_t)x + sy * (ptrdiff_t)y; + /* compress partial or full block */ + if (nx - x < 4u || ny - y < 4u) + _t2(zfp_encode_partial_block_strided, Scalar, 2)(&s, p, MIN(nx - x, 4u), MIN(ny - y, 4u), sx, sy); + else + _t2(zfp_encode_block_strided, Scalar, 2)(&s, p, sx, sy); + } + } + + /* concatenate per-thread streams */ + compress_finish_par(stream, bs, chunks); +} + +/* compress 3d strided array in parallel */ +static void +_t2(compress_strided_omp, Scalar, 3)(zfp_stream* stream, const zfp_field* field) +{ + /* array metadata */ + const Scalar* data = (const Scalar*)field->data; + size_t nx = field->nx; + size_t ny = field->ny; + size_t nz = field->nz; + ptrdiff_t sx = field->sx ? field->sx : 1; + ptrdiff_t sy = field->sy ? field->sy : (ptrdiff_t)nx; + ptrdiff_t sz = field->sz ? field->sz : (ptrdiff_t)(nx * ny); + + /* number of omp threads, blocks, and chunks */ + uint threads = thread_count_omp(stream); + size_t bx = (nx + 3) / 4; + size_t by = (ny + 3) / 4; + size_t bz = (nz + 3) / 4; + size_t blocks = bx * by * bz; + size_t chunks = chunk_count_omp(stream, blocks, threads); + int chunk; /* OpenMP 2.0 requires int loop counter */ + + /* allocate per-thread streams */ + bitstream** bs = compress_init_par(stream, field, chunks, blocks); + if (!bs) + return; + + /* compress chunks of blocks in parallel */ + #pragma omp parallel for num_threads(threads) + for (chunk = 0; chunk < (int)chunks; chunk++) { + /* determine range of block indices assigned to this thread */ + size_t bmin = chunk_offset(blocks, chunks, chunk + 0); + size_t bmax = chunk_offset(blocks, chunks, chunk + 1); + size_t block; + /* set up thread-local bit stream */ + zfp_stream s = *stream; + zfp_stream_set_bit_stream(&s, bs[chunk]); + /* compress sequence of blocks */ + for (block = bmin; block < bmax; block++) { + /* determine block origin (x, y, z) within array */ + const Scalar* p = data; + size_t b = block; + size_t x, y, z; + x = 4 * (b % bx); b /= bx; + y = 4 * (b % by); b /= by; + z = 4 * b; + p += sx * (ptrdiff_t)x + sy * (ptrdiff_t)y + sz * (ptrdiff_t)z; + /* compress partial or full block */ + if (nx - x < 4u || ny - y < 4u || nz - z < 4u) + _t2(zfp_encode_partial_block_strided, Scalar, 3)(&s, p, MIN(nx - x, 4u), MIN(ny - y, 4u), MIN(nz - z, 4u), sx, sy, sz); + else + _t2(zfp_encode_block_strided, Scalar, 3)(&s, p, sx, sy, sz); + } + } + + /* concatenate per-thread streams */ + compress_finish_par(stream, bs, chunks); +} + +/* compress 4d strided array in parallel */ +static void +_t2(compress_strided_omp, Scalar, 4)(zfp_stream* stream, const zfp_field* field) +{ + /* array metadata */ + const Scalar* data = field->data; + size_t nx = field->nx; + size_t ny = field->ny; + size_t nz = field->nz; + size_t nw = field->nw; + ptrdiff_t sx = field->sx ? field->sx : 1; + ptrdiff_t sy = field->sy ? field->sy : (ptrdiff_t)nx; + ptrdiff_t sz = field->sz ? field->sz : (ptrdiff_t)(nx * ny); + ptrdiff_t sw = field->sw ? field->sw : (ptrdiff_t)(nx * ny * nz); + + /* number of omp threads, blocks, and chunks */ + uint threads = thread_count_omp(stream); + size_t bx = (nx + 3) / 4; + size_t by = (ny + 3) / 4; + size_t bz = (nz + 3) / 4; + size_t bw = (nw + 3) / 4; + size_t blocks = bx * by * bz * bw; + size_t chunks = chunk_count_omp(stream, blocks, threads); + int chunk; /* OpenMP 2.0 requires int loop counter */ + + /* allocate per-thread streams */ + bitstream** bs = compress_init_par(stream, field, chunks, blocks); + if (!bs) + return; + + /* compress chunks of blocks in parallel */ + #pragma omp parallel for num_threads(threads) + for (chunk = 0; chunk < (int)chunks; chunk++) { + /* determine range of block indices assigned to this thread */ + size_t bmin = chunk_offset(blocks, chunks, chunk + 0); + size_t bmax = chunk_offset(blocks, chunks, chunk + 1); + size_t block; + /* set up thread-local bit stream */ + zfp_stream s = *stream; + zfp_stream_set_bit_stream(&s, bs[chunk]); + /* compress sequence of blocks */ + for (block = bmin; block < bmax; block++) { + /* determine block origin (x, y, z, w) within array */ + const Scalar* p = data; + size_t b = block; + size_t x, y, z, w; + x = 4 * (b % bx); b /= bx; + y = 4 * (b % by); b /= by; + z = 4 * (b % bz); b /= bz; + w = 4 * b; + p += sx * (ptrdiff_t)x + sy * (ptrdiff_t)y + sz * (ptrdiff_t)z + sw * (ptrdiff_t)w; + /* compress partial or full block */ + if (nx - x < 4u || ny - y < 4u || nz - z < 4u || nw - w < 4u) + _t2(zfp_encode_partial_block_strided, Scalar, 4)(&s, p, MIN(nx - x, 4u), MIN(ny - y, 4u), MIN(nz - z, 4u), MIN(nw - w, 4u), sx, sy, sz, sw); + else + _t2(zfp_encode_block_strided, Scalar, 4)(&s, p, sx, sy, sz, sw); + } + } + + /* concatenate per-thread streams */ + compress_finish_par(stream, bs, chunks); +} + +#endif diff --git a/src/blosc2/plugins/codecs/zfp/src/template/revcodecf.c b/src/blosc2/plugins/codecs/zfp/src/template/revcodecf.c new file mode 100644 index 0000000..ad5f788 --- /dev/null +++ b/src/blosc2/plugins/codecs/zfp/src/template/revcodecf.c @@ -0,0 +1,11 @@ +/* inverse block-floating-point transform from signed integers */ +static void +_t1(rev_inv_cast, Scalar)(const Int* iblock, Scalar* fblock, uint n, int emax) +{ + /* test for all-zero block, which needs special treatment */ + if (emax != -EBIAS) + _t1(inv_cast, Scalar)(iblock, fblock, n, emax); + else + while (n--) + *fblock++ = 0; +} diff --git a/src/blosc2/plugins/codecs/zfp/src/template/revdecode.c b/src/blosc2/plugins/codecs/zfp/src/template/revdecode.c new file mode 100644 index 0000000..07205f5 --- /dev/null +++ b/src/blosc2/plugins/codecs/zfp/src/template/revdecode.c @@ -0,0 +1,52 @@ +static void _t2(rev_inv_xform, Int, DIMS)(Int* p); + +/* private functions ------------------------------------------------------- */ + +/* reversible inverse lifting transform of 4-vector */ +static void +_t1(rev_inv_lift, Int)(Int* p, uint s) +{ + Int x, y, z, w; + x = *p; p += s; + y = *p; p += s; + z = *p; p += s; + w = *p; p += s; + + /* + ** high-order Lorenzo transform (P4 Pascal matrix) + ** ( 1 0 0 0) (x) + ** ( 1 1 0 0) (y) + ** ( 1 2 1 0) (z) + ** ( 1 3 3 1) (w) + */ + w += z; + z += y; w += z; + y += x; z += y; w += z; + + p -= s; *p = w; + p -= s; *p = z; + p -= s; *p = y; + p -= s; *p = x; +} + +/* decode block of integers using reversible algorithm */ +static uint +_t2(rev_decode_block, Int, DIMS)(bitstream* stream, int minbits, int maxbits, Int* iblock) +{ + /* decode number of significant bits */ + int bits = PBITS; + int prec = (int)stream_read_bits(stream, PBITS) + 1; + cache_align_(UInt ublock[BLOCK_SIZE]); + /* decode integer coefficients */ + bits += _t1(decode_ints, UInt)(stream, maxbits - bits, prec, ublock, BLOCK_SIZE); + /* read at least minbits bits */ + if (bits < minbits) { + stream_skip(stream, minbits - bits); + bits = minbits; + } + /* reorder unsigned coefficients and convert to signed integer */ + _t1(inv_order, Int)(ublock, iblock, PERM, BLOCK_SIZE); + /* perform decorrelating transform */ + _t2(rev_inv_xform, Int, DIMS)(iblock); + return bits; +} diff --git a/src/blosc2/plugins/codecs/zfp/src/template/revdecode1.c b/src/blosc2/plugins/codecs/zfp/src/template/revdecode1.c new file mode 100644 index 0000000..018a101 --- /dev/null +++ b/src/blosc2/plugins/codecs/zfp/src/template/revdecode1.c @@ -0,0 +1,9 @@ +/* private functions ------------------------------------------------------- */ + +/* reversible inverse decorrelating 1D transform */ +static void +_t2(rev_inv_xform, Int, 1)(Int* p) +{ + /* transform along x */ + _t1(rev_inv_lift, Int)(p, 1); +} diff --git a/src/blosc2/plugins/codecs/zfp/src/template/revdecode2.c b/src/blosc2/plugins/codecs/zfp/src/template/revdecode2.c new file mode 100644 index 0000000..9cb8669 --- /dev/null +++ b/src/blosc2/plugins/codecs/zfp/src/template/revdecode2.c @@ -0,0 +1,14 @@ +/* private functions ------------------------------------------------------- */ + +/* reversible inverse decorrelating 2D transform */ +static void +_t2(rev_inv_xform, Int, 2)(Int* p) +{ + uint x, y; + /* transform along y */ + for (x = 0; x < 4; x++) + _t1(rev_inv_lift, Int)(p + 1 * x, 4); + /* transform along x */ + for (y = 0; y < 4; y++) + _t1(rev_inv_lift, Int)(p + 4 * y, 1); +} diff --git a/src/blosc2/plugins/codecs/zfp/src/template/revdecode3.c b/src/blosc2/plugins/codecs/zfp/src/template/revdecode3.c new file mode 100644 index 0000000..1219443 --- /dev/null +++ b/src/blosc2/plugins/codecs/zfp/src/template/revdecode3.c @@ -0,0 +1,20 @@ +/* private functions ------------------------------------------------------- */ + +/* reversible inverse decorrelating 3D transform */ +static void +_t2(rev_inv_xform, Int, 3)(Int* p) +{ + uint x, y, z; + /* transform along z */ + for (y = 0; y < 4; y++) + for (x = 0; x < 4; x++) + _t1(rev_inv_lift, Int)(p + 1 * x + 4 * y, 16); + /* transform along y */ + for (x = 0; x < 4; x++) + for (z = 0; z < 4; z++) + _t1(rev_inv_lift, Int)(p + 16 * z + 1 * x, 4); + /* transform along x */ + for (z = 0; z < 4; z++) + for (y = 0; y < 4; y++) + _t1(rev_inv_lift, Int)(p + 4 * y + 16 * z, 1); +} diff --git a/src/blosc2/plugins/codecs/zfp/src/template/revdecode4.c b/src/blosc2/plugins/codecs/zfp/src/template/revdecode4.c new file mode 100644 index 0000000..98a98f8 --- /dev/null +++ b/src/blosc2/plugins/codecs/zfp/src/template/revdecode4.c @@ -0,0 +1,28 @@ +/* private functions ------------------------------------------------------- */ + +/* reversible inverse decorrelating 4D transform */ +static void +_t2(rev_inv_xform, Int, 4)(Int* p) +{ + uint x, y, z, w; + /* transform along w */ + for (z = 0; z < 4; z++) + for (y = 0; y < 4; y++) + for (x = 0; x < 4; x++) + _t1(rev_inv_lift, Int)(p + 1 * x + 4 * y + 16 * z, 64); + /* transform along z */ + for (y = 0; y < 4; y++) + for (x = 0; x < 4; x++) + for (w = 0; w < 4; w++) + _t1(rev_inv_lift, Int)(p + 64 * w + 1 * x + 4 * y, 16); + /* transform along y */ + for (x = 0; x < 4; x++) + for (w = 0; w < 4; w++) + for (z = 0; z < 4; z++) + _t1(rev_inv_lift, Int)(p + 16 * z + 64 * w + 1 * x, 4); + /* transform along x */ + for (w = 0; w < 4; w++) + for (z = 0; z < 4; z++) + for (y = 0; y < 4; y++) + _t1(rev_inv_lift, Int)(p + 4 * y + 16 * z + 64 * w, 1); +} diff --git a/src/blosc2/plugins/codecs/zfp/src/template/revdecodef.c b/src/blosc2/plugins/codecs/zfp/src/template/revdecodef.c new file mode 100644 index 0000000..6777ce0 --- /dev/null +++ b/src/blosc2/plugins/codecs/zfp/src/template/revdecodef.c @@ -0,0 +1,59 @@ +#include + +/* private functions ------------------------------------------------------- */ + +/* reinterpret two's complement integers as floating values */ +static void +_t1(rev_inv_reinterpret, Scalar)(Int* iblock, Scalar* fblock, uint n) +{ + /* convert two's complement integers to sign-magnitude integers */ + uint i; + for (i = 0; i < n; i++) { + Int x = iblock[i]; + if (x < 0) + iblock[i] = (Int)((UInt)x ^ TCMASK); + } + /* reinterpret sign-magnitude integers as floating values */ + memcpy(fblock, iblock, n * sizeof(*fblock)); +} + +/* decode contiguous floating-point block using reversible algorithm */ +static uint +_t2(rev_decode_block, Scalar, DIMS)(zfp_stream* zfp, Scalar* fblock) +{ + uint bits = 0; + cache_align_(Int iblock[BLOCK_SIZE]); + /* test whether block is all-zero */ + bits++; + if (stream_read_bit(zfp->stream)) { + /* non-zero block; test whether to use block-floating-point transform */ + bits++; + if (stream_read_bit(zfp->stream)) { + /* decode integer block */ + bits += _t2(rev_decode_block, Int, DIMS)(zfp->stream, zfp->minbits - bits, zfp->maxbits - bits, iblock); + /* reinterpret integers as floating values */ + _t1(rev_inv_reinterpret, Scalar)(iblock, fblock, BLOCK_SIZE); + } + else { + /* decode common exponent */ + int emax; + bits += EBITS; + emax = (int)stream_read_bits(zfp->stream, EBITS) - EBIAS; + /* decode integer block */ + bits += _t2(rev_decode_block, Int, DIMS)(zfp->stream, zfp->minbits - bits, zfp->maxbits - bits, iblock); + /* perform inverse block-floating-point transform */ + _t1(rev_inv_cast, Scalar)(iblock, fblock, BLOCK_SIZE, emax); + } + } + else { + /* all-zero block; set all values to zero */ + uint i; + for (i = 0; i < BLOCK_SIZE; i++) + *fblock++ = 0; + if (zfp->minbits > bits) { + stream_skip(zfp->stream, zfp->minbits - bits); + bits = zfp->minbits; + } + } + return bits; +} diff --git a/src/blosc2/plugins/codecs/zfp/src/template/revencode.c b/src/blosc2/plugins/codecs/zfp/src/template/revencode.c new file mode 100644 index 0000000..3473cdd --- /dev/null +++ b/src/blosc2/plugins/codecs/zfp/src/template/revencode.c @@ -0,0 +1,76 @@ +static void _t2(rev_fwd_xform, Int, DIMS)(Int* p); + +/* private functions ------------------------------------------------------- */ + +/* reversible forward lifting transform of 4-vector */ +static void +_t1(rev_fwd_lift, Int)(Int* p, uint s) +{ + Int x, y, z, w; + x = *p; p += s; + y = *p; p += s; + z = *p; p += s; + w = *p; p += s; + + /* + ** high-order Lorenzo transform + ** ( 1 0 0 0) (x) + ** (-1 1 0 0) (y) + ** ( 1 -2 1 0) (z) + ** (-1 3 -3 1) (w) + */ + w -= z; z -= y; y -= x; + w -= z; z -= y; + w -= z; + + p -= s; *p = w; + p -= s; *p = z; + p -= s; *p = y; + p -= s; *p = x; +} + +/* return precision required to encode block reversibly */ +static uint +_t1(rev_precision, UInt)(const UInt* block, uint n) +{ + uint p = 0; + uint s; + /* compute bitwise OR of all values */ + UInt m = 0; + while (n--) + m |= *block++; + /* count trailing zeros via binary search */ + for (s = (uint)(CHAR_BIT * sizeof(UInt)); m; s /= 2) + if ((UInt)(m << (s - 1))) { + m <<= s - 1; + m <<= 1; + p += s; + } + return p; +} + +/* encode block of integers using reversible algorithm */ +static uint +_t2(rev_encode_block, Int, DIMS)(bitstream* stream, int minbits, int maxbits, int maxprec, Int* iblock) +{ + int bits = PBITS; + int prec; + cache_align_(UInt ublock[BLOCK_SIZE]); + /* perform decorrelating transform */ + _t2(rev_fwd_xform, Int, DIMS)(iblock); + /* reorder signed coefficients and convert to unsigned integer */ + _t1(fwd_order, Int)(ublock, iblock, PERM, BLOCK_SIZE); + /* determine and encode number of significant bits */ + prec = _t1(rev_precision, UInt)(ublock, BLOCK_SIZE); + prec = MIN(prec, maxprec); + prec = MAX(prec, 1); + stream_write_bits(stream, prec - 1, PBITS); + /* encode integer coefficients */ + bits += _t1(encode_ints, UInt)(stream, maxbits - bits, prec, ublock, BLOCK_SIZE); + /* write at least minbits bits by padding with zeros */ + if (bits < minbits) { + stream_pad(stream, minbits - bits); + bits = minbits; + } + return bits; +} diff --git a/src/blosc2/plugins/codecs/zfp/src/template/revencode1.c b/src/blosc2/plugins/codecs/zfp/src/template/revencode1.c new file mode 100644 index 0000000..1e7b29a --- /dev/null +++ b/src/blosc2/plugins/codecs/zfp/src/template/revencode1.c @@ -0,0 +1,9 @@ +/* private functions ------------------------------------------------------- */ + +/* reversible forward decorrelating 1D transform */ +static void +_t2(rev_fwd_xform, Int, 1)(Int* p) +{ + /* transform along x */ + _t1(rev_fwd_lift, Int)(p, 1); +} diff --git a/src/blosc2/plugins/codecs/zfp/src/template/revencode2.c b/src/blosc2/plugins/codecs/zfp/src/template/revencode2.c new file mode 100644 index 0000000..621fc0c --- /dev/null +++ b/src/blosc2/plugins/codecs/zfp/src/template/revencode2.c @@ -0,0 +1,14 @@ +/* private functions ------------------------------------------------------- */ + +/* reversible forward decorrelating 2D transform */ +static void +_t2(rev_fwd_xform, Int, 2)(Int* p) +{ + uint x, y; + /* transform along x */ + for (y = 0; y < 4; y++) + _t1(rev_fwd_lift, Int)(p + 4 * y, 1); + /* transform along y */ + for (x = 0; x < 4; x++) + _t1(rev_fwd_lift, Int)(p + 1 * x, 4); +} diff --git a/src/blosc2/plugins/codecs/zfp/src/template/revencode3.c b/src/blosc2/plugins/codecs/zfp/src/template/revencode3.c new file mode 100644 index 0000000..64d49f1 --- /dev/null +++ b/src/blosc2/plugins/codecs/zfp/src/template/revencode3.c @@ -0,0 +1,20 @@ +/* private functions ------------------------------------------------------- */ + +/* reversible forward decorrelating 3D transform */ +static void +_t2(rev_fwd_xform, Int, 3)(Int* p) +{ + uint x, y, z; + /* transform along x */ + for (z = 0; z < 4; z++) + for (y = 0; y < 4; y++) + _t1(rev_fwd_lift, Int)(p + 4 * y + 16 * z, 1); + /* transform along y */ + for (x = 0; x < 4; x++) + for (z = 0; z < 4; z++) + _t1(rev_fwd_lift, Int)(p + 16 * z + 1 * x, 4); + /* transform along z */ + for (y = 0; y < 4; y++) + for (x = 0; x < 4; x++) + _t1(rev_fwd_lift, Int)(p + 1 * x + 4 * y, 16); +} diff --git a/src/blosc2/plugins/codecs/zfp/src/template/revencode4.c b/src/blosc2/plugins/codecs/zfp/src/template/revencode4.c new file mode 100644 index 0000000..46e2a46 --- /dev/null +++ b/src/blosc2/plugins/codecs/zfp/src/template/revencode4.c @@ -0,0 +1,28 @@ +/* private functions ------------------------------------------------------- */ + +/* reversible forward decorrelating 4D transform */ +static void +_t2(rev_fwd_xform, Int, 4)(Int* p) +{ + uint x, y, z, w; + /* transform along x */ + for (w = 0; w < 4; w++) + for (z = 0; z < 4; z++) + for (y = 0; y < 4; y++) + _t1(rev_fwd_lift, Int)(p + 4 * y + 16 * z + 64 * w, 1); + /* transform along y */ + for (x = 0; x < 4; x++) + for (w = 0; w < 4; w++) + for (z = 0; z < 4; z++) + _t1(rev_fwd_lift, Int)(p + 16 * z + 64 * w + 1 * x, 4); + /* transform along z */ + for (y = 0; y < 4; y++) + for (x = 0; x < 4; x++) + for (w = 0; w < 4; w++) + _t1(rev_fwd_lift, Int)(p + 64 * w + 1 * x + 4 * y, 16); + /* transform along w */ + for (z = 0; z < 4; z++) + for (y = 0; y < 4; y++) + for (x = 0; x < 4; x++) + _t1(rev_fwd_lift, Int)(p + 1 * x + 4 * y + 16 * z, 64); +} diff --git a/src/blosc2/plugins/codecs/zfp/src/template/revencodef.c b/src/blosc2/plugins/codecs/zfp/src/template/revencodef.c new file mode 100644 index 0000000..789f9b9 --- /dev/null +++ b/src/blosc2/plugins/codecs/zfp/src/template/revencodef.c @@ -0,0 +1,80 @@ +#include + +/* private functions ------------------------------------------------------- */ + +/* test if block-floating-point encoding is reversible */ +static int +_t1(rev_fwd_reversible, Scalar)(const Int* iblock, const Scalar* fblock, uint n, int emax) +{ + /* reconstruct block */ + cache_align_(Scalar gblock[BLOCK_SIZE]); + _t1(rev_inv_cast, Scalar)(iblock, gblock, n, emax); + /* perform bit-wise comparison */ + return !memcmp(fblock, gblock, n * sizeof(*fblock)); +} + +/* forward block-floating-point transform to signed integers */ +static void +_t1(rev_fwd_cast, Scalar)(Int* iblock, const Scalar* fblock, uint n, int emax) +{ + /* test for all-zero block, which needs special treatment */ + if (emax != -EBIAS) + _t1(fwd_cast, Scalar)(iblock, fblock, BLOCK_SIZE, emax); + else + while (n--) + *iblock++ = 0; +} + +/* reinterpret floating values as two's complement integers */ +static void +_t1(rev_fwd_reinterpret, Scalar)(Int* iblock, const Scalar* fblock, uint n) +{ + /* reinterpret floating values as sign-magnitude integers */ + memcpy(iblock, fblock, n * sizeof(*iblock)); + /* convert sign-magnitude integers to two's complement integers */ + while (n--) { + Int x = *iblock; + if (x < 0) + *iblock = (Int)((UInt)x ^ TCMASK); + iblock++; + } +} + +/* encode contiguous floating-point block using reversible algorithm */ +static uint +_t2(rev_encode_block, Scalar, DIMS)(zfp_stream* zfp, const Scalar* fblock) +{ + uint bits = 0; + cache_align_(Int iblock[BLOCK_SIZE]); + /* compute maximum exponent */ + int emax = _t1(exponent_block, Scalar)(fblock, BLOCK_SIZE); + /* perform forward block-floating-point transform */ + _t1(rev_fwd_cast, Scalar)(iblock, fblock, BLOCK_SIZE, emax); + /* test if block-floating-point transform is reversible */ + if (_t1(rev_fwd_reversible, Scalar)(iblock, fblock, BLOCK_SIZE, emax)) { + /* transform is reversible; test if block has any non-zeros */ + uint e = emax + EBIAS; + if (e) { + /* encode common exponent */ + bits += 2; + stream_write_bits(zfp->stream, 1, 2); + bits += EBITS; + stream_write_bits(zfp->stream, e, EBITS); + } + else { + /* emit single bit for all-zero block */ + bits++; + stream_write_bit(zfp->stream, 0); + return bits; + } + } + else { + /* transform is irreversible; reinterpret floating values as integers */ + _t1(rev_fwd_reinterpret, Scalar)(iblock, fblock, BLOCK_SIZE); + bits += 2; + stream_write_bits(zfp->stream, 3, 2); + } + /* losslessly encode integers */ + bits += _t2(rev_encode_block, Int, DIMS)(zfp->stream, zfp->minbits - bits, zfp->maxbits - bits, zfp->maxprec, iblock); + return bits; +} diff --git a/src/blosc2/plugins/codecs/zfp/src/template/template.h b/src/blosc2/plugins/codecs/zfp/src/template/template.h new file mode 100644 index 0000000..fd5becf --- /dev/null +++ b/src/blosc2/plugins/codecs/zfp/src/template/template.h @@ -0,0 +1,12 @@ +#ifndef TEMPLATE_H +#define TEMPLATE_H + +/* concatenation */ +#define _cat2(x, y) x ## _ ## y +#define _cat3(x, y, z) x ## _ ## y ## _ ## z + +/* 1- and 2-argument function templates */ +#define _t1(function, arg) _cat2(function, arg) +#define _t2(function, type, dims) _cat3(function, type, dims) + +#endif diff --git a/src/blosc2/plugins/codecs/zfp/src/traitsd.h b/src/blosc2/plugins/codecs/zfp/src/traitsd.h new file mode 100644 index 0000000..05110d5 --- /dev/null +++ b/src/blosc2/plugins/codecs/zfp/src/traitsd.h @@ -0,0 +1,14 @@ +/* double-precision floating-point traits */ + +#define Scalar double /* floating-point type */ +#define Int int64 /* corresponding signed integer type */ +#define UInt uint64 /* corresponding unsigned integer type */ +#define EBITS 11 /* number of exponent bits */ +#define PBITS 6 /* number of bits needed to encode precision */ +#define NBMASK UINT64C(0xaaaaaaaaaaaaaaaa) /* negabinary mask */ +#define TCMASK UINT64C(0x7fffffffffffffff) /* two's complement mask */ +#define SCALAR_MIN DBL_MIN /* smallest positive normal number */ + +#define FABS(x) fabs(x) +#define FREXP(x, e) frexp(x, e) +#define LDEXP(x, e) ldexp(x, e) diff --git a/src/blosc2/plugins/codecs/zfp/src/traitsf.h b/src/blosc2/plugins/codecs/zfp/src/traitsf.h new file mode 100644 index 0000000..7e85299 --- /dev/null +++ b/src/blosc2/plugins/codecs/zfp/src/traitsf.h @@ -0,0 +1,20 @@ +/* single-precision floating-point traits */ + +#define Scalar float /* floating-point type */ +#define Int int32 /* corresponding signed integer type */ +#define UInt uint32 /* corresponding unsigned integer type */ +#define EBITS 8 /* number of exponent bits */ +#define PBITS 5 /* number of bits needed to encode precision */ +#define NBMASK 0xaaaaaaaau /* negabinary mask */ +#define TCMASK 0x7fffffffu /* two's complement mask */ +#define SCALAR_MIN FLT_MIN /* smallest positive normal number */ + +#if __STDC_VERSION__ >= 199901L + #define FABS(x) fabsf(x) + #define FREXP(x, e) frexpf(x, e) + #define LDEXP(x, e) ldexpf(x, e) +#else + #define FABS(x) (float)fabs(x) + #define FREXP(x, e) (void)frexp(x, e) + #define LDEXP(x, e) (float)ldexp(x, e) +#endif diff --git a/src/blosc2/plugins/codecs/zfp/src/traitsi.h b/src/blosc2/plugins/codecs/zfp/src/traitsi.h new file mode 100644 index 0000000..a540fb4 --- /dev/null +++ b/src/blosc2/plugins/codecs/zfp/src/traitsi.h @@ -0,0 +1,7 @@ +/* 32-bit integer traits */ + +#define Scalar int32 /* integer type */ +#define Int int32 /* corresponding signed integer type */ +#define UInt uint32 /* corresponding unsigned integer type */ +#define PBITS 5 /* number of bits needed to encode precision */ +#define NBMASK 0xaaaaaaaau /* negabinary mask */ diff --git a/src/blosc2/plugins/codecs/zfp/src/traitsl.h b/src/blosc2/plugins/codecs/zfp/src/traitsl.h new file mode 100644 index 0000000..ffa01e0 --- /dev/null +++ b/src/blosc2/plugins/codecs/zfp/src/traitsl.h @@ -0,0 +1,7 @@ +/* 64-bit integer traits */ + +#define Scalar int64 /* integer type */ +#define Int int64 /* corresponding signed integer type */ +#define UInt uint64 /* corresponding unsigned integer type */ +#define PBITS 6 /* number of bits needed to encode precision */ +#define NBMASK UINT64C(0xaaaaaaaaaaaaaaaa) /* negabinary mask */ diff --git a/src/blosc2/plugins/codecs/zfp/src/zfp.c b/src/blosc2/plugins/codecs/zfp/src/zfp.c new file mode 100644 index 0000000..68d12fe --- /dev/null +++ b/src/blosc2/plugins/codecs/zfp/src/zfp.c @@ -0,0 +1,1209 @@ +#include +#include +#include +#include +#include "zfp.h" +#include "zfp/macros.h" +#include "zfp/version.h" +#include "template/template.h" + +/* public data ------------------------------------------------------------- */ + +const uint zfp_codec_version = ZFP_CODEC; +const uint zfp_library_version = ZFP_VERSION; +const char* const zfp_version_string = "zfp version " ZFP_VERSION_STRING " (May 5, 2019)"; + +/* private functions ------------------------------------------------------- */ + +static size_t +field_index_span(const zfp_field* field, ptrdiff_t* min, ptrdiff_t* max) +{ + /* compute strides */ + ptrdiff_t sx = field->sx ? field->sx : 1; + ptrdiff_t sy = field->sy ? field->sy : (ptrdiff_t)field->nx; + ptrdiff_t sz = field->sz ? field->sz : (ptrdiff_t)(field->nx * field->ny); + ptrdiff_t sw = field->sw ? field->sw : (ptrdiff_t)(field->nx * field->ny * field->nz); + /* compute largest offsets from base pointer */ + ptrdiff_t dx = field->nx ? sx * (ptrdiff_t)(field->nx - 1) : 0; + ptrdiff_t dy = field->ny ? sy * (ptrdiff_t)(field->ny - 1) : 0; + ptrdiff_t dz = field->nz ? sz * (ptrdiff_t)(field->nz - 1) : 0; + ptrdiff_t dw = field->nw ? sw * (ptrdiff_t)(field->nw - 1) : 0; + /* compute lowest and highest offset */ + ptrdiff_t imin = MIN(dx, 0) + MIN(dy, 0) + MIN(dz, 0) + MIN(dw, 0); + ptrdiff_t imax = MAX(dx, 0) + MAX(dy, 0) + MAX(dz, 0) + MAX(dw, 0); + if (min) + *min = imin; + if (max) + *max = imax; + return imax - imin + 1; +} + +static zfp_bool +is_reversible(const zfp_stream* zfp) +{ + return zfp->minexp < ZFP_MIN_EXP; +} + +/* shared code across template instances ------------------------------------*/ + +#include "share/parallel.c" +#include "share/omp.c" + +/* template instantiation of integer and float compressor -------------------*/ + +#define Scalar int32 +#include "template/compress.c" +#include "template/decompress.c" +#include "template/ompcompress.c" +#undef Scalar + +#define Scalar int64 +#include "template/compress.c" +#include "template/decompress.c" +#include "template/ompcompress.c" +#undef Scalar + +#define Scalar float +#include "template/compress.c" +#include "template/decompress.c" +#include "template/ompcompress.c" +#undef Scalar + +#define Scalar double +#include "template/compress.c" +#include "template/decompress.c" +#include "template/ompcompress.c" +#undef Scalar + +/* public functions: miscellaneous ----------------------------------------- */ + +size_t +zfp_type_size(zfp_type type) +{ + switch (type) { + case zfp_type_int32: + return sizeof(int32); + case zfp_type_int64: + return sizeof(int64); + case zfp_type_float: + return sizeof(float); + case zfp_type_double: + return sizeof(double); + default: + return 0; + } +} + +/* public functions: fields ------------------------------------------------ */ + +zfp_field* +zfp_field_alloc() +{ + zfp_field* field = (zfp_field*)malloc(sizeof(zfp_field)); + if (field) { + field->type = zfp_type_none; + field->nx = field->ny = field->nz = field->nw = 0; + field->sx = field->sy = field->sz = field->sw = 0; + field->data = 0; + } + return field; +} + +zfp_field* +zfp_field_1d(void* data, zfp_type type, size_t nx) +{ + zfp_field* field = zfp_field_alloc(); + if (field) { + field->type = type; + field->nx = nx; + field->data = data; + } + return field; +} + +zfp_field* +zfp_field_2d(void* data, zfp_type type, size_t nx, size_t ny) +{ + zfp_field* field = zfp_field_alloc(); + if (field) { + field->type = type; + field->nx = nx; + field->ny = ny; + field->data = data; + } + return field; +} + +zfp_field* +zfp_field_3d(void* data, zfp_type type, size_t nx, size_t ny, size_t nz) +{ + zfp_field* field = zfp_field_alloc(); + if (field) { + field->type = type; + field->nx = nx; + field->ny = ny; + field->nz = nz; + field->data = data; + } + return field; +} + +zfp_field* +zfp_field_4d(void* data, zfp_type type, size_t nx, size_t ny, size_t nz, size_t nw) +{ + zfp_field* field = zfp_field_alloc(); + if (field) { + field->type = type; + field->nx = nx; + field->ny = ny; + field->nz = nz; + field->nw = nw; + field->data = data; + } + return field; +} + +void +zfp_field_free(zfp_field* field) +{ + free(field); +} + +void* +zfp_field_pointer(const zfp_field* field) +{ + return field->data; +} + +void* +zfp_field_begin(const zfp_field* field) +{ + if (field->data) { + ptrdiff_t min; + field_index_span(field, &min, NULL); + return (void*)((uchar*)field->data + min * (ptrdiff_t)zfp_type_size(field->type)); + } + else + return NULL; +} + +zfp_type +zfp_field_type(const zfp_field* field) +{ + return field->type; +} + +uint +zfp_field_precision(const zfp_field* field) +{ + return (uint)(CHAR_BIT * zfp_type_size(field->type)); +} + +uint +zfp_field_dimensionality(const zfp_field* field) +{ + return field->nx ? field->ny ? field->nz ? field->nw ? 4 : 3 : 2 : 1 : 0; +} + +size_t +zfp_field_size(const zfp_field* field, size_t* size) +{ + if (size) + switch (zfp_field_dimensionality(field)) { + case 4: + size[3] = field->nw; + /* FALLTHROUGH */ + case 3: + size[2] = field->nz; + /* FALLTHROUGH */ + case 2: + size[1] = field->ny; + /* FALLTHROUGH */ + case 1: + size[0] = field->nx; + break; + } + return MAX(field->nx, 1u) * MAX(field->ny, 1u) * MAX(field->nz, 1u) * MAX(field->nw, 1u); +} + +size_t +zfp_field_size_bytes(const zfp_field* field) +{ + return field_index_span(field, NULL, NULL) * zfp_type_size(field->type); +} + +zfp_bool +zfp_field_stride(const zfp_field* field, ptrdiff_t* stride) +{ + if (stride) + switch (zfp_field_dimensionality(field)) { + case 4: + stride[3] = field->sw ? field->sw : (ptrdiff_t)(field->nx * field->ny * field->nz); + /* FALLTHROUGH */ + case 3: + stride[2] = field->sz ? field->sz : (ptrdiff_t)(field->nx * field->ny); + /* FALLTHROUGH */ + case 2: + stride[1] = field->sy ? field->sy : (ptrdiff_t)field->nx; + /* FALLTHROUGH */ + case 1: + stride[0] = field->sx ? field->sx : 1; + break; + } + return field->sx || field->sy || field->sz || field->sw; +} + +zfp_bool +zfp_field_is_contiguous(const zfp_field* field) +{ + return field_index_span(field, NULL, NULL) == zfp_field_size(field, NULL); +} + +uint64 +zfp_field_metadata(const zfp_field* field) +{ + uint64 meta = 0; + /* 48 bits for dimensions */ + switch (zfp_field_dimensionality(field)) { + case 1: + if ((uint64)(field->nx - 1) >> 48) + return ZFP_META_NULL; + meta <<= 48; meta += field->nx - 1; + break; + case 2: + if (((field->nx - 1) >> 24) || + ((field->ny - 1) >> 24)) + return ZFP_META_NULL; + meta <<= 24; meta += field->ny - 1; + meta <<= 24; meta += field->nx - 1; + break; + case 3: + if (((field->nx - 1) >> 16) || + ((field->ny - 1) >> 16) || + ((field->nz - 1) >> 16)) + return ZFP_META_NULL; + meta <<= 16; meta += field->nz - 1; + meta <<= 16; meta += field->ny - 1; + meta <<= 16; meta += field->nx - 1; + break; + case 4: + if (((field->nx - 1) >> 12) || + ((field->ny - 1) >> 12) || + ((field->nz - 1) >> 12) || + ((field->nw - 1) >> 12)) + return ZFP_META_NULL; + meta <<= 12; meta += field->nw - 1; + meta <<= 12; meta += field->nz - 1; + meta <<= 12; meta += field->ny - 1; + meta <<= 12; meta += field->nx - 1; + break; + } + /* 2 bits for dimensionality (1D, 2D, 3D, 4D) */ + meta <<= 2; meta += zfp_field_dimensionality(field) - 1; + /* 2 bits for scalar type */ + meta <<= 2; meta += field->type - 1; + return meta; +} + +void +zfp_field_set_pointer(zfp_field* field, void* data) +{ + field->data = data; +} + +zfp_type +zfp_field_set_type(zfp_field* field, zfp_type type) +{ + switch (type) { + case zfp_type_int32: + case zfp_type_int64: + case zfp_type_float: + case zfp_type_double: + field->type = type; + return type; + default: + return zfp_type_none; + } +} + +void +zfp_field_set_size_1d(zfp_field* field, size_t n) +{ + field->nx = n; + field->ny = 0; + field->nz = 0; + field->nw = 0; +} + +void +zfp_field_set_size_2d(zfp_field* field, size_t nx, size_t ny) +{ + field->nx = nx; + field->ny = ny; + field->nz = 0; + field->nw = 0; +} + +void +zfp_field_set_size_3d(zfp_field* field, size_t nx, size_t ny, size_t nz) +{ + field->nx = nx; + field->ny = ny; + field->nz = nz; + field->nw = 0; +} + +void +zfp_field_set_size_4d(zfp_field* field, size_t nx, size_t ny, size_t nz, size_t nw) +{ + field->nx = nx; + field->ny = ny; + field->nz = nz; + field->nw = nw; +} + +void +zfp_field_set_stride_1d(zfp_field* field, ptrdiff_t sx) +{ + field->sx = sx; + field->sy = 0; + field->sz = 0; + field->sw = 0; +} + +void +zfp_field_set_stride_2d(zfp_field* field, ptrdiff_t sx, ptrdiff_t sy) +{ + field->sx = sx; + field->sy = sy; + field->sz = 0; + field->sw = 0; +} + +void +zfp_field_set_stride_3d(zfp_field* field, ptrdiff_t sx, ptrdiff_t sy, ptrdiff_t sz) +{ + field->sx = sx; + field->sy = sy; + field->sz = sz; + field->sw = 0; +} + +void +zfp_field_set_stride_4d(zfp_field* field, ptrdiff_t sx, ptrdiff_t sy, ptrdiff_t sz, ptrdiff_t sw) +{ + field->sx = sx; + field->sy = sy; + field->sz = sz; + field->sw = sw; +} + +zfp_bool +zfp_field_set_metadata(zfp_field* field, uint64 meta) +{ + uint64 dims; + /* ensure value is in range */ + if (meta >> ZFP_META_BITS) + return zfp_false; + field->type = (zfp_type)((meta & 0x3u) + 1); meta >>= 2; + dims = (meta & 0x3u) + 1; meta >>= 2; + switch (dims) { + case 1: + /* currently dimensions are limited to 2^32 - 1 */ + field->nx = (size_t)(meta & UINT64C(0x0000ffffffff)) + 1; meta >>= 48; + field->ny = 0; + field->nz = 0; + field->nw = 0; + break; + case 2: + field->nx = (size_t)(meta & UINT64C(0xffffff)) + 1; meta >>= 24; + field->ny = (size_t)(meta & UINT64C(0xffffff)) + 1; meta >>= 24; + field->nz = 0; + field->nw = 0; + break; + case 3: + field->nx = (size_t)(meta & UINT64C(0xffff)) + 1; meta >>= 16; + field->ny = (size_t)(meta & UINT64C(0xffff)) + 1; meta >>= 16; + field->nz = (size_t)(meta & UINT64C(0xffff)) + 1; meta >>= 16; + field->nw = 0; + break; + case 4: + field->nx = (size_t)(meta & UINT64C(0xfff)) + 1; meta >>= 12; + field->ny = (size_t)(meta & UINT64C(0xfff)) + 1; meta >>= 12; + field->nz = (size_t)(meta & UINT64C(0xfff)) + 1; meta >>= 12; + field->nw = (size_t)(meta & UINT64C(0xfff)) + 1; meta >>= 12; + break; + } + field->sx = field->sy = field->sz = field->sw = 0; + return zfp_true; +} + +/* public functions: compression mode and parameter settings --------------- */ + +zfp_config +zfp_config_none() +{ + zfp_config config; + config.mode = zfp_mode_null; + return config; +} + +zfp_config +zfp_config_rate( + double rate, + zfp_bool align +) +{ + zfp_config config; + config.mode = zfp_mode_fixed_rate; + config.arg.rate = align ? -rate : +rate; + return config; +} + +zfp_config +zfp_config_precision( + uint precision +) +{ + zfp_config config; + config.mode = zfp_mode_fixed_precision; + config.arg.precision = precision; + return config; +} + +zfp_config +zfp_config_accuracy( + double tolerance +) +{ + zfp_config config; + config.mode = zfp_mode_fixed_accuracy; + config.arg.tolerance = tolerance; + return config; +} + +zfp_config +zfp_config_reversible() +{ + zfp_config config; + config.mode = zfp_mode_reversible; + return config; +} + +zfp_config +zfp_config_expert( + uint minbits, + uint maxbits, + uint maxprec, + int minexp +) +{ + zfp_config config; + config.mode = zfp_mode_expert; + config.arg.expert.minbits = minbits; + config.arg.expert.maxbits = maxbits; + config.arg.expert.maxprec = maxprec; + config.arg.expert.minexp = minexp; + return config; +} + +/* public functions: zfp compressed stream --------------------------------- */ + +zfp_stream* +zfp_stream_open(bitstream* stream) +{ + zfp_stream* zfp = (zfp_stream*)malloc(sizeof(zfp_stream)); + if (zfp) { + zfp->stream = stream; + zfp->minbits = ZFP_MIN_BITS; + zfp->maxbits = ZFP_MAX_BITS; + zfp->maxprec = ZFP_MAX_PREC; + zfp->minexp = ZFP_MIN_EXP; + zfp->exec.policy = zfp_exec_serial; + } + return zfp; +} + +void +zfp_stream_close(zfp_stream* zfp) +{ + free(zfp); +} + +bitstream* +zfp_stream_bit_stream(const zfp_stream* zfp) +{ + return zfp->stream; +} + +zfp_mode +zfp_stream_compression_mode(const zfp_stream* zfp) +{ + if (zfp->minbits > zfp->maxbits || !(0 < zfp->maxprec && zfp->maxprec <= 64)) + return zfp_mode_null; + + /* default values are considered expert mode */ + if (zfp->minbits == ZFP_MIN_BITS && + zfp->maxbits == ZFP_MAX_BITS && + zfp->maxprec == ZFP_MAX_PREC && + zfp->minexp == ZFP_MIN_EXP) + return zfp_mode_expert; + + /* fixed rate? */ + if (zfp->minbits == zfp->maxbits && + 1 <= zfp->maxbits && zfp->maxbits <= ZFP_MAX_BITS && + zfp->maxprec >= ZFP_MAX_PREC && + zfp->minexp == ZFP_MIN_EXP) + return zfp_mode_fixed_rate; + + /* fixed precision? */ + if (zfp->minbits <= ZFP_MIN_BITS && + zfp->maxbits >= ZFP_MAX_BITS && + zfp->maxprec >= 1 && + zfp->minexp == ZFP_MIN_EXP) + return zfp_mode_fixed_precision; + + /* fixed accuracy? */ + if (zfp->minbits <= ZFP_MIN_BITS && + zfp->maxbits >= ZFP_MAX_BITS && + zfp->maxprec >= ZFP_MAX_PREC && + zfp->minexp >= ZFP_MIN_EXP) + return zfp_mode_fixed_accuracy; + + /* reversible? */ + if (zfp->minbits <= ZFP_MIN_BITS && + zfp->maxbits >= ZFP_MAX_BITS && + zfp->maxprec >= ZFP_MAX_PREC && + zfp->minexp < ZFP_MIN_EXP) + return zfp_mode_reversible; + + return zfp_mode_expert; +} + +double +zfp_stream_rate(const zfp_stream* zfp, uint dims) +{ + return (zfp_stream_compression_mode(zfp) == zfp_mode_fixed_rate) + ? (double)zfp->maxbits / (1u << (2 * dims)) + : 0.0; +} + +uint +zfp_stream_precision(const zfp_stream* zfp) +{ + return (zfp_stream_compression_mode(zfp) == zfp_mode_fixed_precision) + ? zfp->maxprec + : 0; +} + +double +zfp_stream_accuracy(const zfp_stream* zfp) +{ + return (zfp_stream_compression_mode(zfp) == zfp_mode_fixed_accuracy) + ? ldexp(1.0, zfp->minexp) + : 0.0; +} + +uint64 +zfp_stream_mode(const zfp_stream* zfp) +{ + uint64 mode = 0; + uint minbits; + uint maxbits; + uint maxprec; + uint minexp; + + /* common configurations mapped to short representation */ + switch (zfp_stream_compression_mode(zfp)) { + case zfp_mode_fixed_rate: + if (zfp->maxbits <= 2048) + /* maxbits is [1, 2048] */ + /* returns [0, 2047] */ + return (zfp->maxbits - 1); + else + break; + + case zfp_mode_fixed_precision: + if (zfp->maxprec <= 128) + /* maxprec is [1, 128] */ + /* returns [2048, 2175] */ + return (zfp->maxprec - 1) + (2048); + else + break; + + case zfp_mode_fixed_accuracy: + if (zfp->minexp <= 843) + /* minexp is [ZFP_MIN_EXP=-1074, 843] */ + /* returns [2177, ZFP_MODE_SHORT_MAX=4094] */ + /* +1 because skipped 2176 */ + return (zfp->minexp - ZFP_MIN_EXP) + (2048 + 128 + 1); + else + break; + + case zfp_mode_reversible: + /* returns 2176 */ + return 2048 + 128; + + default: + break; + } + + /* encode each parameter separately */ + minbits = MAX(1, MIN(zfp->minbits, 0x8000u)) - 1; + maxbits = MAX(1, MIN(zfp->maxbits, 0x8000u)) - 1; + maxprec = MAX(1, MIN(zfp->maxprec, 0x0080u)) - 1; + minexp = MAX(0, MIN(zfp->minexp + 16495, 0x7fff)); + mode <<= 15; mode += minexp; + mode <<= 7; mode += maxprec; + mode <<= 15; mode += maxbits; + mode <<= 15; mode += minbits; + mode <<= 12; mode += 0xfffu; + + return mode; +} + +void +zfp_stream_params(const zfp_stream* zfp, uint* minbits, uint* maxbits, uint* maxprec, int* minexp) +{ + if (minbits) + *minbits = zfp->minbits; + if (maxbits) + *maxbits = zfp->maxbits; + if (maxprec) + *maxprec = zfp->maxprec; + if (minexp) + *minexp = zfp->minexp; +} + +size_t +zfp_stream_compressed_size(const zfp_stream* zfp) +{ + return stream_size(zfp->stream); +} + +size_t +zfp_stream_maximum_size(const zfp_stream* zfp, const zfp_field* field) +{ + zfp_bool reversible = is_reversible(zfp); + uint dims = zfp_field_dimensionality(field); + size_t mx = (MAX(field->nx, 1u) + 3) / 4; + size_t my = (MAX(field->ny, 1u) + 3) / 4; + size_t mz = (MAX(field->nz, 1u) + 3) / 4; + size_t mw = (MAX(field->nw, 1u) + 3) / 4; + size_t blocks = mx * my * mz * mw; + uint values = 1u << (2 * dims); + uint maxbits = 0; + + if (!dims) + return 0; + switch (field->type) { + case zfp_type_int32: + maxbits += reversible ? 5 : 0; + break; + case zfp_type_int64: + maxbits += reversible ? 6 : 0; + break; + case zfp_type_float: + maxbits += reversible ? 1 + 1 + 8 + 5 : 1 + 8; + break; + case zfp_type_double: + maxbits += reversible ? 1 + 1 + 11 + 6 : 1 + 11; + break; + default: + return 0; + } + maxbits += values - 1 + values * MIN(zfp->maxprec, zfp_field_precision(field)); + maxbits = MIN(maxbits, zfp->maxbits); + maxbits = MAX(maxbits, zfp->minbits); + return ((ZFP_HEADER_MAX_BITS + blocks * maxbits + stream_word_bits - 1) & ~(stream_word_bits - 1)) / CHAR_BIT; +} + +void +zfp_stream_set_bit_stream(zfp_stream* zfp, bitstream* stream) +{ + zfp->stream = stream; +} + +void +zfp_stream_set_reversible(zfp_stream* zfp) +{ + zfp->minbits = ZFP_MIN_BITS; + zfp->maxbits = ZFP_MAX_BITS; + zfp->maxprec = ZFP_MAX_PREC; + zfp->minexp = ZFP_MIN_EXP - 1; +} + +double +zfp_stream_set_rate(zfp_stream* zfp, double rate, zfp_type type, uint dims, zfp_bool align) +{ + uint n = 1u << (2 * dims); + uint bits = (uint)floor(n * rate + 0.5); + switch (type) { + case zfp_type_float: + bits = MAX(bits, 1 + 8u); + break; + case zfp_type_double: + bits = MAX(bits, 1 + 11u); + break; + default: + break; + } + if (align) { + /* for write random access, round up to next multiple of stream word size */ + bits += (uint)stream_word_bits - 1; + bits &= ~(stream_word_bits - 1); + } + zfp->minbits = bits; + zfp->maxbits = bits; + zfp->maxprec = ZFP_MAX_PREC; + zfp->minexp = ZFP_MIN_EXP; + return (double)bits / n; +} + +uint +zfp_stream_set_precision(zfp_stream* zfp, uint precision) +{ + zfp->minbits = ZFP_MIN_BITS; + zfp->maxbits = ZFP_MAX_BITS; + zfp->maxprec = precision ? MIN(precision, ZFP_MAX_PREC) : ZFP_MAX_PREC; + zfp->minexp = ZFP_MIN_EXP; + return zfp->maxprec; +} + +double +zfp_stream_set_accuracy(zfp_stream* zfp, double tolerance) +{ + int emin = ZFP_MIN_EXP; + if (tolerance > 0) { + /* tolerance = x * 2^emin, with 0.5 <= x < 1 */ + frexp(tolerance, &emin); + emin--; + /* assert: 2^emin <= tolerance < 2^(emin+1) */ + } + zfp->minbits = ZFP_MIN_BITS; + zfp->maxbits = ZFP_MAX_BITS; + zfp->maxprec = ZFP_MAX_PREC; + zfp->minexp = emin; + return tolerance > 0 ? ldexp(1.0, emin) : 0; +} + +zfp_mode +zfp_stream_set_mode(zfp_stream* zfp, uint64 mode) +{ + uint minbits, maxbits, maxprec; + int minexp; + + if (mode <= ZFP_MODE_SHORT_MAX) { + /* 12-bit (short) encoding of one of four modes */ + if (mode < 2048) { + /* fixed rate */ + minbits = maxbits = (uint)mode + 1; + maxprec = ZFP_MAX_PREC; + minexp = ZFP_MIN_EXP; + } + else if (mode < (2048 + 128)) { + /* fixed precision */ + minbits = ZFP_MIN_BITS; + maxbits = ZFP_MAX_BITS; + maxprec = (uint)mode + 1 - (2048); + minexp = ZFP_MIN_EXP; + } + else if (mode == (2048 + 128)) { + /* reversible */ + minbits = ZFP_MIN_BITS; + maxbits = ZFP_MAX_BITS; + maxprec = ZFP_MAX_PREC; + minexp = ZFP_MIN_EXP - 1; + } + else { + /* fixed accuracy */ + minbits = ZFP_MIN_BITS; + maxbits = ZFP_MAX_BITS; + maxprec = ZFP_MAX_PREC; + minexp = (int)mode + ZFP_MIN_EXP - (2048 + 128 + 1); + } + } + else { + /* 64-bit encoding */ + mode >>= 12; minbits = (uint)(mode & 0x7fffu) + 1; + mode >>= 15; maxbits = (uint)(mode & 0x7fffu) + 1; + mode >>= 15; maxprec = (uint)(mode & 0x007fu) + 1; + mode >>= 7; minexp = (int)(mode & 0x7fffu) - 16495; + } + + if (!zfp_stream_set_params(zfp, minbits, maxbits, maxprec, minexp)) + return zfp_mode_null; + + return zfp_stream_compression_mode(zfp); +} + +zfp_bool +zfp_stream_set_params(zfp_stream* zfp, uint minbits, uint maxbits, uint maxprec, int minexp) +{ + if (minbits > maxbits || !(0 < maxprec && maxprec <= 64)) + return zfp_false; + zfp->minbits = minbits; + zfp->maxbits = maxbits; + zfp->maxprec = maxprec; + zfp->minexp = minexp; + return zfp_true; +} + +size_t +zfp_stream_flush(zfp_stream* zfp) +{ + return stream_flush(zfp->stream); +} + +size_t +zfp_stream_align(zfp_stream* zfp) +{ + return stream_align(zfp->stream); +} + +void +zfp_stream_rewind(zfp_stream* zfp) +{ + stream_rewind(zfp->stream); +} + +/* public functions: execution policy -------------------------------------- */ + +zfp_exec_policy +zfp_stream_execution(const zfp_stream* zfp) +{ + return zfp->exec.policy; +} + +uint +zfp_stream_omp_threads(const zfp_stream* zfp) +{ + return zfp->exec.params.omp.threads; +} + +uint +zfp_stream_omp_chunk_size(const zfp_stream* zfp) +{ + return zfp->exec.params.omp.chunk_size; +} + +zfp_bool +zfp_stream_set_execution(zfp_stream* zfp, zfp_exec_policy policy) +{ + switch (policy) { + case zfp_exec_serial: + break; +#ifdef ZFP_WITH_CUDA + case zfp_exec_cuda: + break; +#endif + case zfp_exec_omp: +#ifdef _OPENMP + if (zfp->exec.policy != policy) { + zfp->exec.params.omp.threads = 0; + zfp->exec.params.omp.chunk_size = 0; + } + break; +#else + return zfp_false; +#endif + default: + return zfp_false; + } + zfp->exec.policy = policy; + return zfp_true; +} + +zfp_bool +zfp_stream_set_omp_threads(zfp_stream* zfp, uint threads) +{ + if (!zfp_stream_set_execution(zfp, zfp_exec_omp)) + return zfp_false; + zfp->exec.params.omp.threads = threads; + return zfp_true; +} + +zfp_bool +zfp_stream_set_omp_chunk_size(zfp_stream* zfp, uint chunk_size) +{ + if (!zfp_stream_set_execution(zfp, zfp_exec_omp)) + return zfp_false; + zfp->exec.params.omp.chunk_size = chunk_size; + return zfp_true; +} + +/* public functions: utility functions --------------------------------------*/ + +void +zfp_promote_int8_to_int32(int32* oblock, const int8* iblock, uint dims) +{ + uint count = 1u << (2 * dims); + while (count--) + *oblock++ = (int32)*iblock++ << 23; +} + +void +zfp_promote_uint8_to_int32(int32* oblock, const uint8* iblock, uint dims) +{ + uint count = 1u << (2 * dims); + while (count--) + *oblock++ = ((int32)*iblock++ - 0x80) << 23; +} + +void +zfp_promote_int16_to_int32(int32* oblock, const int16* iblock, uint dims) +{ + uint count = 1u << (2 * dims); + while (count--) + *oblock++ = (int32)*iblock++ << 15; +} + +void +zfp_promote_uint16_to_int32(int32* oblock, const uint16* iblock, uint dims) +{ + uint count = 1u << (2 * dims); + while (count--) + *oblock++ = ((int32)*iblock++ - 0x8000) << 15; +} + +void +zfp_demote_int32_to_int8(int8* oblock, const int32* iblock, uint dims) +{ + uint count = 1u << (2 * dims); + while (count--) { + int32 i = *iblock++ >> 23; + *oblock++ = (int8)MAX(-0x80, MIN(i, 0x7f)); + } +} + +void +zfp_demote_int32_to_uint8(uint8* oblock, const int32* iblock, uint dims) +{ + uint count = 1u << (2 * dims); + while (count--) { + int32 i = (*iblock++ >> 23) + 0x80; + *oblock++ = (uint8)MAX(0x00, MIN(i, 0xff)); + } +} + +void +zfp_demote_int32_to_int16(int16* oblock, const int32* iblock, uint dims) +{ + uint count = 1u << (2 * dims); + while (count--) { + int32 i = *iblock++ >> 15; + *oblock++ = (int16)MAX(-0x8000, MIN(i, 0x7fff)); + } +} + +void +zfp_demote_int32_to_uint16(uint16* oblock, const int32* iblock, uint dims) +{ + uint count = 1u << (2 * dims); + while (count--) { + int32 i = (*iblock++ >> 15) + 0x8000; + *oblock++ = (uint16)MAX(0x0000, MIN(i, 0xffff)); + } +} + +/* public functions: compression and decompression --------------------------*/ + +size_t +zfp_compress(zfp_stream* zfp, const zfp_field* field) +{ + /* function table [execution][strided][dimensionality][scalar type] */ + void (*ftable[3][2][4][4])(zfp_stream*, const zfp_field*) = { + /* serial */ + {{{ compress_int32_1, compress_int64_1, compress_float_1, compress_double_1 }, + { compress_strided_int32_2, compress_strided_int64_2, compress_strided_float_2, compress_strided_double_2 }, + { compress_strided_int32_3, compress_strided_int64_3, compress_strided_float_3, compress_strided_double_3 }, + { compress_strided_int32_4, compress_strided_int64_4, compress_strided_float_4, compress_strided_double_4 }}, + {{ compress_strided_int32_1, compress_strided_int64_1, compress_strided_float_1, compress_strided_double_1 }, + { compress_strided_int32_2, compress_strided_int64_2, compress_strided_float_2, compress_strided_double_2 }, + { compress_strided_int32_3, compress_strided_int64_3, compress_strided_float_3, compress_strided_double_3 }, + { compress_strided_int32_4, compress_strided_int64_4, compress_strided_float_4, compress_strided_double_4 }}}, + + /* OpenMP */ +#ifdef _OPENMP + {{{ compress_omp_int32_1, compress_omp_int64_1, compress_omp_float_1, compress_omp_double_1 }, + { compress_strided_omp_int32_2, compress_strided_omp_int64_2, compress_strided_omp_float_2, compress_strided_omp_double_2 }, + { compress_strided_omp_int32_3, compress_strided_omp_int64_3, compress_strided_omp_float_3, compress_strided_omp_double_3 }, + { compress_strided_omp_int32_4, compress_strided_omp_int64_4, compress_strided_omp_float_4, compress_strided_omp_double_4 }}, + {{ compress_strided_omp_int32_1, compress_strided_omp_int64_1, compress_strided_omp_float_1, compress_strided_omp_double_1 }, + { compress_strided_omp_int32_2, compress_strided_omp_int64_2, compress_strided_omp_float_2, compress_strided_omp_double_2 }, + { compress_strided_omp_int32_3, compress_strided_omp_int64_3, compress_strided_omp_float_3, compress_strided_omp_double_3 }, + { compress_strided_omp_int32_4, compress_strided_omp_int64_4, compress_strided_omp_float_4, compress_strided_omp_double_4 }}}, +#else + {{{ NULL }}}, +#endif + + /* CUDA */ +#ifdef ZFP_WITH_CUDA + {{{ compress_cuda_int32_1, compress_cuda_int64_1, compress_cuda_float_1, compress_cuda_double_1 }, + { compress_strided_cuda_int32_2, compress_strided_cuda_int64_2, compress_strided_cuda_float_2, compress_strided_cuda_double_2 }, + { compress_strided_cuda_int32_3, compress_strided_cuda_int64_3, compress_strided_cuda_float_3, compress_strided_cuda_double_3 }, + { NULL, NULL, NULL, NULL }}, + {{ compress_strided_cuda_int32_1, compress_strided_cuda_int64_1, compress_strided_cuda_float_1, compress_strided_cuda_double_1 }, + { compress_strided_cuda_int32_2, compress_strided_cuda_int64_2, compress_strided_cuda_float_2, compress_strided_cuda_double_2 }, + { compress_strided_cuda_int32_3, compress_strided_cuda_int64_3, compress_strided_cuda_float_3, compress_strided_cuda_double_3 }, + { NULL, NULL, NULL, NULL }}}, +#else + {{{ NULL }}}, +#endif + }; + uint exec = zfp->exec.policy; + uint strided = zfp_field_stride(field, NULL); + uint dims = zfp_field_dimensionality(field); + uint type = field->type; + void (*compress)(zfp_stream*, const zfp_field*); + + switch (type) { + case zfp_type_int32: + case zfp_type_int64: + case zfp_type_float: + case zfp_type_double: + break; + default: + return 0; + } + + /* return 0 if compression mode is not supported */ + compress = ftable[exec][strided][dims - 1][type - zfp_type_int32]; + if (!compress) + return 0; + + /* compress field and align bit stream on word boundary */ + compress(zfp, field); + stream_flush(zfp->stream); + + return stream_size(zfp->stream); +} + +size_t +zfp_decompress(zfp_stream* zfp, zfp_field* field) +{ + /* function table [execution][strided][dimensionality][scalar type] */ + void (*ftable[3][2][4][4])(zfp_stream*, zfp_field*) = { + /* serial */ + {{{ decompress_int32_1, decompress_int64_1, decompress_float_1, decompress_double_1 }, + { decompress_strided_int32_2, decompress_strided_int64_2, decompress_strided_float_2, decompress_strided_double_2 }, + { decompress_strided_int32_3, decompress_strided_int64_3, decompress_strided_float_3, decompress_strided_double_3 }, + { decompress_strided_int32_4, decompress_strided_int64_4, decompress_strided_float_4, decompress_strided_double_4 }}, + {{ decompress_strided_int32_1, decompress_strided_int64_1, decompress_strided_float_1, decompress_strided_double_1 }, + { decompress_strided_int32_2, decompress_strided_int64_2, decompress_strided_float_2, decompress_strided_double_2 }, + { decompress_strided_int32_3, decompress_strided_int64_3, decompress_strided_float_3, decompress_strided_double_3 }, + { decompress_strided_int32_4, decompress_strided_int64_4, decompress_strided_float_4, decompress_strided_double_4 }}}, + + /* OpenMP; not yet supported */ + {{{ NULL }}}, + + /* CUDA */ +#ifdef ZFP_WITH_CUDA + {{{ decompress_cuda_int32_1, decompress_cuda_int64_1, decompress_cuda_float_1, decompress_cuda_double_1 }, + { decompress_strided_cuda_int32_2, decompress_strided_cuda_int64_2, decompress_strided_cuda_float_2, decompress_strided_cuda_double_2 }, + { decompress_strided_cuda_int32_3, decompress_strided_cuda_int64_3, decompress_strided_cuda_float_3, decompress_strided_cuda_double_3 }, + { NULL, NULL, NULL, NULL }}, + {{ decompress_strided_cuda_int32_1, decompress_strided_cuda_int64_1, decompress_strided_cuda_float_1, decompress_strided_cuda_double_1 }, + { decompress_strided_cuda_int32_2, decompress_strided_cuda_int64_2, decompress_strided_cuda_float_2, decompress_strided_cuda_double_2 }, + { decompress_strided_cuda_int32_3, decompress_strided_cuda_int64_3, decompress_strided_cuda_float_3, decompress_strided_cuda_double_3 }, + { NULL, NULL, NULL, NULL }}}, +#else + {{{ NULL }}}, +#endif + }; + uint exec = zfp->exec.policy; + uint strided = zfp_field_stride(field, NULL); + uint dims = zfp_field_dimensionality(field); + uint type = field->type; + void (*decompress)(zfp_stream*, zfp_field*); + + switch (type) { + case zfp_type_int32: + case zfp_type_int64: + case zfp_type_float: + case zfp_type_double: + break; + default: + return 0; + } + + /* return 0 if decompression mode is not supported */ + decompress = ftable[exec][strided][dims - 1][type - zfp_type_int32]; + if (!decompress) + return 0; + + /* decompress field and align bit stream on word boundary */ + decompress(zfp, field); + stream_align(zfp->stream); + + return stream_size(zfp->stream); +} + +size_t +zfp_write_header(zfp_stream* zfp, const zfp_field* field, uint mask) +{ + size_t bits = 0; + uint64 meta = 0; + + /* first make sure field dimensions fit in header */ + if (mask & ZFP_HEADER_META) { + meta = zfp_field_metadata(field); + if (meta == ZFP_META_NULL) + return 0; + } + + /* 32-bit magic */ + if (mask & ZFP_HEADER_MAGIC) { + stream_write_bits(zfp->stream, 'z', 8); + stream_write_bits(zfp->stream, 'f', 8); + stream_write_bits(zfp->stream, 'p', 8); + stream_write_bits(zfp->stream, zfp_codec_version, 8); + bits += ZFP_MAGIC_BITS; + } + /* 52-bit field metadata */ + if (mask & ZFP_HEADER_META) { + stream_write_bits(zfp->stream, meta, ZFP_META_BITS); + bits += ZFP_META_BITS; + } + /* 12- or 64-bit compression parameters */ + if (mask & ZFP_HEADER_MODE) { + uint64 mode = zfp_stream_mode(zfp); + uint size = mode > ZFP_MODE_SHORT_MAX ? ZFP_MODE_LONG_BITS : ZFP_MODE_SHORT_BITS; + stream_write_bits(zfp->stream, mode, size); + bits += size; + } + + return bits; +} + +size_t +zfp_read_header(zfp_stream* zfp, zfp_field* field, uint mask) +{ + size_t bits = 0; + if (mask & ZFP_HEADER_MAGIC) { + if (stream_read_bits(zfp->stream, 8) != 'z' || + stream_read_bits(zfp->stream, 8) != 'f' || + stream_read_bits(zfp->stream, 8) != 'p' || + stream_read_bits(zfp->stream, 8) != zfp_codec_version) + return 0; + bits += ZFP_MAGIC_BITS; + } + if (mask & ZFP_HEADER_META) { + uint64 meta = stream_read_bits(zfp->stream, ZFP_META_BITS); + if (!zfp_field_set_metadata(field, meta)) + return 0; + bits += ZFP_META_BITS; + } + if (mask & ZFP_HEADER_MODE) { + uint64 mode = stream_read_bits(zfp->stream, ZFP_MODE_SHORT_BITS); + bits += ZFP_MODE_SHORT_BITS; + if (mode > ZFP_MODE_SHORT_MAX) { + uint size = ZFP_MODE_LONG_BITS - ZFP_MODE_SHORT_BITS; + mode += stream_read_bits(zfp->stream, size) << ZFP_MODE_SHORT_BITS; + bits += size; + } + if (zfp_stream_set_mode(zfp, mode) == zfp_mode_null) + return 0; + } + return bits; +} diff --git a/src/blosc2/plugins/codecs/zfp/test_zfp_acc_float.c b/src/blosc2/plugins/codecs/zfp/test_zfp_acc_float.c new file mode 100644 index 0000000..ccda898 --- /dev/null +++ b/src/blosc2/plugins/codecs/zfp/test_zfp_acc_float.c @@ -0,0 +1,252 @@ +/********************************************************************* + Blosc - Blocked Shuffling and Compression Library + + Copyright (C) 2021 The Blosc Developers + https://blosc.org + License: BSD 3-Clause (see LICENSE.txt) + + See LICENSE.txt for details about copyright and rights to use. + + Test program demonstrating use of the Blosc codec from C code. + To compile this program: + + $ gcc -O test_zfp_acc_float.c -o test_zfp_acc_float -lblosc2 + + +**********************************************************************/ + +#include +#include +#include "blosc2.h" +#include "blosc2/codecs-registry.h" +#include "blosc-private.h" +#include + +static int test_zfp_acc_float(blosc2_schunk* schunk) { + + if (schunk->typesize != 4) { + printf("Error: This test is only for doubles.\n"); + return 0; + } + int64_t nchunks = schunk->nchunks; + int32_t chunksize = (int32_t) (schunk->chunksize); + float *data_in = malloc(chunksize); + int decompressed; + int64_t csize; + int64_t dsize; + int64_t csize_f = 0; + uint8_t *data_out = malloc(chunksize + BLOSC2_MAX_OVERHEAD); + float *data_dest = malloc(chunksize); + + /* Create a context for compression */ + int8_t zfp_tol = -2; + blosc2_cparams cparams = BLOSC2_CPARAMS_DEFAULTS; + cparams.splitmode = BLOSC_NEVER_SPLIT; + cparams.typesize = schunk->typesize; + cparams.compcode = BLOSC_CODEC_ZFP_FIXED_ACCURACY; + cparams.compcode_meta = zfp_tol; + cparams.filters[BLOSC2_MAX_FILTERS - 1] = BLOSC_NOFILTER; + cparams.clevel = 5; + cparams.nthreads = 1; + cparams.blocksize = schunk->blocksize; + cparams.schunk = schunk; + blosc2_context *cctx; + cctx = blosc2_create_cctx(cparams); + + blosc2_dparams dparams = BLOSC2_DPARAMS_DEFAULTS; + dparams.nthreads = 1; + dparams.schunk = schunk; + blosc2_context *dctx; + dctx = blosc2_create_dctx(dparams); + + for (int ci = 0; ci < nchunks; ci++) { + + decompressed = blosc2_schunk_decompress_chunk(schunk, ci, data_in, chunksize); + if (decompressed < 0) { + printf("Error decompressing chunk \n"); + return -1; + } + + /* Compress with clevel=5 and shuffle active */ + csize = blosc2_compress_ctx(cctx, data_in, chunksize, data_out, chunksize + BLOSC2_MAX_OVERHEAD); + if (csize == 0) { + printf("Buffer is uncompressible. Giving up.\n"); + return 0; + } else if (csize < 0) { + printf("Compression error. Error code: %" PRId64 "\n", csize); + return (int) csize; + } + csize_f += csize; + + + /* Decompress */ + dsize = blosc2_decompress_ctx(dctx, data_out, chunksize + BLOSC2_MAX_OVERHEAD, data_dest, chunksize); + if (dsize <= 0) { + printf("Decompression error. Error code: %" PRId64 "\n", dsize); + return (int) dsize; + } + double tolerance = exp(zfp_tol); + for (int i = 0; i < (chunksize / cparams.typesize); i++) { + if (fabsf(data_in[i] - data_dest[i]) > tolerance) { + printf("i: %d, data %f, dest %f", i, data_in[i], data_dest[i]); + printf("\n Decompressed data differs from original!\n"); + return -1; + } + } + } + csize_f = csize_f / nchunks; + + free(data_in); + free(data_out); + free(data_dest); + blosc2_free_ctx(cctx); + blosc2_free_ctx(dctx); + + printf("Successful roundtrip!\n"); + printf("Compression: %d -> %" PRId64 " (%.1fx)\n", chunksize, csize_f, (1. * chunksize) / (double) csize_f); + return (int) (chunksize - csize_f); +} + +static int test_zfp_acc_double(blosc2_schunk* schunk) { + + if (schunk->typesize != 8) { + printf("Error: This test is only for doubles.\n"); + return 0; + } + int64_t nchunks = schunk->nchunks; + int32_t chunksize = (int32_t) (schunk->chunksize); + double *data_in = malloc(chunksize); + int decompressed; + int64_t csize; + int64_t dsize; + int64_t csize_f = 0; + uint8_t *data_out = malloc(chunksize + BLOSC2_MAX_OVERHEAD); + double *data_dest = malloc(chunksize); + + /* Create a context for compression */ + int zfp_tol = -2; + blosc2_cparams cparams = BLOSC2_CPARAMS_DEFAULTS; + cparams.splitmode = BLOSC_NEVER_SPLIT; + cparams.typesize = schunk->typesize; + cparams.compcode = BLOSC_CODEC_ZFP_FIXED_ACCURACY; + cparams.compcode_meta = zfp_tol; + cparams.filters[BLOSC2_MAX_FILTERS - 1] = BLOSC_NOFILTER; + cparams.clevel = 5; + cparams.nthreads = 1; + cparams.blocksize = schunk->blocksize; + cparams.schunk = schunk; + blosc2_context *cctx; + cctx = blosc2_create_cctx(cparams); + + blosc2_dparams dparams = BLOSC2_DPARAMS_DEFAULTS; + dparams.nthreads = 1; + dparams.schunk = schunk; + blosc2_context *dctx; + dctx = blosc2_create_dctx(dparams); + + for (int ci = 0; ci < nchunks; ci++) { + + decompressed = blosc2_schunk_decompress_chunk(schunk, ci, data_in, chunksize); + if (decompressed < 0) { + printf("Error decompressing chunk \n"); + return -1; + } + + /* Compress with clevel=5 and shuffle active */ + csize = blosc2_compress_ctx(cctx, data_in, chunksize, data_out, chunksize + BLOSC2_MAX_OVERHEAD); + if (csize == 0) { + printf("Buffer is uncompressible. Giving up.\n"); + return 0; + } else if (csize < 0) { + printf("Compression error. Error code: %" PRId64 "\n", csize); + return (int) csize; + } + csize_f += csize; + + + /* Decompress */ + dsize = blosc2_decompress_ctx(dctx, data_out, chunksize + BLOSC2_MAX_OVERHEAD, data_dest, chunksize); + if (dsize <= 0) { + printf("Decompression error. Error code: %" PRId64 "\n", dsize); + return (int) dsize; + } + double tolerance = exp(zfp_tol); + for (int i = 0; i < (chunksize / cparams.typesize); i++) { + if (fabs(data_in[i] - data_dest[i]) > tolerance) { + printf("i: %d, data %f, dest %f", i, data_in[i], data_dest[i]); + printf("\n Decompressed data differs from original!\n"); + return -1; + } + } + } + csize_f = csize_f / nchunks; + + free(data_in); + free(data_out); + free(data_dest); + blosc2_free_ctx(cctx); + blosc2_free_ctx(dctx); + + printf("Successful roundtrip!\n"); + printf("Compression: %d -> %" PRId64 " (%.1fx)\n", chunksize, csize_f, (1. * chunksize) / (double) csize_f); + return (int) (chunksize - csize_f); +} + + +int float_cyclic() { + blosc2_schunk *schunk = blosc2_schunk_open("example_float_cyclic.caterva"); + BLOSC_ERROR_NULL(schunk, BLOSC2_ERROR_FILE_OPEN); + + /* Run the test. */ + int result = test_zfp_acc_float(schunk); + blosc2_schunk_free(schunk); + return result; +} + +int double_same_cells() { + blosc2_schunk *schunk = blosc2_schunk_open("example_double_same_cells.caterva"); + BLOSC_ERROR_NULL(schunk, BLOSC2_ERROR_FILE_OPEN); + + /* Run the test. */ + int result = test_zfp_acc_double(schunk); + blosc2_schunk_free(schunk); + return result; +} + +int day_month_temp() { + blosc2_schunk *schunk = blosc2_schunk_open("example_day_month_temp.caterva"); + BLOSC_ERROR_NULL(schunk, BLOSC2_ERROR_FILE_OPEN); + + /* Run the test. */ + int result = test_zfp_acc_float(schunk); + blosc2_schunk_free(schunk); + return result; +} + +int item_prices() { + blosc2_schunk *schunk = blosc2_schunk_open("example_item_prices.caterva"); + BLOSC_ERROR_NULL(schunk, BLOSC2_ERROR_FILE_OPEN); + + /* Run the test. */ + int result = test_zfp_acc_float(schunk); + blosc2_schunk_free(schunk); + return result; +} + + +int main(void) { + + int result; + blosc2_init(); // this is mandatory for initiallizing the plugin mechanism + result = float_cyclic(); + printf("float_cyclic: %d obtained \n \n", result); + result = double_same_cells(); + printf("double_same_cells: %d obtained \n \n", result); + result = day_month_temp(); + printf("day_month_temp: %d obtained \n \n", result); + result = item_prices(); + printf("item_prices: %d obtained \n \n", result); + blosc2_destroy(); + + return BLOSC2_ERROR_SUCCESS; +} diff --git a/src/blosc2/plugins/codecs/zfp/test_zfp_acc_int.c b/src/blosc2/plugins/codecs/zfp/test_zfp_acc_int.c new file mode 100644 index 0000000..f476020 --- /dev/null +++ b/src/blosc2/plugins/codecs/zfp/test_zfp_acc_int.c @@ -0,0 +1,148 @@ +/********************************************************************* + Blosc - Blocked Shuffling and Compression Library + + Copyright (C) 2021 The Blosc Developers + https://blosc.org + License: BSD 3-Clause (see LICENSE.txt) + + See LICENSE.txt for details about copyright and rights to use. + + Test program demonstrating use of the Blosc codec from C code. + To compile this program: + + $ gcc -O test_zfp_acc_int.c -o test_zfp_acc_int -lblosc2 + + +**********************************************************************/ + +#include +#include "blosc2.h" +#include "blosc2/codecs-registry.h" +#include "blosc-private.h" +#include + +static int test_zfp(blosc2_schunk* schunk) { + + int64_t nchunks = schunk->nchunks; + int32_t chunksize = (int32_t) (schunk->chunksize); + uint8_t *data_in = malloc(chunksize); + int decompressed; + int64_t csize; + int64_t dsize; + int64_t csize_f = 0; + uint8_t *data_out = malloc(chunksize + BLOSC2_MAX_OVERHEAD); + uint8_t *data_dest = malloc(chunksize); + + /* Create a context for compression */ + blosc2_cparams cparams = BLOSC2_CPARAMS_DEFAULTS; + cparams.splitmode = BLOSC_NEVER_SPLIT; + cparams.typesize = schunk->typesize; + cparams.compcode = BLOSC_CODEC_ZFP_FIXED_ACCURACY; + cparams.compcode_meta = schunk->typesize; + cparams.filters[BLOSC2_MAX_FILTERS - 1] = BLOSC_NOFILTER; + cparams.clevel = 5; + cparams.nthreads = 1; + cparams.blocksize = schunk->blocksize; + cparams.schunk = schunk; + blosc2_context *cctx; + cctx = blosc2_create_cctx(cparams); + + blosc2_dparams dparams = BLOSC2_DPARAMS_DEFAULTS; + dparams.nthreads = 1; + dparams.schunk = schunk; + blosc2_context *dctx; + dctx = blosc2_create_dctx(dparams); + + for (int ci = 0; ci < nchunks; ci++) { + + decompressed = blosc2_schunk_decompress_chunk(schunk, ci, data_in, chunksize); + if (decompressed < 0) { + printf("Error decompressing chunk \n"); + return -1; + } + + /* Compress with clevel=5 and shuffle active */ + csize = blosc2_compress_ctx(cctx, data_in, chunksize, data_out, chunksize + BLOSC2_MAX_OVERHEAD); + if (csize == 0) { + printf("Buffer is uncompressible. Giving up.\n"); + return 0; + } else if (csize < 0) { + printf("Compression error. Error code: %" PRId64 "\n", csize); + return (int) csize; + } + csize_f += csize; + + /* Decompress */ + dsize = blosc2_decompress_ctx(dctx, data_out, chunksize + BLOSC2_MAX_OVERHEAD, data_dest, chunksize); + if (dsize <= 0) { + printf("Decompression error. Error code: %" PRId64 "\n", dsize); + return (int) dsize; + } + + for (int i = 0; i < chunksize; i++) { + if ((data_in[i] - data_dest[i]) > 1) { + printf("i: %d, data %u, dest %u", i, data_in[i], data_dest[i]); + printf("\n Decompressed data differs from original!\n"); + return -1; + } + } + } + csize_f = csize_f / nchunks; + + free(data_in); + free(data_out); + free(data_dest); + blosc2_free_ctx(cctx); + blosc2_free_ctx(dctx); + + printf("Successful roundtrip!\n"); + printf("Compression: %d -> %" PRId64 " (%.1fx)\n", chunksize, csize_f, (1. * chunksize) / (double)csize_f); + return (int) (chunksize - csize_f); +} + + +int rand_() { + blosc2_schunk *schunk = blosc2_schunk_open("example_ndmean_repart_rand.caterva"); + BLOSC_ERROR_NULL(schunk, BLOSC2_ERROR_FILE_OPEN); + + /* Run the test. */ + int result = test_zfp(schunk); + blosc2_schunk_free(schunk); + return result; +} + +int same_cells() { + blosc2_schunk *schunk = blosc2_schunk_open("example_ndmean_repart_same_cells.caterva"); + BLOSC_ERROR_NULL(schunk, BLOSC2_ERROR_FILE_OPEN); + + /* Run the test. */ + int result = test_zfp(schunk); + blosc2_schunk_free(schunk); + return result; +} + +int some_matches() { + blosc2_schunk *schunk = blosc2_schunk_open("example_ndmean_repart_some_matches.caterva"); + BLOSC_ERROR_NULL(schunk, BLOSC2_ERROR_FILE_OPEN); + + /* Run the test. */ + int result = test_zfp(schunk); + blosc2_schunk_free(schunk); + return result; +} + + +int main(void) { + + int result; + blosc2_init(); // this is mandatory for initiallizing the plugin mechanism + result = rand_(); + printf("rand: %d obtained \n \n", result); + result = same_cells(); + printf("same_cells: %d obtained \n \n", result); + result = some_matches(); + printf("some_matches: %d obtained \n \n", result); + blosc2_destroy(); + + return BLOSC2_ERROR_SUCCESS; +} diff --git a/src/blosc2/plugins/codecs/zfp/test_zfp_prec_float.c b/src/blosc2/plugins/codecs/zfp/test_zfp_prec_float.c new file mode 100644 index 0000000..e8f10a7 --- /dev/null +++ b/src/blosc2/plugins/codecs/zfp/test_zfp_prec_float.c @@ -0,0 +1,264 @@ +/********************************************************************* + Blosc - Blocked Shuffling and Compression Library + + Copyright (C) 2021 The Blosc Developers + https://blosc.org + License: BSD 3-Clause (see LICENSE.txt) + + See LICENSE.txt for details about copyright and rights to use. + + Test program demonstrating use of the Blosc codec from C code. + To compile this program: + + $ gcc -O test_zfp_prec_float.c -o test_zfp_prec_float -lblosc2 + + +**********************************************************************/ + +#include +#include +#include "blosc2.h" +#include "blosc2/codecs-registry.h" +#include "blosc-private.h" +#include + +static int test_zfp_prec_float(blosc2_schunk* schunk) { + + if (schunk->typesize != 4) { + printf("Error: This test is only for doubles.\n"); + return 0; + } + int64_t nchunks = schunk->nchunks; + int32_t chunksize = (int32_t) (schunk->chunksize); + float *data_in = malloc(chunksize); + int decompressed; + int64_t csize; + int64_t dsize; + int64_t csize_f = 0; + uint8_t *data_out = malloc(chunksize + BLOSC2_MAX_OVERHEAD); + float *data_dest = malloc(chunksize); + + /* Create a context for compression */ + int8_t zfp_prec = 25; + blosc2_cparams cparams = BLOSC2_CPARAMS_DEFAULTS; + cparams.splitmode = BLOSC_NEVER_SPLIT; + cparams.typesize = schunk->typesize; + cparams.compcode = BLOSC_CODEC_ZFP_FIXED_PRECISION; + cparams.compcode_meta = zfp_prec; + cparams.filters[BLOSC2_MAX_FILTERS - 1] = BLOSC_NOFILTER; + cparams.clevel = 5; + cparams.nthreads = 1; + cparams.blocksize = schunk->blocksize; + cparams.schunk = schunk; + blosc2_context *cctx; + cctx = blosc2_create_cctx(cparams); + + blosc2_dparams dparams = BLOSC2_DPARAMS_DEFAULTS; + dparams.nthreads = 1; + dparams.schunk = schunk; + blosc2_context *dctx; + dctx = blosc2_create_dctx(dparams); + + for (int ci = 0; ci < nchunks; ci++) { + + decompressed = blosc2_schunk_decompress_chunk(schunk, ci, data_in, chunksize); + if (decompressed < 0) { + printf("Error decompressing chunk \n"); + return -1; + } + + /* Compress with clevel=5 and shuffle active */ + csize = blosc2_compress_ctx(cctx, data_in, chunksize, data_out, chunksize + BLOSC2_MAX_OVERHEAD); + if (csize == 0) { + printf("Buffer is uncompressible. Giving up.\n"); + return 0; + } else if (csize < 0) { + printf("Compression error. Error code: %" PRId64 "\n", csize); + return (int) csize; + } + csize_f += csize; + + + /* Decompress */ + dsize = blosc2_decompress_ctx(dctx, data_out, chunksize + BLOSC2_MAX_OVERHEAD, data_dest, chunksize); + if (dsize <= 0) { + printf("Decompression error. Error code: %" PRId64 "\n", dsize); + return (int) dsize; + } + double tolerance = 0.01; + for (int i = 0; i < (chunksize / cparams.typesize); i++) { + if ((data_in[i] == 0) || (data_dest[i] == 0)) { + if (fabsf(data_in[i] - data_dest[i]) > tolerance) { + printf("i: %d, data %.8f, dest %.8f", i, data_in[i], data_dest[i]); + printf("\n Decompressed data differs from original!\n"); + return -1; + } + } else if (fabsf(data_in[i] - data_dest[i]) > tolerance * fmaxf(fabsf(data_in[i]), fabsf(data_dest[i]))) { + printf("i: %d, data %.8f, dest %.8f", i, data_in[i], data_dest[i]); + printf("\n Decompressed data differs from original!\n"); + return -1; + } + } + } + csize_f = csize_f / nchunks; + + free(data_in); + free(data_out); + free(data_dest); + blosc2_free_ctx(cctx); + blosc2_free_ctx(dctx); + + printf("Successful roundtrip!\n"); + printf("Compression: %d -> %" PRId64 " (%.1fx)\n", chunksize, csize_f, (1. * chunksize) / (double) csize_f); + return (int) (chunksize - csize_f); +} + +static int test_zfp_prec_double(blosc2_schunk* schunk) { + + if (schunk->typesize != 8) { + printf("Error: This test is only for doubles.\n"); + return 0; + } + int64_t nchunks = schunk->nchunks; + int32_t chunksize = (int32_t) (schunk->chunksize); + double *data_in = malloc(chunksize); + int decompressed; + int64_t csize; + int64_t dsize; + int64_t csize_f = 0; + uint8_t *data_out = malloc(chunksize + BLOSC2_MAX_OVERHEAD); + double *data_dest = malloc(chunksize); + + /* Create a context for compression */ + int zfp_prec = 25; + blosc2_cparams cparams = BLOSC2_CPARAMS_DEFAULTS; + cparams.splitmode = BLOSC_NEVER_SPLIT; + cparams.typesize = schunk->typesize; + cparams.compcode = BLOSC_CODEC_ZFP_FIXED_PRECISION; + cparams.compcode_meta = zfp_prec; + cparams.filters[BLOSC2_MAX_FILTERS - 1] = BLOSC_NOFILTER; + cparams.clevel = 5; + cparams.nthreads = 1; + cparams.blocksize = schunk->blocksize; + cparams.schunk = schunk; + blosc2_context *cctx; + cctx = blosc2_create_cctx(cparams); + + blosc2_dparams dparams = BLOSC2_DPARAMS_DEFAULTS; + dparams.nthreads = 1; + dparams.schunk = schunk; + blosc2_context *dctx; + dctx = blosc2_create_dctx(dparams); + + for (int ci = 0; ci < nchunks; ci++) { + + decompressed = blosc2_schunk_decompress_chunk(schunk, ci, data_in, chunksize); + if (decompressed < 0) { + printf("Error decompressing chunk \n"); + return -1; + } + + /* Compress with clevel=5 and shuffle active */ + csize = blosc2_compress_ctx(cctx, data_in, chunksize, data_out, chunksize + BLOSC2_MAX_OVERHEAD); + if (csize == 0) { + printf("Buffer is uncompressible. Giving up.\n"); + return 0; + } else if (csize < 0) { + printf("Compression error. Error code: %" PRId64 "\n", csize); + return (int) csize; + } + csize_f += csize; + + + /* Decompress */ + dsize = blosc2_decompress_ctx(dctx, data_out, chunksize + BLOSC2_MAX_OVERHEAD, data_dest, chunksize); + if (dsize <= 0) { + printf("Decompression error. Error code: %" PRId64 "\n", dsize); + return (int) dsize; + } + double tolerance = 0.01; + for (int i = 0; i < (chunksize / cparams.typesize); i++) { + if ((data_in[i] == 0) || (data_dest[i] == 0)) { + if (fabs(data_in[i] - data_dest[i]) > tolerance) { + printf("i: %d, data %.16f, dest %.16f", i, data_in[i], data_dest[i]); + printf("\n Decompressed data differs from original!\n"); + return -1; + } + } else if (fabs(data_in[i] - data_dest[i]) > tolerance * fmax(fabs(data_in[i]), fabs(data_dest[i]))) { + printf("i: %d, data %.16f, dest %.16f", i, data_in[i], data_dest[i]); + printf("\n Decompressed data differs from original!\n"); + return -1; + } + } + } + csize_f = csize_f / nchunks; + + free(data_in); + free(data_out); + free(data_dest); + blosc2_free_ctx(cctx); + blosc2_free_ctx(dctx); + + printf("Successful roundtrip!\n"); + printf("Compression: %d -> %" PRId64 " (%.1fx)\n", chunksize, csize_f, (1. * chunksize) / (double) csize_f); + return (int) (chunksize - csize_f); +} + + +int float_cyclic() { + blosc2_schunk *schunk = blosc2_schunk_open("example_float_cyclic.caterva"); + BLOSC_ERROR_NULL(schunk, BLOSC2_ERROR_FILE_OPEN); + + /* Run the test. */ + int result = test_zfp_prec_float(schunk); + blosc2_schunk_free(schunk); + return result; +} + +int double_same_cells() { + blosc2_schunk *schunk = blosc2_schunk_open("example_double_same_cells.caterva"); + BLOSC_ERROR_NULL(schunk, BLOSC2_ERROR_FILE_OPEN); + + /* Run the test. */ + int result = test_zfp_prec_double(schunk); + blosc2_schunk_free(schunk); + return result; +} + +int day_month_temp() { + blosc2_schunk *schunk = blosc2_schunk_open("example_day_month_temp.caterva"); + BLOSC_ERROR_NULL(schunk, BLOSC2_ERROR_FILE_OPEN); + + /* Run the test. */ + int result = test_zfp_prec_float(schunk); + blosc2_schunk_free(schunk); + return result; +} + +int item_prices() { + blosc2_schunk *schunk = blosc2_schunk_open("example_item_prices.caterva"); + BLOSC_ERROR_NULL(schunk, BLOSC2_ERROR_FILE_OPEN); + + /* Run the test. */ + int result = test_zfp_prec_float(schunk); + blosc2_schunk_free(schunk); + return result; +} + + +int main(void) { + + int result; + blosc2_init(); // this is mandatory for initiallizing the plugin mechanism + result = float_cyclic(); + printf("float_cyclic: %d obtained \n \n", result); + result = double_same_cells(); + printf("double_same_cells: %d obtained \n \n", result); + result = day_month_temp(); + printf("day_month_temp: %d obtained \n \n", result); + result = item_prices(); + printf("item_prices: %d obtained \n \n", result); + blosc2_destroy(); + + return BLOSC2_ERROR_SUCCESS; +} diff --git a/src/blosc2/plugins/codecs/zfp/test_zfp_rate_float.c b/src/blosc2/plugins/codecs/zfp/test_zfp_rate_float.c new file mode 100644 index 0000000..ffd4d5a --- /dev/null +++ b/src/blosc2/plugins/codecs/zfp/test_zfp_rate_float.c @@ -0,0 +1,234 @@ +/********************************************************************* + Blosc - Blocked Shuffling and Compression Library + + Copyright (C) 2021 The Blosc Developers + https://blosc.org + License: BSD 3-Clause (see LICENSE.txt) + + See LICENSE.txt for details about copyright and rights to use. + + Test program demonstrating use of the Blosc codec from C code. + To compile this program: + + $ gcc -O test_zfp_rate_float.c -o test_zfp_rate_float -lblosc2 + + +**********************************************************************/ + +#include +#include "blosc2.h" +#include "blosc2/codecs-registry.h" +#include "blosc-private.h" +#include + +static int test_zfp_rate_float(blosc2_schunk* schunk) { + + if (schunk->typesize != 4) { + printf("Error: This test is only for doubles.\n"); + return 0; + } + int64_t nchunks = schunk->nchunks; + int32_t chunksize = (int32_t) (schunk->chunksize); + float *data_in = malloc(chunksize); + int decompressed; + int64_t csize; + int64_t dsize; + int64_t csize_f = 0; + uint8_t *data_out = malloc(chunksize + BLOSC2_MAX_OVERHEAD); + float *data_dest = malloc(chunksize); + + /* Create a context for compression */ + int8_t zfp_rate = 37; + blosc2_cparams cparams = BLOSC2_CPARAMS_DEFAULTS; + cparams.splitmode = BLOSC_NEVER_SPLIT; + cparams.typesize = schunk->typesize; + cparams.compcode = BLOSC_CODEC_ZFP_FIXED_RATE; + cparams.compcode_meta = zfp_rate; + cparams.filters[BLOSC2_MAX_FILTERS - 1] = BLOSC_NOFILTER; + cparams.clevel = 5; + cparams.nthreads = 1; + cparams.blocksize = schunk->blocksize; + cparams.schunk = schunk; + blosc2_context *cctx; + cctx = blosc2_create_cctx(cparams); + + blosc2_dparams dparams = BLOSC2_DPARAMS_DEFAULTS; + dparams.nthreads = 1; + dparams.schunk = schunk; + blosc2_context *dctx; + dctx = blosc2_create_dctx(dparams); + + for (int ci = 0; ci < nchunks; ci++) { + + decompressed = blosc2_schunk_decompress_chunk(schunk, ci, data_in, chunksize); + if (decompressed < 0) { + printf("Error decompressing chunk \n"); + return -1; + } + + /* Compress with clevel=5 and shuffle active */ + csize = blosc2_compress_ctx(cctx, data_in, chunksize, data_out, chunksize + BLOSC2_MAX_OVERHEAD); + if (csize == 0) { + printf("Buffer is uncompressible. Giving up.\n"); + return 0; + } else if (csize < 0) { + printf("Compression error. Error code: %" PRId64 "\n", csize); + return (int) csize; + } + csize_f += csize; + + /* Decompress */ + dsize = blosc2_decompress_ctx(dctx, data_out, chunksize + BLOSC2_MAX_OVERHEAD, data_dest, chunksize); + if (dsize <= 0) { + printf("Decompression error. Error code: %" PRId64 "\n", dsize); + return (int) dsize; + } + } + csize_f = csize_f / nchunks; + + free(data_in); + free(data_out); + free(data_dest); + blosc2_free_ctx(cctx); + blosc2_free_ctx(dctx); + + printf("Successful roundtrip!\n"); + printf("Compression: %d -> %" PRId64 " (%.1fx)\n", chunksize, csize_f, (1. * chunksize) / (double) csize_f); + return (int) (chunksize - csize_f); +} + +static int test_zfp_rate_double(blosc2_schunk* schunk) { + + if (schunk->typesize != 8) { + printf("Error: This test is only for doubles.\n"); + return 0; + } + int64_t nchunks = schunk->nchunks; + int32_t chunksize = (int32_t) (schunk->chunksize); + double *data_in = malloc(chunksize); + int decompressed; + int64_t csize; + int64_t dsize; + int64_t csize_f = 0; + uint8_t *data_out = malloc(chunksize + BLOSC2_MAX_OVERHEAD); + double *data_dest = malloc(chunksize); + + /* Create a context for compression */ + int zfp_rate = 37; + blosc2_cparams cparams = BLOSC2_CPARAMS_DEFAULTS; + cparams.splitmode = BLOSC_NEVER_SPLIT; + cparams.typesize = schunk->typesize; + cparams.compcode = BLOSC_CODEC_ZFP_FIXED_RATE; + cparams.compcode_meta = zfp_rate; + cparams.filters[BLOSC2_MAX_FILTERS - 1] = BLOSC_NOFILTER; + cparams.clevel = 5; + cparams.nthreads = 1; + cparams.blocksize = schunk->blocksize; + cparams.schunk = schunk; + blosc2_context *cctx; + cctx = blosc2_create_cctx(cparams); + + blosc2_dparams dparams = BLOSC2_DPARAMS_DEFAULTS; + dparams.nthreads = 1; + dparams.schunk = schunk; + blosc2_context *dctx; + dctx = blosc2_create_dctx(dparams); + + for (int ci = 0; ci < nchunks; ci++) { + + decompressed = blosc2_schunk_decompress_chunk(schunk, ci, data_in, chunksize); + if (decompressed < 0) { + printf("Error decompressing chunk \n"); + return -1; + } + + /* Compress with clevel=5 and shuffle active */ + csize = blosc2_compress_ctx(cctx, data_in, chunksize, data_out, chunksize + BLOSC2_MAX_OVERHEAD); + if (csize == 0) { + printf("Buffer is uncompressible. Giving up.\n"); + return 0; + } else if (csize < 0) { + printf("Compression error. Error code: %" PRId64 "\n", csize); + return (int) csize; + } + csize_f += csize; + + /* Decompress */ + dsize = blosc2_decompress_ctx(dctx, data_out, chunksize + BLOSC2_MAX_OVERHEAD, data_dest, chunksize); + if (dsize <= 0) { + printf("Decompression error. Error code: %" PRId64 "\n", dsize); + return (int) dsize; + } + } + + csize_f = csize_f / nchunks; + + free(data_in); + free(data_out); + free(data_dest); + blosc2_free_ctx(cctx); + blosc2_free_ctx(dctx); + + printf("Successful roundtrip!\n"); + printf("Compression: %d -> %" PRId64 " (%.1fx)\n", chunksize, csize_f, (1. * chunksize) / (double) csize_f); + return (int) (chunksize - csize_f); +} + + +int float_cyclic() { + blosc2_schunk *schunk = blosc2_schunk_open("example_float_cyclic.caterva"); + BLOSC_ERROR_NULL(schunk, BLOSC2_ERROR_FILE_OPEN); + + /* Run the test. */ + int result = test_zfp_rate_float(schunk); + blosc2_schunk_free(schunk); + return result; +} + +int double_same_cells() { + blosc2_schunk *schunk = blosc2_schunk_open("example_double_same_cells.caterva"); + BLOSC_ERROR_NULL(schunk, BLOSC2_ERROR_FILE_OPEN); + + /* Run the test. */ + int result = test_zfp_rate_double(schunk); + blosc2_schunk_free(schunk); + return result; +} + +int day_month_temp() { + blosc2_schunk *schunk = blosc2_schunk_open("example_day_month_temp.caterva"); + BLOSC_ERROR_NULL(schunk, BLOSC2_ERROR_FILE_OPEN); + + /* Run the test. */ + int result = test_zfp_rate_float(schunk); + blosc2_schunk_free(schunk); + return result; +} + +int item_prices() { + blosc2_schunk *schunk = blosc2_schunk_open("example_item_prices.caterva"); + BLOSC_ERROR_NULL(schunk, BLOSC2_ERROR_FILE_OPEN); + + /* Run the test. */ + int result = test_zfp_rate_float(schunk); + blosc2_schunk_free(schunk); + return result; +} + + +int main(void) { + + int result; + blosc2_init(); // this is mandatory for initiallizing the plugin mechanism + result = float_cyclic(); + printf("float_cyclic: %d obtained \n \n", result); + result = double_same_cells(); + printf("double_same_cells: %d obtained \n \n", result); + result = day_month_temp(); + printf("day_month_temp: %d obtained \n \n", result); + result = item_prices(); + printf("item_prices: %d obtained \n \n", result); + blosc2_destroy(); + + return BLOSC2_ERROR_SUCCESS; +} diff --git a/src/blosc2/plugins/codecs/zfp/test_zfp_rate_getitem.c b/src/blosc2/plugins/codecs/zfp/test_zfp_rate_getitem.c new file mode 100644 index 0000000..fce26b0 --- /dev/null +++ b/src/blosc2/plugins/codecs/zfp/test_zfp_rate_getitem.c @@ -0,0 +1,299 @@ +/********************************************************************* + Blosc - Blocked Shuffling and Compression Library + + Copyright (C) 2021 The Blosc Developers + https://blosc.org + License: BSD 3-Clause (see LICENSE.txt) + + See LICENSE.txt for details about copyright and rights to use. + + Test program demonstrating use of the Blosc codec from C code. + To compile this program: + + $ gcc -O test_zfp_rate_getitem.c -o test_zfp_rate_getitem -lblosc2 + + +**********************************************************************/ + +#include +#include +#include +#include "blosc2.h" +#include "blosc2/codecs-registry.h" +#include "blosc-private.h" + +static int test_zfp_rate_getitem_float(blosc2_schunk* schunk) { + + if (schunk->typesize != 4) { + printf("Error: This test is only for doubles.\n"); + return 0; + } + int64_t nchunks = schunk->nchunks; + int32_t chunksize = (int32_t) (schunk->chunksize); + float *data_in = malloc(chunksize); + int decompressed; + int64_t csize; + uint8_t *chunk_zfp = malloc(chunksize + BLOSC2_MAX_OVERHEAD); + uint8_t *chunk_blosc = malloc(chunksize + BLOSC2_MAX_OVERHEAD); + float *data_dest = malloc(chunksize); + + /* Create a context for compression */ + int8_t zfp_rate = 37; + blosc2_cparams cparams = BLOSC2_CPARAMS_DEFAULTS; + cparams.splitmode = BLOSC_NEVER_SPLIT; + cparams.typesize = schunk->typesize; + cparams.compcode = BLOSC_CODEC_ZFP_FIXED_RATE; + cparams.compcode_meta = zfp_rate; + cparams.filters[BLOSC2_MAX_FILTERS - 1] = BLOSC_NOFILTER; + cparams.clevel = 5; + cparams.nthreads = 1; + cparams.blocksize = schunk->blocksize; + cparams.schunk = schunk; + blosc2_context *cctx; + cctx = blosc2_create_cctx(cparams); + + blosc2_dparams dparams = BLOSC2_DPARAMS_DEFAULTS; + dparams.nthreads = 1; + dparams.schunk = schunk; + blosc2_context *dctx; + dctx = blosc2_create_dctx(dparams); + int32_t zfp_chunk_nbytes, zfp_chunk_cbytes, blosc_chunk_cbytes; + uint8_t *lossy_chunk = malloc(chunksize + BLOSC2_MAX_OVERHEAD); + + for (int ci = 0; ci < nchunks; ci++) { + decompressed = blosc2_schunk_decompress_chunk(schunk, ci, data_in, chunksize); + if (decompressed < 0) { + printf("Error decompressing chunk \n"); + return -1; + } + + /* Compress using ZFP fixed-rate */ + csize = blosc2_compress_ctx(cctx, data_in, chunksize, chunk_zfp, chunksize + BLOSC2_MAX_OVERHEAD); + if (csize == 0) { + printf("Buffer is uncompressible. Giving up.\n"); + return 0; + } else if (csize < 0) { + printf("Compression error. Error code: %" PRId64 "\n", csize); + return (int) csize; + } + blosc2_cbuffer_sizes(chunk_zfp, &zfp_chunk_nbytes, &zfp_chunk_cbytes, NULL); + + decompressed = blosc2_decompress_ctx(dctx, chunk_zfp, zfp_chunk_cbytes, lossy_chunk, chunksize); + if (decompressed < 0) { + printf("Error decompressing chunk \n"); + return -1; + } + + /* Compress not using ZFP fixed-rate */ + csize = blosc2_compress_ctx(schunk->cctx, lossy_chunk, chunksize, chunk_blosc, chunksize + BLOSC2_MAX_OVERHEAD); + if (csize == 0) { + printf("Buffer is uncompressible. Giving up.\n"); + return 0; + } else if (csize < 0) { + printf("Compression error. Error code: %" PRId64 "\n", csize); + return (int) csize; + } + blosc2_cbuffer_sizes(chunk_blosc, NULL, &blosc_chunk_cbytes, NULL); + + /* Get item */ + int index, dsize_zfp, dsize_blosc; + float item_zfp, item_blosc; + int nelems = schunk->chunksize / schunk->typesize; + for (int i = 0; i < 100; ++i) { + srand(i); + index = rand() % nelems; + // Usual getitem + dsize_blosc = blosc2_getitem_ctx(schunk->dctx, chunk_blosc, blosc_chunk_cbytes, + index, 1, &item_blosc, sizeof(item_blosc)); + // Optimized getitem using ZFP cell machinery + dsize_zfp = blosc2_getitem_ctx(dctx, chunk_zfp, zfp_chunk_cbytes, + index, 1, &item_zfp, sizeof(item_zfp)); + if (dsize_blosc != dsize_zfp) { + printf("Different amount of items gotten\n"); + return -1; + } + if (item_blosc != item_zfp) { + printf("\nIn index %d different items extracted zfp %f blosc %f\n", index, item_zfp, item_blosc); + return -1; + } + } + } + + free(data_in); + free(data_dest); + free(chunk_zfp); + free(chunk_blosc); + free(lossy_chunk); + blosc2_free_ctx(cctx); + blosc2_free_ctx(dctx); + + printf("Successful roundtrip!\n"); + return (int) (BLOSC2_ERROR_SUCCESS); +} + +static int test_zfp_rate_getitem_double(blosc2_schunk* schunk) { + + if (schunk->typesize != 8) { + printf("Error: This test is only for doubles.\n"); + return 0; + } + int64_t nchunks = schunk->nchunks; + int32_t chunksize = (int32_t) (schunk->chunksize); + double *data_in = malloc(chunksize); + int decompressed; + int64_t csize; + uint8_t *chunk_zfp = malloc(chunksize + BLOSC2_MAX_OVERHEAD); + uint8_t *chunk_blosc = malloc(chunksize + BLOSC2_MAX_OVERHEAD); + double *data_dest = malloc(chunksize); + + /* Create a context for compression */ + int zfp_rate = 37; + blosc2_cparams cparams = BLOSC2_CPARAMS_DEFAULTS; + cparams.splitmode = BLOSC_NEVER_SPLIT; + cparams.typesize = schunk->typesize; + cparams.compcode = BLOSC_CODEC_ZFP_FIXED_RATE; + cparams.compcode_meta = zfp_rate; + cparams.filters[BLOSC2_MAX_FILTERS - 1] = BLOSC_NOFILTER; + cparams.clevel = 5; + cparams.nthreads = 1; + cparams.blocksize = schunk->blocksize; + cparams.schunk = schunk; + blosc2_context *cctx; + cctx = blosc2_create_cctx(cparams); + + blosc2_dparams dparams = BLOSC2_DPARAMS_DEFAULTS; + dparams.nthreads = 1; + dparams.schunk = schunk; + blosc2_context *dctx; + dctx = blosc2_create_dctx(dparams); + int32_t zfp_chunk_nbytes, zfp_chunk_cbytes, blosc_chunk_cbytes; + uint8_t *lossy_chunk = malloc(chunksize + BLOSC2_MAX_OVERHEAD); + + for (int ci = 0; ci < nchunks; ci++) { + decompressed = blosc2_schunk_decompress_chunk(schunk, ci, data_in, chunksize); + if (decompressed < 0) { + printf("Error decompressing chunk \n"); + return -1; + } + + /* Compress using ZFP fixed-rate */ + csize = blosc2_compress_ctx(cctx, data_in, chunksize, chunk_zfp, chunksize + BLOSC2_MAX_OVERHEAD); + if (csize == 0) { + printf("Buffer is uncompressible. Giving up.\n"); + return 0; + } else if (csize < 0) { + printf("Compression error. Error code: %" PRId64 "\n", csize); + return (int) csize; + } + blosc2_cbuffer_sizes(chunk_zfp, &zfp_chunk_nbytes, &zfp_chunk_cbytes, NULL); + + decompressed = blosc2_decompress_ctx(dctx, chunk_zfp, zfp_chunk_cbytes, lossy_chunk, chunksize); + if (decompressed < 0) { + printf("Error decompressing chunk \n"); + return -1; + } + + /* Compress not using ZFP fixed-rate */ + csize = blosc2_compress_ctx(schunk->cctx, lossy_chunk, chunksize, chunk_blosc, chunksize + BLOSC2_MAX_OVERHEAD); + if (csize == 0) { + printf("Buffer is uncompressible. Giving up.\n"); + return 0; + } else if (csize < 0) { + printf("Compression error. Error code: %" PRId64 "\n", csize); + return (int) csize; + } + blosc2_cbuffer_sizes(chunk_blosc, NULL, &blosc_chunk_cbytes, NULL); + + /* Get item */ + int index, dsize_zfp, dsize_blosc; + double item_zfp, item_blosc; + int nelems = schunk->chunksize / schunk->typesize; + for (int i = 0; i < 100; ++i) { + srand(i); + index = rand() % nelems; + // Usual getitem + dsize_blosc = blosc2_getitem_ctx(schunk->dctx, chunk_blosc, blosc_chunk_cbytes, + index, 1, &item_blosc, sizeof(item_blosc)); + // Optimized getitem using ZFP cell machinery + dsize_zfp = blosc2_getitem_ctx(dctx, chunk_zfp, zfp_chunk_cbytes, + index, 1, &item_zfp, sizeof(item_zfp)); + if (dsize_blosc != dsize_zfp) { + printf("Different amount of items gotten\n"); + return -1; + } + if (item_blosc != item_zfp) { + printf("\nIn index %d different items extracted zfp %f blosc %f\n", index, item_zfp, item_blosc); + return -1; + } + } + } + + free(data_in); + free(data_dest); + free(chunk_zfp); + free(chunk_blosc); + free(lossy_chunk); + blosc2_free_ctx(cctx); + blosc2_free_ctx(dctx); + + printf("Successful roundtrip!\n"); + return (int) (BLOSC2_ERROR_SUCCESS); +} + + +int float_cyclic() { + blosc2_schunk *schunk = blosc2_schunk_open("example_float_cyclic.caterva"); + BLOSC_ERROR_NULL(schunk, BLOSC2_ERROR_FILE_OPEN); + + /* Run the test. */ + int result = test_zfp_rate_getitem_float(schunk); + blosc2_schunk_free(schunk); + return result; +} + +int double_same_cells() { + blosc2_schunk *schunk = blosc2_schunk_open("example_double_same_cells.caterva"); + BLOSC_ERROR_NULL(schunk, BLOSC2_ERROR_FILE_OPEN); + + /* Run the test. */ + int result = test_zfp_rate_getitem_double(schunk); + blosc2_schunk_free(schunk); + return result; +} + +int day_month_temp() { + blosc2_schunk *schunk = blosc2_schunk_open("example_day_month_temp.caterva"); + BLOSC_ERROR_NULL(schunk, BLOSC2_ERROR_FILE_OPEN); + + /* Run the test. */ + int result = test_zfp_rate_getitem_float(schunk); + blosc2_schunk_free(schunk); + return result; +} + +int item_prices() { + blosc2_schunk *schunk = blosc2_schunk_open("example_item_prices.caterva"); + BLOSC_ERROR_NULL(schunk, BLOSC2_ERROR_FILE_OPEN); + + /* Run the test. */ + int result = test_zfp_rate_getitem_float(schunk); + blosc2_schunk_free(schunk); + return result; +} + + +int main(void) { + + blosc2_init(); // this is mandatory for initiallizing the plugin mechanism + printf("float_cyclic: "); + float_cyclic(); + printf("double_same_cells: "); + double_same_cells(); + printf("day_month_temp: "); + day_month_temp(); + printf("item_prices: "); + item_prices(); + blosc2_destroy(); + + return BLOSC2_ERROR_SUCCESS; +} diff --git a/src/blosc2/plugins/codecs/zfp/zfp-private.h b/src/blosc2/plugins/codecs/zfp/zfp-private.h new file mode 100644 index 0000000..ac71c16 --- /dev/null +++ b/src/blosc2/plugins/codecs/zfp/zfp-private.h @@ -0,0 +1,37 @@ +/********************************************************************* + Blosc - Blocked Shuffling and Compression Library + + Copyright (C) 2021 The Blosc Developers + https://blosc.org + License: BSD 3-Clause (see LICENSE.txt) + + See LICENSE.txt for details about copyright and rights to use. +**********************************************************************/ + + + +#ifndef ZFP_PRIVATE_H +#define ZFP_PRIVATE_H + +#define ZFP_MAX_DIM 4 +#define ZFP_CELL_SHAPE 4 + + +#if defined (__cplusplus) +extern "C" { +#endif +#define XXH_INLINE_ALL + +#define ZFP_ERROR_NULL(pointer) \ + do { \ + if ((pointer) == NULL) { \ + return 0; \ + } \ + } while (0) + + +#if defined (__cplusplus) +} +#endif + +#endif /* ZFP_PRIVATE_H */ diff --git a/src/blosc2/plugins/filters/CMakeLists.txt b/src/blosc2/plugins/filters/CMakeLists.txt new file mode 100644 index 0000000..9b6d414 --- /dev/null +++ b/src/blosc2/plugins/filters/CMakeLists.txt @@ -0,0 +1,4 @@ +add_subdirectory(ndcell) +add_subdirectory(ndmean) + +set(SOURCES ${SOURCES} ../plugins/filters/filters-registry.c PARENT_SCOPE) \ No newline at end of file diff --git a/src/blosc2/plugins/filters/filters-registry.c b/src/blosc2/plugins/filters/filters-registry.c new file mode 100644 index 0000000..e03f27f --- /dev/null +++ b/src/blosc2/plugins/filters/filters-registry.c @@ -0,0 +1,26 @@ +/* + Copyright (C) 2021 The Blosc Developers + http://blosc.org + License: BSD 3-Clause (see LICENSE.txt) +*/ + +#include +#include "blosc2/filters-registry.h" +#include "ndmean/ndmean.h" +#include "ndcell/ndcell.h" + +void register_filters(void) { + + blosc2_filter ndcell; + ndcell.id = BLOSC_FILTER_NDCELL; + ndcell.forward = (blosc2_filter_forward_cb) ndcell_encoder; + ndcell.backward = (blosc2_filter_backward_cb) ndcell_decoder; + register_filter_private(&ndcell); + + blosc2_filter ndmean; + ndmean.id = BLOSC_FILTER_NDMEAN; + ndmean.forward = (blosc2_filter_forward_cb) ndmean_encoder; + ndmean.backward = (blosc2_filter_backward_cb) ndmean_decoder; + register_filter_private(&ndmean); + +} \ No newline at end of file diff --git a/src/blosc2/plugins/filters/ndcell/CMakeLists.txt b/src/blosc2/plugins/filters/ndcell/CMakeLists.txt new file mode 100644 index 0000000..bdfe8ff --- /dev/null +++ b/src/blosc2/plugins/filters/ndcell/CMakeLists.txt @@ -0,0 +1,26 @@ +# sources +set(SOURCES ${SOURCES} ../plugins/filters/ndcell/ndcell.c ../plugins/filters/ndcell/ndcell.h PARENT_SCOPE) + +if(BUILD_TESTS) + + # targets + add_executable(test_ndcell test_ndcell.c) + # Define the BLOSC_TESTING symbol so normally-hidden functions + # aren't hidden from the view of the test programs. + set_property( + TARGET test_ndcell + APPEND PROPERTY COMPILE_DEFINITIONS BLOSC_TESTING) + + target_link_libraries(test_ndcell blosc_testing) + + # tests + add_test(NAME test_plugin_test_ndcell + COMMAND ${CMAKE_CROSSCOMPILING_EMULATOR} $) + + # Copy test files + file(GLOB TESTS_DATA ../../test_data/*.caterva) + foreach (data ${TESTS_DATA}) + file(COPY ${data} + DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/) + endforeach(data) +endif() diff --git a/src/blosc2/plugins/filters/ndcell/README.md b/src/blosc2/plugins/filters/ndcell/README.md new file mode 100644 index 0000000..b4299f6 --- /dev/null +++ b/src/blosc2/plugins/filters/ndcell/README.md @@ -0,0 +1,63 @@ +NDCELL: a multidimensional filter for lossless compression +============================================================================= + +Given an n-dim array or matrix, *NDCELL* is a filter based on the codec *NDLZ* +that divides data into multidimensional cells, reordering the dataset so +that the codec compress cell by cell. + +Plugin motivation +-------------------- + +*NDCELL* was created in order to make easy for codecs to search for patterns repetitions in multidimensional datasets using the Caterva blocking machinery. + +Plugin usage +------------------- + +The codec consists of an encoder called *ndcell_encoder()* to reorder data and +a decoder called *ndcell_decoder()* to recover the original data. + +The parameters used by *NDCELL* are the ones specified in the *blosc2_filter* +structure of *blosc2.h*. +Furthermore, since *NDCELL* goes through dataset blocks dividing them into fixed size cells, +user must specify the parameter meta as the cellshape, so if in a +3-dim dataset user specifies meta = 4, then cellshape will be 4x4x4. + +Plugin behaviour +------------------- + +This filter is meant to leverage multidimensionality for helping codecs to get +better compression ratios. The idea is to order the data so that when the +dataset is traversed the elements of a cell are all in a row. + + + + ------------------------ ----------------------------- + | 0 1 | 2 3 | 4 5 | 6 7 | | 0 1 8 9 | 2 3 10 11 | + | 8 9 |10 11|12 13|14 15| NDCELL encoder ----------------------------- + ------------------------- ------> | 4 5 12 13 | 6 7 14 15 | + |16 17|18 19|20 21|22 23| ----------------------------- + |24 25|26 27|28 29|30 31| | 16 17 24 25 | 18 19 26 27 | + ------------------------- ----------------------------- + | 20 21 28 29 | 22 23 30 31 | + ----------------------------- + + + + +Advantages and disadvantages +------------------------------ + +The particularity of *NDCELL* is that this filter +considers datasets multidimensionality and takes advantage of it instead +of processing all data as serial. + +The main disadvantage of *NDCELL* is that only a few codecs are benefited +by it and only for multidimensional datasets. + + + + + + + + diff --git a/src/blosc2/plugins/filters/ndcell/ndcell.c b/src/blosc2/plugins/filters/ndcell/ndcell.c new file mode 100644 index 0000000..8178a5a --- /dev/null +++ b/src/blosc2/plugins/filters/ndcell/ndcell.c @@ -0,0 +1,255 @@ +/* + Copyright (C) 2021 The Blosc Developers + http://blosc.org + License: BSD 3-Clause (see LICENSE.txt) +*/ + + +#include +#include "ndcell.h" +#include +#include +#include "../plugins/plugin_utils.h" + + +int ndcell_encoder(const uint8_t* input, uint8_t* output, int32_t length, uint8_t meta, blosc2_cparams* cparams) { + + int8_t ndim; + int64_t* shape = malloc(8 * sizeof(int64_t)); + int32_t* chunkshape = malloc(8 * sizeof(int32_t)); + int32_t* blockshape = malloc(8 * sizeof(int32_t)); + uint8_t* smeta; + int32_t smeta_len; + if (blosc2_meta_get(cparams->schunk, "caterva", &smeta, &smeta_len) < 0) { + free(shape); + free(chunkshape); + free(blockshape); + printf("Blosc error"); + return 0; + } + deserialize_meta(smeta, smeta_len, &ndim, shape, chunkshape, blockshape); + free(smeta); + int typesize = cparams->typesize; + + int8_t cell_shape = (int8_t)meta; + const int cell_size = (int) pow(cell_shape, ndim); + + int32_t blocksize = (int32_t) typesize; + for (int i = 0; i < ndim; i++){ + blocksize *= blockshape[i]; + } + + if (length != blocksize) { + free(shape); + free(chunkshape); + free(blockshape); + printf("Length not equal to blocksize %d %d \n", length, blocksize); + return -1; + } + + uint8_t* ip = (uint8_t *) input; + uint8_t* op = (uint8_t *) output; + uint8_t* op_limit = op + length; + + /* input and output buffer cannot be less than cell size */ + if (length < cell_size * typesize) { + free(shape); + free(chunkshape); + free(blockshape); + printf("Incorrect length"); + return 0; + } + + uint8_t* obase = op; + + int64_t i_shape[NDCELL_MAX_DIM]; + for (int i = 0; i < ndim; ++i) { + i_shape[i] = (blockshape[i] + cell_shape - 1) / cell_shape; + } + + int64_t ncells = 1; + for (int i = 0; i < ndim; ++i) { + ncells *= i_shape[i]; + } + + /* main loop */ + int64_t pad_shape[NDCELL_MAX_DIM]; + int64_t ii[NDCELL_MAX_DIM]; + for (int cell_ind = 0; cell_ind < ncells; cell_ind++) { // for each cell + blosc2_unidim_to_multidim(ndim, i_shape, cell_ind, ii); + uint32_t orig = 0; + int64_t nd_aux = (int64_t) cell_shape; + for (int i = ndim - 1; i >= 0; i--) { + orig += ii[i] * nd_aux; + nd_aux *= blockshape[i]; + } + + for (int dim_ind = 0; dim_ind < ndim; dim_ind++) { + if ((blockshape[dim_ind] % cell_shape != 0) && (ii[dim_ind] == i_shape[dim_ind] - 1)) { + pad_shape[dim_ind] = blockshape[dim_ind] % cell_shape; + } else { + pad_shape[dim_ind] = (int64_t) cell_shape; + } + } + int64_t ncopies = 1; + for (int i = 0; i < ndim - 1; ++i) { + ncopies *= pad_shape[i]; + } + int64_t kk[NDCELL_MAX_DIM]; + for (int copy_ind = 0; copy_ind < ncopies; ++copy_ind) { + blosc2_unidim_to_multidim(ndim - 1, pad_shape, copy_ind, kk); + nd_aux = blockshape[ndim - 1]; + int64_t ind = orig; + for (int i = ndim - 2; i >= 0; i--) { + ind += kk[i] * nd_aux; + nd_aux *= blockshape[i]; + } + memcpy(op, &ip[ind * typesize], pad_shape[ndim - 1] * typesize); + op += pad_shape[ndim - 1] * typesize; + } + + if (op > op_limit) { + free(shape); + free(chunkshape); + free(blockshape); + printf("Output too big"); + return 0; + } + } + + if((op - obase) != length) { + free(shape); + free(chunkshape); + free(blockshape); + printf("Output size must be equal to input size \n"); + return 0; + } + + free(shape); + free(chunkshape); + free(blockshape); + + return BLOSC2_ERROR_SUCCESS; +} + + +int ndcell_decoder(const uint8_t* input, uint8_t* output, int32_t length, uint8_t meta, blosc2_dparams* dparams) { + + blosc2_schunk *schunk = dparams->schunk; + int8_t ndim; + int64_t* shape = malloc(8 * sizeof(int64_t)); + int32_t* chunkshape = malloc(8 * sizeof(int32_t)); + int32_t* blockshape = malloc(8 * sizeof(int32_t)); + uint8_t* smeta; + int32_t smeta_len; + if (blosc2_meta_get(schunk, "caterva", &smeta, &smeta_len) < 0) { + free(shape); + free(chunkshape); + free(blockshape); + printf("Blosc error"); + return 0; + } + deserialize_meta(smeta, smeta_len, &ndim, shape, chunkshape, blockshape); + free(smeta); + + int8_t cell_shape = (int8_t)meta; + int cell_size = (int) pow(cell_shape, ndim); + int32_t typesize = schunk->typesize; + uint8_t* ip = (uint8_t*)input; + uint8_t* ip_limit = ip + length; + uint8_t* op = (uint8_t*)output; + int32_t blocksize = (int32_t) typesize; + for (int i = 0; i < ndim; i++){ + blocksize *= blockshape[i]; + } + + if (length != blocksize) { + free(shape); + free(chunkshape); + free(blockshape); + printf("Length not equal to blocksize \n"); + return -1; + } + + /* input and output buffer cannot be less than cell size */ + if (length < cell_size * typesize) { + free(shape); + free(chunkshape); + free(blockshape); + printf("Incorrect length"); + return 0; + } + + int64_t i_shape[NDCELL_MAX_DIM]; + for (int i = 0; i < ndim; ++i) { + i_shape[i] = (blockshape[i] + cell_shape - 1) / cell_shape; + } + + int64_t ncells = 1; + for (int i = 0; i < ndim; ++i) { + ncells *= i_shape[i]; + } + + /* main loop */ + int64_t pad_shape[NDCELL_MAX_DIM] = {0}; + int64_t ii[NDCELL_MAX_DIM]; + int32_t ind = 0; + for (int cell_ind = 0; cell_ind < ncells; cell_ind++) { // for each cell + + if (ip > ip_limit) { + free(shape); + free(chunkshape); + free(blockshape); + printf("Literal copy \n"); + return 0; + } + blosc2_unidim_to_multidim(ndim, i_shape, cell_ind, ii); + uint32_t orig = 0; + int64_t nd_aux = (int64_t) cell_shape; + for (int i = ndim - 1; i >= 0; i--) { + orig += ii[i] * nd_aux; + nd_aux *= blockshape[i]; + } + + for (int dim_ind = 0; dim_ind < ndim; dim_ind++) { + if ((blockshape[dim_ind] % cell_shape != 0) && (ii[dim_ind] == i_shape[dim_ind] - 1)) { + pad_shape[dim_ind] = blockshape[dim_ind] % cell_shape; + } else { + pad_shape[dim_ind] = (int64_t) cell_shape; + } + } + + int64_t ncopies = 1; + for (int i = 0; i < ndim - 1; ++i) { + ncopies *= pad_shape[i]; + } + int64_t kk[NDCELL_MAX_DIM]; + for (int copy_ind = 0; copy_ind < ncopies; ++copy_ind) { + blosc2_unidim_to_multidim(ndim - 1, pad_shape, copy_ind, kk); + nd_aux = blockshape[ndim - 1]; + ind = (int32_t) orig; + for (int i = ndim - 2; i >= 0; i--) { + ind += (int32_t) (kk[i] * nd_aux); + nd_aux *= blockshape[i]; + } + memcpy(&op[ind * typesize], ip, pad_shape[ndim - 1] * typesize); + ip += pad_shape[ndim - 1] * typesize; + } + } + ind += (int32_t) pad_shape[ndim - 1]; + + + if (ind != (int32_t) (blocksize / typesize)) { + free(shape); + free(chunkshape); + free(blockshape); + printf("Output size is not compatible with embedded blockshape ind %d %d \n", ind, (blocksize / typesize)); + return 0; + } + + free(shape); + free(chunkshape); + free(blockshape); + + return BLOSC2_ERROR_SUCCESS; +} diff --git a/src/blosc2/plugins/filters/ndcell/ndcell.h b/src/blosc2/plugins/filters/ndcell/ndcell.h new file mode 100644 index 0000000..7c58895 --- /dev/null +++ b/src/blosc2/plugins/filters/ndcell/ndcell.h @@ -0,0 +1,25 @@ +/********************************************************************* + Blosc - Blocked Shuffling and Compression Library + + Copyright (C) 2021 The Blosc Developers + https://blosc.org + License: BSD 3-Clause (see LICENSE.txt) + + See LICENSE.txt for details about copyright and rights to use. +**********************************************************************/ + +#ifndef CATERVA_NDCELL_H +#define CATERVA_NDCELL_H + +#include + +#define NDCELL_MAX_DIM 8 + + +int ndcell_encoder(const uint8_t* input, uint8_t* output, int32_t length, uint8_t meta, blosc2_cparams* cparams); + +int ndcell_decoder(const uint8_t* input, uint8_t* output, int32_t length, uint8_t meta, blosc2_dparams* dparams); + +#endif //CATERVA_NDCELL_H + + diff --git a/src/blosc2/plugins/filters/ndcell/test_ndcell.c b/src/blosc2/plugins/filters/ndcell/test_ndcell.c new file mode 100644 index 0000000..1f0771e --- /dev/null +++ b/src/blosc2/plugins/filters/ndcell/test_ndcell.c @@ -0,0 +1,162 @@ +/********************************************************************* + Blosc - Blocked Shuffling and Compression Library + + Copyright (C) 2021 The Blosc Developers + https://blosc.org + License: BSD 3-Clause (see LICENSE.txt) + + See LICENSE.txt for details about copyright and rights to use. + + Test program demonstrating use of the Blosc filter from C code. + To compile this program: + + $ gcc -O test_ndcell.c -o test_ndcell -lblosc2 + + To run: + + $ ./test_ndcell + Blosc version info: 2.0.0a6.dev ($Date:: 2018-05-18 #$) + Using 1 thread + Using ZSTD compressor + Successful roundtrip! + Compression: 41472 -> 3992 (10.4x) + rand: 37480 obtained + + Successful roundtrip! + Compression: 1792 -> 979 (1.8x) + same_cells: 813 obtained + + Successful roundtrip! + Compression: 16128 -> 1438 (11.2x) + some_matches: 14690 obtained + +**********************************************************************/ + +#include +#include "ndcell.h" +#include +#include "blosc2/filters-registry.h" + +static int test_ndcell(blosc2_schunk* schunk) { + + int64_t nchunks = schunk->nchunks; + int32_t chunksize = (int32_t) (schunk->chunksize); + // int isize = (int) array->extchunknitems * typesize; + uint8_t *data_in = malloc(chunksize); + int decompressed; + int64_t csize; + int64_t dsize; + int64_t csize_f = 0; + uint8_t *data_out = malloc(chunksize + BLOSC2_MAX_OVERHEAD); + uint8_t *data_dest = malloc(chunksize); + + /* Create a context for compression */ + blosc2_cparams cparams = BLOSC2_CPARAMS_DEFAULTS; + cparams.splitmode = BLOSC_ALWAYS_SPLIT; + cparams.typesize = schunk->typesize; + cparams.compcode = BLOSC_ZSTD; + cparams.filters[4] = BLOSC_FILTER_NDCELL; + cparams.filters_meta[4] =4; + cparams.filters[BLOSC2_MAX_FILTERS - 1] = BLOSC_SHUFFLE; + cparams.clevel = 9; + cparams.nthreads = 1; + cparams.blocksize = schunk->blocksize; + cparams.schunk = schunk; + blosc2_context *cctx; + cctx = blosc2_create_cctx(cparams); + + blosc2_dparams dparams = BLOSC2_DPARAMS_DEFAULTS; + dparams.nthreads = 1; + dparams.schunk = schunk; + blosc2_context *dctx; + dctx = blosc2_create_dctx(dparams); + + for (int ci = 0; ci < nchunks; ci++) { + + decompressed = blosc2_schunk_decompress_chunk(schunk, ci, data_in, chunksize); + if (decompressed < 0) { + printf("Error decompressing chunk \n"); + return -1; + } + + /* Compress with clevel=5 and shuffle active */ + csize = blosc2_compress_ctx(cctx, data_in, chunksize, data_out, chunksize + BLOSC2_MAX_OVERHEAD); + if (csize == 0) { + printf("Buffer is uncompressible. Giving up.\n"); + return 0; + } else if (csize < 0) { + printf("Compression error. Error code: %" PRId64 "\n", csize); + return (int) csize; + } + csize_f += csize; + + /* Decompress */ + dsize = blosc2_decompress_ctx(dctx, data_out, chunksize + BLOSC2_MAX_OVERHEAD, data_dest, chunksize); + if (dsize <= 0) { + printf("Decompression error. Error code: %" PRId64 "\n", dsize); + return (int) dsize; + } + + for (int i = 0; i < chunksize; i++) { + if (data_in[i] != data_dest[i]) { + printf("i: %d, data %u, dest %u", i, data_in[i], data_dest[i]); + printf("\n Decompressed data differs from original!\n"); + return -1; + } + } + } + csize_f = csize_f / nchunks; + + free(data_in); + free(data_out); + free(data_dest); + blosc2_free_ctx(cctx); + blosc2_free_ctx(dctx); + + printf("Successful roundtrip!\n"); + printf("Compression: %d -> %" PRId64 " (%.1fx)\n", chunksize, csize_f, (1. * chunksize) / (double)csize_f); + return (int) (chunksize - csize_f); +} + + +int rand_() { + blosc2_schunk *schunk = blosc2_schunk_open("example_rand.caterva"); + + /* Run the test. */ + int result = test_ndcell(schunk); + blosc2_schunk_free(schunk); + return result; +} + +int same_cells() { + blosc2_schunk *schunk = blosc2_schunk_open("example_same_cells.caterva"); + + /* Run the test. */ + int result = test_ndcell(schunk); + blosc2_schunk_free(schunk); + return result; +} + +int some_matches() { + blosc2_schunk *schunk = blosc2_schunk_open("example_some_matches.caterva"); + + /* Run the test. */ + int result = test_ndcell(schunk); + blosc2_schunk_free(schunk); + return result; +} + + +int main(void) { + + int result; + blosc2_init(); + result = rand_(); + printf("rand: %d obtained \n \n", result); + result = same_cells(); + printf("same_cells: %d obtained \n \n", result); + result = some_matches(); + printf("some_matches: %d obtained \n \n", result); + blosc2_destroy(); + return BLOSC2_ERROR_SUCCESS; +} diff --git a/src/blosc2/plugins/filters/ndmean/CMakeLists.txt b/src/blosc2/plugins/filters/ndmean/CMakeLists.txt new file mode 100644 index 0000000..47ceb3b --- /dev/null +++ b/src/blosc2/plugins/filters/ndmean/CMakeLists.txt @@ -0,0 +1,32 @@ +# sources +set(SOURCES ${SOURCES} ../plugins/filters/ndmean/ndmean.c ../plugins/filters/ndmean/ndmean.h PARENT_SCOPE) + +if(BUILD_TESTS) + # targets + add_executable(test_ndmean_repart test_ndmean_repart.c) + add_executable(test_ndmean_mean test_ndmean_mean.c) + # Define the BLOSC_TESTING symbol so normally-hidden functions + # aren't hidden from the view of the test programs. + set_property( + TARGET test_ndmean_mean + APPEND PROPERTY COMPILE_DEFINITIONS BLOSC_TESTING) + set_property( + TARGET test_ndmean_repart + APPEND PROPERTY COMPILE_DEFINITIONS BLOSC_TESTING) + + target_link_libraries(test_ndmean_repart blosc_testing) + target_link_libraries(test_ndmean_mean blosc_testing) + + # tests + add_test(NAME test_plugin_ndmean_repart + COMMAND ${CMAKE_CROSSCOMPILING_EMULATOR} $) + add_test(NAME test_plugin_ndmean_mean + COMMAND ${CMAKE_CROSSCOMPILING_EMULATOR} $) + + # Copy test files + file(GLOB TESTS_DATA *.caterva) + foreach (data ${TESTS_DATA}) + file(COPY ${data} + DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/) + endforeach(data) +endif() diff --git a/src/blosc2/plugins/filters/ndmean/README.md b/src/blosc2/plugins/filters/ndmean/README.md new file mode 100644 index 0000000..7cac74d --- /dev/null +++ b/src/blosc2/plugins/filters/ndmean/README.md @@ -0,0 +1,64 @@ +NDMEAN: a multidimensional filter for lossy compression +============================================================================= + +Given an n-dim array or matrix, *NDMEAN* is a filter based on *NDCELL* +(https://github.com/Blosc/c-blosc2/tree/main/plugins/filters/ndcell/README.rst) +that not only divides data into multidimensional cells so +that a codec compress cell by cell, but also replaces each cell elements by +the cell mean. + +Plugin motivation +-------------------- + +*NDMEAN* was created in order to add another depth level to *NDCELL*, taking +advantage of multidimensionality and making each cell items equal to earn +better compression ratios for the codecs. + +Plugin usage +------------------- + +The codec consists of an encoder called *ndmean_encoder()* to reorder data and +a decoder called *ndmean_decoder()* to recover the original order of data +but not the original data (lossy compression). + +The parameters used by *NDMEAN* are the ones specified in the *blosc2_filter* +structure of *blosc2.h*. +Furthermore, since *NDMEAN* goes through dataset blocks dividing them into fixed size cells, +user must specify the parameter meta as the cellshape, so if in a +3-dim dataset user specifies meta = 4, then cellshape will be 4x4x4. If user tries to use other value for meta, the codec +will return an error value. + +NDMEAN only works for float or double datasets (depending on the typesize), so if it is used +for a different data type it may return an error code. + +Plugin behaviour +------------------- + +This filter is meant first to order the data so that when the +dataset is traversed the elements of a cell are all in a row. +Furthermore, *NDMEAN* replaces cell elements by the cell mean. + + + + ------------------------ ----------------------------- + | 0 1 | 1 2 | 2 3 | 3 5 | | 1 1 1 1 1 1 | 2 2 2 2 2 2 | + | 2 0 | 3 1 | 4 2 | 3 5 | ----------------------------- + | 1 2 | 2 3 | 3 4 | 5 3 | NDMEAN encoder | 3 3 3 3 3 3 | 4 4 4 4 4 4 | + ------------------------- ------> ----------------------------- + | 1 3 | 1 2 | 2 4 | 4 8 | | 2 2 2 2 2 2 | 1 1 1 1 1 1 | + | 3 1 | 0 1 | 1 5 | 7 5 | ----------------------------- + | 1 3 | 0 2 | 0 6 | 4 8 | | 3 3 3 3 3 3 | 6 6 6 6 6 6 | + ------------------------- ----------------------------- + + + +Advantages and disadvantages +------------------------------ + +The particularity of *NDMEAN* is that this filter not only +considers datasets multidimensionality and takes advantage of it instead +of processing all data as serial, but also makes cell data easier to +compress. + +The main disadvantage of *NDMEAN* is that it changes the original data +and can not recover it (lossy compression). diff --git a/src/blosc2/plugins/filters/ndmean/ndmean.c b/src/blosc2/plugins/filters/ndmean/ndmean.c new file mode 100644 index 0000000..a481e62 --- /dev/null +++ b/src/blosc2/plugins/filters/ndmean/ndmean.c @@ -0,0 +1,316 @@ +/* + Copyright (C) 2021 The Blosc Developers + http://blosc.org + License: BSD 3-Clause (see LICENSE.txt) +*/ + + +#include "ndmean.h" +#include +#include +#include "../plugins/plugin_utils.h" + +int ndmean_encoder(const uint8_t* input, uint8_t* output, int32_t length, uint8_t meta, blosc2_cparams* cparams) { + + int8_t ndim; + int64_t* shape = malloc(8 * sizeof(int64_t)); + int32_t* chunkshape = malloc(8 * sizeof(int32_t)); + int32_t* blockshape = malloc(8 * sizeof(int32_t)); + uint8_t* smeta; + int32_t smeta_len; + if (blosc2_meta_get(cparams->schunk, "caterva", &smeta, &smeta_len) < 0) { + free(shape); + free(chunkshape); + free(blockshape); + printf("Blosc error"); + return 0; + } + deserialize_meta(smeta, smeta_len, &ndim, shape, chunkshape, blockshape); + free(smeta); + int typesize = cparams->typesize; + + if ((typesize != 4) && (typesize != 8)) { + free(shape); + free(chunkshape); + free(blockshape); + fprintf(stderr, "This filter only works for float or double buffers"); + return -1; + } + + int8_t cellshape[8]; + int cell_size = 1; + for (int i = 0; i < 8; ++i) { + if (i < ndim) { + cellshape[i] = (int8_t)meta; + if (cellshape[i] > blockshape[i]) { + cellshape[i] = (int8_t) blockshape[i]; + } + cell_size *= cellshape[i]; + } else { + cellshape[i] = 1; + } + } + int32_t blocksize = (int32_t) typesize; + for (int i = 0; i < ndim; i++){ + blocksize *= blockshape[i]; + } + + if (length != blocksize) { + free(shape); + free(chunkshape); + free(blockshape); + printf("Length not equal to blocksize %d %d \n", length, blocksize); + return -1; + } + + uint8_t* ip = (uint8_t *) input; + float* ip_float = (float *) ip; + double * ip_double = (double *) ip; + uint8_t* op = (uint8_t *) output; + uint8_t* op_limit = op + length; + int64_t cell_length; + float mean_float = 0; + double mean_double = 0; + + + /* input and output buffer cannot be less than cell size */ + if (length < cell_size * typesize) { + free(shape); + free(chunkshape); + free(blockshape); + printf("Incorrect length"); + return 0; + } + + uint8_t* obase = op; + + int64_t i_shape[NDMEAN_MAX_DIM]; + for (int i = 0; i < ndim; ++i) { + i_shape[i] = (blockshape[i] + cellshape[i] - 1) / cellshape[i]; + } + + int64_t ncells = 1; + for (int i = 0; i < ndim; ++i) { + ncells *= i_shape[i]; + } + + /* main loop */ + int64_t pad_shape[NDMEAN_MAX_DIM]; + int64_t ii[NDMEAN_MAX_DIM]; + for (int cell_ind = 0; cell_ind < ncells; cell_ind++) { // for each cell + blosc2_unidim_to_multidim(ndim, i_shape, cell_ind, ii); + uint32_t orig = 0; + int64_t nd_aux = (int64_t) (cellshape[0]); + for (int i = ndim - 1; i >= 0; i--) { + orig += ii[i] * nd_aux; + nd_aux *= blockshape[i]; + } + + for (int dim_ind = 0; dim_ind < ndim; dim_ind++) { + if ((blockshape[dim_ind] % cellshape[dim_ind] != 0) && (ii[dim_ind] == i_shape[dim_ind] - 1)) { + pad_shape[dim_ind] = blockshape[dim_ind] % cellshape[dim_ind]; + } else { + pad_shape[dim_ind] = (int32_t) cellshape[dim_ind]; + } + } + int64_t ncopies = 1; + for (int i = 0; i < ndim - 1; ++i) { + ncopies *= pad_shape[i]; + } + int64_t kk[8]; + mean_float = 0; + mean_double = 0; + for (int copy_ind = 0; copy_ind < ncopies; ++copy_ind) { + blosc2_unidim_to_multidim((int8_t) (ndim - 1), pad_shape, copy_ind, kk); + nd_aux = blockshape[ndim - 1]; + int64_t ind = orig; + for (int i = ndim - 2; i >= 0; i--) { + ind += kk[i] * nd_aux; + nd_aux *= blockshape[i]; + } + + switch (typesize) { + case 4: + for (int i = 0; i < pad_shape[ndim - 1]; i++) { + mean_float += ip_float[ind + i]; + } + break; + case 8: + for (int i = 0; i < pad_shape[ndim - 1]; i++) { + mean_double += ip_double[ind + i]; + } + break; + default : + break; + } + + } + cell_length = ncopies * pad_shape[ndim - 1]; + + switch (typesize) { + case 4: + mean_float /= (float) cell_length; + for (int i = 0; i < cell_length; i++) { + memcpy(op, &mean_float, typesize); + op += typesize; + } + break; + case 8: + mean_double /= (double) cell_length; + for (int i = 0; i < cell_length; i++) { + memcpy(op, &mean_double, typesize); + op += typesize; + } + break; + default : + break; + } + + if (op > op_limit) { + free(shape); + free(chunkshape); + free(blockshape); + printf("Output too big"); + return 0; + } + } + + free(shape); + free(chunkshape); + free(blockshape); + if((op - obase) != length) { + printf("Output size must be equal to input size \n"); + return 0; + } + + return BLOSC2_ERROR_SUCCESS; +} + + +int ndmean_decoder(const uint8_t* input, uint8_t* output, int32_t length, uint8_t meta, blosc2_dparams* dparams) { + + blosc2_schunk *schunk = dparams->schunk; + int8_t ndim; + int64_t* shape = malloc(8 * sizeof(int64_t)); + int32_t* chunkshape = malloc(8 * sizeof(int32_t)); + int32_t* blockshape = malloc(8 * sizeof(int32_t)); + uint8_t* smeta; + int32_t smeta_len; + if (blosc2_meta_get(schunk, "caterva", &smeta, &smeta_len) < 0) { + free(shape); + free(chunkshape); + free(blockshape); + printf("Blosc error"); + return 0; + } + deserialize_meta(smeta, smeta_len, &ndim, shape, chunkshape, blockshape); + free(smeta); + + int8_t cellshape[8]; + int cell_size = 1; + for (int i = 0; i < 8; ++i) { + if (i < ndim) { + cellshape[i] = (int8_t)meta; + if (cellshape[i] > blockshape[i]) { + cellshape[i] = (int8_t) blockshape[i]; + } + cell_size *= cellshape[i]; + } else { + cellshape[i] = 1; + } + } + + int8_t typesize = (int8_t) schunk->typesize; + uint8_t* ip = (uint8_t*)input; + uint8_t* ip_limit = ip + length; + uint8_t* op = (uint8_t*)output; + int32_t blocksize = (int32_t) typesize; + for (int i = 0; i < ndim; i++){ + blocksize *= blockshape[i]; + } + + if (length != blocksize) { + free(shape); + free(chunkshape); + free(blockshape); + printf("Length not equal to blocksize \n"); + return -1; + } + /* input and output buffer cannot be less than cell size */ + if (length < cell_size * typesize) { + free(shape); + free(chunkshape); + free(blockshape); + printf("Incorrect length"); + return 0; + } + + int64_t i_shape[NDMEAN_MAX_DIM]; + for (int i = 0; i < ndim; ++i) { + i_shape[i] = (blockshape[i] + cellshape[i] - 1) / cellshape[i]; + } + + int64_t ncells = 1; + for (int i = 0; i < ndim; ++i) { + ncells *= i_shape[i]; + } + + /* main loop */ + int64_t pad_shape[NDMEAN_MAX_DIM] = {0}; + int64_t ii[NDMEAN_MAX_DIM]; + int32_t ind = 0; + for (int cell_ind = 0; cell_ind < ncells; cell_ind++) { // for each cell + + if (ip > ip_limit) { + free(shape); + free(chunkshape); + free(blockshape); + printf("Literal copy \n"); + return 0; + } + blosc2_unidim_to_multidim(ndim, i_shape, cell_ind, ii); + uint32_t orig = 0; + int64_t nd_aux = (int64_t) (cellshape[0]); + for (int i = ndim - 1; i >= 0; i--) { + orig += ii[i] * nd_aux; + nd_aux *= blockshape[i]; + } + + for (int dim_ind = 0; dim_ind < ndim; dim_ind++) { + if ((blockshape[dim_ind] % cellshape[dim_ind] != 0) && (ii[dim_ind] == i_shape[dim_ind] - 1)) { + pad_shape[dim_ind] = blockshape[dim_ind] % cellshape[dim_ind]; + } else { + pad_shape[dim_ind] = (int64_t) cellshape[dim_ind]; + } + } + + int64_t ncopies = 1; + for (int i = 0; i < ndim - 1; ++i) { + ncopies *= pad_shape[i]; + } + int64_t kk[NDMEAN_MAX_DIM]; + for (int copy_ind = 0; copy_ind < ncopies; ++copy_ind) { + blosc2_unidim_to_multidim((int8_t) (ndim - 1), pad_shape, copy_ind, kk); + nd_aux = blockshape[ndim - 1]; + ind = (int32_t) orig; + for (int i = ndim - 2; i >= 0; i--) { + ind += (int32_t) (kk[i] * nd_aux); + nd_aux *= blockshape[i]; + } + memcpy(&op[ind * typesize], ip, pad_shape[ndim - 1] * typesize); + ip += pad_shape[ndim - 1] * typesize; + } + } + ind += (int32_t) pad_shape[ndim - 1]; + + free(shape); + free(chunkshape); + free(blockshape); + + if (ind != (int32_t) (blocksize / typesize)) { + printf("Output size is not compatible with embedded blockshape ind %d %d \n", ind, (blocksize / typesize)); + return 0; + } + + return BLOSC2_ERROR_SUCCESS; +} diff --git a/src/blosc2/plugins/filters/ndmean/ndmean.h b/src/blosc2/plugins/filters/ndmean/ndmean.h new file mode 100644 index 0000000..f3b7630 --- /dev/null +++ b/src/blosc2/plugins/filters/ndmean/ndmean.h @@ -0,0 +1,25 @@ +/********************************************************************* + Blosc - Blocked Shuffling and Compression Library + + Copyright (C) 2021 The Blosc Developers + https://blosc.org + License: BSD 3-Clause (see LICENSE.txt) + + See LICENSE.txt for details about copyright and rights to use. +**********************************************************************/ + +#ifndef CATERVA_NDMEAN_H +#define CATERVA_NDMEAN_H + +#include "blosc2.h" + +#define NDMEAN_MAX_DIM 8 + + +int ndmean_encoder(const uint8_t* input, uint8_t* output, int32_t length, uint8_t meta, blosc2_cparams* cparams); + +int ndmean_decoder(const uint8_t* input, uint8_t* output, int32_t length, uint8_t meta, blosc2_dparams* dparams); + +#endif //CATERVA_NDMEAN_H + + diff --git a/src/blosc2/plugins/filters/ndmean/test_ndmean_mean.c b/src/blosc2/plugins/filters/ndmean/test_ndmean_mean.c new file mode 100644 index 0000000..015da63 --- /dev/null +++ b/src/blosc2/plugins/filters/ndmean/test_ndmean_mean.c @@ -0,0 +1,269 @@ +/********************************************************************* + Blosc - Blocked Shuffling and Compression Library + + Copyright (C) 2021 The Blosc Developers + https://blosc.org + License: BSD 3-Clause (see LICENSE.txt) + + See LICENSE.txt for details about copyright and rights to use. + + Test program demonstrating use of the Blosc filter from C code. + To compile this program: + + $ gcc -O test_ndmean_mean.c -o test_ndmean_mean -lblosc2 + + To run: + + $ ./test_ndmean_mean + Blosc version info: 2.0.0a6.dev ($Date:: 2018-05-18 #$) + Using 1 thread + Using ZSTD compressor + Successful roundtrip! + Compression: 256 -> 137 (1.9x) + 2_rows_matches: 119 obtained + + Successful roundtrip! + Compression: 128 -> 110 (1.2x) + same_cells: 18 obtained + + Successful roundtrip! + Compression: 448 -> 259 (1.7x) + some_matches: 189 obtained + + +**********************************************************************/ + +#include +#include "ndmean.h" +#include +#include +#include "blosc2/filters-registry.h" +#include "../plugins/plugin_utils.h" + +#define EPSILON (float) (1) + +static bool is_close(double d1, double d2) { + + double aux = 1; + if (fabs(d1) < fabs(d2)) { + if (fabs(d1) > 0) { + aux = fabs(d2); + } + } else { + if (fabs(d2) > 0) { + aux = fabs(d1); + } + } + + return fabs(d1 - d2) < aux * EPSILON; +} + + +static int test_ndmean(blosc2_schunk* schunk) { + + int8_t ndim; + int64_t* shape = malloc(8 * sizeof(int64_t)); + int32_t* chunkshape = malloc(8 * sizeof(int32_t)); + int32_t* blockshape = malloc(8 * sizeof(int32_t)); + uint8_t cellshape = 4; + uint8_t* smeta; + int32_t smeta_len; + if (blosc2_meta_get(schunk, "caterva", &smeta, &smeta_len) < 0) { + printf("Blosc error"); + return 0; + } + deserialize_meta(smeta, smeta_len, &ndim, shape, chunkshape, blockshape); + free(smeta); + + if (ndim != 1) { + fprintf(stderr, "This test only works for ndim = 1"); + return -1; + } + + int32_t typesize = schunk->typesize; + int64_t nchunks = schunk->nchunks; + int32_t chunksize = (int32_t) (schunk->chunksize); + // int isize = (int) array->extchunknitems * typesize; + uint8_t *data_in = malloc(chunksize); + int decompressed; + int64_t csize; + int64_t dsize; + int64_t csize_f = 0; + uint8_t *data_out = malloc(chunksize + BLOSC2_MAX_OVERHEAD); + uint8_t *data_dest = malloc(chunksize); + + /* Create a context for compression */ + blosc2_cparams cparams = BLOSC2_CPARAMS_DEFAULTS; + cparams.splitmode = BLOSC_ALWAYS_SPLIT; + cparams.typesize = typesize; + cparams.compcode = BLOSC_ZSTD; + cparams.filters[4] = BLOSC_FILTER_NDMEAN; + cparams.filters_meta[4] = cellshape; + cparams.filters[BLOSC2_MAX_FILTERS - 1] = BLOSC_SHUFFLE; + cparams.clevel = 9; + cparams.nthreads = 1; + cparams.blocksize = schunk->blocksize; + cparams.schunk = schunk; + blosc2_context *cctx; + cctx = blosc2_create_cctx(cparams); + + blosc2_dparams dparams = BLOSC2_DPARAMS_DEFAULTS; + dparams.nthreads = 1; + dparams.schunk = schunk; + blosc2_context *dctx; + dctx = blosc2_create_dctx(dparams); + + double cell_mean; + for (int ci = 0; ci < nchunks; ci++) { + + decompressed = blosc2_schunk_decompress_chunk(schunk, ci, data_in, chunksize); + if (decompressed < 0) { + printf("Error decompressing chunk \n"); + return -1; + } + + /* + printf("\n data \n"); + for (int i = 0; i < nbytes; i++) { + printf("%u, ", data2[i]); + } + */ + + /* Compress with clevel=5 and shuffle active */ + csize = blosc2_compress_ctx(cctx, data_in, chunksize, data_out, chunksize + BLOSC2_MAX_OVERHEAD); + if (csize == 0) { + printf("Buffer is uncompressible. Giving up.\n"); + return 0; + } else if (csize < 0) { + printf("Compression error. Error code: %" PRId64 "\n", csize); + return (int) csize; + } + csize_f += csize; + + /* Decompress */ + dsize = blosc2_decompress_ctx(dctx, data_out, chunksize + BLOSC2_MAX_OVERHEAD, data_dest, chunksize); + if (dsize <= 0) { + printf("Decompression error. Error code: %" PRId64 "\n", dsize); + return (int) dsize; + } +/* + printf("\n data_dest: \n"); + for (int i = 0; i < chunksize / typesize; i++) { + if (typesize == 4) { + printf("%f, ", ((float *) data_dest)[i]); + } else if (typesize == 8) { + printf("%f, ", ((double *) data_dest)[i]); + } + } +*/ + int chunk_shape = chunkshape[0]; + if (ci == nchunks - 1) { + chunk_shape = (int) (shape[0] % chunkshape[0]); + } + int nblocks = (chunk_shape + blockshape[0] - 1) / blockshape[0]; + + for (int bi = 0; bi < nblocks; bi++) { + int block_shape = blockshape[0]; + if (bi == nblocks - 1) { + block_shape = chunk_shape % blockshape[0]; + } + int ncells = (block_shape + cellshape - 1) / cellshape; + + for (int cei = 0; cei < ncells; cei++) { + int ind = bi * blockshape[0] + cei * cellshape; + cell_mean = 0; + int cell_shape = cellshape; + if (cei == ncells - 1) { + cell_shape = block_shape % cellshape; + } + switch (typesize) { + case 4: + for (int i = 0; i < cell_shape; i++) { + cell_mean += ((float *) data_in)[ind + i]; + } + cell_mean /= cell_shape; + for (int i = 0; i < cell_shape; i++) { + if (!is_close(cell_mean, ((float *) data_dest)[ind + i])) { + printf("i: %d, cell_mean %.9f, dest %.9f", ind + i, cell_mean, ((float *) data_dest)[ind + i]); + printf("\n Decompressed data differs from original!\n"); + return -1; + } + } + break; + case 8: + for (int i = 0; i < cell_shape; i++) { + cell_mean += ((double *) data_in)[ind + i]; + } + cell_mean /= cell_shape; + for (int i = 0; i < cell_shape; i++) { + if (!is_close(cell_mean, ((double *) data_dest)[ind + i])) { + printf("i: %d, cell_mean %.9f, dest %.9f", ind + i, cell_mean, ((double *) data_dest)[ind + i]); + printf("\n Decompressed data differs from original!\n"); + return -1; + } + } + break; + default : + break; + } + } + } + } + csize_f = csize_f / nchunks; + + free(data_in); + free(data_out); + free(data_dest); + blosc2_free_ctx(cctx); + blosc2_free_ctx(dctx); + free(shape); + free(chunkshape); + free(blockshape); + + printf("Successful roundtrip!\n"); + printf("Compression: %d -> %" PRId64 " (%.1fx)\n", chunksize, csize_f, (1. * chunksize) / (double)csize_f); + return (int) (chunksize - csize_f); +} + + +int rows_matches() { + blosc2_schunk *schunk = blosc2_schunk_open("example_ndmean_1dim_2rows.caterva"); + + /* Run the test. */ + int result = test_ndmean(schunk); + blosc2_schunk_free(schunk); + return result; +} + +int same_cells() { + blosc2_schunk *schunk = blosc2_schunk_open("example_ndmean_1dim_same_cells.caterva"); + + /* Run the test. */ + int result = test_ndmean(schunk); + blosc2_schunk_free(schunk); + return result; +} + +int some_matches() { + blosc2_schunk *schunk = blosc2_schunk_open("example_ndmean_1dim_some_matches.caterva"); + + /* Run the test. */ + int result = test_ndmean(schunk); + blosc2_schunk_free(schunk); + return result; +} + + +int main(void) { + + int result; + blosc2_init(); + result = rows_matches(); + printf("2_rows_matches: %d obtained \n \n", result); + result = same_cells(); + printf("same_cells: %d obtained \n \n", result); + result = some_matches(); + printf("some_matches: %d obtained \n \n", result); + blosc2_destroy(); + return BLOSC2_ERROR_SUCCESS; +} diff --git a/src/blosc2/plugins/filters/ndmean/test_ndmean_repart.c b/src/blosc2/plugins/filters/ndmean/test_ndmean_repart.c new file mode 100644 index 0000000..0198016 --- /dev/null +++ b/src/blosc2/plugins/filters/ndmean/test_ndmean_repart.c @@ -0,0 +1,204 @@ +/********************************************************************* + Blosc - Blocked Shuffling and Compression Library + + Copyright (C) 2021 The Blosc Developers + https://blosc.org + License: BSD 3-Clause (see LICENSE.txt) + + See LICENSE.txt for details about copyright and rights to use. + + Test program demonstrating use of the Blosc filter from C code. + To compile this program: + $ gcc -O test_ndmean_repart.c -o test_ndmean_repart -lblosc2 + + To run: + + $ ./test_ndmean_repart + Blosc version info: 2.0.0a6.dev ($Date:: 2018-05-18 #$) + Using 1 thread + Using ZSTD compressor + Successful roundtrip! + Compression: 41472 -> 1015 (40.9x) + rand: 40457 obtained + + Successful roundtrip! + Compression: 1792 -> 323 (5.5x) + same_cells: 1469 obtained + + Successful roundtrip! + Compression: 16128 -> 767 (21.0x) + some_matches: 15361 obtained + +**********************************************************************/ + +#include +#include "ndmean.h" +#include +#include +#include "blosc2/filters-registry.h" + +#define EPSILON (float) (1e-5) + +static bool is_close(double d1, double d2) { + + double aux = 1; + if (fabs(d1) < fabs(d2)) { + if (fabs(d1) > 0) { + aux = fabs(d2); + } + } else { + if (fabs(d2) > 0) { + aux = fabs(d1); + } + } + + return fabs(d1 - d2) < aux * EPSILON; +} + + +static int test_ndmean(blosc2_schunk* schunk) { + + int32_t typesize = schunk->typesize; + int64_t nchunks = schunk->nchunks; + int32_t chunksize = (int32_t) (schunk->chunksize); + // int isize = (int) array->extchunknitems * typesize; + uint8_t *data_in = malloc(chunksize); + int decompressed; + int64_t csize; + int64_t dsize; + int64_t csize_f = 0; + uint8_t *data_out = malloc(chunksize + BLOSC2_MAX_OVERHEAD); + uint8_t *data_dest = malloc(chunksize); + + /* Create a context for compression */ + blosc2_cparams cparams = BLOSC2_CPARAMS_DEFAULTS; + cparams.splitmode = BLOSC_ALWAYS_SPLIT; + cparams.typesize = typesize; + cparams.compcode = BLOSC_ZSTD; + cparams.filters[4] = BLOSC_FILTER_NDMEAN; + cparams.filters_meta[4] = 4; + cparams.filters[BLOSC2_MAX_FILTERS - 1] = BLOSC_SHUFFLE; + cparams.clevel = 9; + cparams.nthreads = 1; + cparams.blocksize = schunk->blocksize; + cparams.schunk = schunk; + blosc2_context *cctx; + cctx = blosc2_create_cctx(cparams); + + blosc2_dparams dparams = BLOSC2_DPARAMS_DEFAULTS; + dparams.nthreads = 1; + dparams.schunk = schunk; + blosc2_context *dctx; + dctx = blosc2_create_dctx(dparams); + + for (int ci = 0; ci < nchunks; ci++) { + + decompressed = blosc2_schunk_decompress_chunk(schunk, ci, data_in, chunksize); + if (decompressed < 0) { + printf("Error decompressing chunk \n"); + return -1; + } + + /* + printf("\n data \n"); + for (int i = 0; i < nbytes; i++) { + printf("%u, ", data2[i]); + } + */ + + /* Compress with clevel=5 and shuffle active */ + csize = blosc2_compress_ctx(cctx, data_in, chunksize, data_out, chunksize + BLOSC2_MAX_OVERHEAD); + if (csize == 0) { + printf("Buffer is uncompressible. Giving up.\n"); + return 0; + } else if (csize < 0) { + printf("Compression error. Error code: %" PRId64 "\n", csize); + return (int) csize; + } + csize_f += csize; + + /* Decompress */ + dsize = blosc2_decompress_ctx(dctx, data_out, chunksize + BLOSC2_MAX_OVERHEAD, data_dest, chunksize); + if (dsize <= 0) { + printf("Decompression error. Error code: %" PRId64 "\n", dsize); + return (int) dsize; + } + + switch (typesize) { + case 4: + for (int i = 0; i < chunksize / typesize; i++) { + if (!is_close(((float *) data_in)[i], ((float *) data_dest)[i])) { + printf("i: %d, data %.9f, dest %.9f", i, ((float *) data_in)[i], ((float *) data_dest)[i]); + printf("\n Decompressed data differs from original!\n"); + return -1; + } + } + break; + case 8: + for (int i = 0; i < chunksize / typesize; i++) { + if (!is_close(((double *) data_in)[i], ((double *) data_dest)[i])) { + printf("i: %d, data %.9f, dest %.9f", i, ((double *) data_in)[i], ((double *) data_dest)[i]); + printf("\n Decompressed data differs from original!\n"); + return -1; + } + } + break; + default : + break; + } + } + csize_f = csize_f / nchunks; + + free(data_in); + free(data_out); + free(data_dest); + blosc2_free_ctx(cctx); + blosc2_free_ctx(dctx); + + printf("Successful roundtrip!\n"); + printf("Compression: %d -> %" PRId64 " (%.1fx)\n", chunksize, csize_f, (1. * chunksize) / (double)csize_f); + return (int) (chunksize - csize_f); +} + + +int rand_() { + blosc2_schunk *schunk = blosc2_schunk_open("example_ndmean_repart_rand.caterva"); + + /* Run the test. */ + int result = test_ndmean(schunk); + blosc2_schunk_free(schunk); + return result; +} + +int same_cells() { + blosc2_schunk *schunk = blosc2_schunk_open("example_ndmean_repart_same_cells.caterva"); + + /* Run the test. */ + int result = test_ndmean(schunk); + blosc2_schunk_free(schunk); + return result; +} + +int some_matches() { + blosc2_schunk *schunk = blosc2_schunk_open("example_ndmean_repart_some_matches.caterva"); + + /* Run the test. */ + int result = test_ndmean(schunk); + blosc2_schunk_free(schunk); + return result; +} + + +int main(void) { + + int result; + blosc2_init(); + result = rand_(); + printf("rand: %d obtained \n \n", result); + result = same_cells(); + printf("same_cells: %d obtained \n \n", result); + result = some_matches(); + printf("some_matches: %d obtained \n \n", result); + blosc2_destroy(); + return BLOSC2_ERROR_SUCCESS; +} diff --git a/src/blosc2/plugins/plugin_utils.c b/src/blosc2/plugins/plugin_utils.c new file mode 100644 index 0000000..d53581e --- /dev/null +++ b/src/blosc2/plugins/plugin_utils.c @@ -0,0 +1,102 @@ +/* + Copyright (C) 2021 The Blosc Developers + http://blosc.org + License: BSD 3-Clause (see LICENSE.txt) +*/ + +#include +#include "blosc2.h" +#include "plugin_utils.h" + +#define BLOSC_PLUGINS_MAX_DIM 8 + +void swap_store(void *dest, const void *pa, int size) { + uint8_t *pa_ = (uint8_t *) pa; + uint8_t *pa2_ = malloc((size_t) size); + int i = 1; /* for big/little endian detection */ + char *p = (char *) &i; + + if (p[0] == 1) { + /* little endian */ + switch (size) { + case 8: + pa2_[0] = pa_[7]; + pa2_[1] = pa_[6]; + pa2_[2] = pa_[5]; + pa2_[3] = pa_[4]; + pa2_[4] = pa_[3]; + pa2_[5] = pa_[2]; + pa2_[6] = pa_[1]; + pa2_[7] = pa_[0]; + break; + case 4: + pa2_[0] = pa_[3]; + pa2_[1] = pa_[2]; + pa2_[2] = pa_[1]; + pa2_[3] = pa_[0]; + break; + case 2: + pa2_[0] = pa_[1]; + pa2_[1] = pa_[0]; + break; + case 1: + pa2_[0] = pa_[0]; + break; + default: + fprintf(stderr, "Unhandled nitems: %d\n", size); + } + } + memcpy(dest, pa2_, size); + free(pa2_); +} + +int32_t deserialize_meta(uint8_t *smeta, int32_t smeta_len, int8_t *ndim, int64_t *shape, + int32_t *chunkshape, int32_t *blockshape) { + BLOSC_UNUSED_PARAM(smeta_len); + uint8_t *pmeta = smeta; + + // Check that we have an array with 5 entries (version, ndim, shape, chunkshape, blockshape) + pmeta += 1; + + // version entry + int8_t version = (int8_t)pmeta[0]; // positive fixnum (7-bit positive integer) + BLOSC_UNUSED_PARAM(version); + pmeta += 1; + + // ndim entry + *ndim = (int8_t)pmeta[0]; + int8_t ndim_aux = *ndim; // positive fixnum (7-bit positive integer) + pmeta += 1; + + // shape entry + // Initialize to ones, as required by Caterva + for (int i = 0; i < BLOSC_PLUGINS_MAX_DIM; i++) shape[i] = 1; + pmeta += 1; + for (int8_t i = 0; i < ndim_aux; i++) { + pmeta += 1; + swap_store(shape + i, pmeta, sizeof(int64_t)); + pmeta += sizeof(int64_t); + } + + // chunkshape entry + // Initialize to ones, as required by Caterva + for (int i = 0; i < BLOSC_PLUGINS_MAX_DIM; i++) chunkshape[i] = 1; + pmeta += 1; + for (int8_t i = 0; i < ndim_aux; i++) { + pmeta += 1; + swap_store(chunkshape + i, pmeta, sizeof(int32_t)); + pmeta += sizeof(int32_t); + } + + // blockshape entry + // Initialize to ones, as required by Caterva + for (int i = 0; i < BLOSC_PLUGINS_MAX_DIM; i++) blockshape[i] = 1; + pmeta += 1; + for (int8_t i = 0; i < ndim_aux; i++) { + pmeta += 1; + swap_store(blockshape + i, pmeta, sizeof(int32_t)); + pmeta += sizeof(int32_t); + } + int32_t slen = (int32_t)(pmeta - smeta); + return slen; +} diff --git a/src/blosc2/plugins/plugin_utils.h b/src/blosc2/plugins/plugin_utils.h new file mode 100644 index 0000000..15f54d1 --- /dev/null +++ b/src/blosc2/plugins/plugin_utils.h @@ -0,0 +1,10 @@ +/* + Copyright (C) 2021 The Blosc Developers + http://blosc.org + License: BSD 3-Clause (see LICENSE.txt) +*/ + +void swap_store(void *dest, const void *pa, int size); + +int32_t deserialize_meta(uint8_t *smeta, int32_t smeta_len, int8_t *ndim, int64_t *shape, + int32_t *chunkshape, int32_t *blockshape);