diff --git a/deps/zlib/CMakeLists.txt b/deps/zlib/CMakeLists.txt index 66f7d04966afa5..59d77c3a413628 100644 --- a/deps/zlib/CMakeLists.txt +++ b/deps/zlib/CMakeLists.txt @@ -24,6 +24,7 @@ check_include_file(stddef.h HAVE_STDDEF_H) option(ENABLE_SIMD_OPTIMIZATIONS "Enable all SIMD optimizations" OFF) option(ENABLE_SIMD_AVX512 "Enable SIMD AXV512 optimizations" OFF) option(USE_ZLIB_RABIN_KARP_HASH "Enable bitstream compatibility with canonical zlib" OFF) +option(ENABLE_INTEL_QAT_COMPRESSION "Enable Intel Quick Assist Technology use for compression" OFF) option(BUILD_UNITTESTS "Enable standalone unit tests build" OFF) option(BUILD_MINIZIP_BIN "Enable building minzip_bin tool" OFF) option(BUILD_ZPIPE "Enable building zpipe tool" OFF) @@ -228,6 +229,22 @@ if (ENABLE_SIMD_OPTIMIZATIONS) endif() endif() +if (ENABLE_INTEL_QAT_COMPRESSION) + list(APPEND ZLIB_SRCS ${CMAKE_CURRENT_SOURCE_DIR}/contrib/qat/deflate_qat.cpp) + list(APPEND ZLIB_SRCS ${CMAKE_CURRENT_SOURCE_DIR}/contrib/qat/qatzpp/io_buffers.cpp) + list(APPEND ZLIB_SRCS ${CMAKE_CURRENT_SOURCE_DIR}/contrib/qat/qatzpp/memory.cpp) + list(APPEND ZLIB_SRCS ${CMAKE_CURRENT_SOURCE_DIR}/contrib/qat/qatzpp/qat_buffer_list.cpp) + list(APPEND ZLIB_SRCS ${CMAKE_CURRENT_SOURCE_DIR}/contrib/qat/qatzpp/qat.cpp) + list(APPEND ZLIB_SRCS ${CMAKE_CURRENT_SOURCE_DIR}/contrib/qat/qatzpp/qat_instance.cpp) + list(APPEND ZLIB_SRCS ${CMAKE_CURRENT_SOURCE_DIR}/contrib/qat/qatzpp/session.cpp) + list(APPEND ZLIB_SRCS ${CMAKE_CURRENT_SOURCE_DIR}/contrib/qat/qatzpp/qat_task.cpp) + + # TODO(gustavoa): Find a way to include the qatzpp headers without having the + # presubmit check throw errors. + include_directories(${CMAKE_CURRENT_SOURCE_DIR}/contrib/qat/qatzpp) + add_compile_definitions(QAT_COMPRESSION_ENABLED) +endif() + # parse the full version number from zlib.h and include in ZLIB_FULL_VERSION file(READ ${CMAKE_CURRENT_SOURCE_DIR}/zlib.h _zlib_h_contents) string(REGEX REPLACE ".*#define[ \t]+ZLIB_VERSION[ \t]+\"([-0-9A-Za-z.]+)\".*" @@ -254,6 +271,15 @@ add_library(zlibstatic STATIC ${ZLIB_SRCS} ${ZLIB_PUBLIC_HDRS} ${ZLIB_PRIVATE_HD set_target_properties(zlib PROPERTIES DEFINE_SYMBOL ZLIB_DLL) set_target_properties(zlib PROPERTIES SOVERSION 1) +if (ENABLE_INTEL_QAT_COMPRESSION) + target_include_directories(zlib PUBLIC ${QATZPP_INCLUDE_DIRS}) + target_link_libraries(zlib ${QATZPP_LIBRARY}) + target_link_libraries(zlib qat) + target_include_directories(zlibstatic PUBLIC ${QATZPP_INCLUDE_DIRS}) + target_link_libraries(zlibstatic ${QATZPP_LIBRARY}) + target_link_libraries(zlibstatic qat) +endif() + if(NOT CYGWIN) # This property causes shared libraries on Linux to have the full version # encoded into their final filename. We disable this on Cygwin because diff --git a/deps/zlib/contrib/bench/zlib_bench.cc b/deps/zlib/contrib/bench/zlib_bench.cc index b65f9291bff500..6df296c8721056 100644 --- a/deps/zlib/contrib/bench/zlib_bench.cc +++ b/deps/zlib/contrib/bench/zlib_bench.cc @@ -71,10 +71,6 @@ Data read_file_data_or_exit(const char* name) { return data; } -size_t zlib_estimate_compressed_size(size_t input_size) { - return compressBound(input_size); -} - enum zlib_wrapper { kWrapperNONE, kWrapperZLIB, @@ -128,10 +124,6 @@ void zlib_compress( std::string* output, bool resize_output = false) { - if (resize_output) - output->resize(zlib_estimate_compressed_size(input_size)); - size_t output_size = output->size(); - z_stream stream; memset(&stream, 0, sizeof(stream)); @@ -140,6 +132,11 @@ void zlib_compress( if (result != Z_OK) error_exit("deflateInit2 failed", result); + if (resize_output) { + output->resize(deflateBound(&stream, input_size)); + } + size_t output_size = output->size(); + stream.next_out = (Bytef*)string_data(output); stream.avail_out = (uInt)output_size; stream.next_in = (z_const Bytef*)input; @@ -299,7 +296,7 @@ void zlib_file(const char* name, // Pre-grow the output buffer so we don't measure string resize time. for (int b = 0; b < blocks; ++b) - compressed[b].resize(zlib_estimate_compressed_size(block_size)); + zlib_compress(type, input[b], input_length[b], &compressed[b], true); auto start = now(); for (int b = 0; b < blocks; ++b) @@ -307,11 +304,6 @@ void zlib_file(const char* name, zlib_compress(type, input[b], input_length[b], &compressed[b]); ctime[run] = std::chrono::duration(now() - start).count(); - // Compress again, resizing compressed, so we don't leave junk at the - // end of the compressed string that could confuse zlib_uncompress(). - for (int b = 0; b < blocks; ++b) - zlib_compress(type, input[b], input_length[b], &compressed[b], true); - for (int b = 0; b < blocks; ++b) output[b].resize(input_length[b]); diff --git a/deps/zlib/contrib/minizip/README.chromium b/deps/zlib/contrib/minizip/README.chromium index b5895f2a5181b0..ceaff34f0f3fa6 100644 --- a/deps/zlib/contrib/minizip/README.chromium +++ b/deps/zlib/contrib/minizip/README.chromium @@ -2,10 +2,11 @@ Name: ZIP file API for reading file entries in a ZIP archive Short Name: minizip URL: https://github.com/madler/zlib/tree/master/contrib/minizip Version: 1.3.0.1 +Revision: 643e17b7498d12ab8d15565662880579692f769d License: Zlib License File: //third_party/zlib/LICENSE -Security Critical: yes Shipped: yes +Security Critical: yes CPEPrefix: cpe:/a:minizip_project:minizip Description: diff --git a/deps/zlib/contrib/qat/deflate_qat.cpp b/deps/zlib/contrib/qat/deflate_qat.cpp new file mode 100644 index 00000000000000..bfe45472bb51b1 --- /dev/null +++ b/deps/zlib/contrib/qat/deflate_qat.cpp @@ -0,0 +1,312 @@ +/* + * Copyright (C) 2024 Intel Corporation. All rights reserved. + * Authors: + * Gustavo A Espinoza + * + * + * For conditions of distribution and use, see copyright notice in zlib.h + */ +#include "deflate_qat.h" +#include "deflate.h" + +#include "session.hpp" +#include "qat_instance.hpp" +#include "qat_buffer_list.hpp" +#include "qat.hpp" + +#include + +/* +* TODO(gustavoa): Make the input size adjustable from the memlevel +* attribute on deflateInit. +*/ +static constexpr size_t kInputSize = 1024 * 1024; + +/* QAT Instances obtained available from the library. */ +static std::vector> qat_instances; + +/* +* TODO(gustavoa): Verify if the ordering of the struct fields won't create +* unnecessary holes in the structure that requires extraneous padding. +*/ +struct qat_deflate { + std::unique_ptr qat_session; + + /* QAT requires contiguous physical pages. Cannot be allocated using + * malloc/new. + */ + uint8_t *input_buffer; + uint8_t *output_buffer; + + /* Pointer to the next byte in the output buffer. */ + uint8_t *pending_out; + + unsigned input_buffer_size; + unsigned output_buffer_size; + + unsigned pending_in_count; + unsigned pending_out_count; +}; + +static std::unique_ptr qat_create_session(int level, int wrap) +{ + CpaDcChecksum checksum = CPA_DC_NONE; + + switch(wrap) { + case 1: + checksum = CPA_DC_ADLER32; + break; + case 2: + checksum = CPA_DC_CRC32; + break; + } + + return std::make_unique( + qat_instances[0], + (CpaDcCompLvl)level, + checksum, + 0 + ); +} + + +int qat_deflate_init() +{ + return (qat::Initialize()) ? Z_ERRNO : Z_OK; +} + +struct qat_deflate* qat_deflate_state_init(int level, int wrap) +{ + if (qat_instances.empty()) { + qat_instances = qat::Instance::Create(); + } + if (qat_instances.empty()) { + return nullptr; + } + + struct qat_deflate *qat_deflate = new struct qat_deflate; + if (!qat_deflate) { + return nullptr; + } + + /* TODO(gustavoa): Find a way to utilize all the available instances for the same + * process. + */ + qat_instances[0]->Start(); + + qat_deflate->qat_session = qat_create_session(level, wrap); + + qat_deflate->input_buffer_size = kInputSize; + qat_deflate->input_buffer = qat::AllocBlockArray(kInputSize, 0); + qat_deflate->output_buffer_size = + qat_deflate->qat_session->GetDeflateBound(qat_deflate->input_buffer_size); + qat_deflate->pending_out = qat_deflate->output_buffer = + qat::AllocBlockArray(qat_deflate->output_buffer_size, 0); + + qat_deflate->pending_in_count = qat_deflate->pending_out_count = 0; + + if (!qat_deflate->input_buffer || !qat_deflate->output_buffer) { + return nullptr; + } + + return qat_deflate; +} + +static unsigned qat_read_buf(z_streamp strm, struct qat_deflate* qat, unsigned size) +{ + unsigned len = strm->avail_in; + + if (len > size) { + len = size; + } + if (len == 0) return 0; + + strm->avail_in -= len; + strm->total_in += len; + + zmemcpy( + qat->input_buffer + qat->pending_in_count, + strm->next_in, + len + ); + + strm->next_in += len; + qat->pending_in_count += len; + + return len; +} + +void qat_flush_pending(deflate_state* s) +{ + unsigned len; + z_streamp strm = s->strm; + struct qat_deflate* qat = s->qat_s; + + len = qat->pending_out_count; + if (len > strm->avail_out) len = strm->avail_out; + if (len == 0) return; + + zmemcpy(strm->next_out, qat->pending_out, len); + + qat->pending_out += len; + qat->pending_out_count -= len; + strm->next_out += len; + strm->avail_out -= len; + strm->total_out += len; + if (qat->pending_out_count == 0) { + qat->pending_out = qat->output_buffer; + } +} + +static int qat_compress_pending(deflate_state*s, int flush) +{ + struct qat_deflate* qat = s->qat_s; + uint32_t metadata_size; + + /* TODO(gustavoa): find a way to make qatzpp setup this number internally. */ + cpaDcBufferListGetMetaSize(qat->qat_session->getInstance()->GetHandle(), 1, &metadata_size); + + auto job = qat->qat_session->Deflate( + std::make_unique( + std::make_unique( + qat->input_buffer, + qat->pending_in_count, + metadata_size + ), + std::make_unique( + qat->output_buffer, + qat->output_buffer_size, + metadata_size + ) + ), (flush == Z_FINISH && s->strm->avail_in == 0) + ); + + job->WaitCompletion(); + + /* + * TODO(gustavoa): make QAT perform the checksum combine. + */ + if (s->wrap == 2) { + s->strm->adler = crc32_combine( + s->strm->adler, + job->GetResults()->checksum, + job->GetResults()->consumed + ); + } else if (s->wrap == 1) { + s->strm->adler = adler32( + s->strm->adler, + qat->input_buffer, + job->GetResults()->consumed + ); + } + + qat->pending_out_count = job->GetResults()->produced; + qat->pending_in_count -= job->GetResults()->consumed; + + if(qat->pending_in_count != 0) { + /* Copy any remaining bytes to the beginning of the buffer. */ + zmemcpy( + qat->input_buffer, + qat->input_buffer + job->GetResults()->consumed, + qat->pending_in_count + ); + } + + return 0; +} + +qat_block_state qat_deflate_step(deflate_state* s, int flush) +{ + z_streamp strm = s->strm; + struct qat_deflate* qat_state = s->qat_s; + + for (;;) { + if (qat_state->pending_in_count < qat_state->input_buffer_size) { + qat_read_buf( + strm, + qat_state, + qat_state->input_buffer_size - qat_state->pending_in_count + ); + if (qat_state->pending_in_count < qat_state->input_buffer_size && flush == Z_NO_FLUSH) { + return qat_block_need_more; + } else { + qat_compress_pending(s, flush); + } + if (strm->avail_in == 0) { + break; + } + } else { + qat_compress_pending(s, flush); + } + + qat_flush_pending(s); + if (strm->avail_out == 0) { + return (flush == Z_FINISH) ? qat_block_finish_started : qat_block_need_more; + } + } + + if (flush == Z_FINISH) { + qat_flush_pending(s); + if (strm->avail_out == 0) { + return qat_block_finish_started; + } else { + return qat_block_finish_done; + } + } + + qat_flush_pending(s); + if (strm->avail_out == 0) { + return qat_block_done; + } + + return qat_block_need_more; +} + +int qat_deflate_state_free(deflate_state* s) +{ + struct qat_deflate* qat_state = s->qat_s; + if (qat_state->input_buffer) { + qat::Free(qat_state->input_buffer); + } + if (qat_state->output_buffer) { + qat::Free(qat_state->output_buffer); + } + + qat_state->qat_session.reset(); + delete qat_state; + s->qat_s = nullptr; + + return Z_OK; +} + +struct qat_deflate *qat_deflate_copy(deflate_state *ss) +{ + struct qat_deflate *sqat = ss->qat_s; + struct qat_deflate *dqat = nullptr; + + if (!sqat) { + return nullptr; + } + + dqat = new struct qat_deflate; + + dqat->qat_session = qat_create_session(ss->level, ss->wrap); + + dqat->input_buffer_size = sqat->input_buffer_size; + dqat->input_buffer = qat::AllocBlockArray(dqat->input_buffer_size, 0); + + dqat->output_buffer_size = sqat->output_buffer_size; + dqat->output_buffer = qat::AllocBlockArray(dqat->output_buffer_size, 0); + + dqat->pending_in_count = sqat->pending_in_count; + dqat->pending_out_count = sqat->pending_out_count; + + dqat->pending_out = + dqat->output_buffer + (sqat->pending_out - sqat->output_buffer); + + zmemcpy(dqat->input_buffer, sqat->input_buffer, dqat->input_buffer_size); + zmemcpy(dqat->output_buffer, sqat->output_buffer, dqat->output_buffer_size); + + return dqat; +} + diff --git a/deps/zlib/contrib/qat/deflate_qat.h b/deps/zlib/contrib/qat/deflate_qat.h new file mode 100644 index 00000000000000..3c7aa116b7dc70 --- /dev/null +++ b/deps/zlib/contrib/qat/deflate_qat.h @@ -0,0 +1,54 @@ +/* + * Copyright (C) 2024 Intel Corporation. All rights reserved. + * Authors: + * Gustavo A Espinoza + * + * + * For conditions of distribution and use, see copyright notice in zlib.h + */ +#ifndef DEFLATE_QAT_H +#define DEFLATE_QAT_H + +#include "deflate.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/* This is a 1:1 mapping of the block states that deflate_fast, deflate_slow, + * deflate_rle, etc.. return. + * The added 'qat_failure' value is used for signaling the caller to revert + * back into software mode. + */ +typedef enum { + qat_block_need_more, + qat_block_done, + qat_block_finish_started, + qat_block_finish_done, + qat_failure +} qat_block_state; + +/* Initialize QAT for the calling process if it has not been yet initialized. */ +int qat_deflate_init(); + +/* Initialize a QAT stream state for a deflate_state object. */ +struct qat_deflate *qat_deflate_state_init(int level, int wra); + +/* Flush QAT output buffer into the zstream.next_out pointer. */ +void qat_flush_pending(deflate_state*); + +/* Compresses/copies/flushes any data in the internal QAT state + * input/output buffers. +*/ +qat_block_state qat_deflate_step(deflate_state*, int flush); + +/* Frees all the QAT-related buffers and objects for a given deflate_state. */ +int qat_deflate_state_free(deflate_state*); + +struct qat_deflate *qat_deflate_copy(deflate_state *ss); + +#ifdef __cplusplus +} +#endif + +#endif \ No newline at end of file diff --git a/deps/zlib/contrib/qat/qatzpp/io_buffers.cpp b/deps/zlib/contrib/qat/qatzpp/io_buffers.cpp new file mode 100644 index 00000000000000..2870292be17251 --- /dev/null +++ b/deps/zlib/contrib/qat/qatzpp/io_buffers.cpp @@ -0,0 +1,31 @@ +/* + * Copyright (C) 2024 Intel Corporation. All rights reserved. + * Authors: + * Gustavo A Espinoza + * + * + * For conditions of distribution and use, see copyright notice in zlib.h + */ +#include +#include + +#include "io_buffers.h" +#include "qat_instance.hpp" + +namespace qat +{ + +IOBuffers::IOBuffers() +{ +} + +IOBuffers::IOBuffers(std::unique_ptr&& src_list, std::unique_ptr&& dst_list): + src_buffer_list_(std::move(src_list)), dst_buffer_list_(std::move(dst_list)) +{ +} + +IOBuffers::~IOBuffers() +{ +} + +} diff --git a/deps/zlib/contrib/qat/qatzpp/io_buffers.h b/deps/zlib/contrib/qat/qatzpp/io_buffers.h new file mode 100644 index 00000000000000..9fe8bfdbc336c8 --- /dev/null +++ b/deps/zlib/contrib/qat/qatzpp/io_buffers.h @@ -0,0 +1,62 @@ +/* + * Copyright (C) 2024 Intel Corporation. All rights reserved. + * Authors: + * Gustavo A Espinoza + * + * + * For conditions of distribution and use, see copyright notice in zlib.h + */ +#ifndef QATZPP_IO_BUFFERS_H +#define QATZPP_IO_BUFFERS_H + +#include + +#include +#include +#include +#include +#include + +#include "memory.hpp" +#include "qat_instance.hpp" + +namespace qat +{ + +struct BaseBufferList +{ + virtual ~BaseBufferList() {} + + CpaBufferList list; + std::vector flat_buffers; + +protected: + BaseBufferList() {} +}; + +class IOBuffers +{ +public: + IOBuffers( + std::unique_ptr &&src_list, + std::unique_ptr &&dst_list + ); + virtual ~IOBuffers(); + + BaseBufferList *GetSrc() const { + return src_buffer_list_.get(); + } + + BaseBufferList *GetDst() const { + return dst_buffer_list_.get(); + } +protected: + IOBuffers(); + + std::unique_ptr src_buffer_list_; + std::unique_ptr dst_buffer_list_; +}; + +} + +#endif \ No newline at end of file diff --git a/deps/zlib/contrib/qat/qatzpp/memory.cpp b/deps/zlib/contrib/qat/qatzpp/memory.cpp new file mode 100644 index 00000000000000..6a97ffe2fdfcdb --- /dev/null +++ b/deps/zlib/contrib/qat/qatzpp/memory.cpp @@ -0,0 +1,30 @@ +/* + * Copyright (C) 2024 Intel Corporation. All rights reserved. + * Authors: + * Gustavo A Espinoza + * + * + * For conditions of distribution and use, see copyright notice in zlib.h + */ +#include + +#include +#include + +#include "memory.hpp" +#include "qat.hpp" + +namespace qat +{ + +void *Alloc(size_t size_bytes, uint32_t numa_node) +{ + return qaeMemAllocNUMA(size_bytes, numa_node, 1); +} + +void Free(void *ptr) +{ + qaeMemFreeNUMA(&ptr); +} + +} \ No newline at end of file diff --git a/deps/zlib/contrib/qat/qatzpp/memory.hpp b/deps/zlib/contrib/qat/qatzpp/memory.hpp new file mode 100644 index 00000000000000..191516ca75dd20 --- /dev/null +++ b/deps/zlib/contrib/qat/qatzpp/memory.hpp @@ -0,0 +1,40 @@ +/* + * Copyright (C) 2024 Intel Corporation. All rights reserved. + * Authors: + * Gustavo A Espinoza + * + * + * For conditions of distribution and use, see copyright notice in zlib.h + */ +#ifndef QATZPP_MEMORY_HPP +#define QATZPP_MEMORY_HPP + +#include +#include + +namespace qat +{ + +void *Alloc(size_t sizeBytes, uint32_t numa_node); + +template +T *AllocBlock(int32_t numa_node) +{ + return static_cast(Alloc(sizeof(T), numa_node)); +} + +template +T *AllocBlockArray(size_t count, int32_t numa_node) +{ + if (count <= 0) { + return nullptr; + } + + return static_cast(Alloc(sizeof(T) * count, numa_node)); +} + +void Free(void *ptr); + +} + +#endif \ No newline at end of file diff --git a/deps/zlib/contrib/qat/qatzpp/qat.cpp b/deps/zlib/contrib/qat/qatzpp/qat.cpp new file mode 100644 index 00000000000000..80468d395151f3 --- /dev/null +++ b/deps/zlib/contrib/qat/qatzpp/qat.cpp @@ -0,0 +1,73 @@ +/* + * Copyright (C) 2024 Intel Corporation. All rights reserved. + * Authors: + * Gustavo A Espinoza + * + * + * For conditions of distribution and use, see copyright notice in zlib.h + */ +#include "qat.hpp" + +#include +#include +#include + +#include +#include +#include +#include + +namespace qat +{ + +static bool g_qat_not_available = false; +static bool g_qat_initialized = false; +static std::mutex g_qat_initialization_mutex; + +class QATContext +{ +public: + explicit QATContext() {} + + QATContext(const QATContext &) = delete; + QATContext &operator=(const QATContext &) = delete; + + QATContext(QATContext &&) = delete; + QATContext &operator=(QATContext &&) = delete; + + ~QATContext() + { + std::lock_guard lock(g_qat_initialization_mutex); + + if (g_qat_not_available) return; + + if (g_qat_initialized) { + icp_sal_userStop(); + g_qat_initialized = false; + } + } +}; + +static std::unique_ptr qat_context; + +int Initialize() +{ + std::lock_guard lock(g_qat_initialization_mutex); + uint32_t cpa_state; + if (g_qat_not_available) { + return CPA_STATUS_FAIL; + } + if (g_qat_initialized) { + return CPA_STATUS_SUCCESS; + } + + cpa_state = icp_sal_userStartMultiProcess("SSL", CPA_FALSE); + + g_qat_not_available = (cpa_state != CPA_STATUS_SUCCESS); + g_qat_initialized = (cpa_state == CPA_STATUS_SUCCESS); + + qat_context = std::make_unique(); + return cpa_state; +} + +} diff --git a/deps/zlib/contrib/qat/qatzpp/qat.hpp b/deps/zlib/contrib/qat/qatzpp/qat.hpp new file mode 100644 index 00000000000000..8ee7746b5efd04 --- /dev/null +++ b/deps/zlib/contrib/qat/qatzpp/qat.hpp @@ -0,0 +1,19 @@ +/* + * Copyright (C) 2024 Intel Corporation. All rights reserved. + * Authors: + * Gustavo A Espinoza + * + * + * For conditions of distribution and use, see copyright notice in zlib.h + */ +#ifndef QATZPP_QAT_HPP +#define QATZPP_QAT_HPP + +namespace qat +{ + +int Initialize(); + +} + +#endif \ No newline at end of file diff --git a/deps/zlib/contrib/qat/qatzpp/qat_buffer_list.cpp b/deps/zlib/contrib/qat/qatzpp/qat_buffer_list.cpp new file mode 100644 index 00000000000000..f0eea4908a54ff --- /dev/null +++ b/deps/zlib/contrib/qat/qatzpp/qat_buffer_list.cpp @@ -0,0 +1,34 @@ +/* + * Copyright (C) 2024 Intel Corporation. All rights reserved. + * Authors: + * Gustavo A Espinoza + * + * + * For conditions of distribution and use, see copyright notice in zlib.h + */ +#include "qat_buffer_list.hpp" + +namespace qat +{ + +BufferListUser::BufferListUser( + uint8_t *data, + size_t size, + size_t metadata_size) +{ + flat_buffers = std::vector(1); + flat_buffers[0].pData = data; + flat_buffers[0].dataLenInBytes = size; + list.pPrivateMetaData = AllocBlockArray(metadata_size, 0); + list.numBuffers = 1; + list.pBuffers = flat_buffers.data(); +} + +BufferListUser::~BufferListUser() +{ + if (list.pPrivateMetaData) { + Free(list.pPrivateMetaData); + } +} + +} diff --git a/deps/zlib/contrib/qat/qatzpp/qat_buffer_list.hpp b/deps/zlib/contrib/qat/qatzpp/qat_buffer_list.hpp new file mode 100644 index 00000000000000..2a28175e18dc58 --- /dev/null +++ b/deps/zlib/contrib/qat/qatzpp/qat_buffer_list.hpp @@ -0,0 +1,32 @@ +/* + * Copyright (C) 2024 Intel Corporation. All rights reserved. + * Authors: + * Gustavo A Espinoza + * + * + * For conditions of distribution and use, see copyright notice in zlib.h + */ +#ifndef QATZPP_QAT_BUFFER_LIST_HPP +#define QATZPP_QAT_BUFFER_LIST_HPP + +#include + +#include "io_buffers.h" + +namespace qat +{ + +struct BufferListUser final : public BaseBufferList +{ + BufferListUser( + uint8_t *data, + size_t size, + size_t metadata_size + ); + + ~BufferListUser() override; +}; + +} + +#endif \ No newline at end of file diff --git a/deps/zlib/contrib/qat/qatzpp/qat_instance.cpp b/deps/zlib/contrib/qat/qatzpp/qat_instance.cpp new file mode 100644 index 00000000000000..5b833c2ce7e2dd --- /dev/null +++ b/deps/zlib/contrib/qat/qatzpp/qat_instance.cpp @@ -0,0 +1,135 @@ +/* + * Copyright (C) 2024 Intel Corporation. All rights reserved. + * Authors: + * Gustavo A Espinoza + * + * + * For conditions of distribution and use, see copyright notice in zlib.h + */ +#include + +#include +#include + +#include "memory.hpp" +#include "qat_instance.hpp" +#include "session.hpp" + +#define MAX_SAMPLE_BUFFER_SIZE (4*1024*1024) + +namespace qat +{ + +static std::mutex g_instance_mutex; +static std::vector> instances; + +static CpaPhysicalAddr virt2Phys(void *virt_addr) +{ + return (CpaPhysicalAddr)qaeVirtToPhysNUMA(virt_addr); +} + +Instance::Instance(CpaInstanceHandle instance): + instance_(instance), + num_intermediate_buffer_lists_(0), + intermediate_buffer_array_(nullptr), + started_(false) +{ + CpaDcInstanceCapabilities caps{}; + cpaDcQueryCapabilities(instance_, &caps); + + if (!caps.statelessDeflateCompression || !caps.statelessDeflateDecompression || + !caps.checksumAdler32 || !caps.dynamicHuffman) + { + return; + } + + if (caps.dynamicHuffmanBufferReq) { + uint32_t buffer_metadata_size; + cpaDcBufferListGetMetaSize(instance_, 1, &buffer_metadata_size); + cpaDcGetNumIntermediateBuffers(instance_, &num_intermediate_buffer_lists_); + + if(num_intermediate_buffer_lists_) { + intermediate_buffer_array_ = AllocBlockArray(num_intermediate_buffer_lists_, 0); + } + for (int i = 0; i < num_intermediate_buffer_lists_; ++i) { + intermediate_buffer_array_[i] = AllocBlock(0); + intermediate_buffer_array_[i]->pPrivateMetaData = + AllocBlockArray(buffer_metadata_size, 0); + intermediate_buffer_array_[i]->pBuffers = AllocBlock(0); + intermediate_buffer_array_[i]->pBuffers->pData = + AllocBlockArray(MAX_SAMPLE_BUFFER_SIZE, 0); + intermediate_buffer_array_[i]->pBuffers->dataLenInBytes = MAX_SAMPLE_BUFFER_SIZE; + } + } + + cpaDcSetAddressTranslation(instance_, virt2Phys); +} + +Instance::~Instance() +{ +} + +CpaDcInstanceCapabilities Instance::GetCapabilities() +{ + CpaDcInstanceCapabilities caps{}; + cpaDcQueryCapabilities(instance_, &caps); + + return caps; +} + +CpaInstanceInfo2 Instance::GetInfo() +{ + CpaInstanceInfo2 info{}; + cpaDcInstanceGetInfo2(instance_, &info); + + return info; +} + +int Instance::Start() +{ + std::lock_guard lock(mutex_); + + if (started_) { + return 0; + } + + int ret = cpaDcStartInstance + ( + instance_, + num_intermediate_buffer_lists_, + intermediate_buffer_array_ + ); + if (ret) { + return -1; + } + started_ = true; + return 0; +} + +std::vector> Instance::Create() +{ + std::lock_guard lock(g_instance_mutex); + uint16_t num_instances = 0; + + if (!instances.empty()) { + return instances; + } + + cpaDcGetNumInstances(&num_instances); + + if (!num_instances) { + std::cerr << "No instances found\n"; + return {}; + } + + std::vector handles(num_instances); + cpaDcGetInstances(num_instances, handles.data()); + + for(auto& handle: handles) { + instances.emplace_back(std::make_shared(handle)); + } + + return instances; +} + +} diff --git a/deps/zlib/contrib/qat/qatzpp/qat_instance.hpp b/deps/zlib/contrib/qat/qatzpp/qat_instance.hpp new file mode 100644 index 00000000000000..1a2b4afcab10f1 --- /dev/null +++ b/deps/zlib/contrib/qat/qatzpp/qat_instance.hpp @@ -0,0 +1,45 @@ +/* + * Copyright (C) 2024 Intel Corporation. All rights reserved. + * Authors: + * Gustavo A Espinoza + * + * + * For conditions of distribution and use, see copyright notice in zlib.h + */ +#ifndef QATZPP_QAT_INSTANCE_HPP +#define QATZPP_QAT_INSTANCE_HPP + +#include + +#include +#include +#include + +namespace qat +{ + +class Instance +{ +public: + Instance(CpaInstanceHandle); + ~Instance(); + + CpaInstanceHandle GetHandle() { return instance_; } + CpaDcInstanceCapabilities GetCapabilities(); + CpaInstanceInfo2 GetInfo(); + + int Start(void); + static std::vector> Create(); +private: + + CpaInstanceHandle instance_; + uint16_t num_intermediate_buffer_lists_; + CpaBufferList **intermediate_buffer_array_; + bool started_; + + std::mutex mutex_; +}; + +} + +#endif \ No newline at end of file diff --git a/deps/zlib/contrib/qat/qatzpp/qat_task.cpp b/deps/zlib/contrib/qat/qatzpp/qat_task.cpp new file mode 100644 index 00000000000000..a53ea94ac95a16 --- /dev/null +++ b/deps/zlib/contrib/qat/qatzpp/qat_task.cpp @@ -0,0 +1,58 @@ +/* + * Copyright (C) 2024 Intel Corporation. All rights reserved. + * Authors: + * Gustavo A Espinoza + * + * + * For conditions of distribution and use, see copyright notice in zlib.h + */ +#include +#include + +#include "qat_task.hpp" + +namespace qat +{ + +QATTask::QATTask(std::shared_ptr &qat_instance, + std::unique_ptr &&buffers, + std::unique_ptr &&dc_results): + qat_instance_(qat_instance), + io_buffers_(std::move(buffers)), + dc_results_(std::move(dc_results)), + completed_(false) +{ +} + +void QATTask::WaitCompletion() +{ + if (completed_) { + return; + } + + while (!completed_) { + icp_sal_DcPollInstance(qat_instance_->GetHandle(), 0); + } +} + +IOBuffers *QATTask::GetBuffers() +{ + return io_buffers_.get(); +} + +CpaDcRqResults *QATTask::GetResults() +{ + return dc_results_.get(); +} + +void dc_callback(void *callback_tag, CpaStatus status) +{ + if (!callback_tag) { + return; + } + // Ugly and dangerous + QATTask* task = static_cast(callback_tag); + task->completed_ = true; +} + +} \ No newline at end of file diff --git a/deps/zlib/contrib/qat/qatzpp/qat_task.hpp b/deps/zlib/contrib/qat/qatzpp/qat_task.hpp new file mode 100644 index 00000000000000..3950502f50d7e7 --- /dev/null +++ b/deps/zlib/contrib/qat/qatzpp/qat_task.hpp @@ -0,0 +1,54 @@ +/* + * Copyright (C) 2024 Intel Corporation. All rights reserved. + * Authors: + * Gustavo A Espinoza + * + * + * For conditions of distribution and use, see copyright notice in zlib.h + */ +#ifndef QATZPP_WORK_HPP +#define QATZPP_WORK_HPP + +#include + +#include + +#include "io_buffers.h" + +namespace qat +{ + +class QATTask +{ +public: + explicit QATTask(std::shared_ptr &qat_instance, + std::unique_ptr &&, + std::unique_ptr &&dc_results); + + QATTask(QATTask &&) = delete; + QATTask& operator=(QATTask &&) = delete; + + QATTask(const QATTask &) = delete; + QATTask &operator=(const QATTask &) = delete; + + void WaitCompletion(); + + IOBuffers *GetBuffers(); + CpaDcRqResults *GetResults(); + +private: + bool completed_; + + std::shared_ptr qat_instance_; + + std::unique_ptr dc_results_; + std::unique_ptr io_buffers_; + + friend void dc_callback(void *, CpaStatus); +}; + +void dc_callback(void*, CpaStatus); + +} + +#endif \ No newline at end of file diff --git a/deps/zlib/contrib/qat/qatzpp/session.cpp b/deps/zlib/contrib/qat/qatzpp/session.cpp new file mode 100644 index 00000000000000..b4cefb31e85008 --- /dev/null +++ b/deps/zlib/contrib/qat/qatzpp/session.cpp @@ -0,0 +1,129 @@ +/* + * Copyright (C) 2024 Intel Corporation. All rights reserved. + * Authors: + * Gustavo A Espinoza + * + * + * For conditions of distribution and use, see copyright notice in zlib.h + */ +#include +#include + +#include "memory.hpp" +#include "session.hpp" + +namespace qat +{ + +constexpr CpaDcHuffType kHuffType = CPA_DC_HT_FULL_DYNAMIC; + +DeflateSession::DeflateSession( + std::shared_ptr &qat_instance, + CpaDcCompLvl comp_level, CpaDcChecksum checksum, + uint32_t numa_node): + qat_instance_(qat_instance) +{ + uint32_t session_size = 0; + uint32_t ctx_size = 0; + + CpaDcSessionSetupData sd{}; + sd.compLevel = comp_level; + sd.compType = CPA_DC_DEFLATE; + sd.huffType = kHuffType; + sd.autoSelectBestHuffmanTree = CPA_DC_ASB_UNCOMP_STATIC_DYNAMIC_WITH_STORED_HDRS; + sd.sessDirection = CPA_DC_DIR_COMBINED; + sd.sessState = CPA_DC_STATELESS; + sd.checksum = checksum; + + cpaDcGetSessionSize(qat_instance_->GetHandle(), &sd, &session_size, &ctx_size); + session_ = AllocBlockArray(session_size, numa_node); + + cpaDcInitSession( + qat_instance_->GetHandle(), + session_, + &sd, + nullptr, // No context for stateless operations + &dc_callback + ); + +} + +DeflateSession::~DeflateSession() +{ + if (session_) { + cpaDcRemoveSession(qat_instance_->GetHandle(), session_); + Free(session_); + } + + session_ = nullptr; +} + +std::unique_ptr DeflateSession::Deflate( + std::unique_ptr &&buffers, + bool flush_final) +{ + CpaDcOpData op_data{}; + op_data.flushFlag = (flush_final) ? + CPA_DC_FLUSH_FINAL : CPA_DC_FLUSH_FULL; + op_data.compressAndVerify = CPA_TRUE; + op_data.inputSkipData.skipMode = CPA_DC_SKIP_DISABLED; + op_data.outputSkipData.skipMode = CPA_DC_SKIP_DISABLED; + + auto task = std::make_unique( + qat_instance_, std::move(buffers), + std::make_unique() + ); + + cpaDcCompressData2( + qat_instance_->GetHandle(), + session_, + &task->GetBuffers()->GetSrc()->list, + &task->GetBuffers()->GetDst()->list, + &op_data, + task->GetResults(), + static_cast(task.get()) + ); + + return std::move(task); +} + +std::unique_ptr DeflateSession::Inflate(std::unique_ptr &&buffers) +{ + CpaDcOpData op_data = {}; + op_data.flushFlag = CPA_DC_FLUSH_FINAL; + op_data.compressAndVerify = CPA_TRUE; + op_data.inputSkipData.skipMode = CPA_DC_SKIP_DISABLED; + op_data.outputSkipData.skipMode = CPA_DC_SKIP_DISABLED; + + auto task = std::make_unique( + qat_instance_, std::move(buffers), + std::make_unique() + ); + + cpaDcDecompressData2( + qat_instance_->GetHandle(), + session_, + &task->GetBuffers()->GetSrc()->list, + &task->GetBuffers()->GetDst()->list, + &op_data, + task->GetResults(), + static_cast(task.get()) + ); + + return std::move(task); +} + +uint32_t DeflateSession::GetDeflateBound(uint32_t input_size) +{ + uint32_t output_size = 0; + + cpaDcDeflateCompressBound( + qat_instance_->GetHandle(), + kHuffType, + input_size, &output_size + ); + + return output_size; +} + +} diff --git a/deps/zlib/contrib/qat/qatzpp/session.hpp b/deps/zlib/contrib/qat/qatzpp/session.hpp new file mode 100644 index 00000000000000..c8af47c27c2231 --- /dev/null +++ b/deps/zlib/contrib/qat/qatzpp/session.hpp @@ -0,0 +1,45 @@ +/* + * Copyright (C) 2024 Intel Corporation. All rights reserved. + * Authors: + * Gustavo A Espinoza + * + * + * For conditions of distribution and use, see copyright notice in zlib.h + */ +#ifndef QATZPP_SESSION_HPP +#define QATZPP_SESSION_HPP + +#include +#include + +#include + +#include "io_buffers.h" +#include "qat_task.hpp" + +namespace qat +{ + +class DeflateSession +{ +public: + DeflateSession( + std::shared_ptr &, CpaDcCompLvl, + CpaDcChecksum, uint32_t numa_node); + ~DeflateSession(); + + std::unique_ptr Deflate(std::unique_ptr &&buffers, bool flush_final); + std::unique_ptr Inflate(std::unique_ptr &&buffers); + + uint32_t GetDeflateBound(uint32_t input_size); + + std::shared_ptr getInstance() { return qat_instance_; } + +private: + std::shared_ptr qat_instance_; + CpaDcSessionHandle session_; +}; + +} + +#endif \ No newline at end of file diff --git a/deps/zlib/contrib/tests/fuzzers/BUILD.gn b/deps/zlib/contrib/tests/fuzzers/BUILD.gn index 16e918a720ece9..d7db4b3459bae2 100644 --- a/deps/zlib/contrib/tests/fuzzers/BUILD.gn +++ b/deps/zlib/contrib/tests/fuzzers/BUILD.gn @@ -34,6 +34,11 @@ fuzzer_test("zlib_deflate_set_dictionary_fuzzer") { deps = [ "../../../:zlib" ] } +fuzzer_test("zlib_compress_fuzzer") { + sources = [ "compress_fuzzer.cc" ] + deps = [ "../../../:zlib" ] +} + fuzzer_test("zlib_deflate_fuzzer") { sources = [ "deflate_fuzzer.cc" ] deps = [ "../../../:zlib" ] diff --git a/deps/zlib/contrib/tests/fuzzers/compress_fuzzer.cc b/deps/zlib/contrib/tests/fuzzers/compress_fuzzer.cc new file mode 100644 index 00000000000000..3afc78122be2ba --- /dev/null +++ b/deps/zlib/contrib/tests/fuzzers/compress_fuzzer.cc @@ -0,0 +1,46 @@ +// Copyright 2024 The Chromium Authors +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include + +#include + +#include "zlib.h" + +// Fuzzer builds often have NDEBUG set, so roll our own assert macro. +#define ASSERT(cond) \ + do { \ + if (!(cond)) { \ + fprintf(stderr, "%s:%d Assert failed: %s\n", __FILE__, __LINE__, #cond); \ + exit(1); \ + } \ + } while (0) + +extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { + FuzzedDataProvider fdp(data, size); + const int level = fdp.PickValueInArray({-1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9}); + const std::vector src = fdp.ConsumeRemainingBytes(); + + const unsigned long compress_bound = compressBound(src.size()); + std::vector compressed; + compressed.resize(compress_bound); + + unsigned long compressed_size = compress_bound; + int ret = compress2(compressed.data(), &compressed_size, src.data(), + src.size(), level); + ASSERT(ret == Z_OK); + ASSERT(compressed_size <= compress_bound); + compressed.resize(compressed_size); + + std::vector uncompressed; + uncompressed.resize(src.size()); + unsigned long uncompressed_size = uncompressed.size(); + ret = uncompress(uncompressed.data(), &uncompressed_size, compressed.data(), + compressed.size()); + ASSERT(ret == Z_OK); + ASSERT(uncompressed_size == src.size()); + ASSERT(uncompressed == src); + + return 0; +} diff --git a/deps/zlib/crc32.c b/deps/zlib/crc32.c index 4177e920a479df..204aa1ad0c445a 100644 --- a/deps/zlib/crc32.c +++ b/deps/zlib/crc32.c @@ -1168,6 +1168,11 @@ ZLIB_INTERNAL void crc_reset(deflate_state *const s) ZLIB_INTERNAL void crc_finalize(deflate_state *const s) { +#ifdef QAT_COMPRESSION_ENABLED + if (s->qat_s) { + return; + } +#endif #ifdef CRC32_SIMD_SSE42_PCLMUL if (x86_cpu_enable_simd) s->strm->adler = crc_fold_512to32(s); diff --git a/deps/zlib/crc32_simd.c b/deps/zlib/crc32_simd.c index 1ee7742015da60..1c60ae9bd3c118 100644 --- a/deps/zlib/crc32_simd.c +++ b/deps/zlib/crc32_simd.c @@ -387,9 +387,9 @@ uint32_t ZLIB_INTERNAL crc32_sse42_simd_( /* SSE4.2+PCLMUL */ #endif #if defined(__aarch64__) -#define TARGET_ARMV8_WITH_CRC __attribute__((target("aes,crc"))) +#define TARGET_ARMV8_WITH_CRC __attribute__((target("arch=armv8-a+aes+crc"))) #else // !defined(__aarch64__) -#define TARGET_ARMV8_WITH_CRC __attribute__((target("armv8-a,crc"))) +#define TARGET_ARMV8_WITH_CRC __attribute__((target("crc"))) #endif // defined(__aarch64__) #elif defined(__GNUC__) @@ -398,7 +398,7 @@ uint32_t ZLIB_INTERNAL crc32_sse42_simd_( /* SSE4.2+PCLMUL */ */ #include #include -#define TARGET_ARMV8_WITH_CRC +#define TARGET_ARMV8_WITH_CRC __attribute__((target("arch=armv8-a+crc+crypto"))) #else // !defined(__GNUC__) && !defined(_aarch64__) #error ARM CRC32 SIMD extensions only supported for Clang and GCC #endif diff --git a/deps/zlib/deflate.c b/deps/zlib/deflate.c index b9a312030464c7..8a5281c2b6cd8d 100644 --- a/deps/zlib/deflate.c +++ b/deps/zlib/deflate.c @@ -57,6 +57,10 @@ #include "slide_hash_simd.h" #endif +#if defined(QAT_COMPRESSION_ENABLED) +#include "contrib/qat/deflate_qat.h" +#endif + #include "contrib/optimizations/insert_string.h" #ifdef FASTEST @@ -564,6 +568,13 @@ int ZEXPORT deflateInit2_(z_streamp strm, int level, int method, s->strategy = strategy; s->method = (Byte)method; +#if defined(QAT_COMPRESSION_ENABLED) + s->qat_s = NULL; + if (s->level && qat_deflate_init() == Z_OK) { + s->qat_s = qat_deflate_state_init(s->level, s->wrap); + } +#endif + return deflateReset(strm); } @@ -962,6 +973,12 @@ local void flush_pending(z_streamp strm) { unsigned len; deflate_state *s = strm->state; +#if defined(QAT_COMPRESSION_ENABLED) + if (s->qat_s) { + qat_flush_pending(s); + } +#endif + _tr_flush_bits(s); len = s->pending; if (len > strm->avail_out) len = strm->avail_out; @@ -1315,6 +1332,12 @@ int ZEXPORT deflateEnd(z_streamp strm) { TRY_FREE(strm, strm->state->prev); TRY_FREE(strm, strm->state->window); +#if defined(QAT_COMPRESSION_ENABLED) + if (strm->state->qat_s) { + qat_deflate_state_free(strm->state); + } +#endif + ZFREE(strm, strm->state); strm->state = Z_NULL; @@ -1389,6 +1412,14 @@ int ZEXPORT deflateCopy(z_streamp dest, z_streamp source) { ds->d_desc.dyn_tree = ds->dyn_dtree; ds->bl_desc.dyn_tree = ds->bl_tree; +#if defined(QAT_COMPRESSION_ENABLED) + if(ss->qat_s) { + ds->qat_s = qat_deflate_copy(ss); + if (!ds->qat_s) + return Z_MEM_ERROR; + } +#endif + return Z_OK; #endif /* MAXSEG_64K */ } @@ -1880,6 +1911,24 @@ local block_state deflate_fast(deflate_state *s, int flush) { IPos hash_head; /* head of the hash chain */ int bflush; /* set if current block must be flushed */ +#if defined(QAT_COMPRESSION_ENABLED) + if (s->qat_s) { + qat_block_state qat_block = qat_deflate_step(s, flush); + switch (qat_block) { + case qat_block_need_more: + return need_more; + case qat_block_done: + return block_done; + case qat_block_finish_started: + return finish_started; + case qat_block_finish_done: + return finish_done; + case qat_failure: + break; + } + } +#endif + for (;;) { /* Make sure that we always have enough lookahead, except * at the end of the input file. We need MAX_MATCH bytes @@ -1982,6 +2031,24 @@ local block_state deflate_slow(deflate_state *s, int flush) { IPos hash_head; /* head of hash chain */ int bflush; /* set if current block must be flushed */ +#if defined(QAT_COMPRESSION_ENABLED) + if (s->qat_s) { + qat_block_state qat_block = qat_deflate_step(s, flush); + switch (qat_block) { + case qat_block_need_more: + return need_more; + case qat_block_done: + return block_done; + case qat_block_finish_started: + return finish_started; + case qat_block_finish_done: + return finish_done; + case qat_failure: + break; + } + } +#endif + /* Process the input block. */ for (;;) { /* Make sure that we always have enough lookahead, except diff --git a/deps/zlib/deflate.h b/deps/zlib/deflate.h index eb7f0724015cc7..099d35943192bf 100644 --- a/deps/zlib/deflate.h +++ b/deps/zlib/deflate.h @@ -282,6 +282,13 @@ typedef struct internal_state { * hash is enabled. */ +#if defined(QAT_COMPRESSION_ENABLED) + /* Pointer to a struct that contains the current state of the QAT + * stream. + */ + struct qat_deflate *qat_s; +#endif + } FAR deflate_state; /* Output a byte on the stream. diff --git a/deps/zlib/google/zip_internal.cc b/deps/zlib/google/zip_internal.cc index d15c6cfe4f5de2..aa49f4546caa0e 100644 --- a/deps/zlib/google/zip_internal.cc +++ b/deps/zlib/google/zip_internal.cc @@ -8,12 +8,12 @@ #include #include +#include #include "base/containers/fixed_flat_set.h" #include "base/files/file_path.h" #include "base/logging.h" #include "base/notreached.h" -#include "base/strings/string_piece.h" #include "base/strings/string_util.h" #include "base/strings/utf_string_conversions.h" @@ -398,64 +398,64 @@ Compression GetCompressionMethod(const base::FilePath& path) { if (ext.empty()) return kDeflated; - using StringPiece = base::FilePath::StringPieceType; // Skip the leading dot. - StringPiece ext_without_dot = ext; + base::FilePath::StringPieceType ext_without_dot = ext; DCHECK_EQ(ext_without_dot.front(), FILE_PATH_LITERAL('.')); ext_without_dot.remove_prefix(1); // Well known filename extensions of files that a likely to be already // compressed. The extensions are in lower case without the leading dot. - static constexpr auto kExts = base::MakeFixedFlatSet({ - FILE_PATH_LITERAL("3g2"), // - FILE_PATH_LITERAL("3gp"), // - FILE_PATH_LITERAL("7z"), // - FILE_PATH_LITERAL("7zip"), // - FILE_PATH_LITERAL("aac"), // - FILE_PATH_LITERAL("avi"), // - FILE_PATH_LITERAL("bz"), // - FILE_PATH_LITERAL("bz2"), // - FILE_PATH_LITERAL("crx"), // - FILE_PATH_LITERAL("gif"), // - FILE_PATH_LITERAL("gz"), // - FILE_PATH_LITERAL("jar"), // - FILE_PATH_LITERAL("jpeg"), // - FILE_PATH_LITERAL("jpg"), // - FILE_PATH_LITERAL("lz"), // - FILE_PATH_LITERAL("m2v"), // - FILE_PATH_LITERAL("m4p"), // - FILE_PATH_LITERAL("m4v"), // - FILE_PATH_LITERAL("mng"), // - FILE_PATH_LITERAL("mov"), // - FILE_PATH_LITERAL("mp2"), // - FILE_PATH_LITERAL("mp3"), // - FILE_PATH_LITERAL("mp4"), // - FILE_PATH_LITERAL("mpe"), // - FILE_PATH_LITERAL("mpeg"), // - FILE_PATH_LITERAL("mpg"), // - FILE_PATH_LITERAL("mpv"), // - FILE_PATH_LITERAL("ogg"), // - FILE_PATH_LITERAL("ogv"), // - FILE_PATH_LITERAL("png"), // - FILE_PATH_LITERAL("qt"), // - FILE_PATH_LITERAL("rar"), // - FILE_PATH_LITERAL("taz"), // - FILE_PATH_LITERAL("tb2"), // - FILE_PATH_LITERAL("tbz"), // - FILE_PATH_LITERAL("tbz2"), // - FILE_PATH_LITERAL("tgz"), // - FILE_PATH_LITERAL("tlz"), // - FILE_PATH_LITERAL("tz"), // - FILE_PATH_LITERAL("tz2"), // - FILE_PATH_LITERAL("vob"), // - FILE_PATH_LITERAL("webm"), // - FILE_PATH_LITERAL("wma"), // - FILE_PATH_LITERAL("wmv"), // - FILE_PATH_LITERAL("xz"), // - FILE_PATH_LITERAL("z"), // - FILE_PATH_LITERAL("zip"), // - }); + static constexpr auto kExts = + base::MakeFixedFlatSet({ + FILE_PATH_LITERAL("3g2"), // + FILE_PATH_LITERAL("3gp"), // + FILE_PATH_LITERAL("7z"), // + FILE_PATH_LITERAL("7zip"), // + FILE_PATH_LITERAL("aac"), // + FILE_PATH_LITERAL("avi"), // + FILE_PATH_LITERAL("bz"), // + FILE_PATH_LITERAL("bz2"), // + FILE_PATH_LITERAL("crx"), // + FILE_PATH_LITERAL("gif"), // + FILE_PATH_LITERAL("gz"), // + FILE_PATH_LITERAL("jar"), // + FILE_PATH_LITERAL("jpeg"), // + FILE_PATH_LITERAL("jpg"), // + FILE_PATH_LITERAL("lz"), // + FILE_PATH_LITERAL("m2v"), // + FILE_PATH_LITERAL("m4p"), // + FILE_PATH_LITERAL("m4v"), // + FILE_PATH_LITERAL("mng"), // + FILE_PATH_LITERAL("mov"), // + FILE_PATH_LITERAL("mp2"), // + FILE_PATH_LITERAL("mp3"), // + FILE_PATH_LITERAL("mp4"), // + FILE_PATH_LITERAL("mpe"), // + FILE_PATH_LITERAL("mpeg"), // + FILE_PATH_LITERAL("mpg"), // + FILE_PATH_LITERAL("mpv"), // + FILE_PATH_LITERAL("ogg"), // + FILE_PATH_LITERAL("ogv"), // + FILE_PATH_LITERAL("png"), // + FILE_PATH_LITERAL("qt"), // + FILE_PATH_LITERAL("rar"), // + FILE_PATH_LITERAL("taz"), // + FILE_PATH_LITERAL("tb2"), // + FILE_PATH_LITERAL("tbz"), // + FILE_PATH_LITERAL("tbz2"), // + FILE_PATH_LITERAL("tgz"), // + FILE_PATH_LITERAL("tlz"), // + FILE_PATH_LITERAL("tz"), // + FILE_PATH_LITERAL("tz2"), // + FILE_PATH_LITERAL("vob"), // + FILE_PATH_LITERAL("webm"), // + FILE_PATH_LITERAL("wma"), // + FILE_PATH_LITERAL("wmv"), // + FILE_PATH_LITERAL("xz"), // + FILE_PATH_LITERAL("z"), // + FILE_PATH_LITERAL("zip"), // + }); if (kExts.count(ext_without_dot)) { return kStored; diff --git a/deps/zlib/google/zip_reader.cc b/deps/zlib/google/zip_reader.cc index 34a815e5f52e9f..182a802ad84000 100644 --- a/deps/zlib/google/zip_reader.cc +++ b/deps/zlib/google/zip_reader.cc @@ -5,6 +5,7 @@ #include "third_party/zlib/google/zip_reader.h" #include +#include #include #include "base/check.h" @@ -15,7 +16,6 @@ #include "base/logging.h" #include "base/numerics/safe_conversions.h" #include "base/strings/strcat.h" -#include "base/strings/string_piece.h" #include "base/strings/string_util.h" #include "base/strings/utf_string_conversions.h" #include "base/task/sequenced_task_runner.h" @@ -267,7 +267,7 @@ bool ZipReader::OpenEntry() { return true; } -void ZipReader::Normalize(base::StringPiece16 in) { +void ZipReader::Normalize(std::u16string_view in) { entry_.is_unsafe = true; // Directory entries in ZIP have a path ending with "/". @@ -281,15 +281,16 @@ void ZipReader::Normalize(base::StringPiece16 in) { for (;;) { // Consume initial path separators. - const base::StringPiece16::size_type i = in.find_first_not_of(u'/'); - if (i == base::StringPiece16::npos) + const std::u16string_view::size_type i = in.find_first_not_of(u'/'); + if (i == std::u16string_view::npos) { break; + } in.remove_prefix(i); DCHECK(!in.empty()); // Isolate next path component. - const base::StringPiece16 part = in.substr(0, in.find_first_of(u'/')); + const std::u16string_view part = in.substr(0, in.find_first_of(u'/')); DCHECK(!part.empty()); in.remove_prefix(part.size()); diff --git a/deps/zlib/google/zip_reader.h b/deps/zlib/google/zip_reader.h index b7680cc839386a..0dbf50b87b6b91 100644 --- a/deps/zlib/google/zip_reader.h +++ b/deps/zlib/google/zip_reader.h @@ -10,6 +10,7 @@ #include #include #include +#include #include "base/files/file.h" #include "base/files/file_path.h" @@ -281,7 +282,7 @@ class ZipReader { // Normalizes the given path passed as UTF-16 string piece. Sets entry_.path, // entry_.is_directory and entry_.is_unsafe. - void Normalize(base::StringPiece16 in); + void Normalize(std::u16string_view in); // Runs the ListenerCallback at a throttled rate. void ReportProgress(ListenerCallback listener_callback, uint64_t bytes) const; diff --git a/deps/zlib/google/zip_reader_unittest.cc b/deps/zlib/google/zip_reader_unittest.cc index 9eb7d7d2b10e05..9d1406feff9887 100644 --- a/deps/zlib/google/zip_reader_unittest.cc +++ b/deps/zlib/google/zip_reader_unittest.cc @@ -10,6 +10,7 @@ #include #include +#include #include #include "base/check.h" @@ -22,7 +23,6 @@ #include "base/i18n/time_formatting.h" #include "base/path_service.h" #include "base/run_loop.h" -#include "base/strings/string_piece.h" #include "base/strings/stringprintf.h" #include "base/strings/utf_string_conversions.h" #include "base/test/bind.h" @@ -172,7 +172,7 @@ class ZipReaderTest : public PlatformTest { } static Paths GetPaths(const base::FilePath& zip_path, - base::StringPiece encoding = {}) { + std::string_view encoding = {}) { Paths paths; if (ZipReader reader; reader.Open(zip_path)) { @@ -422,7 +422,7 @@ TEST_F(ZipReaderTest, EncryptedFile_WrongPassword) { EXPECT_EQ("This is not encrypted.\n", contents); } - for (const base::StringPiece path : { + for (const std::string_view path : { "Encrypted AES-128.txt", "Encrypted AES-192.txt", "Encrypted AES-256.txt", @@ -458,7 +458,7 @@ TEST_F(ZipReaderTest, EncryptedFile_RightPassword) { } // TODO(crbug.com/1296838) Support AES encryption. - for (const base::StringPiece path : { + for (const std::string_view path : { "Encrypted AES-128.txt", "Encrypted AES-192.txt", "Encrypted AES-256.txt", @@ -713,12 +713,12 @@ TEST_F(ZipReaderTest, ExtractCurrentEntryToString) { if (i > 0) { // Exact byte read limit: must pass. EXPECT_TRUE(reader.ExtractCurrentEntryToString(i, &contents)); - EXPECT_EQ(std::string(base::StringPiece("0123456", i)), contents); + EXPECT_EQ(std::string(std::string_view("0123456", i)), contents); } // More than necessary byte read limit: must pass. EXPECT_TRUE(reader.ExtractCurrentEntryToString(&contents)); - EXPECT_EQ(std::string(base::StringPiece("0123456", i)), contents); + EXPECT_EQ(std::string(std::string_view("0123456", i)), contents); } reader.Close(); } diff --git a/deps/zlib/google/zip_unittest.cc b/deps/zlib/google/zip_unittest.cc index 922d38303ce845..58bafb809d6bf9 100644 --- a/deps/zlib/google/zip_unittest.cc +++ b/deps/zlib/google/zip_unittest.cc @@ -2,12 +2,15 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +#include "third_party/zlib/google/zip.h" + #include #include #include #include #include +#include #include #include #include @@ -29,7 +32,6 @@ #include "testing/gmock/include/gmock/gmock.h" #include "testing/gtest/include/gtest/gtest.h" #include "testing/platform_test.h" -#include "third_party/zlib/google/zip.h" #include "third_party/zlib/google/zip_internal.h" #include "third_party/zlib/google/zip_reader.h" @@ -1290,7 +1292,7 @@ TEST_F(ZipTest, Compressed) { EXPECT_TRUE(base::CreateDirectory(src_dir)); // Create some dummy source files. - for (const base::StringPiece s : {"foo", "bar.txt", ".hidden"}) { + for (const std::string_view s : {"foo", "bar.txt", ".hidden"}) { base::File f(src_dir.AppendASCII(s), base::File::FLAG_CREATE | base::File::FLAG_WRITE); ASSERT_TRUE(f.SetLength(5000)); diff --git a/src/zlib_version.h b/src/zlib_version.h index c7e5a26f7a0a78..f9db56d4f51d37 100644 --- a/src/zlib_version.h +++ b/src/zlib_version.h @@ -2,5 +2,5 @@ // Refer to tools/dep_updaters/update-zlib.sh #ifndef SRC_ZLIB_VERSION_H_ #define SRC_ZLIB_VERSION_H_ -#define ZLIB_VERSION "1.3.0.1-motley-209717d" +#define ZLIB_VERSION "1.3.0.1-motley-71660e1" #endif // SRC_ZLIB_VERSION_H_