forked from osmocom/wireshark
Kafka: include zstd compression in Kafka message batches
Change-Id: I1d06486ccf7b174ee9aa621fa3d8acb8b3673777 Reviewed-on: https://code.wireshark.org/review/34222 Petri-Dish: Anders Broman <a.broman58@gmail.com> Tested-by: Petri Dish Buildbot Reviewed-by: Anders Broman <a.broman58@gmail.com>
This commit is contained in:
Piotr Smolinski
Anders Broman
parent
8b8ce52abc
commit
ad94c4d459
|
|
@ -1138,6 +1138,9 @@ ws_find_package(LZ4 ENABLE_LZ4 HAVE_LZ4)
|
|||
# Snappy compression
|
||||
ws_find_package(SNAPPY ENABLE_SNAPPY HAVE_SNAPPY)
|
||||
|
||||
# zstd compression
|
||||
ws_find_package(ZSTD ENABLE_ZSTD HAVE_ZSTD)
|
||||
|
||||
# Enhanced HTTP/2 dissection
|
||||
ws_find_package(NGHTTP2 ENABLE_NGHTTP2 HAVE_NGHTTP2)
|
||||
|
||||
|
|
@ -1594,6 +1597,11 @@ set_package_properties(SNAPPY PROPERTIES
|
|||
URL "https://google.github.io/snappy/"
|
||||
PURPOSE "Snappy decompression in CQL and Kafka dissectors"
|
||||
)
|
||||
set_package_properties(ZSTD PROPERTIES
|
||||
DESCRIPTION "A compressor/decompressor from Facebook providing better compression than Snappy at a cost of speed"
|
||||
URL "https://facebook.github.io/zstd/"
|
||||
PURPOSE "Zstd decompression in Kafka dissector"
|
||||
)
|
||||
set_package_properties(NGHTTP2 PROPERTIES
|
||||
DESCRIPTION "HTTP/2 C library and tools"
|
||||
URL "https://nghttp2.org"
|
||||
|
|
@ -1818,6 +1826,9 @@ if(WIN32)
|
|||
list (APPEND OPTIONAL_DLLS "${LZ4_DLL_DIR}/${LZ4_DLL}")
|
||||
list (APPEND OPTIONAL_PDBS "${LZ4_DLL_DIR}/${LZ4_PDB}")
|
||||
endif(LZ4_FOUND)
|
||||
if (ZSTD_FOUND)
|
||||
list (APPEND OPTIONAL_DLLS "${ZSTD_DLL_DIR}/${ZSTD_DLL}")
|
||||
endif(ZSTD_FOUND)
|
||||
if (NGHTTP2_FOUND)
|
||||
list (APPEND OPTIONAL_DLLS "${NGHTTP2_DLL_DIR}/${NGHTTP2_DLL}")
|
||||
list (APPEND OPTIONAL_PDBS "${NGHTTP2_DLL_DIR}/${NGHTTP2_PDB}")
|
||||
|
|
|
|||
|
|
@ -69,6 +69,7 @@ option(ENABLE_MINIZIP "Build with zip file compression support" ON)
|
|||
option(ENABLE_LZ4 "Build with LZ4 compression support" ON)
|
||||
option(ENABLE_BROTLI "Build with brotli compression support" ON)
|
||||
option(ENABLE_SNAPPY "Build with Snappy compression support" ON)
|
||||
option(ENABLE_ZSTD "Build with Facebook zstd compression support" ON)
|
||||
option(ENABLE_NGHTTP2 "Build with HTTP/2 header decompression support" ON)
|
||||
option(ENABLE_LUA "Build with Lua dissector support" ON)
|
||||
option(ENABLE_SMI "Build with libsmi snmp support" ON)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,58 @@
|
|||
#
|
||||
# - Find zstd
|
||||
# Find Zstd includes and library
|
||||
#
|
||||
# ZSTD_INCLUDE_DIRS - where to find zstd.h, etc.
|
||||
# ZSTD_LIBRARIES - List of libraries when using Zstd.
|
||||
# ZSTD_FOUND - True if Zstd found.
|
||||
# ZSTD_DLL_DIR - (Windows) Path to the Zstd DLL
|
||||
# ZSTD_DLL - (Windows) Name of the Zstd DLL
|
||||
|
||||
include( FindWSWinLibs )
|
||||
FindWSWinLibs( "zstd-.*" "ZSTD_HINTS" )
|
||||
|
||||
if( NOT WIN32)
|
||||
find_package(PkgConfig)
|
||||
pkg_search_module(ZSTD libzstd)
|
||||
endif()
|
||||
|
||||
find_path(ZSTD_INCLUDE_DIR
|
||||
NAMES zstd.h
|
||||
HINTS "${ZSTD_INCLUDEDIR}" "${ZSTD_HINTS}/include"
|
||||
/usr/include
|
||||
/usr/local/include
|
||||
)
|
||||
|
||||
find_library(ZSTD_LIBRARY
|
||||
NAMES zstd
|
||||
HINTS "${ZSTD_LIBDIR}" "${ZSTD_HINTS}/lib"
|
||||
PATHS
|
||||
/usr/lib
|
||||
/usr/local/lib
|
||||
)
|
||||
|
||||
include(FindPackageHandleStandardArgs)
|
||||
find_package_handle_standard_args( ZSTD DEFAULT_MSG ZSTD_LIBRARY ZSTD_INCLUDE_DIR )
|
||||
|
||||
if( ZSTD_FOUND )
|
||||
set( ZSTD_INCLUDE_DIRS ${ZSTD_INCLUDE_DIR} )
|
||||
set( ZSTD_LIBRARIES ${ZSTD_LIBRARY} )
|
||||
if (WIN32)
|
||||
set ( ZSTD_DLL_DIR "${ZSTD_HINTS}/bin"
|
||||
CACHE PATH "Path to Zstd DLL"
|
||||
)
|
||||
file( GLOB _zstd_dll RELATIVE "${ZSTD_DLL_DIR}"
|
||||
"${ZSTD_DLL_DIR}/zstd*.dll"
|
||||
)
|
||||
set ( ZSTD_DLL ${_zstd_dll}
|
||||
# We're storing filenames only. Should we use STRING instead?
|
||||
CACHE FILEPATH "Zstd DLL file name"
|
||||
)
|
||||
mark_as_advanced( ZSTD_DLL_DIR ZSTD_DLL )
|
||||
endif()
|
||||
else()
|
||||
set( ZSTD_INCLUDE_DIRS )
|
||||
set( ZSTD_LIBRARIES )
|
||||
endif()
|
||||
|
||||
mark_as_advanced( ZSTD_LIBRARIES ZSTD_INCLUDE_DIRS )
|
||||
|
|
@ -151,6 +151,9 @@
|
|||
/* Define to use snappy library */
|
||||
#cmakedefine HAVE_SNAPPY 1
|
||||
|
||||
/* Define to use zstd library */
|
||||
#cmakedefine HAVE_ZSTD 1
|
||||
|
||||
/* Define to 1 if you have the <linux/sockios.h> header file. */
|
||||
#cmakedefine HAVE_LINUX_SOCKIOS_H 1
|
||||
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ Build-Depends:
|
|||
libmaxminddb-dev, dpkg-dev (>= 1.16.1~), libsystemd-dev | libsystemd-journal-dev,
|
||||
libnl-genl-3-dev [linux-any], libnl-route-3-dev [linux-any], asciidoctor,
|
||||
cmake (>= 3.5) | cmake3, libsbc-dev, libnghttp2-dev, libssh-gcrypt-dev,
|
||||
liblz4-dev, libsnappy-dev, libspandsp-dev, libxml2-dev, libbrotli-dev,
|
||||
liblz4-dev, libsnappy-dev, libzstd-dev, libspandsp-dev, libxml2-dev, libbrotli-dev,
|
||||
libspeexdsp-dev
|
||||
Build-Conflicts: libsnmp4.2-dev, libsnmp-dev
|
||||
Vcs-Git: https://salsa.debian.org/debian/wireshark -b debian/master
|
||||
|
|
|
|||
|
|
@ -27,6 +27,7 @@ include_directories(
|
|||
${SMI_INCLUDE_DIRS}
|
||||
${SNAPPY_INCLUDE_DIRS}
|
||||
${ZLIB_INCLUDE_DIRS}
|
||||
${ZSTD_INCLUDE_DIRS}
|
||||
${LIBXML2_INCLUDE_DIRS}
|
||||
)
|
||||
|
||||
|
|
@ -352,6 +353,7 @@ target_link_libraries(epan
|
|||
${WIN_PSAPI_LIBRARY}
|
||||
${LIBXML2_LIBRARIES}
|
||||
${ZLIB_LIBRARIES}
|
||||
${ZSTD_LIBRARIES}
|
||||
)
|
||||
|
||||
target_include_directories(epan
|
||||
|
|
|
|||
|
|
@ -26,6 +26,9 @@
|
|||
#include <lz4.h>
|
||||
#include <lz4frame.h>
|
||||
#endif
|
||||
#ifdef HAVE_ZSTD
|
||||
#include <zstd.h>
|
||||
#endif
|
||||
#include "packet-tcp.h"
|
||||
#include "packet-tls.h"
|
||||
|
||||
|
|
@ -1522,12 +1525,51 @@ decompress_snappy(tvbuff_t *tvb _U_, packet_info *pinfo, int offset _U_, int len
|
|||
}
|
||||
#endif /* HAVE_SNAPPY */
|
||||
|
||||
#ifdef HAVE_ZSTD
|
||||
static int
|
||||
decompress_zstd(tvbuff_t *tvb, packet_info *pinfo, int offset, int length, tvbuff_t **decompressed_tvb, int *decompressed_offset)
|
||||
{
|
||||
ZSTD_inBuffer input = { tvb_memdup(wmem_packet_scope(), tvb, offset, length), length, 0 };
|
||||
ZSTD_DStream *zds = ZSTD_createDStream();
|
||||
size_t rc = 0;
|
||||
tvbuff_t *composite_tvb = tvb_new_composite();
|
||||
int ret = 0;
|
||||
|
||||
do {
|
||||
ZSTD_outBuffer output = { wmem_alloc(pinfo->pool, ZSTD_DStreamOutSize()), ZSTD_DStreamOutSize(), 0 };
|
||||
rc = ZSTD_decompressStream(zds, &output, &input);
|
||||
// rc holds either the number of decompressed offsets or the error code.
|
||||
// Both values are positive, one has to use ZSTD_isError to determine if the call succeeded.
|
||||
if (ZSTD_isError(rc)) {
|
||||
goto end;
|
||||
}
|
||||
tvb_composite_append(composite_tvb,
|
||||
tvb_new_child_real_data(tvb, (guint8*)output.dst, (guint)output.pos, (gint)output.pos));
|
||||
// rc == 0 means there is nothing more to decompress, but there could be still something in the data
|
||||
} while (rc > 0);
|
||||
tvb_composite_finalize(composite_tvb);
|
||||
*decompressed_tvb = composite_tvb;
|
||||
*decompressed_offset = 0;
|
||||
composite_tvb = NULL;
|
||||
ret = 1;
|
||||
end:
|
||||
ZSTD_freeDStream(zds);
|
||||
if (composite_tvb != NULL) {
|
||||
tvb_free_chain(composite_tvb);
|
||||
}
|
||||
if (ret == 0) {
|
||||
col_append_str(pinfo->cinfo, COL_INFO, " [zstd decompression failed]");
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
#else
|
||||
static int
|
||||
decompress_zstd(tvbuff_t *tvb _U_, packet_info *pinfo, int offset _U_, int length _U_, tvbuff_t **decompressed_tvb _U_, int *decompressed_offset _U_)
|
||||
{
|
||||
col_append_str(pinfo->cinfo, COL_INFO, " [zstd compression unsupported]");
|
||||
return 0;
|
||||
}
|
||||
#endif /* HAVE_ZSTD */
|
||||
|
||||
static int
|
||||
decompress(tvbuff_t *tvb, packet_info *pinfo, int offset, int length, int codec, tvbuff_t **decompressed_tvb, int *decompressed_offset)
|
||||
|
|
|
|||
|
|
@ -799,6 +799,14 @@ epan_get_compiled_version_info(GString *str)
|
|||
g_string_append(str, "without LZ4");
|
||||
#endif /* HAVE_LZ4 */
|
||||
|
||||
/* Zstandard */
|
||||
g_string_append(str, ", ");
|
||||
#ifdef HAVE_ZSTD
|
||||
g_string_append(str, "with Zstandard");
|
||||
#else
|
||||
g_string_append(str, "without Zstandard");
|
||||
#endif /* HAVE_ZSTD */
|
||||
|
||||
/* Snappy */
|
||||
g_string_append(str, ", ");
|
||||
#ifdef HAVE_SNAPPY
|
||||
|
|
|
|||
|
|
@ -141,7 +141,7 @@ foreach(_dll ${GLIB2_DLLS} ${CARES_DLL} ${GCRYPT_DLLS}
|
|||
${GNUTLS_DLLS} ${KERBEROS_DLLS} ${LIBSSH_DLL} ${LUA_DLL}
|
||||
${LZ4_DLL} ${NGHTTP2_DLL} ${SBC_DLL} ${SMI_DLL} ${SNAPPY_DLL}
|
||||
${SPANDSP_DLL} ${BCG729_DLL} ${LIBXML2_DLLS} ${WINSPARKLE_DLL}
|
||||
${ZLIB_DLL} ${BROTLI_DLLS}
|
||||
${ZLIB_DLL} ${BROTLI_DLLS} ${ZSTD_DLL}
|
||||
# Needed for mmdbresolve
|
||||
${MAXMINDDB_DLL}
|
||||
)
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@
|
|||
%bcond_with sdjournal
|
||||
%bcond_with guides
|
||||
%bcond_with brotli
|
||||
|
||||
%bcond_with zstd
|
||||
|
||||
# Set at most one of these two:
|
||||
# Note that setcap requires rpmbuild 4.7.0 or later.
|
||||
|
|
@ -161,6 +161,15 @@ Requires: brotli
|
|||
%endif
|
||||
%endif
|
||||
|
||||
%if %{with zstd}
|
||||
BuildRequires: libzstd-devel
|
||||
%if 0%{?suse_version}
|
||||
Requires: libzstd1
|
||||
%else
|
||||
Requires: libzstd
|
||||
%endif
|
||||
%endif
|
||||
|
||||
# Uncomment these if you want to be sure you get them...
|
||||
#BuildRequires: krb5-devel
|
||||
#BuildRequires: libsmi-devel
|
||||
|
|
@ -497,6 +506,9 @@ update-mime-database %{_datadir}/mime &> /dev/null || :
|
|||
%{_libdir}/pkgconfig/wireshark.pc
|
||||
|
||||
%changelog
|
||||
* Wed Aug 15 2019 Gerald Combs
|
||||
- Add zstd
|
||||
|
||||
* Mon Apr 22 2019 Daniel Bakai
|
||||
- Added brotli (as an option, defaulting to not required).
|
||||
|
||||
|
|
|
|||
|
|
@ -120,7 +120,7 @@ foreach(_dll ${GLIB2_DLLS} ${CARES_DLL} ${GCRYPT_DLLS}
|
|||
${GNUTLS_DLLS} ${KERBEROS_DLLS} ${LIBSSH_DLL} ${LUA_DLL}
|
||||
${LZ4_DLL} ${NGHTTP2_DLL} ${SBC_DLL} ${SMI_DLL} ${SNAPPY_DLL}
|
||||
${SPANDSP_DLL} ${BCG729_DLL} ${LIBXML2_DLLS} ${WINSPARKLE_DLL}
|
||||
${ZLIB_DLL} ${BROTLI_DLLS}
|
||||
${ZLIB_DLL} ${BROTLI_DLLS} ${ZSTD_DLL}
|
||||
# Required for mmdbresolve
|
||||
${MAXMINDDB_DLL}
|
||||
)
|
||||
|
|
@ -148,7 +148,7 @@ foreach(_dll ${GLIB2_DLLS} ${CARES_DLL} ${GCRYPT_DLLS}
|
|||
${GNUTLS_DLLS} ${KERBEROS_DLLS} ${LIBSSH_DLL} ${LUA_DLL}
|
||||
${LZ4_DLL} ${NGHTTP2_DLL} ${SBC_DLL} ${SMI_DLL} ${SNAPPY_DLL}
|
||||
${SPANDSP_DLL} ${BCG729_DLL} ${LIBXML2_DLLS} ${WINSPARKLE_DLL}
|
||||
${ZLIB_DLL} ${BROTLI_DLLS}
|
||||
${ZLIB_DLL} ${BROTLI_DLLS} ${ZSTD_DLL}
|
||||
# mmdbresolve
|
||||
${MAXMINDDB_DLL}
|
||||
)
|
||||
|
|
|
|||
|
|
@ -76,6 +76,7 @@ ADDITIONAL_LIST="libnl-3-dev \
|
|||
libcap-dev \
|
||||
liblz4-dev \
|
||||
libsnappy-dev \
|
||||
libzstd-dev \
|
||||
libspandsp-dev \
|
||||
libxml2-dev \
|
||||
libminizip-dev \
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@
|
|||
brew update
|
||||
|
||||
#install some libs needed by Wireshark
|
||||
brew install c-ares glib libgcrypt gnutls lua@5.1 cmake python nghttp2 snappy lz4 libxml2 ninja libmaxminddb doxygen libsmi spandsp brotli minizip
|
||||
brew install c-ares glib libgcrypt gnutls lua@5.1 cmake python nghttp2 snappy lz4 libxml2 ninja libmaxminddb doxygen libsmi spandsp brotli minizip zstd
|
||||
|
||||
#install Qt5
|
||||
brew install qt5
|
||||
|
|
|
|||
|
|
@ -151,6 +151,7 @@ fi
|
|||
# features present in all three versions)
|
||||
LUA_VERSION=5.2.4
|
||||
SNAPPY_VERSION=1.1.4
|
||||
ZSTD_VERSION=1.4.2
|
||||
LIBXML2_VERSION=2.9.9
|
||||
LZ4_VERSION=1.7.5
|
||||
SBC_VERSION=1.3
|
||||
|
|
@ -1241,6 +1242,41 @@ uninstall_snappy() {
|
|||
fi
|
||||
}
|
||||
|
||||
install_zstd() {
|
||||
if [ "$ZSTD_VERSION" -a ! -f zstd-$ZSTD_VERSION-done ] ; then
|
||||
echo "Downloading, building, and installing zstd:"
|
||||
[ -f zstd-$ZSTD_VERSION.tar.gz ] || curl -L -O https://github.com/facebook/zstd/releases/download/v$ZSTD_VERSION/zstd-$ZSTD_VERSION.tar.gz || exit 1
|
||||
$no_build && echo "Skipping installation" && return
|
||||
gzcat zstd-$ZSTD_VERSION.tar.gz | tar xf - || exit 1
|
||||
cd zstd-$ZSTD_VERSION
|
||||
make $MAKE_BUILD_OPTS || exit 1
|
||||
$DO_MAKE_INSTALL || exit 1
|
||||
cd ..
|
||||
touch zstd-$ZSTD_VERSION-done
|
||||
fi
|
||||
}
|
||||
|
||||
uninstall_zstd() {
|
||||
if [ ! -z "$installed_zstd_version" ] ; then
|
||||
echo "Uninstalling zstd:"
|
||||
cd zstd-$installed_zstd_version
|
||||
$DO_MAKE_UNINSTALL || exit 1
|
||||
make distclean || exit 1
|
||||
cd ..
|
||||
rm zstd-$installed_zstd_version-done
|
||||
|
||||
if [ "$#" -eq 1 -a "$1" = "-r" ] ; then
|
||||
#
|
||||
# Get rid of the previously downloaded and unpacked version.
|
||||
#
|
||||
rm -rf zstd-$installed_zstd_version
|
||||
rm -rf zstd-$installed_zstd_version.tar.gz
|
||||
fi
|
||||
|
||||
installed_zstd_version=""
|
||||
fi
|
||||
}
|
||||
|
||||
install_libxml2() {
|
||||
if [ "$LIBXML2_VERSION" -a ! -f libxml2-$LIBXML2_VERSION-done ] ; then
|
||||
echo "Downloading, building, and installing libxml2:"
|
||||
|
|
@ -2227,6 +2263,8 @@ install_all() {
|
|||
|
||||
install_snappy
|
||||
|
||||
install_zstd
|
||||
|
||||
install_libxml2
|
||||
|
||||
install_lz4
|
||||
|
|
@ -2286,6 +2324,8 @@ uninstall_all() {
|
|||
|
||||
uninstall_snappy
|
||||
|
||||
uninstall_zstd
|
||||
|
||||
uninstall_libxml2
|
||||
|
||||
uninstall_lz4
|
||||
|
|
@ -2459,6 +2499,7 @@ then
|
|||
installed_gnutls_version=`ls gnutls-*-done 2>/dev/null | sed 's/gnutls-\(.*\)-done/\1/'`
|
||||
installed_lua_version=`ls lua-*-done 2>/dev/null | sed 's/lua-\(.*\)-done/\1/'`
|
||||
installed_snappy_version=`ls snappy-*-done 2>/dev/null | sed 's/snappy-\(.*\)-done/\1/'`
|
||||
installed_zstd_version=`ls zstd-*-done 2>/dev/null | sed 's/zstd-\(.*\)-done/\1/'`
|
||||
installed_libxml2_version=`ls libxml2-*-done 2>/dev/null | sed 's/libxml2-\(.*\)-done/\1/'`
|
||||
installed_lz4_version=`ls lz4-*-done 2>/dev/null | sed 's/lz4-\(.*\)-done/\1/'`
|
||||
installed_sbc_version=`ls sbc-*-done 2>/dev/null | sed 's/sbc-\(.*\)-done/\1/'`
|
||||
|
|
|
|||
|
|
@ -183,6 +183,8 @@ echo "nghttp2 is unavailable" >&2
|
|||
add_package ADDITIONAL_LIST snappy || add_package ADDITIONAL_LIST libsnappy1 ||
|
||||
echo "snappy is unavailable" >&2
|
||||
|
||||
add_package ADDITIONAL_LIST libzstd-devel || echo "zstd is unavailable" >&2
|
||||
|
||||
add_package ADDITIONAL_LIST lz4-devel || add_package ADDITIONAL_LIST liblz4-devel ||
|
||||
echo "lz4 devel is unavailable" >&2
|
||||
|
||||
|
|
|
|||
|
|
@ -94,6 +94,7 @@ $Win64Archives = @{
|
|||
"vcpkg-export-20190318-win64ws.zip" = "72c2c43594b0581de2bc86517870a561cc40df294662502536b2a6c06cace87e";
|
||||
"WinSparkle-0.5.7.zip" = "56d396ef0c4e8b0589ea74134e484376ca6459d972cd1ab1da6b9624d82e6d04";
|
||||
"WpdPack_4_1_2.zip" = "ea799cf2f26e4afb1892938070fd2b1ca37ce5cf75fec4349247df12b784edbd";
|
||||
"zstd-1.4.0-win64ws.zip" = "154199227bdfdfa608972bcdcea38e20768937085e5a59a8fa06c72d07b00d6b";
|
||||
}
|
||||
|
||||
$Win32Archives = @{
|
||||
|
|
@ -117,6 +118,7 @@ $Win32Archives = @{
|
|||
"vcpkg-export-20190318-win32ws.zip" = "5f9eb78b1ea9e6762c2a4104e0126f1f5453919dc9df66fef2b1e0be8d8c5829";
|
||||
"WinSparkle-0.5.7.zip" = "56d396ef0c4e8b0589ea74134e484376ca6459d972cd1ab1da6b9624d82e6d04";
|
||||
"WpdPack_4_1_2.zip" = "ea799cf2f26e4afb1892938070fd2b1ca37ce5cf75fec4349247df12b784edbd";
|
||||
"zstd-1.4.0-win32ws.zip" = "9141716d4d749e67dad40d4aab6bbb3206085bf68e5acb03baf1e5667aa0b6f5";
|
||||
}
|
||||
|
||||
# Subdirectory to extract an archive to
|
||||
|
|
@ -183,6 +185,7 @@ $CleanupItems = @(
|
|||
"zlib-1.2.5"
|
||||
"zlib-1.2.8"
|
||||
"zlib-1.2.*-ws"
|
||||
"zstd-*-win??ws"
|
||||
"AirPcap_Devpack_4_1_0_1622"
|
||||
"GeoIP-1.*-win??ws"
|
||||
"WinSparkle-0.3-44-g2c8d9d3-win??ws"
|
||||
|
|
|
|||
Loading…
Reference in New Issue