From ad94c4d459d243c0cbbb9b222d5f7cdf8189ab86 Mon Sep 17 00:00:00 2001 From: Piotr Smolinski Date: Mon, 12 Aug 2019 23:57:45 +0200 Subject: [PATCH] Kafka: include zstd compression in Kafka message batches Change-Id: I1d06486ccf7b174ee9aa621fa3d8acb8b3673777 Reviewed-on: https://code.wireshark.org/review/34222 Petri-Dish: Anders Broman Tested-by: Petri Dish Buildbot Reviewed-by: Anders Broman --- CMakeLists.txt | 11 +++++++ CMakeOptions.txt | 1 + cmake/modules/FindZSTD.cmake | 58 +++++++++++++++++++++++++++++++++ cmakeconfig.h.in | 3 ++ debian/control | 2 +- epan/CMakeLists.txt | 2 ++ epan/dissectors/packet-kafka.c | 42 ++++++++++++++++++++++++ epan/epan.c | 8 +++++ packaging/nsis/CMakeLists.txt | 2 +- packaging/rpm/wireshark.spec.in | 14 +++++++- packaging/wix/CMakeLists.txt | 4 +-- tools/debian-setup.sh | 1 + tools/macos-setup-brew.sh | 2 +- tools/macos-setup.sh | 41 +++++++++++++++++++++++ tools/rpm-setup.sh | 2 ++ tools/win-setup.ps1 | 3 ++ 16 files changed, 190 insertions(+), 6 deletions(-) create mode 100644 cmake/modules/FindZSTD.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 725c0c2fc0..01d68b8890 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1138,6 +1138,9 @@ ws_find_package(LZ4 ENABLE_LZ4 HAVE_LZ4) # Snappy compression ws_find_package(SNAPPY ENABLE_SNAPPY HAVE_SNAPPY) +# zstd compression +ws_find_package(ZSTD ENABLE_ZSTD HAVE_ZSTD) + # Enhanced HTTP/2 dissection ws_find_package(NGHTTP2 ENABLE_NGHTTP2 HAVE_NGHTTP2) @@ -1594,6 +1597,11 @@ set_package_properties(SNAPPY PROPERTIES URL "https://google.github.io/snappy/" PURPOSE "Snappy decompression in CQL and Kafka dissectors" ) +set_package_properties(ZSTD PROPERTIES + DESCRIPTION "A compressor/decompressor from Facebook providing better compression than Snappy at a cost of speed" + URL "https://facebook.github.io/zstd/" + PURPOSE "Zstd decompression in Kafka dissector" +) set_package_properties(NGHTTP2 PROPERTIES DESCRIPTION "HTTP/2 C library and tools" URL "https://nghttp2.org" @@ -1818,6 +1826,9 @@ if(WIN32) list (APPEND OPTIONAL_DLLS "${LZ4_DLL_DIR}/${LZ4_DLL}") list (APPEND OPTIONAL_PDBS "${LZ4_DLL_DIR}/${LZ4_PDB}") endif(LZ4_FOUND) + if (ZSTD_FOUND) + list (APPEND OPTIONAL_DLLS "${ZSTD_DLL_DIR}/${ZSTD_DLL}") + endif(ZSTD_FOUND) if (NGHTTP2_FOUND) list (APPEND OPTIONAL_DLLS "${NGHTTP2_DLL_DIR}/${NGHTTP2_DLL}") list (APPEND OPTIONAL_PDBS "${NGHTTP2_DLL_DIR}/${NGHTTP2_PDB}") diff --git a/CMakeOptions.txt b/CMakeOptions.txt index ff71677eb1..8745f2da90 100644 --- a/CMakeOptions.txt +++ b/CMakeOptions.txt @@ -69,6 +69,7 @@ option(ENABLE_MINIZIP "Build with zip file compression support" ON) option(ENABLE_LZ4 "Build with LZ4 compression support" ON) option(ENABLE_BROTLI "Build with brotli compression support" ON) option(ENABLE_SNAPPY "Build with Snappy compression support" ON) +option(ENABLE_ZSTD "Build with Facebook zstd compression support" ON) option(ENABLE_NGHTTP2 "Build with HTTP/2 header decompression support" ON) option(ENABLE_LUA "Build with Lua dissector support" ON) option(ENABLE_SMI "Build with libsmi snmp support" ON) diff --git a/cmake/modules/FindZSTD.cmake b/cmake/modules/FindZSTD.cmake new file mode 100644 index 0000000000..d2d1f6207a --- /dev/null +++ b/cmake/modules/FindZSTD.cmake @@ -0,0 +1,58 @@ +# +# - Find zstd +# Find Zstd includes and library +# +# ZSTD_INCLUDE_DIRS - where to find zstd.h, etc. +# ZSTD_LIBRARIES - List of libraries when using Zstd. +# ZSTD_FOUND - True if Zstd found. +# ZSTD_DLL_DIR - (Windows) Path to the Zstd DLL +# ZSTD_DLL - (Windows) Name of the Zstd DLL + +include( FindWSWinLibs ) +FindWSWinLibs( "zstd-.*" "ZSTD_HINTS" ) + +if( NOT WIN32) + find_package(PkgConfig) + pkg_search_module(ZSTD libzstd) +endif() + +find_path(ZSTD_INCLUDE_DIR + NAMES zstd.h + HINTS "${ZSTD_INCLUDEDIR}" "${ZSTD_HINTS}/include" + /usr/include + /usr/local/include +) + +find_library(ZSTD_LIBRARY + NAMES zstd + HINTS "${ZSTD_LIBDIR}" "${ZSTD_HINTS}/lib" + PATHS + /usr/lib + /usr/local/lib +) + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args( ZSTD DEFAULT_MSG ZSTD_LIBRARY ZSTD_INCLUDE_DIR ) + +if( ZSTD_FOUND ) + set( ZSTD_INCLUDE_DIRS ${ZSTD_INCLUDE_DIR} ) + set( ZSTD_LIBRARIES ${ZSTD_LIBRARY} ) + if (WIN32) + set ( ZSTD_DLL_DIR "${ZSTD_HINTS}/bin" + CACHE PATH "Path to Zstd DLL" + ) + file( GLOB _zstd_dll RELATIVE "${ZSTD_DLL_DIR}" + "${ZSTD_DLL_DIR}/zstd*.dll" + ) + set ( ZSTD_DLL ${_zstd_dll} + # We're storing filenames only. Should we use STRING instead? + CACHE FILEPATH "Zstd DLL file name" + ) + mark_as_advanced( ZSTD_DLL_DIR ZSTD_DLL ) + endif() +else() + set( ZSTD_INCLUDE_DIRS ) + set( ZSTD_LIBRARIES ) +endif() + +mark_as_advanced( ZSTD_LIBRARIES ZSTD_INCLUDE_DIRS ) diff --git a/cmakeconfig.h.in b/cmakeconfig.h.in index bcf16d6597..bf1130b617 100644 --- a/cmakeconfig.h.in +++ b/cmakeconfig.h.in @@ -151,6 +151,9 @@ /* Define to use snappy library */ #cmakedefine HAVE_SNAPPY 1 +/* Define to use zstd library */ +#cmakedefine HAVE_ZSTD 1 + /* Define to 1 if you have the header file. */ #cmakedefine HAVE_LINUX_SOCKIOS_H 1 diff --git a/debian/control b/debian/control index df74d74a9f..4541abb0ec 100644 --- a/debian/control +++ b/debian/control @@ -17,7 +17,7 @@ Build-Depends: libmaxminddb-dev, dpkg-dev (>= 1.16.1~), libsystemd-dev | libsystemd-journal-dev, libnl-genl-3-dev [linux-any], libnl-route-3-dev [linux-any], asciidoctor, cmake (>= 3.5) | cmake3, libsbc-dev, libnghttp2-dev, libssh-gcrypt-dev, - liblz4-dev, libsnappy-dev, libspandsp-dev, libxml2-dev, libbrotli-dev, + liblz4-dev, libsnappy-dev, libzstd-dev, libspandsp-dev, libxml2-dev, libbrotli-dev, libspeexdsp-dev Build-Conflicts: libsnmp4.2-dev, libsnmp-dev Vcs-Git: https://salsa.debian.org/debian/wireshark -b debian/master diff --git a/epan/CMakeLists.txt b/epan/CMakeLists.txt index 5bf71f4e2b..03f4939d2f 100644 --- a/epan/CMakeLists.txt +++ b/epan/CMakeLists.txt @@ -27,6 +27,7 @@ include_directories( ${SMI_INCLUDE_DIRS} ${SNAPPY_INCLUDE_DIRS} ${ZLIB_INCLUDE_DIRS} + ${ZSTD_INCLUDE_DIRS} ${LIBXML2_INCLUDE_DIRS} ) @@ -352,6 +353,7 @@ target_link_libraries(epan ${WIN_PSAPI_LIBRARY} ${LIBXML2_LIBRARIES} ${ZLIB_LIBRARIES} + ${ZSTD_LIBRARIES} ) target_include_directories(epan diff --git a/epan/dissectors/packet-kafka.c b/epan/dissectors/packet-kafka.c index 6fc5b37607..d3e2671bcf 100644 --- a/epan/dissectors/packet-kafka.c +++ b/epan/dissectors/packet-kafka.c @@ -26,6 +26,9 @@ #include #include #endif +#ifdef HAVE_ZSTD +#include +#endif #include "packet-tcp.h" #include "packet-tls.h" @@ -1522,12 +1525,51 @@ decompress_snappy(tvbuff_t *tvb _U_, packet_info *pinfo, int offset _U_, int len } #endif /* HAVE_SNAPPY */ +#ifdef HAVE_ZSTD +static int +decompress_zstd(tvbuff_t *tvb, packet_info *pinfo, int offset, int length, tvbuff_t **decompressed_tvb, int *decompressed_offset) +{ + ZSTD_inBuffer input = { tvb_memdup(wmem_packet_scope(), tvb, offset, length), length, 0 }; + ZSTD_DStream *zds = ZSTD_createDStream(); + size_t rc = 0; + tvbuff_t *composite_tvb = tvb_new_composite(); + int ret = 0; + + do { + ZSTD_outBuffer output = { wmem_alloc(pinfo->pool, ZSTD_DStreamOutSize()), ZSTD_DStreamOutSize(), 0 }; + rc = ZSTD_decompressStream(zds, &output, &input); + // rc holds either the number of decompressed offsets or the error code. + // Both values are positive, one has to use ZSTD_isError to determine if the call succeeded. + if (ZSTD_isError(rc)) { + goto end; + } + tvb_composite_append(composite_tvb, + tvb_new_child_real_data(tvb, (guint8*)output.dst, (guint)output.pos, (gint)output.pos)); + // rc == 0 means there is nothing more to decompress, but there could be still something in the data + } while (rc > 0); + tvb_composite_finalize(composite_tvb); + *decompressed_tvb = composite_tvb; + *decompressed_offset = 0; + composite_tvb = NULL; + ret = 1; +end: + ZSTD_freeDStream(zds); + if (composite_tvb != NULL) { + tvb_free_chain(composite_tvb); + } + if (ret == 0) { + col_append_str(pinfo->cinfo, COL_INFO, " [zstd decompression failed]"); + } + return ret; +} +#else static int decompress_zstd(tvbuff_t *tvb _U_, packet_info *pinfo, int offset _U_, int length _U_, tvbuff_t **decompressed_tvb _U_, int *decompressed_offset _U_) { col_append_str(pinfo->cinfo, COL_INFO, " [zstd compression unsupported]"); return 0; } +#endif /* HAVE_ZSTD */ static int decompress(tvbuff_t *tvb, packet_info *pinfo, int offset, int length, int codec, tvbuff_t **decompressed_tvb, int *decompressed_offset) diff --git a/epan/epan.c b/epan/epan.c index ad51670088..cc4327d270 100644 --- a/epan/epan.c +++ b/epan/epan.c @@ -799,6 +799,14 @@ epan_get_compiled_version_info(GString *str) g_string_append(str, "without LZ4"); #endif /* HAVE_LZ4 */ + /* Zstandard */ + g_string_append(str, ", "); +#ifdef HAVE_ZSTD + g_string_append(str, "with Zstandard"); +#else + g_string_append(str, "without Zstandard"); +#endif /* HAVE_ZSTD */ + /* Snappy */ g_string_append(str, ", "); #ifdef HAVE_SNAPPY diff --git a/packaging/nsis/CMakeLists.txt b/packaging/nsis/CMakeLists.txt index 9ccb9e1139..0e3236cfa4 100644 --- a/packaging/nsis/CMakeLists.txt +++ b/packaging/nsis/CMakeLists.txt @@ -141,7 +141,7 @@ foreach(_dll ${GLIB2_DLLS} ${CARES_DLL} ${GCRYPT_DLLS} ${GNUTLS_DLLS} ${KERBEROS_DLLS} ${LIBSSH_DLL} ${LUA_DLL} ${LZ4_DLL} ${NGHTTP2_DLL} ${SBC_DLL} ${SMI_DLL} ${SNAPPY_DLL} ${SPANDSP_DLL} ${BCG729_DLL} ${LIBXML2_DLLS} ${WINSPARKLE_DLL} - ${ZLIB_DLL} ${BROTLI_DLLS} + ${ZLIB_DLL} ${BROTLI_DLLS} ${ZSTD_DLL} # Needed for mmdbresolve ${MAXMINDDB_DLL} ) diff --git a/packaging/rpm/wireshark.spec.in b/packaging/rpm/wireshark.spec.in index e411c6f5c8..6e600186fb 100644 --- a/packaging/rpm/wireshark.spec.in +++ b/packaging/rpm/wireshark.spec.in @@ -20,7 +20,7 @@ %bcond_with sdjournal %bcond_with guides %bcond_with brotli - +%bcond_with zstd # Set at most one of these two: # Note that setcap requires rpmbuild 4.7.0 or later. @@ -161,6 +161,15 @@ Requires: brotli %endif %endif +%if %{with zstd} +BuildRequires: libzstd-devel +%if 0%{?suse_version} +Requires: libzstd1 +%else +Requires: libzstd +%endif +%endif + # Uncomment these if you want to be sure you get them... #BuildRequires: krb5-devel #BuildRequires: libsmi-devel @@ -497,6 +506,9 @@ update-mime-database %{_datadir}/mime &> /dev/null || : %{_libdir}/pkgconfig/wireshark.pc %changelog +* Wed Aug 15 2019 Gerald Combs +- Add zstd + * Mon Apr 22 2019 Daniel Bakai - Added brotli (as an option, defaulting to not required). diff --git a/packaging/wix/CMakeLists.txt b/packaging/wix/CMakeLists.txt index f261b15b26..c5bba1a17b 100644 --- a/packaging/wix/CMakeLists.txt +++ b/packaging/wix/CMakeLists.txt @@ -120,7 +120,7 @@ foreach(_dll ${GLIB2_DLLS} ${CARES_DLL} ${GCRYPT_DLLS} ${GNUTLS_DLLS} ${KERBEROS_DLLS} ${LIBSSH_DLL} ${LUA_DLL} ${LZ4_DLL} ${NGHTTP2_DLL} ${SBC_DLL} ${SMI_DLL} ${SNAPPY_DLL} ${SPANDSP_DLL} ${BCG729_DLL} ${LIBXML2_DLLS} ${WINSPARKLE_DLL} - ${ZLIB_DLL} ${BROTLI_DLLS} + ${ZLIB_DLL} ${BROTLI_DLLS} ${ZSTD_DLL} # Required for mmdbresolve ${MAXMINDDB_DLL} ) @@ -148,7 +148,7 @@ foreach(_dll ${GLIB2_DLLS} ${CARES_DLL} ${GCRYPT_DLLS} ${GNUTLS_DLLS} ${KERBEROS_DLLS} ${LIBSSH_DLL} ${LUA_DLL} ${LZ4_DLL} ${NGHTTP2_DLL} ${SBC_DLL} ${SMI_DLL} ${SNAPPY_DLL} ${SPANDSP_DLL} ${BCG729_DLL} ${LIBXML2_DLLS} ${WINSPARKLE_DLL} - ${ZLIB_DLL} ${BROTLI_DLLS} + ${ZLIB_DLL} ${BROTLI_DLLS} ${ZSTD_DLL} # mmdbresolve ${MAXMINDDB_DLL} ) diff --git a/tools/debian-setup.sh b/tools/debian-setup.sh index 589cb7e467..2f3a3579e2 100755 --- a/tools/debian-setup.sh +++ b/tools/debian-setup.sh @@ -76,6 +76,7 @@ ADDITIONAL_LIST="libnl-3-dev \ libcap-dev \ liblz4-dev \ libsnappy-dev \ + libzstd-dev \ libspandsp-dev \ libxml2-dev \ libminizip-dev \ diff --git a/tools/macos-setup-brew.sh b/tools/macos-setup-brew.sh index fe77814a47..b93052bd0c 100755 --- a/tools/macos-setup-brew.sh +++ b/tools/macos-setup-brew.sh @@ -13,7 +13,7 @@ brew update #install some libs needed by Wireshark -brew install c-ares glib libgcrypt gnutls lua@5.1 cmake python nghttp2 snappy lz4 libxml2 ninja libmaxminddb doxygen libsmi spandsp brotli minizip +brew install c-ares glib libgcrypt gnutls lua@5.1 cmake python nghttp2 snappy lz4 libxml2 ninja libmaxminddb doxygen libsmi spandsp brotli minizip zstd #install Qt5 brew install qt5 diff --git a/tools/macos-setup.sh b/tools/macos-setup.sh index f9343dd989..5f61c2b41e 100755 --- a/tools/macos-setup.sh +++ b/tools/macos-setup.sh @@ -151,6 +151,7 @@ fi # features present in all three versions) LUA_VERSION=5.2.4 SNAPPY_VERSION=1.1.4 +ZSTD_VERSION=1.4.2 LIBXML2_VERSION=2.9.9 LZ4_VERSION=1.7.5 SBC_VERSION=1.3 @@ -1241,6 +1242,41 @@ uninstall_snappy() { fi } +install_zstd() { + if [ "$ZSTD_VERSION" -a ! -f zstd-$ZSTD_VERSION-done ] ; then + echo "Downloading, building, and installing zstd:" + [ -f zstd-$ZSTD_VERSION.tar.gz ] || curl -L -O https://github.com/facebook/zstd/releases/download/v$ZSTD_VERSION/zstd-$ZSTD_VERSION.tar.gz || exit 1 + $no_build && echo "Skipping installation" && return + gzcat zstd-$ZSTD_VERSION.tar.gz | tar xf - || exit 1 + cd zstd-$ZSTD_VERSION + make $MAKE_BUILD_OPTS || exit 1 + $DO_MAKE_INSTALL || exit 1 + cd .. + touch zstd-$ZSTD_VERSION-done + fi +} + +uninstall_zstd() { + if [ ! -z "$installed_zstd_version" ] ; then + echo "Uninstalling zstd:" + cd zstd-$installed_zstd_version + $DO_MAKE_UNINSTALL || exit 1 + make distclean || exit 1 + cd .. + rm zstd-$installed_zstd_version-done + + if [ "$#" -eq 1 -a "$1" = "-r" ] ; then + # + # Get rid of the previously downloaded and unpacked version. + # + rm -rf zstd-$installed_zstd_version + rm -rf zstd-$installed_zstd_version.tar.gz + fi + + installed_zstd_version="" + fi +} + install_libxml2() { if [ "$LIBXML2_VERSION" -a ! -f libxml2-$LIBXML2_VERSION-done ] ; then echo "Downloading, building, and installing libxml2:" @@ -2227,6 +2263,8 @@ install_all() { install_snappy + install_zstd + install_libxml2 install_lz4 @@ -2286,6 +2324,8 @@ uninstall_all() { uninstall_snappy + uninstall_zstd + uninstall_libxml2 uninstall_lz4 @@ -2459,6 +2499,7 @@ then installed_gnutls_version=`ls gnutls-*-done 2>/dev/null | sed 's/gnutls-\(.*\)-done/\1/'` installed_lua_version=`ls lua-*-done 2>/dev/null | sed 's/lua-\(.*\)-done/\1/'` installed_snappy_version=`ls snappy-*-done 2>/dev/null | sed 's/snappy-\(.*\)-done/\1/'` + installed_zstd_version=`ls zstd-*-done 2>/dev/null | sed 's/zstd-\(.*\)-done/\1/'` installed_libxml2_version=`ls libxml2-*-done 2>/dev/null | sed 's/libxml2-\(.*\)-done/\1/'` installed_lz4_version=`ls lz4-*-done 2>/dev/null | sed 's/lz4-\(.*\)-done/\1/'` installed_sbc_version=`ls sbc-*-done 2>/dev/null | sed 's/sbc-\(.*\)-done/\1/'` diff --git a/tools/rpm-setup.sh b/tools/rpm-setup.sh index 6e50427e67..a988e402f1 100755 --- a/tools/rpm-setup.sh +++ b/tools/rpm-setup.sh @@ -183,6 +183,8 @@ echo "nghttp2 is unavailable" >&2 add_package ADDITIONAL_LIST snappy || add_package ADDITIONAL_LIST libsnappy1 || echo "snappy is unavailable" >&2 +add_package ADDITIONAL_LIST libzstd-devel || echo "zstd is unavailable" >&2 + add_package ADDITIONAL_LIST lz4-devel || add_package ADDITIONAL_LIST liblz4-devel || echo "lz4 devel is unavailable" >&2 diff --git a/tools/win-setup.ps1 b/tools/win-setup.ps1 index b072bea23e..c00ab6b230 100644 --- a/tools/win-setup.ps1 +++ b/tools/win-setup.ps1 @@ -94,6 +94,7 @@ $Win64Archives = @{ "vcpkg-export-20190318-win64ws.zip" = "72c2c43594b0581de2bc86517870a561cc40df294662502536b2a6c06cace87e"; "WinSparkle-0.5.7.zip" = "56d396ef0c4e8b0589ea74134e484376ca6459d972cd1ab1da6b9624d82e6d04"; "WpdPack_4_1_2.zip" = "ea799cf2f26e4afb1892938070fd2b1ca37ce5cf75fec4349247df12b784edbd"; + "zstd-1.4.0-win64ws.zip" = "154199227bdfdfa608972bcdcea38e20768937085e5a59a8fa06c72d07b00d6b"; } $Win32Archives = @{ @@ -117,6 +118,7 @@ $Win32Archives = @{ "vcpkg-export-20190318-win32ws.zip" = "5f9eb78b1ea9e6762c2a4104e0126f1f5453919dc9df66fef2b1e0be8d8c5829"; "WinSparkle-0.5.7.zip" = "56d396ef0c4e8b0589ea74134e484376ca6459d972cd1ab1da6b9624d82e6d04"; "WpdPack_4_1_2.zip" = "ea799cf2f26e4afb1892938070fd2b1ca37ce5cf75fec4349247df12b784edbd"; + "zstd-1.4.0-win32ws.zip" = "9141716d4d749e67dad40d4aab6bbb3206085bf68e5acb03baf1e5667aa0b6f5"; } # Subdirectory to extract an archive to @@ -183,6 +185,7 @@ $CleanupItems = @( "zlib-1.2.5" "zlib-1.2.8" "zlib-1.2.*-ws" + "zstd-*-win??ws" "AirPcap_Devpack_4_1_0_1622" "GeoIP-1.*-win??ws" "WinSparkle-0.3-44-g2c8d9d3-win??ws"