From 9ce60b173bf17b7d20695d9dc1de050989019664 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1niel=20Bakai?= Date: Wed, 10 Apr 2019 12:52:52 +0200 Subject: [PATCH] Add brotli decompression support for HTTP and HTTP2 dissectors. Change-Id: I9c09f55673187f6fee723fcd72798fb6b9958b03 Reviewed-on: https://code.wireshark.org/review/32745 Petri-Dish: Peter Wu Tested-by: Petri Dish Buildbot Reviewed-by: Peter Wu --- CMakeLists.txt | 11 ++ CMakeOptions.txt | 1 + INSTALL | 7 ++ cmake/modules/FindBROTLI.cmake | 50 ++++++++ cmakeconfig.h.in | 3 + debian/control | 2 +- debian/libwireshark0.symbols | 2 + docbook/release-notes.adoc | 2 +- epan/CMakeLists.txt | 3 + epan/dissectors/packet-http.c | 17 ++- epan/dissectors/packet-http2.c | 46 +++++-- epan/epan.c | 18 +++ epan/tvbuff.h | 18 +++ epan/tvbuff_brotli.c | 192 ++++++++++++++++++++++++++++++ packaging/rpm/wireshark.spec.in | 24 ++++ test/captures/http-brotli.pcapng | Bin 0 -> 1840 bytes test/captures/http2-brotli.pcapng | Bin 0 -> 6484 bytes test/fixtures_ws.py | 1 + test/suite_dissection.py | 26 ++++ tools/debian-setup.sh | 4 + tools/macos-setup-brew.sh | 2 +- tools/macos-setup.sh | 65 ++++++++++ tools/rpm-setup.sh | 16 +++ tools/win-setup.ps1 | 2 + 24 files changed, 496 insertions(+), 16 deletions(-) create mode 100644 cmake/modules/FindBROTLI.cmake create mode 100644 epan/tvbuff_brotli.c create mode 100644 test/captures/http-brotli.pcapng create mode 100644 test/captures/http2-brotli.pcapng diff --git a/CMakeLists.txt b/CMakeLists.txt index 6e3478f06b..df62819651 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1104,6 +1104,9 @@ ws_find_package(CARES ENABLE_CARES HAVE_C_ARES "1.5.0") # Zlib compression ws_find_package(ZLIB ENABLE_ZLIB HAVE_ZLIB) +# Brotli compression +ws_find_package(BROTLI ENABLE_BROTLI HAVE_BROTLI) + # LZ4 compression ws_find_package(LZ4 ENABLE_LZ4 HAVE_LZ4) @@ -1809,6 +1812,11 @@ if(WIN32) list (APPEND OPTIONAL_DLLS "${ZLIB_DLL_DIR}/${ZLIB_DLL}") list (APPEND OPTIONAL_PDBS "${ZLIB_DLL_DIR}/${ZLIB_PDB}") endif(ZLIB_FOUND) + if (BROTLI_FOUND) + foreach( _dll ${BROTLI_DLLS} ) + list (APPEND OPTIONAL_DLLS "${BROTLI_DLL_DIR}/${_dll}") + endforeach(_dll) + endif(BROTLI_FOUND) # With libs downloaded to c:/wireshark-win64-libs this currently # (early 2018) expands to about 1900 characters. @@ -2756,6 +2764,9 @@ if(RPMBUILD_EXECUTABLE) if (SYSTEMD_FOUND) list(APPEND _rpmbuild_with_args --with sdjournal) endif() + if (BROTLI_FOUND) + list(APPEND _rpmbuild_with_args --with brotli) + endif() execute_process( COMMAND ${PERL_EXECUTABLE} diff --git a/CMakeOptions.txt b/CMakeOptions.txt index f4d4ce1f31..b99e949c44 100644 --- a/CMakeOptions.txt +++ b/CMakeOptions.txt @@ -65,6 +65,7 @@ option(ENABLE_PCAP_NG_DEFAULT "Enable pcapng as default file format" ON) option(ENABLE_ZLIB "Build with zlib compression support" ON) option(ENABLE_LZ4 "Build with LZ4 compression support" ON) +option(ENABLE_BROTLI "Build with brotli compression support" ON) option(ENABLE_SNAPPY "Build with Snappy compression support" ON) option(ENABLE_NGHTTP2 "Build with HTTP/2 header decompression support" ON) option(ENABLE_LUA "Build with Lua dissector support" ON) diff --git a/INSTALL b/INSTALL index f173ae01c4..cc7a4875db 100644 --- a/INSTALL +++ b/INSTALL @@ -153,6 +153,13 @@ README.windows for those instructions. the capture-file utilities that come in this package, use this switch. + -DENABLE_BROTLI=OFF + By default, if 'configure' finds brotli, the wiretap library + will be built so that it can read brotli compressed capture + files. If you have brotli but do not wish to build it into + the wiretap library, used by Wireshark, TShark, and the + capture-file utilities that come in this package, use this switch. + -DENABLE_PLUGINS=OFF By default, if your system can support run-time loadable modules, the packet analyzers are build with support for plugins. diff --git a/cmake/modules/FindBROTLI.cmake b/cmake/modules/FindBROTLI.cmake new file mode 100644 index 0000000000..4ab8dbb41c --- /dev/null +++ b/cmake/modules/FindBROTLI.cmake @@ -0,0 +1,50 @@ +# +# - Find brotli +# Find brotli includes and libraries +# +# BROTLI_INCLUDE_DIRS - where to find brotli header files +# BROTLI_LIBRARIES - List of libraries when using brotli. +# BROTLI_FOUND - True if brotli found. +# BROTLI_DLL_DIR - (Windows) Path to the brotli DLLs +# BROTLI_DLLS - (Windows) Name of the brotli DLLs + +include( FindWSWinLibs ) +FindWSWinLibs( "brotli-.*" "BROTLI_HINTS" ) + +find_path(BROTLI_INCLUDE_DIR + NAMES "brotli/decode.h" + HINTS "${BROTLI_HINTS}/include" +) + +find_library(BROTLIDEC_LIBRARY + NAMES brotlidec + HINTS "${BROTLI_HINTS}/lib" +) + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args( BROTLI DEFAULT_MSG BROTLIDEC_LIBRARY BROTLI_INCLUDE_DIR ) + +if( BROTLI_FOUND ) + set( BROTLI_INCLUDE_DIRS ${BROTLI_INCLUDE_DIR} ) + set( BROTLI_LIBRARIES ${BROTLIDEC_LIBRARY} ) + + if (WIN32) + set ( BROTLI_DLL_DIR "${BROTLI_HINTS}/bin" + CACHE PATH "Path to the brotli DLLs" + ) + file( GLOB _brotli_dlls RELATIVE "${BROTLI_DLL_DIR}" + "${BROTLI_DLL_DIR}/brotlicommon*.dll" + "${BROTLI_DLL_DIR}/brotlidec*.dll" + ) + set ( BROTLI_DLLS ${_brotli_dlls} + # We're storing filenames only. Should we use STRING instead? + CACHE FILEPATH "brotli DLL list" + ) + mark_as_advanced( BROTLI_DLL_DIR BROTLI_DLLS ) + endif() +else() + set( BROTLI_INCLUDE_DIRS ) + set( BROTLI_LIBRARIES ) +endif() + +mark_as_advanced( BROTLI_LIBRARIES BROTLI_INCLUDE_DIRS ) diff --git a/cmakeconfig.h.in b/cmakeconfig.h.in index 94f94feaf5..924d9347eb 100644 --- a/cmakeconfig.h.in +++ b/cmakeconfig.h.in @@ -136,6 +136,9 @@ /* Define to use zlib library */ #cmakedefine HAVE_ZLIB 1 +/* Define to use brotli library */ +#cmakedefine HAVE_BROTLI 1 + /* Define to use lz4 library */ #cmakedefine HAVE_LZ4 1 diff --git a/debian/control b/debian/control index e77637c971..5667fed82c 100644 --- a/debian/control +++ b/debian/control @@ -17,7 +17,7 @@ Build-Depends: lsb-release, libmaxminddb-dev, dpkg-dev (>= 1.16.1~), libsystemd-dev | libsystemd-journal-dev, libnl-genl-3-dev [linux-any], libnl-route-3-dev [linux-any], asciidoctor, cmake (>= 3.5) | cmake3, libsbc-dev, libnghttp2-dev, libssh-gcrypt-dev, - liblz4-dev, libsnappy-dev, libspandsp-dev, libxml2-dev + liblz4-dev, libsnappy-dev, libspandsp-dev, libxml2-dev, libbrotli-dev Build-Conflicts: libsnmp4.2-dev, libsnmp-dev Vcs-Svn: svn://svn.debian.org/svn/collab-maint/ext-maint/wireshark/trunk Vcs-Browser: http://svn.debian.org/wsvn/collab-maint/ext-maint/wireshark/trunk/ diff --git a/debian/libwireshark0.symbols b/debian/libwireshark0.symbols index 02fccde73f..3844b5fa32 100644 --- a/debian/libwireshark0.symbols +++ b/debian/libwireshark0.symbols @@ -1680,6 +1680,7 @@ libwireshark.so.0 libwireshark0 #MINVER# tvb_captured_length@Base 1.12.0~rc1 tvb_captured_length_remaining@Base 1.12.0~rc1 tvb_child_uncompress@Base 1.12.0~rc1 + tvb_child_uncompress_brotli@Base 3.1.0 tvb_clone@Base 1.12.0~rc1 tvb_clone_offset_len@Base 1.12.0~rc1 tvb_composite_append@Base 1.9.1 @@ -1801,6 +1802,7 @@ libwireshark.so.0 libwireshark0 #MINVER# tvb_strnlen@Base 1.9.1 tvb_strsize@Base 1.9.1 tvb_uncompress@Base 1.9.1 + tvb_uncompress_brotli@Base 3.1.0 tvb_unicode_strsize@Base 1.9.1 tvb_ws_mempbrk_pattern_guint8@Base 1.99.3 tvbparse_casestring@Base 1.9.1 diff --git a/docbook/release-notes.adoc b/docbook/release-notes.adoc index 4cc82eec27..3ad46bd5fa 100644 --- a/docbook/release-notes.adoc +++ b/docbook/release-notes.adoc @@ -34,7 +34,7 @@ Features” section below for more details. The following features are new (or have been significantly updated) since version 3.0.0: -* Watch this space. +* Brotli decompression support in HTTP/HTTP2 (requires the brotli library). // === Removed Features and Support diff --git a/epan/CMakeLists.txt b/epan/CMakeLists.txt index bd13510591..2cd73eaceb 100644 --- a/epan/CMakeLists.txt +++ b/epan/CMakeLists.txt @@ -16,6 +16,7 @@ if (HAVE_HFI_SECTION_INIT) endif() include_directories( + ${BROTLI_INCLUDE_DIRS} ${GLIB2_INCLUDE_DIRS} ${CARES_INCLUDE_DIRS} ${GCRYPT_INCLUDE_DIRS} @@ -265,6 +266,7 @@ set(LIBWIRESHARK_NONGENERATED_FILES tvbparse.c tvbuff.c tvbuff_base64.c + tvbuff_brotli.c tvbuff_composite.c tvbuff_real.c tvbuff_subset.c @@ -378,6 +380,7 @@ target_link_libraries(epan wsutil ${GLIB2_LIBRARIES} PRIVATE + ${BROTLI_LIBRARIES} ${CARES_LIBRARIES} ${GCRYPT_LIBRARIES} ${GIO2_LIBRARIES} diff --git a/epan/dissectors/packet-http.c b/epan/dissectors/packet-http.c index 4ee11d234e..0f3698508d 100644 --- a/epan/dissectors/packet-http.c +++ b/epan/dissectors/packet-http.c @@ -237,9 +237,9 @@ static gboolean http_desegment_body = TRUE; static gboolean http_dechunk_body = TRUE; /* - * Decompression of zlib encoded entities. + * Decompression of zlib or brotli encoded entities. */ -#ifdef HAVE_ZLIB +#if defined(HAVE_ZLIB) || defined(HAVE_BROTLI) static gboolean http_decompress_body = TRUE; #else static gboolean http_decompress_body = FALSE; @@ -1704,6 +1704,7 @@ dissect_http_message(tvbuff_t *tvb, int offset, packet_info *pinfo, proto_item *e_ti = NULL; proto_tree *e_tree = NULL; +#ifdef HAVE_ZLIB if (http_decompress_body && (g_ascii_strcasecmp(headers.content_encoding, "gzip") == 0 || g_ascii_strcasecmp(headers.content_encoding, "deflate") == 0 || @@ -1713,6 +1714,16 @@ dissect_http_message(tvbuff_t *tvb, int offset, packet_info *pinfo, uncomp_tvb = tvb_child_uncompress(tvb, next_tvb, 0, tvb_captured_length(next_tvb)); } +#endif + +#ifdef HAVE_BROTLI + if (http_decompress_body && + g_ascii_strcasecmp(headers.content_encoding, "br") == 0) + { + uncomp_tvb = tvb_child_uncompress_brotli(tvb, next_tvb, 0, + tvb_captured_length(next_tvb)); + } +#endif /* * Add the encoded entity to the protocol tree @@ -4017,7 +4028,7 @@ proto_register_http(void) "Whether to reassemble bodies of entities that are transferred " "using the \"Transfer-Encoding: chunked\" method", &http_dechunk_body); -#ifdef HAVE_ZLIB +#if defined(HAVE_ZLIB) || defined(HAVE_BROTLI) prefs_register_bool_preference(http_module, "decompress_body", "Uncompress entity bodies", "Whether to uncompress entity bodies that are compressed " diff --git a/epan/dissectors/packet-http2.c b/epan/dissectors/packet-http2.c index 331ec4ac37..6f3332a0f8 100644 --- a/epan/dissectors/packet-http2.c +++ b/epan/dissectors/packet-http2.c @@ -65,9 +65,9 @@ VALUE_STRING_ENUM(http2_header_repr_type); VALUE_STRING_ARRAY(http2_header_repr_type); /* - * Decompression of zlib encoded entities. + * Decompression of zlib or brotli encoded entities. */ -#ifdef HAVE_ZLIB +#if defined(HAVE_ZLIB) || defined(HAVE_BROTLI) static gboolean http2_decompress_body = TRUE; #else static gboolean http2_decompress_body = FALSE; @@ -1985,18 +1985,36 @@ dissect_frame_prio(tvbuff_t *tvb, proto_tree *http2_tree, guint offset, guint8 f } #ifdef HAVE_NGHTTP2 -static int -can_uncompress_body(packet_info *pinfo) +enum body_uncompression { + BODY_UNCOMPRESSION_NONE, + BODY_UNCOMPRESSION_ZLIB, + BODY_UNCOMPRESSION_BROTLI +}; + +static enum body_uncompression +get_body_uncompression_info(packet_info *pinfo) { http2_data_stream_body_info_t *body_info = get_data_stream_body_info(pinfo); gchar *content_encoding = body_info->content_encoding; /* Check we have a content-encoding header appropriate as well as checking if this is partial content. * We can't decompress part of a gzip encoded entity */ - return http2_decompress_body - && body_info->is_partial_content == FALSE - && content_encoding != NULL - && (strncmp(content_encoding, "gzip", 4) == 0 || strncmp(content_encoding, "deflate", 7) == 0); + if (!http2_decompress_body || body_info->is_partial_content == TRUE || content_encoding == NULL) { + return BODY_UNCOMPRESSION_NONE; + } + +#ifdef HAVE_ZLIB + if (strncmp(content_encoding, "gzip", 4) == 0 || strncmp(content_encoding, "deflate", 7) == 0) { + return BODY_UNCOMPRESSION_ZLIB; + } +#endif +#ifdef HAVE_BROTLI + if (strncmp(content_encoding, "br", 2) == 0) { + return BODY_UNCOMPRESSION_BROTLI; + } +#endif + + return BODY_UNCOMPRESSION_NONE; } /* Try to dissect reassembled http2.data.data according to content_type. */ @@ -2027,9 +2045,17 @@ dissect_http2_data_full_body(tvbuff_t *tvb, packet_info *pinfo, proto_tree *http gint datalen = tvb_reported_length(tvb); - if (can_uncompress_body(pinfo)) { + enum body_uncompression uncompression = get_body_uncompression_info(pinfo); + if (uncompression != BODY_UNCOMPRESSION_NONE) { proto_item *compressed_proto_item = NULL; - tvbuff_t *uncompressed_tvb = tvb_child_uncompress(tvb, tvb, 0, datalen); + + tvbuff_t *uncompressed_tvb = NULL; + if (uncompression == BODY_UNCOMPRESSION_ZLIB) { + uncompressed_tvb = tvb_child_uncompress(tvb, tvb, 0, datalen); + } else if (uncompression == BODY_UNCOMPRESSION_BROTLI) { + uncompressed_tvb = tvb_child_uncompress_brotli(tvb, tvb, 0, datalen); + } + http2_data_stream_body_info_t *body_info = get_data_stream_body_info(pinfo); gchar *compression_method = body_info->content_encoding; diff --git a/epan/epan.c b/epan/epan.c index 2c475e64b0..04d9d08cef 100644 --- a/epan/epan.c +++ b/epan/epan.c @@ -80,6 +80,10 @@ #include #endif +#ifdef HAVE_BROTLI +#include +#endif + #ifdef HAVE_LIBXML2 #include #include @@ -769,6 +773,14 @@ epan_get_compiled_version_info(GString *str) g_string_append(str, "without nghttp2"); #endif /* HAVE_NGHTTP2 */ + /* brotli */ + g_string_append(str, ", "); +#ifdef HAVE_BROTLI + g_string_append(str, "with brotli"); +#else + g_string_append(str, "without brotli"); +#endif /* HAVE_BROTLI */ + /* LZ4 */ g_string_append(str, ", "); #ifdef HAVE_LZ4 @@ -808,6 +820,12 @@ epan_get_runtime_version_info(GString *str) /* Gcrypt */ g_string_append_printf(str, ", with Gcrypt %s", gcry_check_version(NULL)); + + /* brotli */ +#ifdef HAVE_BROTLI + g_string_append_printf(str, ", with brotli %d.%d.%d", BrotliDecoderVersion() >> 24, + (BrotliDecoderVersion() >> 12) & 0xFFF, BrotliDecoderVersion() & 0xFFF); +#endif } /* diff --git a/epan/tvbuff.h b/epan/tvbuff.h index 763a659762..7b93f98772 100644 --- a/epan/tvbuff.h +++ b/epan/tvbuff.h @@ -907,6 +907,24 @@ WS_DLL_PUBLIC tvbuff_t *tvb_uncompress(tvbuff_t *tvb, const int offset, WS_DLL_PUBLIC tvbuff_t *tvb_child_uncompress(tvbuff_t *parent, tvbuff_t *tvb, const int offset, int comprlen); +/* From tvbuff_brotli.c */ + +/** + * Uncompresses a brotli compressed packet inside a tvbuff at offset with + * length comprlen. Returns an uncompressed tvbuffer if uncompression + * succeeded or NULL if uncompression failed. + */ +WS_DLL_PUBLIC tvbuff_t *tvb_uncompress_brotli(tvbuff_t *tvb, const int offset, + int comprlen); + +/** + * Uncompresses a brotli compressed packet inside a tvbuff at offset with + * length comprlen. Returns an uncompressed tvbuffer attached to tvb if + * uncompression succeeded or NULL if uncompression failed. + */ +WS_DLL_PUBLIC tvbuff_t *tvb_child_uncompress_brotli(tvbuff_t *parent, tvbuff_t *tvb, + const int offset, int comprlen); + /* From tvbuff_base64.c */ /** Return a tvb that contains the binary representation of a base64 diff --git a/epan/tvbuff_brotli.c b/epan/tvbuff_brotli.c new file mode 100644 index 0000000000..3e38c6508b --- /dev/null +++ b/epan/tvbuff_brotli.c @@ -0,0 +1,192 @@ +/* tvbuff_brotli.c + * + * Wireshark - Network traffic analyzer + * By Gerald Combs + * Copyright 1998 Gerald Combs + * + * SPDX-License-Identifier: GPL-2.0-or-later + */ + +#include + +#include + +#include + +#ifdef HAVE_BROTLI +#include +#endif + +#include "tvbuff.h" + +#ifdef HAVE_BROTLI + +/* + * 512KiB is the buffer size used by the brotli tool, so we + * use that too. + */ +#define TVB_BROTLI_BUFSIZ (1 << 19) + +static void* +brotli_g_malloc_wrapper(void *opaque _U_, size_t size) +{ + return g_malloc(size); +} + +static void +brotli_g_free_wrapper(void *opaque _U_, void *address) +{ + g_free(address); +} + +/* + * Uncompresses a brotli compressed packet inside a message of tvb at offset with + * length comprlen. Returns an uncompressed tvbuffer if uncompression + * succeeded or NULL if uncompression failed. + */ + +tvbuff_t * +tvb_uncompress_brotli(tvbuff_t *tvb, const int offset, int comprlen) +{ + guint8 *compr; + guint8 *uncompr = NULL; + tvbuff_t *uncompr_tvb; + BrotliDecoderState *decoder; + guint8 *strmbuf; + const size_t bufsiz = TVB_BROTLI_BUFSIZ; + size_t available_in; + const guint8 *next_in; + size_t available_out; + guint8 *next_out; + size_t total_out; + guint needs_more_output; + guint finished; + + if (tvb == NULL || comprlen <= 0) { + return NULL; + } + + compr = (guint8 *)tvb_memdup(NULL, tvb, offset, comprlen); + if (compr == NULL) { + return NULL; + } + + decoder = BrotliDecoderCreateInstance( + &brotli_g_malloc_wrapper /*alloc_func*/, + &brotli_g_free_wrapper /*free_func*/, + NULL /*opaque*/); + if (decoder == NULL) { + wmem_free(NULL, compr); + return NULL; + } + strmbuf = (guint8 *)g_malloc(bufsiz); + + available_in = comprlen; + next_in = compr; + total_out = 0; + needs_more_output = 0; + finished = 0; + while (available_in > 0 || needs_more_output) { + needs_more_output = 0; + available_out = bufsiz; + next_out = strmbuf; + + BrotliDecoderResult result = BrotliDecoderDecompressStream( + decoder, &available_in, &next_in, &available_out, &next_out, &total_out); + switch (result) { + case BROTLI_DECODER_RESULT_SUCCESS: + if (available_in > 0) { + goto cleanup; + } + finished = 1; + break; + case BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT: + needs_more_output = 1; + break; + case BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT: + /* + * It's possible that not enough frames were captured + * to decompress this fully, so return what we've done + * so far, if any. + */ + break; + case BROTLI_DECODER_RESULT_ERROR: + default: + goto cleanup; + } + + /* + * Check if decompressed size is too large. + */ + if (total_out > G_MAXINT) { + goto cleanup; + } + + /* + * BrotliDecoderDecompressStream sets available_out to the number of bytes + * left unused from the buffer. But we are interested in the bytes it wrote + * to the buffer in this pass, so we calculate pass_out. + */ + size_t pass_out = bufsiz - available_out; + if (pass_out > 0) { + uncompr = (guint8 *)g_realloc(uncompr, total_out); + memcpy(uncompr + (total_out - pass_out), strmbuf, pass_out); + } + } + + if (uncompr == NULL) { + /* + * This is for the case when the validly decompressed + * length is 0. + */ + if (finished) { + uncompr = (guint8 *)g_strdup(""); + } else { + goto cleanup; + } + } + + uncompr_tvb = tvb_new_real_data((guint8 *)uncompr, (guint)total_out, (gint)total_out); + tvb_set_free_cb(uncompr_tvb, g_free); + + g_free(strmbuf); + wmem_free(NULL, compr); + BrotliDecoderDestroyInstance(decoder); + return uncompr_tvb; + +cleanup: + g_free(strmbuf); + g_free(uncompr); + wmem_free(NULL, compr); + BrotliDecoderDestroyInstance(decoder); + return NULL; +} +#else +tvbuff_t * +tvb_uncompress_brotli(tvbuff_t *tvb _U_, const int offset _U_, int comprlen _U_) +{ + return NULL; +} +#endif + +tvbuff_t * +tvb_child_uncompress_brotli(tvbuff_t *parent, tvbuff_t *tvb, const int offset, int comprlen) +{ + tvbuff_t *new_tvb = tvb_uncompress_brotli(tvb, offset, comprlen); + if (new_tvb) + tvb_set_child_real_data_tvbuff(parent, new_tvb); + return new_tvb; +} + +/* + * Editor modelines - https://www.wireshark.org/tools/modelines.html + * + * Local variables: + * c-basic-offset: 4 + * tab-width: 8 + * indent-tabs-mode: nil + * End: + * + * vi: set shiftwidth=4 tabstop=8 expandtab: + * :indentSize=4:tabSize=8:noTabs=true: + */ diff --git a/packaging/rpm/wireshark.spec.in b/packaging/rpm/wireshark.spec.in index 56e1caecce..eab250cddf 100644 --- a/packaging/rpm/wireshark.spec.in +++ b/packaging/rpm/wireshark.spec.in @@ -19,6 +19,7 @@ %bcond_with nghttp2 %bcond_with sdjournal %bcond_with guides +%bcond_with brotli # Set at most one of these two: @@ -145,6 +146,21 @@ Requires: systemd-libs %endif %endif +%if %{with brotli} +%if 0%{?suse_version} +BuildRequires: libbrotli-devel +# On SUSE libbrotli-devel installs the libbrotlidec.so, libbrotlienc.so +# and libbrotlicommon.so symlinks, but only installs the libbrotlicommon1 and libbrotlienc1 +# packages, leaving the libbrotlidec.so symlink broken, so we have to include libbrotlidec1 +# as a build-time dependency... +BuildRequires: libbrotlidec1 +Requires: libbrotlidec1 +%else +BuildRequires: brotli-devel +Requires: brotli +%endif +%endif + # Uncomment these if you want to be sure you get them... #BuildRequires: krb5-devel #BuildRequires: libsmi-devel @@ -314,6 +330,11 @@ cmake3 \ -DBUILD_sdjournal=ON \ %else -DBUILD_sdjournal=OFF \ +%endif +%if %{with brotli} + -DENABLE_BROTLI=ON \ +%else + -DENABLE_BROTLI=OFF \ %endif -DDISABLE_WERROR=ON \ %if %{with ninja} @@ -476,6 +497,9 @@ update-mime-database %{_datadir}/mime &> /dev/null || : %{_libdir}/pkgconfig/wireshark.pc %changelog +* Mon Apr 22 2019 Daniel Bakai +- Added brotli (as an option, defaulting to not required). + * Fri Sep 28 2018 Gerald Combs - Add sdjournal diff --git a/test/captures/http-brotli.pcapng b/test/captures/http-brotli.pcapng new file mode 100644 index 0000000000000000000000000000000000000000..5d7a499d1faf0d18f75f4dfcd9673279ede97238 GIT binary patch literal 1840 zcmb7_O>7%Q6vv$RKqLWINt!2$I^Awg;msL4V$L@qXE)uTGSsK4iLRV2zN+Q z^eE{a87JLD5yR3z^bGNz((MT{HrB6-N&r0221B}$nNF9Xy;2>No9nS_=h{ugzNW+yc3WPpT3S(J!B?~GPL64Y=fh6+(tQIvQ9 zcMHJw&h~x0G_q)JsE52kZ3(aK!Ml5K^#0F1;VVwe1{~9Df$iqcXZL$szxBR=bJpRJ zrvaKqW1xL{zY+QY4zHc-hVF)MKD+ku85j1Jr4umn%nXR^;d%GH-{fWhcU}ELox73C zef{Kf?7GY2d8o*!Wi7w+U71y+6p2L^0Anpi8He#;Wa&g}%hC$+g7Xhgk-$QQ^_6w! zGUHXVk_?ZH>H7Wg@4e6Bob?^L)U|gVU0>Gd3LDDoQ-plSt4EUTh1tZ7qa;BVi_WBNZWW zi|2R_x65=eJ==43!T)19XFKYn`tC73A04Zy&nnhucBTmmH;)$elxj0j$=L;FXlyKJ zzrMWUeGS*F*D_qTZ<8@T+`lX102Hz-)r_qy5k->7kwKmtqgI|;5n`ltW18g;b3**% z@tk4j0zB8J*)&4NH2V;d!lWl-;frz?k=2NzMkJLC499q`*TnCHVGFU-8GP8L({?bE z)^wwCXNVdpdm=*G6`mW?vbHdc>3v0~$=zd1QbY=sV;eMK>!uMQCn?PcT3XLjp6ick zSe@4Pgw`3CXh;aDtrc8#`HBS=Ml~Zvvsls7=owbj$9mxI;^Z%q{#YJKp>#g(!TcKBLdn{|AB zsm9l{tu42Fcy|6;d{FJ{iVwDX?eh=r{2OD=VJz&**oTb!|AjHzTqrYYy#}i9Z^zHE zYOme$p`Yhw_Cevs-m=%O;J0R*pSSDSG%JiPcjmF1BL@hFr= znN+l=qD^{iMUrf7e&pgRQuXFBuX711D`#s<7K4N6DSS!HBjQEGg738LMfY(8g79~YiCEjFoFB*fj`AJxI~D*4i*JQ+5FV~- z&+2@p@L7oGlqnrR)w}dB*43blPYCZSb%l)+aB>m@w5FoYKy(5Bul9RDQza$Bh?o&!F*n7*B_% z!BgSM^29(6$O0!p2G|HT0B_(0On@;ASwj#D_JA0$n@8vA@w9kqJV_o2q=C7>0PN!F z^R#*DyjeUMo+!u$r$8o12cBR7m=ET`5TpYdz`${k0+PWo=n8CL4$OEoo-R+5r^=Jy z5kV?21^OTfuz)a6nkNF}fefGkGLOt705LEM6=8rWfJgvPD>3i~moB$MKX$x1flCF zz}xz>-~H@rP$qye6aTL-iz6U7P`vzg>o5Kr5&%dhVvwG(jw6ii68{b3M9Vk^$^-H8 z1Ojk{nye)d6at-zPYh|heL-U`S z>eL%Yx26kInnP!MTHX?0>GAG!b9XoorfzxUPhi}iC@`rLpbJ14lJ68i+5kwNfaH!v zlh$F;1glJe&xnFYDPYRZa|NjwSx8hX#wuorjKfVK0nCCn1rrq_I1q(o@GJ+K3?_vx zM8w7HU_sKg3*rWO(3G(m=%$#sBC33Wu!2xmc)NRg2W_S<4GO0&3UbG$E0AH~Vq%6c zyE2%Ds};y}bP1EP;PU?pCK^BUX9aLj^=P~#z@qVD_-H&16gz_9o5A5!zq_jp<=Wm` zR_i1txQO?zO-cAh8eudS*}G}!^=Q6axVr94K*mGsCLO7*wSyvCvaAn132RY3B;{*+ zm)HSHBm&YM?mwTk)R*3Qe|2MH$@11+p�gr^`qsB<$$v4e_fjA`)ccl5b}$_@MD2 z>90kEJz4e!F0z}mE)>QbdugTKnD`)%t-EP`d~S-(`|7=R^*atdyX}-%N)BE-dgKOm zO&>cP9E>>gQKxHK&7oAj|E8%a)a?VY?)*+qyrx7=fXuYCrw#8zbVL)qZ@wW>=nvzT zL@Rxi^$%OZh)8fNl>ZjEV^qm#-*L#)3%avx8v`N&3&q%th=)uw^K26 zetYvMv#p=Z;((aq%jb zsPgdNw$bunbg?aXp}S#dcWm0pw}0Y9H$PGg4Y)yEZpLpuk}K_z_55*4;b`VkX|B^_ z{G7iVQUgA^ia$&FB3tj2-KwSYru1&rutVi$KF4B?obl_85v2tkqTTBfXKr~x?$K8D z+~{huaMlIVgH>RC_ssL3-ioGThvQBSo-i@$)Jd!Cm@Cm%t7xa(T!SaT*vPWOmSKNF zkNs1EC{L748?d_L@H#NEqR(>iyTz$$S8N7P2psy)SwuxlKJxcJkVOBN!Qn=UB#Z#3 zpWP0|4RkvkI7-?PGtjcw{aN`PFN*#XKs-LYl_pOKlQr&ew%xYayDRc_gc^Bx+Evl* zvE5H}d$%az9)%p);O?uRmH;2PyrE5lT9__E(_Gdb5)9+T_kW4C( zyVNw8biL5{q>+?&e5P?q5Oso<~cEm6UXZ z?)ZEzzD=UmPs_BV!vHjS=3GWi;pTEd{piVF|kx7@)PiQ+D*yPl%Q=%?8 zP2tf^bu;{QrEV2`ya|&MB;@XtrJA#gRwr_#$!S*6;42O5>??{%EH_u}vsYZ+Pxn=t z9_;b(l}m9!#i7;{PK)L>R~~L#xAbmJEdM~`k=mXpZgVj8MfMVvultVa=GcnUq(^2I zjyhad2^>zJIb>-vkeFI}IIa!v%OkpT*6ynW^a)0AZeQ!>q#WPkGZ zv0DD@;KMan%r8pEq7_CiXRe>6j(xoeF>LYdX03O!5lYI}Ox! zjQ2Guu;##~`A6?WAqdKipx$Jx&g1vO3V~!|y}`d?oj@2-BEQxf@`@idZ887Xdh_l{ zk*RA!Ls%5=hRw7lQq-LECs6@k>xRt4muiIn(Pc6BcC`P4U90-tzjRmX2M2_xOTf~B z-y{z3-UOh?CBAob#qV>D&#~3DYOG{v64KTmT>8#jsi=pzQ>QMwJUf4M-*DMkc0jbk zpX>Dc%IekAB#2MV-1Bcr_+#-k_x9-?yhYA703JvMU=g?6ZK|Co5wwfLHAo13IH$Y-U6>wH}Mye^_B|K+-}b^K|=bu02mwx_sy z#XWaD!VHnz6g*Tj)oscK7q+Re@4$zJU%qK33!T~JA$iZBOy?Ppx{`SLqKb~)E@rWI zkEHpU=5K8n?{+1xJVv@we(Sw*J&LOGk=$tK1yt((EJnk+($J#bz^B~pPfPbN3w3I4 z%=3TMeXrrxnW(DsukNya5{la@XDZ*!jf;w&@w!|o^P_&DLhm_>QBV?zl%eE4#d!&6 z4>&3Ep_BX0AKRNBP!}^*N^5(bdNxiaugFQYUhDFFGG%sgeQ?IevhqthIWxDEyF4Xy zIoaoivRI3%wDJlHW(GF}dL1aqau!-ubf-AE{xf-h|IpDA<-8e@cy3^&<-H2EtEBTz zd9RX^tkde+V@LOU9ch^_Y2RY|CMq5aWArvv9S^gS%8Ak1QRnSCu&xtiV1 zyjuxg^vh)DI;%euYSUP#LO$>|m+c&5^;R)9cXo5$OwlHOy7s$kl+xLr8`NxkMLcB= z%@vww_w}6Lg^Q>sB1)hLKyffJ<{WqYk1>}mh`EBvF-IZ7nCrh{`rl#>wM}TC9`AnC zbAAh4BbgM}_Z@Qt806n#DB=bP>&G!tQCuT^)>8m*C+Q zaJ;olr|0-4Z(gAJNv%w%FEX<9dW}OS(i_R~@pjA=_)qWz{~66^Qnp4y|B+}N_n&lR zL&Oss+skc>8y@<3%;C)-|)w<+k@(ac91k;WYX7Qu}M28)~MH+buaGa z+TUg)4ru?%prC=ctl@m)C&D$7Np{-hcv`?^#RM)%Kez}FPUd1W#-+3@Vc(YI9cj-i z9NvA9Xn*y0M}K>P?s=UH^e(&VgpieKPkp~UIP^_>Wp$FxHf^p9XL#Yu&+nyP*Xypk ze|YmB3q1PfAOFr3agIg&;6R*xvcK`Y;2Oy!&Z`~s1)L)%aBly>Id|1$&dOt)%bH?` z6ZFqAC^^H&YJ-n%4OLpNQH*)Dm1rS+#N+38a7_c;H8Xe*$)p%%veFTco#!AAWyIzC z*%f80T~wDid065V|kz}H7zNrY*u&Yz1;^R&sYRF#f(UvY;7}cdKkkXyx({3!uqupVFTM@>1Q^H zX1T3hyNBXA{c-cNks_A^OPAkR7NI$y(P1yEqehbRzqjqhkz+#ZB`B;A1>IF?2}-Bq zgp)5045>Vf+r<~VW;~bQ8rV(TxORJ9&Ft=W=I84v-Bra6Td`|gRcW6Jxdb6onb>Sc z>rc~yc?-)#D)^b7%N$*0DQ64W7_tSj`$Z^dp#AR^CSF5vg=CUV;#rvjyF31B6Dd7` z-Jwk$!jsCM?T&2XFlLiCZVPD*LDjN(F-`A#F*)T8DK~npud45yTtREQn9Fyt{xrt-3*v7VWs`iNAH4IXdwISpCxjQ3)e^{ zVc0nq2rzd476azwqp&26k_!k(z7 zkKUAVJtyrElO;H7-HFaxe~JyMShz>pky4=l@~dC=DC1u}e~!PFHI#9TNqRCJ3k7;+ zP0&*t+H!BC6tv}wU-kSw#vTjlag5*SaZ2E8XD8^f7wVydV*K*UaXqA9jF0K@AB=v> AApigX literal 0 HcmV?d00001 diff --git a/test/fixtures_ws.py b/test/fixtures_ws.py index a7d633ac35..161da3f3ce 100644 --- a/test/fixtures_ws.py +++ b/test/fixtures_ws.py @@ -169,6 +169,7 @@ def features(cmd_tshark, make_env): have_libgcrypt17=gcry_m and float(gcry_m.group(1)) >= 1.7, have_gnutls='with GnuTLS' in tshark_v, have_pkcs11='and PKCS #11 support' in tshark_v, + have_brotli='with brotli' in tshark_v, ) diff --git a/test/suite_dissection.py b/test/suite_dissection.py index 9a62e933cc..c9234245cc 100644 --- a/test/suite_dissection.py +++ b/test/suite_dissection.py @@ -14,6 +14,20 @@ import subprocesstest import unittest import fixtures +@fixtures.mark_usefixtures('test_env') +@fixtures.uses_fixtures +class case_dissect_http(subprocesstest.SubprocessTestCase): + def test_http_brotli_decompression(self, cmd_tshark, features, dirs, capture_file): + '''HTTP brotli decompression''' + if not features.have_brotli: + self.skipTest('Requires brotli.') + self.assertRun((cmd_tshark, + '-r', capture_file('http-brotli.pcapng'), + '-Y', 'http.response.code==200', + '-Tfields', '-etext', + )) + self.assertTrue(self.grepOutput('This is a test file for testing brotli decompression in Wireshark')) + @fixtures.mark_usefixtures('test_env') @fixtures.uses_fixtures class case_dissect_http2(subprocesstest.SubprocessTestCase): @@ -30,6 +44,18 @@ class case_dissect_http2(subprocesstest.SubprocessTestCase): )) self.assertTrue(self.grepOutput('DATA')) + def test_http2_brotli_decompression(self, cmd_tshark, features, dirs, capture_file): + '''HTTP2 brotli decompression''' + if not features.have_nghttp2: + self.skipTest('Requires nghttp2.') + if not features.have_brotli: + self.skipTest('Requires brotli.') + self.assertRun((cmd_tshark, + '-r', capture_file('http2-brotli.pcapng'), + '-Y', 'http2.data.data matches "This is a test file for testing brotli decompression in Wireshark"', + )) + self.assertTrue(self.grepOutput('DATA')) + @fixtures.mark_usefixtures('test_env') @fixtures.uses_fixtures class case_dissect_tcp(subprocesstest.SubprocessTestCase): diff --git a/tools/debian-setup.sh b/tools/debian-setup.sh index 931d869139..3cb3738d95 100755 --- a/tools/debian-setup.sh +++ b/tools/debian-setup.sh @@ -137,6 +137,10 @@ echo "libgnutls28-dev is unavailable" >&2 add_package ADDITIONAL_LIST libmaxminddb-dev || echo "libmaxminddb-dev is unavailable" >&2 +# Debian >= stretch-backports, Ubuntu >= 16.04 +add_package ADDITIONAL_LIST libbrotli-dev || +echo "libbrotli-dev is unavailable" >&2 + # libsystemd-journal-dev: Ubuntu 14.04 # libsystemd-dev: Ubuntu >= 16.04 add_package DEBDEPS_LIST libsystemd-dev || diff --git a/tools/macos-setup-brew.sh b/tools/macos-setup-brew.sh index 648c445e69..15cd58478b 100755 --- a/tools/macos-setup-brew.sh +++ b/tools/macos-setup-brew.sh @@ -13,7 +13,7 @@ brew update #install some libs needed by Wireshark -brew install c-ares glib libgcrypt gnutls lua@5.1 cmake python nghttp2 snappy lz4 libxml2 ninja libmaxminddb doxygen libsmi spandsp +brew install c-ares glib libgcrypt gnutls lua@5.1 cmake python nghttp2 snappy lz4 libxml2 ninja libmaxminddb doxygen libsmi spandsp brotli #install Qt5 brew install qt5 diff --git a/tools/macos-setup.sh b/tools/macos-setup.sh index 6fcf9a926c..ee39a0fdba 100755 --- a/tools/macos-setup.sh +++ b/tools/macos-setup.sh @@ -172,6 +172,7 @@ if [ "$SPANDSP_VERSION" ]; then fi BCG729_VERSION=1.0.2 PYTHON3_VERSION=3.7.1 +BROTLI_VERSION=1.0.7 # # GNU autotools; they're provided with releases up to Snow Leopard, but @@ -1711,11 +1712,70 @@ uninstall_python3() { fi } +install_brotli() { + if [ "$BROTLI_VERSION" -a ! -f brotli-$BROTLI_VERSION-done ] ; then + echo "Downloading, building, and installing brotli:" + [ -f brotli-$BROTLI_VERSION.tar.gz ] || curl -L -o brotli-$BROTLI_VERSION.tar.gz https://github.com/google/brotli/archive/v$BROTLI_VERSION.tar.gz || exit 1 + $no_build && echo "Skipping installation" && return + gzcat brotli-$BROTLI_VERSION.tar.gz | tar xf - || exit 1 + cd brotli-$BROTLI_VERSION + mkdir build_dir + cd build_dir + MACOSX_DEPLOYMENT_TARGET=$min_osx_target SDKROOT="$SDKPATH" cmake ../ || exit 1 + make $MAKE_BUILD_OPTS || exit 1 + $DO_MAKE_INSTALL || exit 1 + cd ../.. + touch brotli-$BROTLI_VERSION-done + fi +} + +uninstall_brotli() { + if [ ! -z "$installed_brotli_version" ] ; then + echo "Uninstalling brotli:" + cd brotli-$installed_brotli_version + # + # brotli uses cmake on macOS and doesn't support "make uninstall" + # + # $DO_MAKE_UNINSTALL || exit 1 + sudo rm -rf /usr/local/bin/brotli + sudo rm -rf /usr/local/lib/libbrotli* + sudo rm -rf /usr/local/include/brotli + sudo rm -rf /usr/local/lib/pkgconfig/libbrotli* + # + # brotli uses cmake on macOS and doesn't support "make distclean" + # + # make distclean || exit 1 + cd .. + rm brotli-$installed_brotli_version-done + + if [ "$#" -eq 1 -a "$1" = "-r" ] ; then + # + # Get rid of the previously downloaded and unpacked version. + # + rm -rf brotli-$installed_brotli_version + rm -rf brotli-$installed_brotli_version.tar.gz + fi + + installed_brotli_version="" + fi +} + install_all() { # # Check whether the versions we have installed are the versions # requested; if not, uninstall the installed versions. # + if [ ! -z "$installed_brotli_version" -a \ + "$installed_brotli_version" != "$BROTLI_VERSION" ] ; then + echo "Installed brotli version is $installed_brotli_version" + if [ -z "$BROTLI_VERSION" ] ; then + echo "brotli is not requested" + else + echo "Requested brotli version is $BROTLI_VERSION" + fi + uninstall_brotli -r + fi + if [ ! -z "$installed_python3_version" -a \ "$installed_python3_version" != "$PYTHON3_VERSION" ] ; then echo "Installed python3 version is $installed_python3_version" @@ -2188,6 +2248,8 @@ install_all() { install_bcg729 install_python3 + + install_brotli } uninstall_all() { @@ -2204,6 +2266,8 @@ uninstall_all() { # We also do a "make distclean", so that we don't have leftovers from # old configurations. # + uninstall_brotli + uninstall_python3 uninstall_bcg729 @@ -2406,6 +2470,7 @@ then installed_spandsp_version=`ls spandsp-*-done 2>/dev/null | sed 's/spandsp-\(.*\)-done/\1/'` installed_bcg729_version=`ls bcg729-*-done 2>/dev/null | sed 's/bcg729-\(.*\)-done/\1/'` installed_python3_version=`ls python3-*-done 2>/dev/null | sed 's/python3-\(.*\)-done/\1/'` + installed_brotli_version=`ls brotli-*-done 2>/dev/null | sed 's/brotli-\(.*\)-done/\1/'` cd $topdir fi diff --git a/tools/rpm-setup.sh b/tools/rpm-setup.sh index ba07180e82..75b9a766a0 100755 --- a/tools/rpm-setup.sh +++ b/tools/rpm-setup.sh @@ -106,6 +106,19 @@ add_package() { eval "${list}=\"\${${list}} \${pkgname}\"" } +# Adds packages $2-$n to list variable $1 if all the packages are found +add_packages() { + local list="$1" pkgnames="${@:2}" + + # fail if any package is not known + for pkgname in $pkgnames; do + $PM $PM_SEARCH "$pkgname" &> /dev/null || return 1 + done + + # all packages are found, append it to list + eval "${list}=\"\${${list}} \${pkgnames}\"" +} + # python3: OpenSUSE 43.3, Fedora 26 # python34: Centos 7 add_package BASIC_LIST python3 || add_package BASIC_LIST python34 || @@ -197,6 +210,9 @@ echo "ninja is unavailable" >&2 add_package ADDITIONAL_LIST libxslt || add_package ADDITIONAL_LIST libxslt1 || echo "xslt is unavailable" >&2 +add_package ADDITIONAL_LIST brotli-devel || add_packages ADDITIONAL_LIST libbrotli-devel libbrotlidec1 || +echo "brotli is unavailable" >&2 + ACTUAL_LIST=$BASIC_LIST # Now arrange for optional support libraries diff --git a/tools/win-setup.ps1 b/tools/win-setup.ps1 index 6cfd06d162..62eea5c938 100644 --- a/tools/win-setup.ps1 +++ b/tools/win-setup.ps1 @@ -76,6 +76,7 @@ $Win32CurrentTag = "2019-04-08" $Win64Archives = @{ "AirPcap_Devpack_4_1_0_1622.zip" = "09d637f28a79b1d2ecb09f35436271a90c0f69bd0a1ee82b803abaaf63c18a69"; "bcg729-1.0.4-win64ws.zip" = "9a095fda4c39860d96f0c568830faa6651cd17635f68e27aa6de46c689aa0ee2"; + "brotli-1.0.2-4-win64ws.zip" = "f60636764f0e2539ad86d37826e90445a346291f8d046bd4f5c998301f200195"; "c-ares-1.15.0-win64ws.zip" = "ade864fd08e887d353a9c939fa6e68b0bf3e08761b6e81f60ce15e6543256552"; "gnutls-3.6.3-1-win64ws.zip" = "994ac2578e7b4ca01e589ab2598927d53f7370bc3ff679f3006b0e6bb7a06df4"; "krb5-1.17-1-win64ws.zip" = "1f4a7ab86ae331ea9e58c9776a60def81ae9fe622882b2e8da2ad6ce6f6fb1d8"; @@ -97,6 +98,7 @@ $Win64Archives = @{ $Win32Archives = @{ "AirPcap_Devpack_4_1_0_1622.zip" = "09d637f28a79b1d2ecb09f35436271a90c0f69bd0a1ee82b803abaaf63c18a69"; "bcg729-1.0.4-win32ws.zip" = "b785ec78dec6bca8252130eb884bfa28c1140001dd7369a535579176de9e4271"; + "brotli-1.0.2-4-win32ws.zip" = "193a9b35e42a73f77dabe2c99c57173d7f2bb215118c1cd027a6cf522cc3dc14"; "c-ares-1.15.0-win32ws.zip" = "a54151203a631b478470aaa21b3a1fde6178f2fea9f15a1a6da4bfcecc92cfcd"; "gnutls-3.6.3-1-win32ws.zip" = "42d8313ffb888f525d6c39330c39bcc2182e68ee8433a09dd85e1f1e1474f592"; "krb5-1.17-1-win32ws.zip" = "f90cac08355ccfe624652d3e05f8e2e077b8830382315d4ea0a6fa52af08260b";