Add new Secrets API and allow TLS to use pcapng decryption secrets

Add a new secrets API to the core, one that can outlive the lifetime of
a single capture file. Expose decryption secrets from wiretap through a
callback and let the secrets API route it to a dissector.

Bug: 15252
Change-Id: Ie2f1867bdfd265bad11fc58f1e8d8e7295c0d1e7
Reviewed-on: https://code.wireshark.org/review/30705
Petri-Dish: Peter Wu <peter@lekensteyn.nl>
Tested-by: Petri Dish Buildbot
Reviewed-by: Anders Broman <a.broman58@gmail.com>

debian/libwireshark0.symbols

@@ -1412,6 +1412,7 @@ libwireshark.so.0 libwireshark0 #MINVER#
  scsi_ssc_vals_ext@Base 1.12.0~rc1
  scsistat_param@Base 2.5.0
  sctp_port_to_display@Base 1.99.2
+ secrets_wtap_callback@Base 2.9.0
  sequence_analysis_create_sai_with_addresses@Base 2.5.0
  sequence_analysis_dump_to_file@Base 2.5.0
  sequence_analysis_find_by_name@Base 2.5.0

debian/libwiretap0.symbols

@@ -131,6 +131,7 @@ libwiretap.so.0 libwiretap0 #MINVER#
  wtap_seek_read@Base 1.9.1
  wtap_sequential_close@Base 1.9.1
  wtap_set_bytes_dumped@Base 1.9.1
+ wtap_set_cb_new_secrets@Base 2.9.0
  wtap_set_cb_new_ipv4@Base 1.9.1
  wtap_set_cb_new_ipv6@Base 1.9.1
  wtap_short_string_to_encap@Base 1.9.1

epan/CMakeLists.txt

@@ -133,6 +133,7 @@ set(LIBWIRESHARK_PUBLIC_HEADERS
 	rtd_table.h
 	rtp_pt.h
 	sctpppids.h
+	secrets.h
 	show_exception.h
 	slow_protocol_subtypes.h
 	sminmpec.h
@@ -221,6 +222,7 @@ set(LIBWIRESHARK_NONGENERATED_FILES
 	register.c
 	req_resp_hdrs.c
 	rtd_table.c
+	secrets.c
 	sequence_analysis.c
 	show_exception.c
 	srt_table.c

epan/dissectors/packet-tls-utils.c

@@ -5189,7 +5189,7 @@ typedef struct ssl_master_key_match_group {
 } ssl_master_key_match_group_t;
 
 void
-tls_keylog_process_lines(const ssl_master_key_map_t *mk_map, const char *lines)
+tls_keylog_process_lines(const ssl_master_key_map_t *mk_map, const guint8 *data, guint datalen)
 {
     ssl_master_key_match_group_t mk_groups[] = {
         { "encrypted_pmk",  mk_map->pre_master },
@@ -5254,23 +5254,24 @@ tls_keylog_process_lines(const ssl_master_key_map_t *mk_map, const char *lines)
     if (!regex)
         return;
 
-    const char *next_line = lines;
-    while (next_line && next_line[0]) {
+    const char *next_line = (const char *)data;
+    const char *line_end = next_line + datalen;
+    while (next_line && next_line < line_end) {
         const char *line = next_line;
-        next_line = strchr(line, '\n');
+        next_line = (const char *)memchr(line, '\n', line_end - line);
         gssize linelen;
 
         if (next_line) {
             linelen = next_line - line;
             next_line++;    /* drop LF */
         } else {
-            linelen = (gssize)strlen(line);
+            linelen = (gssize)(line_end - line);
         }
         if (linelen > 0 && line[linelen - 1] == '\r') {
             linelen--;      /* drop CR */
         }
 
-        ssl_debug_printf("  checking keylog line: %s\n", line);
+        ssl_debug_printf("  checking keylog line: %.*s\n", (int)linelen, line);
         GMatchInfo *mi;
         if (g_regex_match_full(regex, line, linelen, 0, G_REGEX_MATCH_ANCHORED, &mi, NULL)) {
             gchar *hex_key, *hex_pre_ms_or_ms;
@@ -5370,7 +5371,7 @@ ssl_load_keyfile(const gchar *tls_keylog_filename, FILE **keylog_file,
             }
             break;
         }
-        tls_keylog_process_lines(mk_map, line);
+        tls_keylog_process_lines(mk_map, (guint8 *)line, (int)strlen(line));
     }
 }
 /** SSL keylog file handling. }}} */

epan/dissectors/packet-tls-utils.h

@@ -650,7 +650,7 @@ ssl_common_cleanup(ssl_master_key_map_t *master_key_map, FILE **ssl_keylog_file,
 
 /* Process lines from the TLS key log and populate the secrets map. */
 extern void
-tls_keylog_process_lines(const ssl_master_key_map_t *mk_map, const char *lines);
+tls_keylog_process_lines(const ssl_master_key_map_t *mk_map, const guint8 *data, guint len);
 
 /* tries to update the secrets cache from the given filename */
 extern void

epan/dissectors/packet-tls.c

@@ -75,6 +75,8 @@
 #include <epan/exported_pdu.h>
 #include <epan/proto_data.h>
 #include <epan/decode_as.h>
+#include <epan/secrets.h>
+#include <wiretap/secrets-types.h>
 
 #include <wsutil/utf8_entities.h>
 #include <wsutil/str_util.h>
@@ -3754,6 +3756,12 @@ ssl_both_prompt(packet_info *pinfo, gchar *result)
     g_snprintf(result, MAX_DECODE_AS_PROMPT_LEN, "both (%u%s%u)", srcport, UTF8_LEFT_RIGHT_ARROW, destport);
 }
 
+static void
+tls_secrets_block_callback(const void *secrets, guint size)
+{
+    tls_keylog_process_lines(&ssl_master_key_map, (const guint8 *)secrets, size);
+}
+
 /*********************************************************************
  *
  * Standard Wireshark Protocol Registration and housekeeping
@@ -4171,6 +4179,7 @@ proto_register_tls(void)
 
     register_follow_stream(proto_tls, "tls", tcp_follow_conv_filter, tcp_follow_index_filter, tcp_follow_address_filter,
                             tcp_port_to_display, ssl_follow_tap_listener);
+    secrets_register_type(SECRETS_TYPE_TLS, tls_secrets_block_callback);
 }
 
 static int dissect_tls_sct_ber(tvbuff_t *tvb, packet_info *pinfo, proto_tree *tree, void *data _U_)

epan/epan.c

@@ -55,6 +55,7 @@
 #include "reassemble.h"
 #include "srt_table.h"
 #include "stats_tree.h"
+#include "secrets.h"
 #include <dtd.h>
 
 #ifdef HAVE_PLUGINS
@@ -228,6 +229,7 @@ epan_init(register_cb cb, gpointer client_data, gboolean load_plugins)
 		prefs_init();
 		expert_init();
 		packet_init();
+		secrets_init();
 		conversation_init();
 		capture_dissector_init();
 		reassembly_tables_init();
@@ -316,6 +318,7 @@ epan_cleanup(void)
 	prefs_cleanup();
 	proto_cleanup();
 
+	secrets_cleanup();
 	conversation_filters_cleanup();
 	reassembly_table_cleanup();
 	tap_cleanup();

epan/secrets.c

@@ -0,0 +1,58 @@
+/* secrets.c
+ * Secrets management and processing.
+ * Copyright 2018, Peter Wu <peter@lekensteyn.nl>
+ *
+ * Wireshark - Network traffic analyzer
+ * By Gerald Combs <gerald@wireshark.org>
+ * Copyright 1998 Gerald Combs
+ *
+ * SPDX-License-Identifier: GPL-2.0-or-later
+ */
+
+#include "secrets.h"
+#include <wiretap/wtap.h>
+
+/** Maps guint32 secrets_type -> secrets_block_callback_t. */
+static GHashTable *secrets_callbacks;
+
+void
+secrets_init(void)
+{
+    secrets_callbacks = g_hash_table_new(g_direct_hash, g_direct_equal);
+}
+
+void
+secrets_cleanup(void)
+{
+    g_hash_table_destroy(secrets_callbacks);
+    secrets_callbacks = NULL;
+}
+
+void
+secrets_register_type(guint32 secrets_type, secrets_block_callback_t cb)
+{
+    g_hash_table_insert(secrets_callbacks, GUINT_TO_POINTER(secrets_type), (gpointer)cb);
+}
+
+void
+secrets_wtap_callback(guint32 secrets_type, const void *secrets, guint size)
+{
+    secrets_block_callback_t cb = (secrets_block_callback_t)g_hash_table_lookup(
+            secrets_callbacks, GUINT_TO_POINTER(secrets_type));
+    if (cb) {
+        cb(secrets, size);
+    }
+}
+
+/*
+ * Editor modelines  -  https://www.wireshark.org/tools/modelines.html
+ *
+ * Local variables:
+ * c-basic-offset: 4
+ * tab-width: 8
+ * indent-tabs-mode: nil
+ * End:
+ *
+ * vi: set shiftwidth=4 tabstop=8 expandtab:
+ * :indentSize=4:tabSize=8:noTabs=true:
+ */

epan/secrets.h

@@ -0,0 +1,68 @@
+/* secrets.h
+ * Secrets management and processing.
+ * Copyright 2018, Peter Wu <peter@lekensteyn.nl>
+ *
+ * Wireshark - Network traffic analyzer
+ * By Gerald Combs <gerald@wireshark.org>
+ * Copyright 1998 Gerald Combs
+ *
+ * SPDX-License-Identifier: GPL-2.0-or-later
+ */
+
+#ifndef __SECRETS_H__
+#define __SECRETS_H__
+
+#include <glib.h>
+#include "ws_symbol_export.h"
+
+/**
+ * Interfaces for management and processing of secrets provided by external
+ * sources (wiretap, key files, HSMs, etc.). Dissectors can register themselves
+ * as consumers of these secrets.
+ *
+ * Future idea: provide helper functions to manage external files. Typically
+ * these secrets can be erased when the file is truncated or deleted+created.
+ * Additionally, these secrets are not tied to the lifetime of a capture file.
+ *
+ * Future idea: add a method for dissectors to mark secrets as "in use" such
+ * that unused entries can be removed when saving those secrets to file.
+ * Intended use case: read large TLS key log file (which is infrequently
+ * truncated by the user) and store only the bare minimum keys.
+ */
+
+void secrets_init(void);
+void secrets_cleanup(void);
+
+#if 0
+/**
+ * Lifetime of provided secrets.
+ * HSM: tie information to epan scope? (but if disconnected, clear state?)
+ * wiretap pcang DSB: scoped to (capture) file.
+ * tls.keylog_file pref: epan-scoped (but if the file is deleted, clear it)
+ */
+enum secrets_scope {
+    SECRETS_SCOPE_EPAN,
+    SECRETS_SCOPE_FILE,
+};
+#endif
+
+/**
+ * Callback for the wiretap secrets provider (wtap_new_secrets_callback_t).
+ */
+WS_DLL_PUBLIC void
+secrets_wtap_callback(guint32 secrets_type, const void *secrets, guint size);
+
+/**
+ * Receives a new block of secrets from an external source (wiretap or files).
+ */
+typedef void (*secrets_block_callback_t)(const void *secrets, guint size);
+
+/**
+ * Registers a consumer for pcapng Decryption Secrets Block (DSB). Only one
+ * dissector can register a type.
+ *
+ * @param secrets_type A Secrets Type as defined in wiretap/secrets-types.h
+ * @param cb Callback to be invoked for new secrets.
+ */
+void secrets_register_type(guint32 secrets_type, secrets_block_callback_t cb);
+#endif /* __SECRETS_H__ */

file.c

@@ -41,6 +41,7 @@
 #include <epan/strutil.h>
 #include <epan/addr_resolv.h>
 #include <epan/color_filters.h>
+#include <epan/secrets.h>
 
 #include "cfile.h"
 #include "file.h"
@@ -323,6 +324,7 @@ cf_open(capture_file *cf, const char *fname, unsigned int type, gboolean is_temp
 
   wtap_set_cb_new_ipv4(cf->provider.wth, add_ipv4_name);
   wtap_set_cb_new_ipv6(cf->provider.wth, (wtap_new_ipv6_callback_t) add_ipv6_name);
+  wtap_set_cb_new_secrets(cf->provider.wth, secrets_wtap_callback);
 
   return CF_OK;
 

sharkd.c

@@ -52,6 +52,7 @@
 #include <epan/epan_dissect.h>
 #include <epan/tap.h>
 #include <epan/uat-int.h>
+#include <epan/secrets.h>
 
 #include <codecs/codecs.h>
 
@@ -444,6 +445,7 @@ cf_open(capture_file *cf, const char *fname, unsigned int type, gboolean is_temp
 
   wtap_set_cb_new_ipv4(cf->provider.wth, add_ipv4_name);
   wtap_set_cb_new_ipv6(cf->provider.wth, (wtap_new_ipv6_callback_t) add_ipv6_name);
+  wtap_set_cb_new_secrets(cf->provider.wth, secrets_wtap_callback);
 
   return CF_OK;
 

test/suite_decryption.py

@@ -283,6 +283,16 @@ class case_decrypt_tls(subprocesstest.SubprocessTestCase):
             r'13||Request for /second, version TLSv1.3, Early data: yes\n',
         ], proc.stdout_str.splitlines())
 
+    def test_tls12_dsb(self, cmd_tshark, capture_file):
+        '''TLS 1.2 with master secrets in pcapng Decryption Secrets Blocks.'''
+        output = self.runProcess((cmd_tshark,
+                '-r', capture_file('tls12-dsb.pcapng'),
+                '-Tfields',
+                '-e', 'http.host',
+                '-e', 'http.response.code',
+                '-Y', 'http',
+            )).stdout_str.replace('\r\n', '\n')
+        self.assertEqual('example.com\t\n\t200\nexample.net\t\n\t200\n', output)
 
 
 @fixtures.mark_usefixtures('test_env')

tshark.c

@@ -94,6 +94,7 @@
 #include <epan/rtd_table.h>
 #include <epan/ex-opt.h>
 #include <epan/exported_pdu.h>
+#include <epan/secrets.h>
 
 #include "capture_opts.h"
 
@@ -4057,6 +4058,7 @@ cf_open(capture_file *cf, const char *fname, unsigned int type, gboolean is_temp
 
   wtap_set_cb_new_ipv4(cf->provider.wth, add_ipv4_name);
   wtap_set_cb_new_ipv6(cf->provider.wth, (wtap_new_ipv6_callback_t) add_ipv6_name);
+  wtap_set_cb_new_secrets(cf->provider.wth, secrets_wtap_callback);
 
   return CF_OK;
 

wiretap/pcapng.c

@@ -2646,6 +2646,12 @@ pcapng_process_idb(wtap *wth, pcapng_t *pcapng, wtapng_block_t *wblock)
 static void
 pcapng_process_dsb(wtap *wth, wtapng_block_t *wblock)
 {
+    const wtapng_dsb_mandatory_t *dsb = (wtapng_dsb_mandatory_t*)wtap_block_get_mandatory_data(wblock->block);
+
+    if (wth->add_new_secrets) {
+        wth->add_new_secrets(dsb->secrets_type, dsb->secrets_data, dsb->secrets_len);
+    }
+
     /* Store DSB such that it can be saved by the dumper. */
     g_array_append_val(wth->dsbs, wblock->block);
 }

wiretap/wtap-int.h

@@ -69,6 +69,7 @@ struct wtap {
                                                 */
     wtap_new_ipv4_callback_t    add_new_ipv4;
     wtap_new_ipv6_callback_t    add_new_ipv6;
+    wtap_new_secrets_callback_t add_new_secrets;
     GPtrArray                   *fast_seek;
 };
 

wiretap/wtap.c

@@ -1258,6 +1258,11 @@ void wtap_set_cb_new_ipv6(wtap *wth, wtap_new_ipv6_callback_t add_new_ipv6) {
 		wth->add_new_ipv6 = add_new_ipv6;
 }
 
+void wtap_set_cb_new_secrets(wtap *wth, wtap_new_secrets_callback_t add_new_secrets) {
+	if (wth)
+		wth->add_new_secrets = add_new_secrets;
+}
+
 gboolean
 wtap_read(wtap *wth, int *err, gchar **err_info, gint64 *data_offset)
 {