CMake: Add ENABLE_VLD option for MSVC

Calling cmake with -DENABLE_VLD=ON when building with Visual Studio,
results in debug configuration being linked to Visual Leak Detector.
By default, Visual Leak Detector outputs the leak summary to Visual
Studio debug window. When ENABLE_VLD is active, VLD is linked to all
wireshark libraries and executables.

CMakeLists.txt

@@ -933,6 +933,18 @@ if(ENABLE_FUZZER)
 	set(CMAKE_CXX_FLAGS "-fsanitize=fuzzer-no-link ${CMAKE_CXX_FLAGS}")
 endif()
 
+if(MSVC)
+	set(ENABLE_VLD OFF CACHE BOOL "Whether enable Visual Leak Detect in Debug configuration")
+	if(ENABLE_VLD)
+		include(FindVLD)
+	    if(NOT VLD_FOUND)
+			message(FATAL_ERROR "ENABLE_VLD was requested, but not found!")
+		endif()
+		message(STATUS "Enabling Visual Leak Detector in Debug configuration")
+		set(WS_MSVC_DEBUG_LINK_FLAGS ${VLD_LINK_FLAGS})
+	endif()
+endif()
+
 set(WERROR_COMMON_FLAGS "")
 if(NOT DISABLE_WERROR AND NOT ENABLE_EXTRA_COMPILER_WARNINGS)
 	if(CMAKE_C_COMPILER_ID MATCHES "MSVC")
@@ -1913,6 +1925,15 @@ if(WIN32)
 		COMMAND_EXPAND_LISTS
 	)
 
+	if (MSVC AND VLD_FOUND)
+		add_custom_command(TARGET copy_cli_dlls PRE_BUILD
+			COMMAND ${CMAKE_COMMAND} -E "$<IF:$<CONFIG:Debug>,copy_if_different,true>"
+			"${VLD_FILES}"
+			"${_dll_output_dir}"
+			COMMAND_EXPAND_LISTS
+		)
+	endif()
+
 	# Optional DLLs and PDBs.
 	set (OPTIONAL_DLLS)
 	set (OPTIONAL_PDBS)
@@ -2400,6 +2421,9 @@ if(BUILD_wireshark AND QT_FOUND)
 		FOLDER "Executables"
 		INSTALL_RPATH "${EXECUTABLE_INSTALL_RPATH}"
 	)
+	if(MSVC)
+		set_target_properties(wireshark PROPERTIES LINK_FLAGS_DEBUG "${WS_MSVC_DEBUG_LINK_FLAGS}")
+	endif()
 	if(ENABLE_APPLICATION_BUNDLE OR WIN32)
 		set_target_properties(wireshark PROPERTIES OUTPUT_NAME Wireshark)
 	endif()
@@ -2470,6 +2494,9 @@ macro(set_extra_executable_properties _executable _folder)
 		FOLDER ${_folder}
 		INSTALL_RPATH "${EXECUTABLE_INSTALL_RPATH}"
 	)
+	if(MSVC)
+		set_target_properties(${_executable} PROPERTIES LINK_FLAGS_DEBUG "${WS_MSVC_DEBUG_LINK_FLAGS}")
+	endif()
 
 	set(PROGLIST ${PROGLIST} ${_executable})
 

capture/CMakeLists.txt

@@ -66,6 +66,9 @@ endif(WIN32)
 set_target_properties(caputils PROPERTIES
 	LINK_FLAGS "${WS_LINK_FLAGS}"
 	FOLDER "Libs")
+if(MSVC)
+	set_target_properties(caputils PROPERTIES LINK_FLAGS_DEBUG "${WS_MSVC_DEBUG_LINK_FLAGS}")
+endif()
 
 set(CAPCHILD_SRC
 	capture_ifinfo.c
@@ -93,6 +96,9 @@ target_link_libraries(capchild
 set_target_properties(capchild PROPERTIES
 	LINK_FLAGS "${WS_LINK_FLAGS}"
 	FOLDER "Libs")
+if(MSVC)
+	set_target_properties(capchild PROPERTIES LINK_FLAGS_DEBUG "${WS_MSVC_DEBUG_LINK_FLAGS}")
+endif()
 
 CHECKAPI(
 	NAME

cmake/modules/FindVLD.cmake

@@ -0,0 +1,62 @@
+#
+# - Find Visual Leak Detector
+#
+#  VLD_LINK_FLAGS - MSVC linker flags that enable VLD
+#  VLD_FILES - files that should be copied to dll output directory
+#  VLD_VALID - TRUE if Visual Leak Detector was found
+
+set(_PF86 "PROGRAMFILES(x86)")
+set(_vld_arch Win64)
+set(_vld_dll vld_x64.dll)
+set(_vld_pdb vld_x64.pdb)
+if(WIRESHARK_TARGET_PLATFORM MATCHES "win32")
+	set(_vld_arch Win32)
+	set(_vld_dll vld_x86.dll)
+	set(_vld_pdb vld_x86.pdb)
+endif()
+
+find_library(VLD_LIBRARY
+	NAMES
+		vld
+	HINTS
+		"$ENV{PROGRAMFILES}/Visual Leak Detector/lib/${_vld_arch}"
+		"$ENV{${_PF86}}/Visual Leak Detector/lib/${_vld_arch}"
+		"$ENV{PROGRAMW6432}/Visual Leak Detector/lib/${_vld_arch}"
+)
+
+function(find_vld_file _vld_file _filename)
+	find_file(${_vld_file}
+		NAMES
+			${_filename}
+		HINTS
+			"$ENV{PROGRAMFILES}/Visual Leak Detector/bin/${_vld_arch}"
+			"$ENV{${_PF86}}/Visual Leak Detector/bin/${_vld_arch}"
+			"$ENV{PROGRAMW6432}/Visual Leak Detector/bin/${_vld_arch}"
+	)
+	if(${${_vld_file}} STREQUAL "${_vld_file}-NOTFOUND")
+		set(${_vld_file} "" PARENT_SCOPE)
+	endif()
+endfunction()
+
+find_vld_file(VLD_DLL ${_vld_dll})
+find_vld_file(VLD_DBGHELP_DLL "dbghelp.dll")
+find_vld_file(VLD_MANIFEST "Microsoft.DTfW.DHL.manifest")
+find_vld_file(VLD_PDB ${_vld_pdb})
+
+#library, dlls and manifest are mandatory, while pdb is optional
+IF(VLD_LIBRARY AND (EXISTS ${VLD_DLL}) AND (EXISTS ${VLD_DBGHELP_DLL}) AND (EXISTS ${VLD_MANIFEST}))
+	# Link against VLD library and force it to be linked by referencing symbol
+	# Adding VLD_LINK_FLAGS to linker flags enables Visual Leak Detector
+	set(VLD_LINK_FLAGS "\"${VLD_LIBRARY}\" /include:__imp_?g_vld@@3VVisualLeakDetector@@A")
+	file(GLOB VLD_FILES
+		"${VLD_DLL}"
+		"${VLD_DBGHELP_DLL}"
+		"${VLD_MANIFEST}"
+		"${VLD_PDB}"
+	)
+	set(VLD_FOUND TRUE)
+else()
+	set(VLD_LINK_FLAGS)
+	set(VLD_FILES)
+	set(VLD_FOUND FALSE)
+endif()

cmake/modules/WiresharkPlugin.cmake

@@ -39,6 +39,9 @@ macro(ADD_PLUGIN_LIBRARY _plugin _subfolder)
 		LINK_FLAGS "${WS_LINK_FLAGS}"
 		FOLDER "Plugins"
 	)
+	if(MSVC)
+		set_target_properties(${_plugin} PROPERTIES LINK_FLAGS_DEBUG "${WS_MSVC_DEBUG_LINK_FLAGS}")
+	endif()
 
 	set_target_properties(${_plugin} PROPERTIES
 		LIBRARY_OUTPUT_DIRECTORY ${PLUGIN_DIR}/${_subfolder}

epan/CMakeLists.txt

@@ -313,6 +313,9 @@ set_target_properties(epan PROPERTIES
 	PREFIX "lib"
 	FOLDER "DLLs"
 )
+if(MSVC)
+	set_target_properties(epan PROPERTIES LINK_FLAGS_DEBUG "${WS_MSVC_DEBUG_LINK_FLAGS}")
+endif()
 
 target_link_libraries(epan
 	PUBLIC

extcap/CMakeLists.txt

@@ -34,6 +34,9 @@ macro(set_extcap_executable_properties _executable)
 		FOLDER "Executables/Extcaps"
 		INSTALL_RPATH "${EXTCAP_INSTALL_RPATH}"
 	)
+	if(MSVC)
+		set_target_properties(${_executable} PROPERTIES LINK_FLAGS_DEBUG "${WS_MSVC_DEBUG_LINK_FLAGS}")
+	endif()
 
 	set(PROGLIST ${PROGLIST} ${_executable})
 

randpkt_core/CMakeLists.txt

@@ -27,6 +27,9 @@ add_library(randpkt_core STATIC
 set_target_properties(randpkt_core PROPERTIES
 	LINK_FLAGS "${WS_LINK_FLAGS}"
 	FOLDER "Libs")
+if(MSVC)
+	set_target_properties(randpkt_core PROPERTIES LINK_FLAGS_DEBUG "${WS_MSVC_DEBUG_LINK_FLAGS}")
+endif()
 
 target_link_libraries(randpkt_core PUBLIC ui)
 

ui/CMakeLists.txt

@@ -86,6 +86,9 @@ set_target_properties(ui PROPERTIES
 	LINK_FLAGS "${WS_LINK_FLAGS}"
 	FOLDER "UI"
 )
+if(MSVC)
+	set_target_properties(ui PROPERTIES LINK_FLAGS_DEBUG "${WS_MSVC_DEBUG_LINK_FLAGS}")
+endif()
 
 target_link_libraries(ui wsutil caputils)
 

ui/qt/CMakeLists.txt

@@ -737,6 +737,9 @@ set_target_properties(qtui PROPERTIES
 	# Ensure .qm files are generated before autogenerating i18n.qrc
 	AUTOGEN_TARGET_DEPENDS "${WIRESHARK_QT_QM}"
 )
+if(MSVC)
+	set_target_properties(qtui PROPERTIES LINK_FLAGS_DEBUG "${WS_MSVC_DEBUG_LINK_FLAGS}")
+endif()
 
 CHECKAPI(
 	NAME

wiretap/CMakeLists.txt

@@ -173,6 +173,9 @@ set_target_properties(wiretap PROPERTIES
 	FOLDER "DLLs"
 	INSTALL_RPATH "${LIBRARY_INSTALL_RPATH}"
 )
+if(MSVC)
+	set_target_properties(wiretap PROPERTIES LINK_FLAGS_DEBUG "${WS_MSVC_DEBUG_LINK_FLAGS}")
+endif()
 
 target_link_libraries(wiretap
 	PUBLIC

wsutil/CMakeLists.txt

@@ -255,6 +255,9 @@ set_target_properties(wsutil PROPERTIES
 	FOLDER "DLLs"
 	INSTALL_RPATH "${LIBRARY_INSTALL_RPATH}"
 )
+if(MSVC)
+	set_target_properties(wsutil PROPERTIES LINK_FLAGS_DEBUG "${WS_MSVC_DEBUG_LINK_FLAGS}")
+endif()
 
 target_link_libraries(wsutil
 	PUBLIC