CMake: Prevent WiX installations over existing NSIS installations

Use a custom action to look for Uninstall.exe in the user
selected installation prefix.

Its presence indicates a previous NSIS installation.
Inform the user and request manual resolution of the issue.

CMakeCPack.cmake

@@ -198,6 +198,17 @@ if(EXISTS "${CMAKE_ROOT}/Modules/CPack.cmake")
 
   set(CPACK_WIX_UPGRADE_GUID "8ffd1d72-b7f1-11e2-8ee5-00238bca4991")
 
+  if(MSVC AND NOT "$ENV{WIX}" STREQUAL "")
+    set(WIX_CUSTOM_ACTION_ENABLED TRUE)
+    if(CMAKE_CONFIGURATION_TYPES)
+      set(WIX_CUSTOM_ACTION_MULTI_CONFIG TRUE)
+    else()
+      set(WIX_CUSTOM_ACTION_MULTI_CONFIG FALSE)
+    endif()
+  else()
+    set(WIX_CUSTOM_ACTION_ENABLED FALSE)
+  endif()
+
   # Set the options file that needs to be included inside CMakeCPackOptions.cmake
   set(QT_DIALOG_CPACK_OPTIONS_FILE ${CMake_BINARY_DIR}/Source/QtDialog/QtDialogCPack.cmake)
   configure_file("${CMake_SOURCE_DIR}/CMakeCPackOptions.cmake.in"

CMakeCPackOptions.cmake.in

@@ -246,6 +246,29 @@ if("${CPACK_GENERATOR}" STREQUAL "WIX")
     "@CMake_SOURCE_DIR@/Utilities/Release/WiX/cmake_extra_dialog.wxs"
   )
 
+  set(_WIX_CUSTOM_ACTION_ENABLED "@WIX_CUSTOM_ACTION_ENABLED@")
+  if(_WIX_CUSTOM_ACTION_ENABLED)
+    list(APPEND CPACK_WIX_EXTRA_SOURCES
+      "@CMake_SOURCE_DIR@/Utilities/Release/WiX/cmake_nsis_overwrite_dialog.wxs"
+      )
+    list(APPEND CPACK_WIX_CANDLE_EXTRA_FLAGS -dCHECK_NSIS=1)
+
+    set(_WIX_CUSTOM_ACTION_MULTI_CONFIG "@WIX_CUSTOM_ACTION_MULTI_CONFIG@")
+    if(_WIX_CUSTOM_ACTION_MULTI_CONFIG)
+      if(CPACK_BUILD_CONFIG)
+        set(_WIX_CUSTOM_ACTION_CONFIG "${CPACK_BUILD_CONFIG}")
+      else()
+        set(_WIX_CUSTOM_ACTION_CONFIG "Release")
+      endif()
+
+      list(APPEND CPACK_WIX_EXTRA_SOURCES
+        "@CMake_BINARY_DIR@/Utilities/Release/WiX/custom_action_dll-${_WIX_CUSTOM_ACTION_CONFIG}.wxs")
+    else()
+      list(APPEND CPACK_WIX_EXTRA_SOURCES
+        "@CMake_BINARY_DIR@/Utilities/Release/WiX/custom_action_dll.wxs")
+    endif()
+  endif()
+
   set(CPACK_WIX_UI_REF "CMakeUI_InstallDir")
 
   set(CPACK_WIX_PATCH_FILE
@@ -261,8 +284,7 @@ if("${CPACK_GENERATOR}" STREQUAL "WIX")
   if(BUILD_QtDialog)
     list(APPEND CPACK_WIX_PATCH_FILE
       "@CMake_SOURCE_DIR@/Utilities/Release/WiX/patch_desktop_shortcut.xml"
-    )
+      )
-
+    list(APPEND CPACK_WIX_CANDLE_EXTRA_FLAGS -dBUILD_QtDialog=1)
-    set(CPACK_WIX_CANDLE_EXTRA_FLAGS "-dBUILD_QtDialog=1")
   endif()
 endif()

Utilities/CMakeLists.txt

@@ -33,3 +33,7 @@ else()
   # Normal documentation build.
   add_subdirectory(Sphinx)
 endif()
+
+if(WIX_CUSTOM_ACTION_ENABLED)
+  add_subdirectory(Release/WiX)
+endif()

Utilities/Release/WiX/CMakeLists.txt

@@ -0,0 +1,12 @@
+add_subdirectory(CustomAction)
+
+if(CMAKE_CONFIGURATION_TYPES)
+  set(CUSTOM_ACTION_OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/custom_action_dll-$<CONFIG>.wxs")
+else()
+  set(CUSTOM_ACTION_OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/custom_action_dll.wxs")
+endif()
+
+file(GENERATE
+  OUTPUT "${CUSTOM_ACTION_OUTPUT}"
+  INPUT "${CMAKE_CURRENT_SOURCE_DIR}/custom_action_dll.wxs.in"
+  )

Utilities/Release/WiX/CustomAction/CMakeLists.txt

@@ -0,0 +1,13 @@
+foreach(CONFIG DEBUG MINSIZEREL RELEASE RELWITHDEBINFO)
+  string(REPLACE "/MD" "/MT"
+    "CMAKE_CXX_FLAGS_${CONFIG}"
+    "${CMAKE_CXX_FLAGS_${CONFIG}}"
+  )
+endforeach()
+
+add_library(CMakeWiXCustomActions MODULE
+  detect_nsis_overwrite.cpp
+  exports.def
+)
+
+target_link_libraries(CMakeWiXCustomActions PRIVATE msi)

Utilities/Release/WiX/CustomAction/detect_nsis_overwrite.cpp

@@ -0,0 +1,45 @@
+#include <windows.h>
+#include <msi.h>
+#include <msiquery.h>
+
+#include <string>
+#include <vector>
+
+std::wstring get_property(MSIHANDLE msi_handle, std::wstring const& name)
+{
+  DWORD size = 0;
+
+  UINT status = MsiGetPropertyW(msi_handle, name.c_str(), L"", &size);
+
+  if(status == ERROR_MORE_DATA)
+    {
+    std::vector<wchar_t> buffer(size + 1);
+    MsiGetPropertyW(msi_handle, name.c_str(), &buffer[0], &size);
+    return std::wstring(&buffer[0]);
+    }
+  else
+    {
+    return std::wstring();
+    }
+}
+
+void set_property(MSIHANDLE msi_handle,
+  std::wstring const& name, std::wstring const& value)
+{
+  MsiSetPropertyW(msi_handle, name.c_str(), value.c_str());
+}
+
+extern "C" UINT __stdcall DetectNsisOverwrite(MSIHANDLE msi_handle)
+{
+  std::wstring install_root = get_property(msi_handle, L"INSTALL_ROOT");
+
+  std::wstring uninstall_exe = install_root + L"\\uninstall.exe";
+
+  bool uninstall_exe_exists =
+      GetFileAttributesW(uninstall_exe.c_str()) != INVALID_FILE_ATTRIBUTES;
+
+  set_property(msi_handle, L"CMAKE_NSIS_OVERWRITE_DETECTED",
+    uninstall_exe_exists ? L"1" : L"0");
+
+  return ERROR_SUCCESS;
+}

Utilities/Release/WiX/CustomAction/exports.def

@@ -0,0 +1,2 @@
+EXPORTS
+	DetectNsisOverwrite=DetectNsisOverwrite

Utilities/Release/WiX/cmake_nsis_overwrite_dialog.wxs

@@ -0,0 +1,21 @@
+<Wix xmlns="http://schemas.microsoft.com/wix/2006/wi">
+	<Fragment>
+		<UI>
+	            <Dialog Id="CMakeNsisOverwriteDialog" Width="310" Height="120" Title="NSIS Installation Conflict">
+			<Control Id="OK" Type="PushButton" X="122" Y="90" Width="56" Height="17" Default="yes" Cancel="yes" Text="!(loc.WixUIOK)">
+	                    <Publish Event="EndDialog" Value="Return">1</Publish>
+	                </Control>
+	                <Control Id="Text" Type="Text" X="48" Y="22" Width="260" Height="60">
+				<Text>
+					Uninstall.exe was detected in your chosen installation prefix.
+					This indicates a conflicting NSIS based installation of CMake.
+
+					Please uninstall your old CMake installation or choose a different
+					installation directory.
+				</Text>
+			</Control>
+	                <Control Id="Icon" Type="Icon" X="15" Y="15" Width="24" Height="24" ToolTip="!(loc.InvalidDirDlgIconTooltip)" FixedSize="yes" IconSize="32" Text="!(loc.InvalidDirDlgIcon)" />
+	            </Dialog>
+		</UI>
+	</Fragment>
+</Wix>

Utilities/Release/WiX/custom_action_dll.wxs.in

@@ -0,0 +1,6 @@
+<Wix xmlns="http://schemas.microsoft.com/wix/2006/wi">
+	<Fragment>
+		<Binary Id="CMakeCustomActionsDll"
+			SourceFile="$<TARGET_FILE:CMakeWiXCustomActions>"/>
+	</Fragment>
+</Wix>

Utilities/Release/WiX/install_dir.wxs

@@ -9,6 +9,9 @@
 			<Property Id="WixUI_Mode" Value="InstallDir" />
 
 			<DialogRef Id="CMakeExtraDialog" />
+			<?ifdef CHECK_NSIS ?>
+				<DialogRef Id="CMakeNsisOverwriteDialog" />
+			<?endif ?>
 
 			<DialogRef Id="BrowseDlg" />
 			<DialogRef Id="DiskCostDlg" />
@@ -36,7 +39,11 @@
 			<Publish Dialog="InstallDirDlg" Control="Next" Event="SetTargetPath" Value="[WIXUI_INSTALLDIR]" Order="1">1</Publish>
 			<Publish Dialog="InstallDirDlg" Control="Next" Event="DoAction" Value="WixUIValidatePath" Order="2">NOT WIXUI_DONTVALIDATEPATH</Publish>
 			<Publish Dialog="InstallDirDlg" Control="Next" Event="SpawnDialog" Value="InvalidDirDlg" Order="3"><![CDATA[NOT WIXUI_DONTVALIDATEPATH AND WIXUI_INSTALLDIR_VALID<>"1"]]></Publish>
-			<Publish Dialog="InstallDirDlg" Control="Next" Event="NewDialog" Value="VerifyReadyDlg" Order="4">WIXUI_DONTVALIDATEPATH OR WIXUI_INSTALLDIR_VALID="1"</Publish>
+			<?ifdef CHECK_NSIS ?>
+				<Publish Dialog="InstallDirDlg" Control="Next" Event="DoAction" Value="CMakeDetectNsisOverwrite" Order="4">1</Publish>
+				<Publish Dialog="InstallDirDlg" Control="Next" Event="SpawnDialog" Value="CMakeNsisOverwriteDialog" Order="5">CMAKE_NSIS_OVERWRITE_DETECTED="1"</Publish>
+			<?endif ?>
+			<Publish Dialog="InstallDirDlg" Control="Next" Event="NewDialog" Value="VerifyReadyDlg" Order="6"><![CDATA[(WIXUI_DONTVALIDATEPATH OR WIXUI_INSTALLDIR_VALID="1") AND CMAKE_NSIS_OVERWRITE_DETECTED<>1]]></Publish>
 			<Publish Dialog="InstallDirDlg" Control="ChangeFolder" Property="_BrowseProperty" Value="[WIXUI_INSTALLDIR]" Order="1">1</Publish>
 			<Publish Dialog="InstallDirDlg" Control="ChangeFolder" Event="SpawnDialog" Value="BrowseDlg" Order="2">1</Publish>
 
@@ -57,5 +64,9 @@
 		</UI>
 
 		<UIRef Id="WixUI_Common" />
+
+		<?ifdef CHECK_NSIS ?>
+			<CustomAction Id="CMakeDetectNsisOverwrite" BinaryKey="CMakeCustomActionsDll" DllEntry="DetectNsisOverwrite"/>
+		<?endif ?>
 	</Fragment>
 </Wix>