Tests: Add RunCMake tests for CPack{Deb,RPM}

Add script structure for easier addition of new CPack related tests.

Tests/RunCMake/CMakeLists.txt

@@ -26,6 +26,39 @@ macro(add_RunCMake_test test)
     )
 endmacro()
 
+function(add_RunCMake_test_group test types)
+  # create directory for common content
+  set(TEST_CONFIG_DIR "${CMAKE_CURRENT_BINARY_DIR}/${test}/conf")
+  file(REMOVE_RECURSE "${TEST_CONFIG_DIR}")
+  file(MAKE_DIRECTORY "${TEST_CONFIG_DIR}")
+
+  foreach(type IN LISTS types)
+    # generate prerequirements config file in cmake as ctest doesn't have as
+    # much system information so it is easier to set programs and environment
+    # values here
+    unset(${test}_${type}_FOUND_PREREQUIREMENTS)
+    include("${CMAKE_CURRENT_SOURCE_DIR}/${test}/${type}/Prerequirements.cmake")
+    get_test_prerequirements("${test}_${type}_FOUND_PREREQUIREMENTS"
+        "${TEST_CONFIG_DIR}/${type}_config.cmake")
+
+    # only add the test if prerequirements are met
+    if(${test}_${type}_FOUND_PREREQUIREMENTS)
+      add_test(NAME RunCMake.${test}_${type} COMMAND ${CMAKE_CMAKE_COMMAND}
+        -DTEST_TYPE=${type}
+        -DCMAKE_MODULE_PATH=${CMAKE_CURRENT_SOURCE_DIR}
+        -DRunCMake_GENERATOR=${CMAKE_GENERATOR}
+        -DRunCMake_GENERATOR_PLATFORM=${CMAKE_GENERATOR_PLATFORM}
+        -DRunCMake_GENERATOR_TOOLSET=${CMAKE_GENERATOR_TOOLSET}
+        -DRunCMake_MAKE_PROGRAM=${CMake_TEST_EXPLICIT_MAKE_PROGRAM}
+        -DRunCMake_SOURCE_DIR=${CMAKE_CURRENT_SOURCE_DIR}/${test}
+        -DRunCMake_BINARY_DIR=${CMAKE_CURRENT_BINARY_DIR}/${type}/${test}
+        -Dconfig_file=${TEST_CONFIG_DIR}/${type}_config.cmake
+        -P "${CMAKE_CURRENT_SOURCE_DIR}/${test}/RunCMakeTest.cmake"
+        )
+    endif()
+  endforeach()
+endfunction()
+
 if(XCODE_VERSION AND "${XCODE_VERSION}" VERSION_LESS 3)
   set(GeneratorToolset_ARGS -DXCODE_BELOW_3=1)
 endif()
@@ -233,3 +266,5 @@ if("${CMAKE_GENERATOR}" MATCHES "Make|Ninja")
   add_executable(pseudo_iwyu pseudo_iwyu.c)
   add_RunCMake_test(IncludeWhatYouUse -DPSEUDO_IWYU=$<TARGET_FILE:pseudo_iwyu>)
 endif()
+
+add_RunCMake_test_group(CPack "DEB;RPM")

Tests/RunCMake/CPack/CMakeLists.txt

@@ -0,0 +1,12 @@
+cmake_minimum_required(VERSION 3.1 FATAL_ERROR)
+
+project(${RunCMake_TEST} NONE)
+include(${RunCMake_TEST}.cmake)
+
+# include test generator specifics
+if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/${GENERATOR_TYPE}/${RunCMake_TEST}-specifics.cmake")
+  include("${GENERATOR_TYPE}/${RunCMake_TEST}-specifics.cmake")
+endif()
+
+set(CPACK_GENERATOR "${GENERATOR_TYPE}")
+include(CPack)

Tests/RunCMake/CPack/CPackTestHelpers.cmake

@@ -0,0 +1,76 @@
+cmake_policy(SET CMP0057 NEW)
+
+function(run_cpack_test TEST_NAME types build)
+  if(TEST_TYPE IN_LIST types)
+    set(RunCMake_TEST_NO_CLEAN TRUE)
+    set(RunCMake_TEST_BINARY_DIR "${RunCMake_BINARY_DIR}/${TEST_NAME}-build")
+
+     # TODO this should be executed only once per ctest run (not per generator)
+    file(REMOVE_RECURSE "${RunCMake_TEST_BINARY_DIR}")
+    file(MAKE_DIRECTORY "${RunCMake_TEST_BINARY_DIR}")
+
+    # execute cmake
+    execute_process(
+      COMMAND "${CMAKE_COMMAND}" -DRunCMake_TEST=${TEST_NAME}
+          -DGENERATOR_TYPE=${TEST_TYPE} "${RunCMake_SOURCE_DIR}"
+      WORKING_DIRECTORY "${RunCMake_TEST_BINARY_DIR}"
+      RESULT_VARIABLE res
+      OUTPUT_FILE "${RunCMake_TEST_BINARY_DIR}/test_output.txt"
+      ERROR_FILE "${RunCMake_TEST_BINARY_DIR}/test_error.txt"
+      )
+
+    if(res)
+      run_cmake_command(
+        ${TEST_TYPE}/${TEST_NAME}
+        "${CMAKE_COMMAND}"
+          -DRunCMake_TEST_STEP=configure
+          -Dreturn_code=${res}
+          "-Dbin_dir=${RunCMake_TEST_BINARY_DIR}"
+          -P "${RunCMake_SOURCE_DIR}/PreTestError.cmake"
+        )
+      return()
+    endif()
+
+    # execute optional build step
+    if(build)
+      execute_process(
+        COMMAND "${CMAKE_COMMAND}" --build "${RunCMake_TEST_BINARY_DIR}"
+        RESULT_VARIABLE res
+        OUTPUT_FILE "${RunCMake_TEST_BINARY_DIR}/test_output.txt"
+        ERROR_FILE "${RunCMake_TEST_BINARY_DIR}/test_error.txt"
+        )
+    endif()
+
+    if(res)
+      run_cmake_command(
+        ${TEST_TYPE}/${TEST_NAME}
+        "${CMAKE_COMMAND}"
+          -DRunCMake_TEST_STEP=build
+          -Dreturn_code=${res}
+          "-Dbin_dir=${RunCMake_TEST_BINARY_DIR}"
+          -P "${RunCMake_SOURCE_DIR}/PreTestError.cmake"
+        )
+      return()
+    endif()
+
+    # execute cpack
+    execute_process(
+      COMMAND "${CMAKE_CPACK_COMMAND}"
+      WORKING_DIRECTORY "${RunCMake_TEST_BINARY_DIR}"
+      OUTPUT_FILE "${RunCMake_TEST_BINARY_DIR}/test_output.txt"
+      ERROR_FILE "${RunCMake_TEST_BINARY_DIR}/test_error.txt"
+      )
+
+    # verify result
+    run_cmake_command(
+      ${TEST_TYPE}/${TEST_NAME}
+      "${CMAKE_COMMAND}"
+        -DRunCMake_TEST=${TEST_NAME}
+        -DGENERATOR_TYPE=${TEST_TYPE}
+        "-Dsrc_dir=${RunCMake_SOURCE_DIR}"
+        "-Dbin_dir=${RunCMake_TEST_BINARY_DIR}"
+        "-Dconfig_file=${config_file}"
+        -P "${RunCMake_SOURCE_DIR}/VerifyResult.cmake"
+      )
+  endif()
+endfunction()

Tests/RunCMake/CPack/DEB/Helpers.cmake

@@ -0,0 +1,10 @@
+set(ALL_FILES_GLOB "*.deb")
+
+function(getPackageContent FILE RESULT_VAR)
+  execute_process(COMMAND ${DPKG_EXECUTABLE} -c ${FILE}
+          OUTPUT_VARIABLE package_content_
+          ERROR_QUIET
+          OUTPUT_STRIP_TRAILING_WHITESPACE)
+
+  set(${RESULT_VAR} "${package_content_}" PARENT_SCOPE)
+endfunction()

Tests/RunCMake/CPack/DEB/MINIMAL-ExpectedFiles.cmake

@@ -0,0 +1,5 @@
+set(whitespaces_ "[\t\n\r ]*")
+
+set(EXPECTED_FILES_COUNT "1")
+set(EXPECTED_FILE_1 "minimal*.deb")
+set(EXPECTED_FILE_CONTENT_1 "^.*/usr/${whitespaces_}.*/usr/foo/${whitespaces_}.*/usr/foo/CMakeLists.txt$")

Tests/RunCMake/CPack/DEB/MINIMAL-specifics.cmake

@@ -0,0 +1 @@
+set(CPACK_PACKAGE_CONTACT "someone")

Tests/RunCMake/CPack/DEB/Prerequirements.cmake

@@ -0,0 +1,8 @@
+function(get_test_prerequirements found_var config_file)
+  find_program(DPKG_EXECUTABLE dpkg)
+
+  if(DPKG_EXECUTABLE)
+    file(WRITE "${config_file}" "set(DPKG_EXECUTABLE \"${DPKG_EXECUTABLE}\")")
+    set(${found_var} true PARENT_SCOPE)
+  endif()
+endfunction()

Tests/RunCMake/CPack/MINIMAL.cmake

@@ -0,0 +1,3 @@
+install(FILES CMakeLists.txt DESTINATION foo COMPONENT test)
+
+set(CPACK_PACKAGE_NAME "minimal")

Tests/RunCMake/CPack/PreTestError.cmake

@@ -0,0 +1,7 @@
+file(READ "${bin_dir}/test_output.txt" output)
+file(READ "${bin_dir}/test_error.txt" error)
+
+message(FATAL_ERROR "Error in pre-test phase '${RunCMake_TEST_STEP}'!\n"
+        "Return code: '${return_code}'\n"
+        "Info output: '${output}'\n"
+        "Error output: '${error}'")

Tests/RunCMake/CPack/README.txt

@@ -0,0 +1,99 @@
+RunCMake.CPack is a test module that is intended for testing of package
+generators that can be validated from command line.
+
+-------------
+Adding a test
+-------------
+
+CPack test root directory: 'Tests/RunCMake/CPack'.
+
+All phases are executed separately for each generator that is bound to a test.
+Tests for each generator are subtests of test 'RunCMake.CPack_<generator_name>'.
+
+Each test must also be added to 'RunCMakeTest.cmake' script located in CPack
+test root directory.
+Line that adds a test is:
+run_cpack_test(<test_name> "<generator_name>")
+
+<generator_name> may be one generator e.g. "RPM" or multiple e.g. "RPM;DEB" and
+will be run for all listed generators.
+
+Test consists of
+- CMake execution phase
+- CPack execution phase
+- verification of generated files
+
+CMake execution phase:
+----------------------
+
+To add a new CPack test we first create a <test_name>.cmake script that contains
+CMake commands that should be used as a preparation script for generation of
+different types of packages. This script is placed into CPack test root
+directory even if it will be used for only one of the generators.
+
+If test will be used for multiple generators but some of them require some
+generator speciffic commands then those commands should be added to a separate
+file that should be located in '<generator_name>/<test_name>-specifics.cmake'
+in CPack test root directory.
+
+CPack execution phase:
+----------------------
+
+Only exececutes CPack for content that was generated during CMake execution
+phase.
+
+Verification of generated files:
+--------------------------------
+
+Verification of generated files consists of two phases
+- mandatory verification phase
+- optional verification phase
+
+Mandatory verification phase checks that expected files were generated and
+contain expected files.
+Mandatory verification phase also checks that no other unexpected package files
+were generated (this is executed even if EXPECTED_FILES_COUNT contains 0 in
+order to verify that no files were generated).
+CMake script '<generator_name>/<test_name>-ExpectedFiles.cmake' is required by
+this step and must contain
+- EXPECTED_FILES_COUNT variable that contains the number of expected files that
+  will be generated (0 or more)
+- EXPECTED_FILE_<file_number_starting_with_1> that contains globing expression
+  that uniquely defines expected file name (will be used to find expected file)
+  and should be present once for each expected file
+- EXPECTED_FILE_CONTENT_<file_number_starting_with_1> that contains regular
+  expression of files that should be present in generated file and should be
+  present once for each expected file
+
+Optional verification phase is generator specific and is optionaly executed.
+This phase is executed if '<generator_name>/<test_name>-VerifyResult.cmake'
+script exists.
+In case that the script doesn't exist VerifyResult.cmake script automatically
+prints out standard output and standard error from CPack execution phase that
+is compared with '<generator_name>/<test_name>-stdout.txt' regular expression
+and '<generator_name>/<test_name>-stderr.txt' regular expresson respectively.
+
+----------------------
+Adding a new generator
+----------------------
+
+To add a new generator we must
+- add new generator directory (e.g. RPM for RPM generator) to CPack test root
+  directory that contains 'Helpers.cmake' script. In this script a function
+  named 'getPackageContent' must exist. This function should list files that
+  are contained in a package. Function should accept two parameters
+  + FILE variable that will contain path to file for which the content should be
+    listed
+  + RESULT_VAR that will tell the function which variable in parent scope should
+    contain the result (list of files inside package file)
+- add 'Prerequirements.cmake' script to generator directory. In this script a
+  function named 'get_test_prerequirements' must exist. This function should
+  set a variable in parent scope (name of the variable is the first parameter)
+  that tells if prerequirements for test execution are met (certain programs,
+  OS specifics, ...) and create a config file (name of the variable which
+  contains file name and path is provided with the second parameter) that
+  should contain 'set' commands for variables that will later be used in tests
+  (e.g. location of dpkg program for DEB packages)
+- add tests the same way as described above
+- add generator to 'add_RunCMake_test_group' function call that is located in
+  RunCMake CMakeLists.txt file

Tests/RunCMake/CPack/RPM/Helpers.cmake

@@ -0,0 +1,10 @@
+set(ALL_FILES_GLOB "*.rpm")
+
+function(getPackageContent FILE RESULT_VAR)
+  execute_process(COMMAND ${RPM_EXECUTABLE} -pql ${FILE}
+          OUTPUT_VARIABLE package_content_
+          ERROR_QUIET
+          OUTPUT_STRIP_TRAILING_WHITESPACE)
+
+  set(${RESULT_VAR} "${package_content_}" PARENT_SCOPE)
+endfunction()

Tests/RunCMake/CPack/RPM/MINIMAL-ExpectedFiles.cmake

@@ -0,0 +1,5 @@
+set(whitespaces_ "[\t\n\r ]*")
+
+set(EXPECTED_FILES_COUNT "1")
+set(EXPECTED_FILE_1 "minimal*.rpm")
+set(EXPECTED_FILE_CONTENT_1 "^/usr/foo${whitespaces_}/usr/foo/CMakeLists.txt$")

Tests/RunCMake/CPack/RPM/MINIMAL-stderr.txt

@@ -0,0 +1 @@
+^CPackRPM: Will use GENERATED spec file: .*/Tests/RunCMake/RPM/CPack/MINIMAL-build/_CPack_Packages/.*/RPM/SPECS/minimal.spec$

Tests/RunCMake/CPack/RPM/Prerequirements.cmake

@@ -0,0 +1,16 @@
+function(get_test_prerequirements found_var config_file)
+  if(CMAKE_CURRENT_BINARY_DIR MATCHES " ")
+    # rpmbuild can't handle spaces in path
+    return()
+  endif()
+
+  find_program(RPM_EXECUTABLE rpm)
+  find_program(RPMBUILD_EXECUTABLE rpmbuild)
+
+  if(RPM_EXECUTABLE AND RPMBUILD_EXECUTABLE)
+    file(WRITE "${config_file}" "set(RPM_EXECUTABLE \"${RPM_EXECUTABLE}\")")
+    file(APPEND "${config_file}"
+        "\nset(RPMBUILD_EXECUTABLE \"${RPMBUILD_EXECUTABLE}\")")
+    set(${found_var} true PARENT_SCOPE)
+  endif()
+endfunction()

Tests/RunCMake/CPack/RunCMakeTest.cmake

@@ -0,0 +1,7 @@
+cmake_minimum_required(VERSION 3.1 FATAL_ERROR)
+
+include(RunCMake)
+include("${RunCMake_SOURCE_DIR}/CPackTestHelpers.cmake")
+
+# args: TEST_NAME "GENERATORS" RUN_CMAKE_BUILD_STEP
+run_cpack_test(MINIMAL "RPM;DEB" false)

Tests/RunCMake/CPack/VerifyResult.cmake

@@ -0,0 +1,91 @@
+cmake_minimum_required(VERSION 3.1 FATAL_ERROR)
+
+include("${config_file}")
+include("${src_dir}/${GENERATOR_TYPE}/Helpers.cmake")
+
+file(READ "${bin_dir}/test_output.txt" output)
+file(READ "${bin_dir}/test_error.txt" error)
+file(READ "${config_file}" config_file_content)
+
+set(output_error_message
+    "\nCPack output: '${output}'\nCPack error: '${error}';\nconfig file: '${config_file_content}'")
+
+# check that expected generated files exist and contain expected content
+include("${src_dir}/${GENERATOR_TYPE}/${RunCMake_TEST}-ExpectedFiles.cmake")
+
+if(NOT EXPECTED_FILES_COUNT EQUAL 0)
+  foreach(file_no_ RANGE 1 ${EXPECTED_FILES_COUNT})
+    file(GLOB foundFile_ RELATIVE "${bin_dir}" "${EXPECTED_FILE_${file_no_}}")
+    set(foundFiles_ "${foundFiles_};${foundFile_}")
+    list(LENGTH foundFile_ foundFilesCount_)
+
+    if(foundFilesCount_ EQUAL 1)
+      unset(PACKAGE_CONTENT)
+      getPackageContent("${bin_dir}/${foundFile_}" "PACKAGE_CONTENT")
+
+      string(REGEX MATCH "${EXPECTED_FILE_CONTENT_${file_no_}}"
+          expected_content_list "${PACKAGE_CONTENT}")
+
+      if(NOT expected_content_list)
+        message(FATAL_ERROR
+          "Unexpected file content for file No. '${file_no_}'!"
+          " Content: '${PACKAGE_CONTENT}'"
+          "${output_error_message}")
+      endif()
+    else()
+      message(FATAL_ERROR
+        "Found more than one file for file No. '${file_no_}'!"
+        " Found files count '${foundFilesCount_}'."
+        " Files: '${foundFile_}'"
+        "${output_error_message}")
+    endif()
+  endforeach()
+
+  # check that there were no extra files generated
+  foreach(all_files_glob_ IN LISTS ALL_FILES_GLOB)
+    file(GLOB foundAll_ RELATIVE "${bin_dir}" "${all_files_glob_}")
+    set(allFoundFiles_ "${allFoundFiles_};${foundAll_}")
+  endforeach()
+
+  list(LENGTH foundFiles_ foundFilesCount_)
+  list(LENGTH allFoundFiles_ allFoundFilesCount_)
+
+  if(NOT foundFilesCount_ EQUAL allFoundFilesCount_)
+    message(FATAL_ERROR
+        "Found more files than expected! Found files: '${allFoundFiles_}'"
+        "${output_error_message}")
+  endif()
+
+  # sanity check that we didn't accidentaly list wrong files with our regular
+  # expressions
+  foreach(expected_ IN LISTS allFoundFiles_)
+    list(FIND foundFiles_ "${expected_}" found_)
+
+    if(found_ EQUAL -1)
+      message(FATAL_ERROR
+          "Expected files don't match found files! Found files:"
+          " '${allFoundFiles_}'"
+          "${output_error_message}")
+    endif()
+  endforeach()
+else()
+  # there should be no generated files present
+  foreach(missing_file_glob_ IN LISTS ALL_FILES_GLOB)
+    file(GLOB checkMissingFiles_ RELATIVE "${bin_dir}" "${missing_file_glob_}")
+
+    if(checkMissingFiles_)
+      message(FATAL_ERROR "Unexpected files found: '${checkMissingFiles_}'"
+          "${output_error_message}")
+    endif()
+  endforeach()
+endif()
+
+# handle additional result verifications
+if(EXISTS "${src_dir}/${GENERATOR_TYPE}/${RunCMake_TEST}-VerifyResult.cmake")
+  include("${src_dir}/${GENERATOR_TYPE}/${RunCMake_TEST}-VerifyResult.cmake")
+else()
+  # by default only print out output and error so that they can be compared by
+  # regex
+  message(STATUS "${output}")
+  message("${error}")
+endif()