Tests: Add RunCMake tests for CPack{Deb,RPM}

Add script structure for easier addition of new CPack related tests.
This commit is contained in:
Domen Vrankar 2015-06-05 21:03:26 +02:00 committed by Brad King
parent 092f1539af
commit e726fc025a
16 changed files with 386 additions and 0 deletions

View File

@ -26,6 +26,39 @@ macro(add_RunCMake_test test)
) )
endmacro() endmacro()
function(add_RunCMake_test_group test types)
# create directory for common content
set(TEST_CONFIG_DIR "${CMAKE_CURRENT_BINARY_DIR}/${test}/conf")
file(REMOVE_RECURSE "${TEST_CONFIG_DIR}")
file(MAKE_DIRECTORY "${TEST_CONFIG_DIR}")
foreach(type IN LISTS types)
# generate prerequirements config file in cmake as ctest doesn't have as
# much system information so it is easier to set programs and environment
# values here
unset(${test}_${type}_FOUND_PREREQUIREMENTS)
include("${CMAKE_CURRENT_SOURCE_DIR}/${test}/${type}/Prerequirements.cmake")
get_test_prerequirements("${test}_${type}_FOUND_PREREQUIREMENTS"
"${TEST_CONFIG_DIR}/${type}_config.cmake")
# only add the test if prerequirements are met
if(${test}_${type}_FOUND_PREREQUIREMENTS)
add_test(NAME RunCMake.${test}_${type} COMMAND ${CMAKE_CMAKE_COMMAND}
-DTEST_TYPE=${type}
-DCMAKE_MODULE_PATH=${CMAKE_CURRENT_SOURCE_DIR}
-DRunCMake_GENERATOR=${CMAKE_GENERATOR}
-DRunCMake_GENERATOR_PLATFORM=${CMAKE_GENERATOR_PLATFORM}
-DRunCMake_GENERATOR_TOOLSET=${CMAKE_GENERATOR_TOOLSET}
-DRunCMake_MAKE_PROGRAM=${CMake_TEST_EXPLICIT_MAKE_PROGRAM}
-DRunCMake_SOURCE_DIR=${CMAKE_CURRENT_SOURCE_DIR}/${test}
-DRunCMake_BINARY_DIR=${CMAKE_CURRENT_BINARY_DIR}/${type}/${test}
-Dconfig_file=${TEST_CONFIG_DIR}/${type}_config.cmake
-P "${CMAKE_CURRENT_SOURCE_DIR}/${test}/RunCMakeTest.cmake"
)
endif()
endforeach()
endfunction()
if(XCODE_VERSION AND "${XCODE_VERSION}" VERSION_LESS 3) if(XCODE_VERSION AND "${XCODE_VERSION}" VERSION_LESS 3)
set(GeneratorToolset_ARGS -DXCODE_BELOW_3=1) set(GeneratorToolset_ARGS -DXCODE_BELOW_3=1)
endif() endif()
@ -233,3 +266,5 @@ if("${CMAKE_GENERATOR}" MATCHES "Make|Ninja")
add_executable(pseudo_iwyu pseudo_iwyu.c) add_executable(pseudo_iwyu pseudo_iwyu.c)
add_RunCMake_test(IncludeWhatYouUse -DPSEUDO_IWYU=$<TARGET_FILE:pseudo_iwyu>) add_RunCMake_test(IncludeWhatYouUse -DPSEUDO_IWYU=$<TARGET_FILE:pseudo_iwyu>)
endif() endif()
add_RunCMake_test_group(CPack "DEB;RPM")

View File

@ -0,0 +1,12 @@
cmake_minimum_required(VERSION 3.1 FATAL_ERROR)
project(${RunCMake_TEST} NONE)
include(${RunCMake_TEST}.cmake)
# include test generator specifics
if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/${GENERATOR_TYPE}/${RunCMake_TEST}-specifics.cmake")
include("${GENERATOR_TYPE}/${RunCMake_TEST}-specifics.cmake")
endif()
set(CPACK_GENERATOR "${GENERATOR_TYPE}")
include(CPack)

View File

@ -0,0 +1,76 @@
cmake_policy(SET CMP0057 NEW)
function(run_cpack_test TEST_NAME types build)
if(TEST_TYPE IN_LIST types)
set(RunCMake_TEST_NO_CLEAN TRUE)
set(RunCMake_TEST_BINARY_DIR "${RunCMake_BINARY_DIR}/${TEST_NAME}-build")
# TODO this should be executed only once per ctest run (not per generator)
file(REMOVE_RECURSE "${RunCMake_TEST_BINARY_DIR}")
file(MAKE_DIRECTORY "${RunCMake_TEST_BINARY_DIR}")
# execute cmake
execute_process(
COMMAND "${CMAKE_COMMAND}" -DRunCMake_TEST=${TEST_NAME}
-DGENERATOR_TYPE=${TEST_TYPE} "${RunCMake_SOURCE_DIR}"
WORKING_DIRECTORY "${RunCMake_TEST_BINARY_DIR}"
RESULT_VARIABLE res
OUTPUT_FILE "${RunCMake_TEST_BINARY_DIR}/test_output.txt"
ERROR_FILE "${RunCMake_TEST_BINARY_DIR}/test_error.txt"
)
if(res)
run_cmake_command(
${TEST_TYPE}/${TEST_NAME}
"${CMAKE_COMMAND}"
-DRunCMake_TEST_STEP=configure
-Dreturn_code=${res}
"-Dbin_dir=${RunCMake_TEST_BINARY_DIR}"
-P "${RunCMake_SOURCE_DIR}/PreTestError.cmake"
)
return()
endif()
# execute optional build step
if(build)
execute_process(
COMMAND "${CMAKE_COMMAND}" --build "${RunCMake_TEST_BINARY_DIR}"
RESULT_VARIABLE res
OUTPUT_FILE "${RunCMake_TEST_BINARY_DIR}/test_output.txt"
ERROR_FILE "${RunCMake_TEST_BINARY_DIR}/test_error.txt"
)
endif()
if(res)
run_cmake_command(
${TEST_TYPE}/${TEST_NAME}
"${CMAKE_COMMAND}"
-DRunCMake_TEST_STEP=build
-Dreturn_code=${res}
"-Dbin_dir=${RunCMake_TEST_BINARY_DIR}"
-P "${RunCMake_SOURCE_DIR}/PreTestError.cmake"
)
return()
endif()
# execute cpack
execute_process(
COMMAND "${CMAKE_CPACK_COMMAND}"
WORKING_DIRECTORY "${RunCMake_TEST_BINARY_DIR}"
OUTPUT_FILE "${RunCMake_TEST_BINARY_DIR}/test_output.txt"
ERROR_FILE "${RunCMake_TEST_BINARY_DIR}/test_error.txt"
)
# verify result
run_cmake_command(
${TEST_TYPE}/${TEST_NAME}
"${CMAKE_COMMAND}"
-DRunCMake_TEST=${TEST_NAME}
-DGENERATOR_TYPE=${TEST_TYPE}
"-Dsrc_dir=${RunCMake_SOURCE_DIR}"
"-Dbin_dir=${RunCMake_TEST_BINARY_DIR}"
"-Dconfig_file=${config_file}"
-P "${RunCMake_SOURCE_DIR}/VerifyResult.cmake"
)
endif()
endfunction()

View File

@ -0,0 +1,10 @@
set(ALL_FILES_GLOB "*.deb")
function(getPackageContent FILE RESULT_VAR)
execute_process(COMMAND ${DPKG_EXECUTABLE} -c ${FILE}
OUTPUT_VARIABLE package_content_
ERROR_QUIET
OUTPUT_STRIP_TRAILING_WHITESPACE)
set(${RESULT_VAR} "${package_content_}" PARENT_SCOPE)
endfunction()

View File

@ -0,0 +1,5 @@
set(whitespaces_ "[\t\n\r ]*")
set(EXPECTED_FILES_COUNT "1")
set(EXPECTED_FILE_1 "minimal*.deb")
set(EXPECTED_FILE_CONTENT_1 "^.*/usr/${whitespaces_}.*/usr/foo/${whitespaces_}.*/usr/foo/CMakeLists.txt$")

View File

@ -0,0 +1 @@
set(CPACK_PACKAGE_CONTACT "someone")

View File

@ -0,0 +1,8 @@
function(get_test_prerequirements found_var config_file)
find_program(DPKG_EXECUTABLE dpkg)
if(DPKG_EXECUTABLE)
file(WRITE "${config_file}" "set(DPKG_EXECUTABLE \"${DPKG_EXECUTABLE}\")")
set(${found_var} true PARENT_SCOPE)
endif()
endfunction()

View File

@ -0,0 +1,3 @@
install(FILES CMakeLists.txt DESTINATION foo COMPONENT test)
set(CPACK_PACKAGE_NAME "minimal")

View File

@ -0,0 +1,7 @@
file(READ "${bin_dir}/test_output.txt" output)
file(READ "${bin_dir}/test_error.txt" error)
message(FATAL_ERROR "Error in pre-test phase '${RunCMake_TEST_STEP}'!\n"
"Return code: '${return_code}'\n"
"Info output: '${output}'\n"
"Error output: '${error}'")

View File

@ -0,0 +1,99 @@
RunCMake.CPack is a test module that is intended for testing of package
generators that can be validated from command line.
-------------
Adding a test
-------------
CPack test root directory: 'Tests/RunCMake/CPack'.
All phases are executed separately for each generator that is bound to a test.
Tests for each generator are subtests of test 'RunCMake.CPack_<generator_name>'.
Each test must also be added to 'RunCMakeTest.cmake' script located in CPack
test root directory.
Line that adds a test is:
run_cpack_test(<test_name> "<generator_name>")
<generator_name> may be one generator e.g. "RPM" or multiple e.g. "RPM;DEB" and
will be run for all listed generators.
Test consists of
- CMake execution phase
- CPack execution phase
- verification of generated files
CMake execution phase:
----------------------
To add a new CPack test we first create a <test_name>.cmake script that contains
CMake commands that should be used as a preparation script for generation of
different types of packages. This script is placed into CPack test root
directory even if it will be used for only one of the generators.
If test will be used for multiple generators but some of them require some
generator speciffic commands then those commands should be added to a separate
file that should be located in '<generator_name>/<test_name>-specifics.cmake'
in CPack test root directory.
CPack execution phase:
----------------------
Only exececutes CPack for content that was generated during CMake execution
phase.
Verification of generated files:
--------------------------------
Verification of generated files consists of two phases
- mandatory verification phase
- optional verification phase
Mandatory verification phase checks that expected files were generated and
contain expected files.
Mandatory verification phase also checks that no other unexpected package files
were generated (this is executed even if EXPECTED_FILES_COUNT contains 0 in
order to verify that no files were generated).
CMake script '<generator_name>/<test_name>-ExpectedFiles.cmake' is required by
this step and must contain
- EXPECTED_FILES_COUNT variable that contains the number of expected files that
will be generated (0 or more)
- EXPECTED_FILE_<file_number_starting_with_1> that contains globing expression
that uniquely defines expected file name (will be used to find expected file)
and should be present once for each expected file
- EXPECTED_FILE_CONTENT_<file_number_starting_with_1> that contains regular
expression of files that should be present in generated file and should be
present once for each expected file
Optional verification phase is generator specific and is optionaly executed.
This phase is executed if '<generator_name>/<test_name>-VerifyResult.cmake'
script exists.
In case that the script doesn't exist VerifyResult.cmake script automatically
prints out standard output and standard error from CPack execution phase that
is compared with '<generator_name>/<test_name>-stdout.txt' regular expression
and '<generator_name>/<test_name>-stderr.txt' regular expresson respectively.
----------------------
Adding a new generator
----------------------
To add a new generator we must
- add new generator directory (e.g. RPM for RPM generator) to CPack test root
directory that contains 'Helpers.cmake' script. In this script a function
named 'getPackageContent' must exist. This function should list files that
are contained in a package. Function should accept two parameters
+ FILE variable that will contain path to file for which the content should be
listed
+ RESULT_VAR that will tell the function which variable in parent scope should
contain the result (list of files inside package file)
- add 'Prerequirements.cmake' script to generator directory. In this script a
function named 'get_test_prerequirements' must exist. This function should
set a variable in parent scope (name of the variable is the first parameter)
that tells if prerequirements for test execution are met (certain programs,
OS specifics, ...) and create a config file (name of the variable which
contains file name and path is provided with the second parameter) that
should contain 'set' commands for variables that will later be used in tests
(e.g. location of dpkg program for DEB packages)
- add tests the same way as described above
- add generator to 'add_RunCMake_test_group' function call that is located in
RunCMake CMakeLists.txt file

View File

@ -0,0 +1,10 @@
set(ALL_FILES_GLOB "*.rpm")
function(getPackageContent FILE RESULT_VAR)
execute_process(COMMAND ${RPM_EXECUTABLE} -pql ${FILE}
OUTPUT_VARIABLE package_content_
ERROR_QUIET
OUTPUT_STRIP_TRAILING_WHITESPACE)
set(${RESULT_VAR} "${package_content_}" PARENT_SCOPE)
endfunction()

View File

@ -0,0 +1,5 @@
set(whitespaces_ "[\t\n\r ]*")
set(EXPECTED_FILES_COUNT "1")
set(EXPECTED_FILE_1 "minimal*.rpm")
set(EXPECTED_FILE_CONTENT_1 "^/usr/foo${whitespaces_}/usr/foo/CMakeLists.txt$")

View File

@ -0,0 +1 @@
^CPackRPM: Will use GENERATED spec file: .*/Tests/RunCMake/RPM/CPack/MINIMAL-build/_CPack_Packages/.*/RPM/SPECS/minimal.spec$

View File

@ -0,0 +1,16 @@
function(get_test_prerequirements found_var config_file)
if(CMAKE_CURRENT_BINARY_DIR MATCHES " ")
# rpmbuild can't handle spaces in path
return()
endif()
find_program(RPM_EXECUTABLE rpm)
find_program(RPMBUILD_EXECUTABLE rpmbuild)
if(RPM_EXECUTABLE AND RPMBUILD_EXECUTABLE)
file(WRITE "${config_file}" "set(RPM_EXECUTABLE \"${RPM_EXECUTABLE}\")")
file(APPEND "${config_file}"
"\nset(RPMBUILD_EXECUTABLE \"${RPMBUILD_EXECUTABLE}\")")
set(${found_var} true PARENT_SCOPE)
endif()
endfunction()

View File

@ -0,0 +1,7 @@
cmake_minimum_required(VERSION 3.1 FATAL_ERROR)
include(RunCMake)
include("${RunCMake_SOURCE_DIR}/CPackTestHelpers.cmake")
# args: TEST_NAME "GENERATORS" RUN_CMAKE_BUILD_STEP
run_cpack_test(MINIMAL "RPM;DEB" false)

View File

@ -0,0 +1,91 @@
cmake_minimum_required(VERSION 3.1 FATAL_ERROR)
include("${config_file}")
include("${src_dir}/${GENERATOR_TYPE}/Helpers.cmake")
file(READ "${bin_dir}/test_output.txt" output)
file(READ "${bin_dir}/test_error.txt" error)
file(READ "${config_file}" config_file_content)
set(output_error_message
"\nCPack output: '${output}'\nCPack error: '${error}';\nconfig file: '${config_file_content}'")
# check that expected generated files exist and contain expected content
include("${src_dir}/${GENERATOR_TYPE}/${RunCMake_TEST}-ExpectedFiles.cmake")
if(NOT EXPECTED_FILES_COUNT EQUAL 0)
foreach(file_no_ RANGE 1 ${EXPECTED_FILES_COUNT})
file(GLOB foundFile_ RELATIVE "${bin_dir}" "${EXPECTED_FILE_${file_no_}}")
set(foundFiles_ "${foundFiles_};${foundFile_}")
list(LENGTH foundFile_ foundFilesCount_)
if(foundFilesCount_ EQUAL 1)
unset(PACKAGE_CONTENT)
getPackageContent("${bin_dir}/${foundFile_}" "PACKAGE_CONTENT")
string(REGEX MATCH "${EXPECTED_FILE_CONTENT_${file_no_}}"
expected_content_list "${PACKAGE_CONTENT}")
if(NOT expected_content_list)
message(FATAL_ERROR
"Unexpected file content for file No. '${file_no_}'!"
" Content: '${PACKAGE_CONTENT}'"
"${output_error_message}")
endif()
else()
message(FATAL_ERROR
"Found more than one file for file No. '${file_no_}'!"
" Found files count '${foundFilesCount_}'."
" Files: '${foundFile_}'"
"${output_error_message}")
endif()
endforeach()
# check that there were no extra files generated
foreach(all_files_glob_ IN LISTS ALL_FILES_GLOB)
file(GLOB foundAll_ RELATIVE "${bin_dir}" "${all_files_glob_}")
set(allFoundFiles_ "${allFoundFiles_};${foundAll_}")
endforeach()
list(LENGTH foundFiles_ foundFilesCount_)
list(LENGTH allFoundFiles_ allFoundFilesCount_)
if(NOT foundFilesCount_ EQUAL allFoundFilesCount_)
message(FATAL_ERROR
"Found more files than expected! Found files: '${allFoundFiles_}'"
"${output_error_message}")
endif()
# sanity check that we didn't accidentaly list wrong files with our regular
# expressions
foreach(expected_ IN LISTS allFoundFiles_)
list(FIND foundFiles_ "${expected_}" found_)
if(found_ EQUAL -1)
message(FATAL_ERROR
"Expected files don't match found files! Found files:"
" '${allFoundFiles_}'"
"${output_error_message}")
endif()
endforeach()
else()
# there should be no generated files present
foreach(missing_file_glob_ IN LISTS ALL_FILES_GLOB)
file(GLOB checkMissingFiles_ RELATIVE "${bin_dir}" "${missing_file_glob_}")
if(checkMissingFiles_)
message(FATAL_ERROR "Unexpected files found: '${checkMissingFiles_}'"
"${output_error_message}")
endif()
endforeach()
endif()
# handle additional result verifications
if(EXISTS "${src_dir}/${GENERATOR_TYPE}/${RunCMake_TEST}-VerifyResult.cmake")
include("${src_dir}/${GENERATOR_TYPE}/${RunCMake_TEST}-VerifyResult.cmake")
else()
# by default only print out output and error so that they can be compared by
# regex
message(STATUS "${output}")
message("${error}")
endif()