Merge topic 'file-download-verify'

4bcd84e Utilities/Release: Enable CMAKE_USE_OPENSSL in nightly binaries
e1c89f0 file(DOWNLOAD): Add options for SSL
073a73a Merge branch 'curl-openssl' into file-download-verify
34567df file(DOWNLOAD): Generalize EXPECTED_MD5 to EXPECTED_HASH

Source/cmFileCommand.cxx

@@ -10,6 +10,7 @@
   See the License for more information.
 ============================================================================*/
 #include "cmFileCommand.h"
+#include "cmCryptoHash.h"
 #include "cmake.h"
 #include "cmHexFileConverter.h"
 #include "cmInstallType.h"
@@ -2666,7 +2667,12 @@ cmFileCommand::HandleDownloadCommand(std::vector<std::string> const& args)
   long inactivity_timeout = 0;
   std::string verboseLog;
   std::string statusVar;
-  std::string expectedMD5sum;
+  std::string caFile;
+  bool checkSSL = false;
+  bool verifySSL = false;
+  std::string expectedHash;
+  std::string hashMatchMSG;
+  cmsys::auto_ptr<cmCryptoHash> hash;
   bool showProgress = false;
 
   while(i != args.end())
@@ -2717,6 +2723,33 @@ cmFileCommand::HandleDownloadCommand(std::vector<std::string> const& args)
         }
       statusVar = *i;
       }
+    else if(*i == "SSL_VERIFY")
+      {
+      ++i;
+      if(i != args.end())
+        {
+        verifySSL = cmSystemTools::IsOn(i->c_str());
+        checkSSL = true;
+        }
+      else
+        {
+        this->SetError("SSL_VERIFY missing bool value.");
+        return false;
+        }
+      }
+    else if(*i == "SSL_CAINFO_FILE")
+      {
+      ++i;
+      if(i != args.end())
+        {
+        caFile = *i;
+        }
+      else
+        {
+        this->SetError("SSL_CAFILE missing file value.");
+        return false;
+        }
+      }
     else if(*i == "EXPECTED_MD5")
       {
       ++i;
@@ -2725,48 +2758,67 @@ cmFileCommand::HandleDownloadCommand(std::vector<std::string> const& args)
         this->SetError("DOWNLOAD missing sum value for EXPECTED_MD5.");
         return false;
         }
-      expectedMD5sum = cmSystemTools::LowerCase(*i);
+      hash = cmsys::auto_ptr<cmCryptoHash>(cmCryptoHash::New("MD5"));
+      hashMatchMSG = "MD5 sum";
+      expectedHash = cmSystemTools::LowerCase(*i);
       }
     else if(*i == "SHOW_PROGRESS")
       {
       showProgress = true;
       }
+    else if(*i == "EXPECTED_HASH")
+      {
+      ++i;
+      if(i != args.end())
+        {
+        hash = cmsys::auto_ptr<cmCryptoHash>(cmCryptoHash::New(i->c_str()));
+        if(!hash.get())
+          {
+          std::string err = "DOWNLOAD bad SHA type: ";
+          err += *i;
+          this->SetError(err.c_str());
+          return false;
+          }
+        hashMatchMSG = *i;
+        hashMatchMSG += " hash";
+
+        ++i;
+        }
+      if(i != args.end())
+        {
+        expectedHash = cmSystemTools::LowerCase(*i);
+        }
+      else
+        {
+        this->SetError("DOWNLOAD missing time for EXPECTED_HASH.");
+        return false;
+        }
+      }
     ++i;
     }
-
+  // If file exists already, and caller specified an expected md5 or sha,
-  // If file exists already, and caller specified an expected md5 sum,
+  // and the existing file already has the expected hash, then simply
-  // and the existing file already has the expected md5 sum, then simply
   // return.
   //
-  if(cmSystemTools::FileExists(file.c_str()) &&
+  if(cmSystemTools::FileExists(file.c_str()) && hash.get())
-    !expectedMD5sum.empty())
     {
-    char computedMD5[32];
+    std::string msg;
-
+    std::string actualHash = hash->HashFile(file.c_str());
-    if (!cmSystemTools::ComputeFileMD5(file.c_str(), computedMD5))
+    if(actualHash == expectedHash)
-      {
-      this->SetError("DOWNLOAD cannot compute MD5 sum on pre-existing file");
-      return false;
-      }
-
-    std::string actualMD5sum = cmSystemTools::LowerCase(
-      std::string(computedMD5, 32));
-
-    if (expectedMD5sum == actualMD5sum)
       {
+      msg = "returning early; file already exists with expected ";
+      msg += hashMatchMSG;
+      msg += "\"";
       if(statusVar.size())
         {
         cmOStringStream result;
-        result << (int)0 << ";\""
+        result << (int)0 << ";\"" << msg;
-          "returning early: file already exists with expected MD5 sum\"";
         this->Makefile->AddDefinition(statusVar.c_str(),
                                       result.str().c_str());
         }
-
       return true;
       }
     }
-
   // Make sure parent directory exists so we can write to the file
   // as we receive downloaded bits from curl...
   //
@@ -2798,7 +2850,6 @@ cmFileCommand::HandleDownloadCommand(std::vector<std::string> const& args)
     }
 
   cURLEasyGuard g_curl(curl);
-
   ::CURLcode res = ::curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
   check_curl_result(res, "DOWNLOAD cannot set url: ");
 
@@ -2814,6 +2865,43 @@ cmFileCommand::HandleDownloadCommand(std::vector<std::string> const& args)
                            cmFileCommandCurlDebugCallback);
   check_curl_result(res, "DOWNLOAD cannot set debug function: ");
 
+  // check to see if SSL verification is requested
+  const char* verifyValue =
+    this->Makefile->GetDefinition("CMAKE_CURLOPT_SSL_VERIFYPEER");
+  // if there is a cmake variable or if the command has SSL_VERIFY requested
+  if(verifyValue || checkSSL)
+    {
+    // the args to the command come first
+    bool verify = verifySSL;
+    if(!verify && verifyValue)
+      {
+      verify = cmSystemTools::IsOn(verifyValue);
+      }
+    if(verify)
+      {
+      res = ::curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 1);
+      check_curl_result(res, "Unable to set SSL Verify on: ");
+      }
+    else
+      {
+      res = ::curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0);
+      check_curl_result(res, "Unable to set SSL Verify off: ");
+      }
+    }
+  // check to see if a CAINFO file has been specified
+  const char* cainfo =
+    this->Makefile->GetDefinition("CMAKE_CURLOPT_CAINFO_FILE");
+  // command arg comes first
+  if(caFile.size())
+    {
+    cainfo = caFile.c_str();
+    }
+  if(cainfo)
+    {
+    res = ::curl_easy_setopt(curl, CURLOPT_CAINFO, cainfo);
+    check_curl_result(res, "Unable to set SSL Verify CAINFO: ");
+    }
+
   cmFileCommandVectorOfChar chunkDebug;
 
   res = ::curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)&fout);
@@ -2888,26 +2976,22 @@ cmFileCommand::HandleDownloadCommand(std::vector<std::string> const& args)
 
   // Verify MD5 sum if requested:
   //
-  if (!expectedMD5sum.empty())
+  if (hash.get())
     {
-    char computedMD5[32];
+    std::string actualHash = hash->HashFile(file.c_str());
-
+    if (actualHash.size() == 0)
-    if (!cmSystemTools::ComputeFileMD5(file.c_str(), computedMD5))
       {
-      this->SetError("DOWNLOAD cannot compute MD5 sum on downloaded file");
+      this->SetError("DOWNLOAD cannot compute hash on downloaded file");
       return false;
       }
 
-    std::string actualMD5sum = cmSystemTools::LowerCase(
+    if (expectedHash != actualHash)
-      std::string(computedMD5, 32));
-
-    if (expectedMD5sum != actualMD5sum)
       {
       cmOStringStream oss;
-      oss << "DOWNLOAD MD5 mismatch" << std::endl
+      oss << "DOWNLOAD HASH mismatch" << std::endl
         << "  for file: [" << file << "]" << std::endl
-        << "    expected MD5 sum: [" << expectedMD5sum << "]" << std::endl
+        << "    expected hash: [" << expectedHash << "]" << std::endl
-        << "      actual MD5 sum: [" << actualMD5sum << "]" << std::endl
+        << "      actual hash: [" << actualHash << "]" << std::endl
         ;
       this->SetError(oss.str().c_str());
       return false;

Source/cmFileCommand.h

@@ -83,7 +83,9 @@ public:
       "  file(TO_NATIVE_PATH path result)\n"
       "  file(DOWNLOAD url file [INACTIVITY_TIMEOUT timeout]\n"
       "       [TIMEOUT timeout] [STATUS status] [LOG log] [SHOW_PROGRESS]\n"
-      "       [EXPECTED_MD5 sum])\n"
+      "       [EXPECTED_HASH MD5|SHA1|SHA224|SHA256|SHA384|SHA512 hash]\n"
+      "       [EXPECTED_MD5 sum]\n"
+      "       [SSL_VERIFY on|off] [SSL_CAINFO_FILE file])\n"
       "  file(UPLOAD filename url [INACTIVITY_TIMEOUT timeout]\n"
       "       [TIMEOUT timeout] [STATUS status] [LOG log] [SHOW_PROGRESS])\n"
       "WRITE will write a message into a file called 'filename'. It "
@@ -168,11 +170,20 @@ public:
       "timeout after time seconds, time should be specified as an integer. "
       "The INACTIVITY_TIMEOUT specifies an integer number of seconds of "
       "inactivity after which the operation should terminate. "
-      "If EXPECTED_MD5 sum is specified, the operation will verify that the "
+      "If EXPECTED_HASH is specified, the operation will verify that the "
-      "downloaded file's actual md5 sum matches the expected value. If it "
+      "downloaded file's actual hash matches the expected value. If it "
       "does not match, the operation fails with an error. "
+      "(EXPECTED_MD5 is short-hand for EXPECTED_HASH MD5.) "
       "If SHOW_PROGRESS is specified, progress information will be printed "
-      "as status messages until the operation is complete."
+      "as status messages until the operation is complete. "
+      "For https URLs CMake must be built with OpenSSL.  "
+      "SSL certificates are not checked by default.  "
+      "Set SSL_VERIFY to ON to check certificates and/or use "
+      "EXPECTED_HASH to verify downloaded content.  "
+      "Set SSL_CAINFO_FILE to specify a custom Certificate Authority file.  "
+      "If either SSL option is not given CMake will check variables "
+      "CMAKE_CURLOPT_SSL_VERIFYPEER and CMAKE_CURLOPT_CAINFO_FILE, "
+      "respectively."
       "\n"
       "UPLOAD will upload the given file to the given URL. "
       "If LOG var is specified a log of the upload will be put in var. "

Tests/CMakeTests/FileDownloadTest.cmake.in

@@ -33,6 +33,59 @@ file(DOWNLOAD
   )
 
 message(STATUS "FileDownload:4")
+file(DOWNLOAD
+  ${url}
+  ${dir}/file3.png
+  TIMEOUT 2
+  STATUS status
+  EXPECTED_HASH SHA1 50c614fc28b39c1281d0517bb6d5858b4359c9b7
+  )
+
+message(STATUS "FileDownload:5")
+file(DOWNLOAD
+  ${url}
+  ${dir}/file3.png
+  TIMEOUT 2
+  STATUS status
+  EXPECTED_HASH SHA224 73cd5f442b04e8320e4f907f8e1b21d4befff98b5bd77bc32526ea68
+  )
+
+message(STATUS "FileDownload:6")
+file(DOWNLOAD
+  ${url}
+  ${dir}/file3.png
+  TIMEOUT 2
+  STATUS status
+  EXPECTED_HASH SHA256 2e067f6c09cbc7cd619c8fbcc44eb64cd6b45a95e4cddb3a585eee1f731c4da9
+  )
+
+message(STATUS "FileDownload:7")
+file(DOWNLOAD
+  ${url}
+  ${dir}/file3.png
+  TIMEOUT 2
+  STATUS status
+  EXPECTED_HASH SHA384 398bf41902a7251c30e522b307e3e41e3fb617c765b3feaa99b2f7d063894708ad399267ccc25d877437a10e5e890d35
+  )
+
+message(STATUS "FileDownload:8")
+file(DOWNLOAD
+  ${url}
+  ${dir}/file3.png
+  TIMEOUT 2
+  STATUS status
+  EXPECTED_HASH SHA512 c51854d21052713968b849c2b4263cf54be03bc3a7e9847a6c71c6c8d1d13cd805fe1b9fa95f9ba1d0a5631513974f6fae21e34ab5b171d94bad48df5f073e48
+  )
+message(STATUS "FileDownload:9")
+file(DOWNLOAD
+  ${url}
+  ${dir}/file3.png
+  TIMEOUT 2
+  STATUS status
+  EXPECTED_HASH MD5 d16778650db435bda3a8c3435c3ff5d1
+  )
+
+message(STATUS "FileDownload:10")
 file(DOWNLOAD
   ${url}
   ${dir}/file3.png

Utilities/Release/dash2win64_release.cmake

@@ -8,6 +8,7 @@ set(CPACK_SOURCE_GENERATORS "ZIP")
 set(MAKE_PROGRAM "make")
 set(MAKE "${MAKE_PROGRAM} -j8")
 set(INITIAL_CACHE "CMAKE_BUILD_TYPE:STRING=Release
+CMAKE_USE_OPENSSL:BOOL=ON
 CMAKE_SKIP_BOOTSTRAP_TEST:STRING=TRUE
 CMAKE_Fortran_COMPILER:FILEPATH=FALSE
 CMAKE_GENERATOR:INTERNAL=Unix Makefiles

Utilities/Release/dashmacmini2_release.cmake

@@ -9,6 +9,10 @@ set(CPACK_BINARY_GENERATORS "PackageMaker TGZ TZ")
 set(INITIAL_CACHE "
 CMAKE_BUILD_TYPE:STRING=Release
 CMAKE_OSX_ARCHITECTURES:STRING=ppc;i386
+CMAKE_USE_OPENSSL:BOOL=ON
+OPENSSL_CRYPTO_LIBRARY:FILEPATH=/Users/kitware/openssl-1.0.1c-install/lib/libcrypto.a
+OPENSSL_INCLUDE_DIR:PATH=/Users/kitware/openssl-1.0.1c-install/include
+OPENSSL_SSL_LIBRARY:FILEPATH=/Users/kitware/openssl-1.0.1c-install/lib/libssl.a
 CMAKE_SKIP_BOOTSTRAP_TEST:STRING=TRUE
 CPACK_SYSTEM_NAME:STRING=Darwin-universal
 BUILD_QtDialog:BOOL=TRUE

Utilities/Release/dashmacmini5_release.cmake

@@ -8,6 +8,10 @@ set(MAKE "${MAKE_PROGRAM} -j5")
 set(CPACK_BINARY_GENERATORS "PackageMaker TGZ TZ")
 set(CPACK_SOURCE_GENERATORS "TGZ TZ")
 set(INITIAL_CACHE "
+CMAKE_USE_OPENSSL:BOOL=ON
+OPENSSL_CRYPTO_LIBRARY:FILEPATH=/Users/kitware/openssl-1.0.1c-install/lib/libcrypto.a
+OPENSSL_INCLUDE_DIR:PATH=/Users/kitware/openssl-1.0.1c-install/include
+OPENSSL_SSL_LIBRARY:FILEPATH=/Users/kitware/openssl-1.0.1c-install/lib/libssl.a
 CMAKE_BUILD_TYPE:STRING=Release
 CMAKE_OSX_ARCHITECTURES:STRING=x86_64;i386
 CMAKE_OSX_DEPLOYMENT_TARGET:STRING=10.5

Utilities/Release/magrathea_release.cmake

@@ -10,6 +10,10 @@ CMAKE_BUILD_TYPE:STRING=Release
 CURSES_LIBRARY:FILEPATH=/usr/i686-gcc-332s/lib/libncurses.a
 CURSES_INCLUDE_PATH:PATH=/usr/i686-gcc-332s/include/ncurses
 FORM_LIBRARY:FILEPATH=/usr/i686-gcc-332s/lib/libform.a
+CMAKE_USE_OPENSSL:BOOL=ON
+OPENSSL_CRYPTO_LIBRARY:FILEPATH=/home/kitware/openssl-1.0.1c-install/lib/libcrypto.a
+OPENSSL_INCLUDE_DIR:PATH=/home/kitware/openssl-1.0.1c-install/include
+OPENSSL_SSL_LIBRARY:FILEPATH=/home/kitware/openssl-1.0.1c-install/lib/libssl.a
 CPACK_SYSTEM_NAME:STRING=Linux-i386
 BUILD_QtDialog:BOOL:=TRUE
 QT_QMAKE_EXECUTABLE:FILEPATH=/home/kitware/qt-4.43-install/bin/qmake