Error out if two blobs compiled with different compiler versions are linked. (#5361)

This PR aims to accomplish the final step of the task to prevent blobs
compiled with different compiler versions from being linked together.
The same compiler is used to compile both blobs in the test, but the
compilerversion part in the first blob is forcefully changed to conflict
with the compilerversion part within the second blob.
The test detects the correct error message, which mentions the names of
the two libraries that differ, and what their differing versions are
exactly.
The PR aims to resolve this issue:
#5310

Author: bob80905

tools/clang/tools/dxcompiler/dxclinker.cpp

@@ -44,6 +44,48 @@ using namespace llvm;
 // This declaration is used for the locally-linked validator.
 HRESULT CreateDxcValidator(_In_ REFIID riid, _Out_ LPVOID *ppv);
 
+struct DeserializedDxilCompilerVersion {
+  const hlsl::DxilCompilerVersion DCV;
+  std::string commitHashStr;
+  std::string versionStr;
+
+  DeserializedDxilCompilerVersion(const DxilCompilerVersion *pDCV)
+      : DCV(*pDCV) {
+    // Assumes pDCV has been checked for safe parsing
+    if (pDCV->VersionStringListSizeInBytes) {
+      const char *pStr = (const char *)(pDCV + 1);
+      commitHashStr.assign(pStr);
+      if (commitHashStr.size() + 1 < pDCV->VersionStringListSizeInBytes)
+        versionStr.assign(pStr + commitHashStr.size() + 1);
+    }
+  }
+
+  DeserializedDxilCompilerVersion(DeserializedDxilCompilerVersion &&other)
+      : DCV(other.DCV), commitHashStr(std::move(other.commitHashStr)),
+        versionStr(std::move(other.versionStr)) {}
+
+  bool operator<(const DeserializedDxilCompilerVersion &rhs) const {
+    return std::tie(DCV.Major, DCV.Minor, DCV.CommitCount,
+                    DCV.VersionStringListSizeInBytes, commitHashStr,
+                    versionStr) < std::tie(rhs.DCV.Major, rhs.DCV.Minor,
+                                           rhs.DCV.CommitCount,
+                                           rhs.DCV.VersionStringListSizeInBytes,
+                                           rhs.commitHashStr, rhs.versionStr);
+  }
+
+  std::string display() const {
+    std::string ret;
+    ret += "Version(" + std::to_string(DCV.Major) + "." +
+           std::to_string(DCV.Minor) + ") ";
+    ret += "commits(" + std::to_string(DCV.CommitCount) + ") ";
+    ret += "sha(" + commitHashStr + ") ";
+    ret += "version string: \"" + versionStr + "\"";
+    ret += "\n";
+
+    return ret;
+  }
+};
+
 class DxcLinker : public IDxcLinker, public IDxcContainerEvent {
 public:
   DXC_MICROCOM_TM_ADDREF_RELEASE_IMPL()
@@ -98,6 +140,30 @@ class DxcLinker : public IDxcLinker, public IDxcContainerEvent {
     m_pLinker.reset(DxilLinker::CreateLinker(m_Ctx, valMajor, valMinor));
   }
 
+  bool AddCompilerVersionMapEntry(LPCWSTR libName,
+                                  const hlsl::DxilCompilerVersion *pDCV,
+                                  uint32_t partSize) {
+    // Make sure it's safe to parse pDCV
+    bool valid = pDCV && partSize >= sizeof(hlsl::DxilCompilerVersion) &&
+                 partSize - sizeof(hlsl::DxilCompilerVersion) >=
+                     pDCV->VersionStringListSizeInBytes;
+    if (valid && pDCV->VersionStringListSizeInBytes) {
+        const char *vStr = (const char *)(pDCV + 1);
+        valid = vStr[pDCV->VersionStringListSizeInBytes - 1] == 0;
+    }
+
+    DXASSERT(valid, "DxilCompilerVersion part malformed");
+    if (!valid)
+        return false;
+
+    CW2A pUtf8LibName(libName, CP_UTF8);
+    std::string libNameStr = std::string(pUtf8LibName);
+    auto result = m_uniqueCompilerVersions.insert(DeserializedDxilCompilerVersion(pDCV));    
+    m_libNameToCompilerVersionPart[libNameStr] = &(*result.first);
+
+    return true;
+  }
+
   ~DxcLinker() {
     // Make sure DxilLinker is released before LLVMContext.
     m_pLinker.reset();
@@ -109,6 +175,8 @@ class DxcLinker : public IDxcLinker, public IDxcContainerEvent {
   std::unique_ptr<DxilLinker> m_pLinker;
   CComPtr<IDxcContainerEventsHandler> m_pDxcContainerEventsHandler;
   std::vector<CComPtr<IDxcBlob>> m_blobs; // Keep blobs live for lazy load.
+  std::map<std::string, const DeserializedDxilCompilerVersion*> m_libNameToCompilerVersionPart;
+  std::set<DeserializedDxilCompilerVersion> m_uniqueCompilerVersions;
 };
 
 HRESULT
@@ -137,6 +205,22 @@ DxcLinker::RegisterLibrary(_In_opt_ LPCWSTR pLibName, // Name of the library.
         pBlob->GetBufferPointer(), pBlob->GetBufferSize(), pModule,
         pDebugModule, m_Ctx, m_Ctx, DiagStream));
 
+
+    // add an entry into the library to compiler version part map
+    const hlsl::DxilContainerHeader *pHeader = hlsl::IsDxilContainerLike(
+        pBlob->GetBufferPointer(), pBlob->GetBufferSize());
+    const DxilPartHeader *pDPH = hlsl::GetDxilPartByType(
+        pHeader, hlsl::DxilFourCC::DFCC_CompilerVersion);
+    if (pDPH) {
+        const hlsl::DxilCompilerVersion *pDCV =
+            (const hlsl::DxilCompilerVersion *)(pDPH + 1);
+        // If the compiler version string is non-empty, add the struct to the
+        // map
+        if (!AddCompilerVersionMapEntry(pLibName, pDCV, pDPH->PartSize)) {
+          return E_INVALIDARG;
+        }
+    }
+    
     if (m_pLinker->RegisterLib(pUtf8LibName.m_psz, std::move(pModule),
                                std::move(pDebugModule))) {
       m_blobs.emplace_back(pBlob);
@@ -224,13 +308,53 @@ HRESULT STDMETHODCALLTYPE DxcLinker::Link(
 
     // Attach libraries.
     bool bSuccess = true;
-    for (unsigned i = 0; i < libCount; i++) {
+    const DeserializedDxilCompilerVersion *cur_version = nullptr;
+    const DeserializedDxilCompilerVersion *first_version = nullptr;
+
+    std::string cur_lib_name;
+    std::string first_lib_name;
+
+    for (UINT32 i = 0; i < libCount; i++) {
       CW2A pUtf8LibName(pLibNames[i], CP_UTF8);
       bSuccess &= m_pLinker->AttachLib(pUtf8LibName.m_psz);
+
+      cur_lib_name = std::string(pUtf8LibName);      
+
+      // only libraries with compiler version parts are in the map
+      auto result = m_libNameToCompilerVersionPart.find(cur_lib_name);
+
+      if (result != m_libNameToCompilerVersionPart.end()) {
+        cur_version = result->second;
+      }
+
+      if (i == 0) {
+        first_lib_name = cur_lib_name;
+        first_version = cur_version;
+      }
+
+      if (cur_version != first_version) {
+
+        std::string errorMsg = "error: Cannot link libraries with "
+                               "conflicting compiler versions.\n";
+
+        std::string firstErrorStr =
+            "note: library \"" + first_lib_name + "\" version: ";
+        firstErrorStr += first_version ? first_version->display()
+                                       : "No version info available";
+
+        std::string secondErrorStr =
+            "note: library \"" + cur_lib_name + "\" version: ";
+        secondErrorStr +=
+            cur_version ? cur_version->display() : "No version info available";
+
+        errorMsg += firstErrorStr + secondErrorStr;
+        DiagStream << errorMsg;
+        bSuccess = false;
+      }
     }
 
     dxilutil::ExportMap exportMap;
-    bSuccess = exportMap.ParseExports(opts.Exports, DiagStream);
+    bSuccess &= exportMap.ParseExports(opts.Exports, DiagStream);
 
     if (opts.ExportShadersOnly)
       exportMap.setExportShadersOnly(true);

tools/clang/unittests/HLSL/LinkerTest.cpp

@@ -21,6 +21,7 @@
 #include "WexTestClass.h"
 #include "dxc/Test/HlslTestUtils.h"
 #include "dxc/Test/DxcTestUtils.h"
+#include "dxc/Support/Global.h" // for IFT macro
 #include "dxc/dxcapi.h"
 #include "dxc/DxilContainer/DxilContainer.h"
 
@@ -40,6 +41,7 @@ class LinkerTest
   TEST_CLASS_SETUP(InitSupport);
 
   TEST_METHOD(RunLinkResource);
+  TEST_METHOD(RunLinkModulesDifferentVersions);
   TEST_METHOD(RunLinkResourceWithBinding);
   TEST_METHOD(RunLinkAllProfiles);
   TEST_METHOD(RunLinkFailNoDefine);
@@ -158,14 +160,14 @@ class LinkerTest
 
   void LinkCheckMsg(LPCWSTR pEntryName, LPCWSTR pShaderModel, IDxcLinker *pLinker,
             ArrayRef<LPCWSTR> libNames, llvm::ArrayRef<LPCSTR> pErrorMsgs,
-            llvm::ArrayRef<LPCWSTR> pArguments = {}) {
+            llvm::ArrayRef<LPCWSTR> pArguments = {}, bool bRegex=false) {
     CComPtr<IDxcOperationResult> pResult;
     VERIFY_SUCCEEDED(pLinker->Link(pEntryName, pShaderModel,
                                    libNames.data(), libNames.size(),
                                    pArguments.data(), pArguments.size(),
                                    &pResult));
     CheckOperationResultMsgs(pResult, pErrorMsgs.data(), pErrorMsgs.size(),
-                             false, false);
+                             false, bRegex);
   }
 };
 
@@ -274,6 +276,97 @@ TEST_F(LinkerTest, RunLinkAllProfiles) {
   Link(L"cs_main", L"cs_6_0", pLinker, {libName, libResName}, {},{});
 }
 
+TEST_F(LinkerTest, RunLinkModulesDifferentVersions) {
+  CComPtr<IDxcLinker> pLinker1, pLinker2, pLinker3;
+  CreateLinker(&pLinker1);
+  CreateLinker(&pLinker2);
+  CreateLinker(&pLinker3);
+
+  LPCWSTR libName = L"entry";
+  LPCWSTR option[] = {L"-Zi", L"-Qembed_debug"};
+  CComPtr<IDxcBlob> pEntryLib;
+  CompileLib(L"..\\CodeGenHLSL\\lib_entries2.hlsl", &pEntryLib, option);
+
+  // the 2nd blob is the "good" blob that stays constant
+  CComPtr<IDxcBlob> pResLib;
+  CompileLib(L"..\\CodeGenHLSL\\lib_resource2.hlsl", &pResLib);
+  LPCWSTR libResName = L"res";
+
+  // modify the first compiled blob to force it to have a different compiler
+  // version
+  CComPtr<IDxcContainerReflection> containerReflection;
+  IFT(m_dllSupport.CreateInstance(CLSID_DxcContainerReflection,
+                                  &containerReflection));
+  IFT(containerReflection->Load(pEntryLib));
+  UINT part_index;
+  VERIFY_SUCCEEDED(containerReflection->FindFirstPartKind(
+      hlsl::DFCC_CompilerVersion, &part_index));
+  CComPtr<IDxcBlob> pBlob;
+  IFT(containerReflection->GetPartContent(part_index, &pBlob));
+  void *pBlobPtr = pBlob->GetBufferPointer();
+
+  std::string commonMismatchStr =
+      "error: Cannot link libraries with conflicting compiler versions.";
+
+  hlsl::DxilCompilerVersion *pDCV = (hlsl::DxilCompilerVersion *)pBlobPtr;
+  if (pDCV->VersionStringListSizeInBytes) {
+    // first just get both strings, the compiler version and the commit hash
+    char *pVersionStringListPtr =
+        (char *)pBlobPtr + sizeof(hlsl::DxilCompilerVersion);
+    std::string commitHashStr(pVersionStringListPtr);
+    std::string oldCommitHashStr = commitHashStr;
+    std::string compilerVersionStr(pVersionStringListPtr +
+                                   commitHashStr.size() + 1);
+    std::string oldCompilerVersionStr = compilerVersionStr;
+
+    // flip a character to change the commit hash
+    *pVersionStringListPtr = commitHashStr[0] == 'a' ? 'b' : 'a';
+    // store the modified compiler version part.
+    RegisterDxcModule(libName, pEntryLib, pLinker1);
+    // and the "good" version part
+    RegisterDxcModule(libResName, pResLib, pLinker1);
+    // 2 blobs with different compiler versions should not be linkable
+    LinkCheckMsg(L"hs_main", L"hs_6_1", pLinker1, {libName, libResName},
+                 {commonMismatchStr.c_str()});
+
+    // reset the modified part back to normal
+    *pVersionStringListPtr = oldCommitHashStr[0];
+
+    // flip a character to change the compiler version
+    *(pVersionStringListPtr + commitHashStr.size() + 1) =
+        compilerVersionStr[0] == '1' ? '2' : '1';
+    // store the modified compiler version part.
+    RegisterDxcModule(libName, pEntryLib, pLinker2);
+    // and the "good" version part
+    RegisterDxcModule(libResName, pResLib, pLinker2);
+    // 2 blobs with different compiler versions should not be linkable
+    LinkCheckMsg(L"hs_main", L"hs_6_1", pLinker2, {libName, libResName},
+                 {commonMismatchStr.c_str()});
+
+    // reset the modified part back to normal
+    *(pVersionStringListPtr + commitHashStr.size() + 1) =
+        oldCompilerVersionStr[0];
+  } else {
+    // The blob can't be changed by adding data, since the
+    // the data after this header could be a subsequent header.
+    // The test should announce that it can't make any version string
+    // modifications
+    WEX::Logging::Log::Warning(
+        L"Cannot modify compiler version string for test");
+  }
+
+  // finally, test that a difference is detected if a member of the struct, say
+  // the major member, is different.
+  pDCV->Major = pDCV->Major == 2 ? 1 : 2;
+  // store the modified compiler version part.
+  RegisterDxcModule(libName, pEntryLib, pLinker3);
+  // and the "good" version part
+  RegisterDxcModule(libResName, pResLib, pLinker3);
+  // 2 blobs with different compiler versions should not be linkable
+  LinkCheckMsg(L"hs_main", L"hs_6_1", pLinker3, {libName, libResName},
+               {commonMismatchStr.c_str()});
+}
+
 TEST_F(LinkerTest, RunLinkFailNoDefine) {
   CComPtr<IDxcBlob> pEntryLib;
   CompileLib(L"..\\CodeGenHLSL\\lib_cs_entry.hlsl", &pEntryLib);