Add back compat lit test suite, and download older dxil.dll's (microsoft#7705)

This PR adds new lit test suites, that build by downloading and
unzipping 5 release packages. The release packages are a selection of
historical packages that will be useful in later testing for validation
back-compatibility. However, this data can also be used for other
testing, and doesn't necessarily need to be restricted to older
packages. For now, 5 releases have been included, including the latest 3
releases at the time of this writing.
This PR then executes these test suites with an environment variable
that points to the dxil.dll that was just downloaded, so that in the
future, DxcDllExtValLoader can grab the dxil.dll path and initialize
appropriately.
This PR effectively sets up a test suite and an environment so that back
compat work is possible.
Adds some final touches on @damyanp's work.
Fixes microsoft#7707

---------

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>

Author: bob80905

azure-pipelines.yml

@@ -13,7 +13,7 @@ stages:
 - stage: Build
   jobs:
   - job: Windows
-    timeoutInMinutes: 120
+    timeoutInMinutes: 165
 
     pool:
       vmImage: windows-2022
@@ -48,7 +48,7 @@ stages:
       displayName: 'DXIL Execution Tests (Nuget WARP)'
 
   - job: Nix
-    timeoutInMinutes: 120
+    timeoutInMinutes: 165
 
     variables:
       macOS: macOS-latest

include/dxc/Test/DxcTestUtils.h

@@ -13,6 +13,7 @@
 
 #include "dxc/Support/WinIncludes.h"
 
+#include "dxc/Support/dxcapi.extval.h"
 #include "dxc/Support/dxcapi.use.h"
 #include "dxc/dxcapi.h"
 #include "llvm/ADT/ArrayRef.h"
@@ -175,7 +176,7 @@ class FileRunTestResult {
                       PluginToolsPaths *pPluginToolsPaths = nullptr,
                       LPCWSTR dumpName = nullptr);
   static FileRunTestResult
-  RunFromFileCommands(LPCWSTR fileName, dxc::DxCompilerDllLoader &dllSupport,
+  RunFromFileCommands(LPCWSTR fileName, dxc::SpecificDllLoader &dllSupport,
                       PluginToolsPaths *pPluginToolsPaths = nullptr,
                       LPCWSTR dumpName = nullptr);
 };

projects/dxilconv/unittests/DxilConvTests.cpp

@@ -67,7 +67,8 @@ class DxilConvTest {
   END_TEST_METHOD()
 
 private:
-  dxc::DxCompilerDllLoader m_dllSupport;
+  // this loader only loads dxilconv.dll
+  dxc::SpecificDllLoader m_dllSupport;
   PluginToolsPaths m_TestToolPaths;
 
   void DxilConvTestCheckFile(LPCWSTR path) {

tools/clang/test/CMakeLists.txt

@@ -108,6 +108,95 @@ set_target_properties(clang-test PROPERTIES FOLDER "Clang tests")
 
 # HLSL Change Begin - Generate lit targets for test subdirectories.
 
+# -------------------------------------------------------------------------
+# DXIL Backward Compatibility Section
+# -------------------------------------------------------------------------
+
+# Define DXC releases as (NAME VERSION) pairs
+set(DXC_RELEASES  
+  dxc_2025_02_20 v1.8.2502
+  dxc_2023_08_14 v1.7.2308
+  dxc_2021_12_08 v1.6.2112
+)
+
+# Download a DXC release
+function(add_released_dxc name version)
+  ExternalProject_Add(${name}
+      URL https://github.com/microsoft/DirectXShaderCompiler/releases/download/${version}/${name}.zip
+      DOWNLOAD_DIR ${CMAKE_BINARY_DIR}/tools/clang/test/dxc_releases
+      SOURCE_DIR ${CMAKE_BINARY_DIR}/tools/clang/test/dxc_releases/${version}/${name}
+      CONFIGURE_COMMAND ""
+      BUILD_COMMAND ""
+      INSTALL_COMMAND ""
+  )
+endfunction()
+
+# Iterate over releases
+list(LENGTH DXC_RELEASES DXC_RELEASES_LENGTH)
+math(EXPR num_pairs "${DXC_RELEASES_LENGTH} / 2")
+math(EXPR loop_end "${num_pairs} - 1")
+foreach(i RANGE 0 ${loop_end})
+  math(EXPR idx_name "${i}*2")
+  math(EXPR idx_version "${i}*2 + 1")  
+
+  list(GET DXC_RELEASES ${idx_name} name)
+  list(GET DXC_RELEASES ${idx_version} version)
+
+  add_released_dxc(${name} ${version})
+
+  if(WIN32)
+  # Determine target architecture for DXC binaries
+    if(CMAKE_GENERATOR MATCHES "Visual Studio")
+        # Multi-config generators (VS/Xcode)
+        if(NOT DEFINED CMAKE_GENERATOR_PLATFORM)
+            message(FATAL_ERROR "CMAKE_GENERATOR_PLATFORM not set for multi-config generator")
+        endif()
+        set(TARGET_ARCH ${CMAKE_GENERATOR_PLATFORM})
+    else()
+        # Single-config generators (Ninja, Makefiles)
+        string(TOLOWER "${CMAKE_SYSTEM_PROCESSOR}" SYSTEM_PROCESSOR_LOWER)
+
+        if(SYSTEM_PROCESSOR_LOWER MATCHES "^(x86|i[3-6]86)$")
+            set(TARGET_ARCH x86)
+        elseif(SYSTEM_PROCESSOR_LOWER MATCHES "^(x64|amd64|x86_64)$")
+            set(TARGET_ARCH x64)
+        elseif(SYSTEM_PROCESSOR_LOWER MATCHES "^(arm64|aarch64)$")
+            set(TARGET_ARCH arm64)
+        elseif(SYSTEM_PROCESSOR_LOWER MATCHES "^arm")
+            set(TARGET_ARCH arm)
+        else()
+            message(FATAL_ERROR "Unknown architecture: ${CMAKE_SYSTEM_PROCESSOR}")
+        endif()
+    endif()
+
+    # Path to dxil.dll for this release
+    set(DXC_PATH ${CMAKE_BINARY_DIR}/tools/clang/test/dxc_releases/${version}/${name}/bin/${TARGET_ARCH}/dxil.dll)
+
+    # Create a lit target for this release
+    add_lit_target("check-clang-${name}" "Running clang regression tests with ${name}\n"
+      ${CMAKE_CURRENT_SOURCE_DIR}
+      PARAMS ${CLANG_TEST_PARAMS} DXC_DXIL_DLL_PATH=${DXC_PATH}
+        skip_taef_exec=True
+      DEPENDS ${CLANG_TEST_DEPS} ${name}
+      ARGS ${CLANG_TEST_EXTRA_ARGS}
+    )
+
+    # Ensure there is a top-level check-clang target
+    if(NOT TARGET check-clang)
+      add_custom_target(check-clang)
+    endif()
+
+    # Make check-clang depend on this particular release's tests
+    add_dependencies(check-clang "check-clang-${name}")
+
+    # Hook into check-all
+    if (WIN32 AND TARGET check-all)
+      add_dependencies(check-all "check-clang-${name}")
+    endif()  
+  endif()
+endforeach()
+# -------------------------------------------------------------------------
+
 set(CLANG_TEST_PARAMS
   ${CLANG_TEST_PARAMS}
   clang_unit_site_config=${CMAKE_CURRENT_BINARY_DIR}/Unit/lit.site.cfg
@@ -122,8 +211,6 @@ add_lit_testsuites(CLANG ${CMAKE_CURRENT_SOURCE_DIR}
   FOLDER "Clang tests/Suites"
 )
 
-# Manually generate targets that we need to expose in visual studio builds.
-
 # The code below here _ONLY_ executes when building with Visual Studio or Xcode.
 if (NOT CMAKE_CONFIGURATION_TYPES)
   return()

tools/clang/test/LitDXILValidation/Environment.test

@@ -0,0 +1,3 @@
+# REQUIRES: dxc_dxil_dll_path
+# RUN: python -c "import os; print(os.environ['DXC_DXIL_DLL_PATH'])" | FileCheck %s
+# CHECK: dxil.dll

tools/clang/test/lit.cfg

@@ -94,6 +94,12 @@ for name in possibly_dangerous_env_vars:
   if name in config.environment:
     del config.environment[name]
 
+dxc_dll_path = lit_config.params.get('DXC_DXIL_DLL_PATH', None)
+if dxc_dll_path and os.path.exists(dxc_dll_path):
+    lit_config.note("Setting DXC_DXIL_DLL_PATH environment variable")
+    config.environment.update({'DXC_DXIL_DLL_PATH' : dxc_dll_path})
+    config.available_features.add("dxc_dxil_dll_path")
+
 # Tweak the PATH to include the tools dir and the scripts dir.
 if clang_obj_root is not None:
     clang_tools_dir = getattr(config, 'clang_tools_dir', None)

tools/clang/unittests/HLSL/ValidationTest.cpp

@@ -34,7 +34,6 @@
 #include "dxc/Support/Global.h"
 
 #include "dxc/DXIL/DxilShaderModel.h"
-#include "dxc/Support/dxcapi.extval.h"
 #include "dxc/Test/DxcTestUtils.h"
 #include "dxc/Test/HlslTestUtils.h"
 

tools/clang/unittests/HLSLExec/CMakeLists.txt

@@ -5,8 +5,6 @@ find_package(D3D12 REQUIRED) # Used for ExecutionTest.cpp.
 
 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /bigobj")
 
-set(DXCSUPPORT_DIR "${CMAKE_SOURCE_DIR}/lib/DxcSupport")
-
 add_clang_library(ExecHLSLTests SHARED
   ExecutionTest.cpp
   ShaderOpTest.cpp

tools/clang/unittests/HLSLTestLib/FileCheckerTest.cpp

@@ -1258,7 +1258,8 @@ FileRunCommandPart::RunFromPath(const std::string &toolPath,
 #endif //_WIN32
 
 class FileRunTestResultImpl : public FileRunTestResult {
-  dxc::DxCompilerDllLoader &m_support;
+  dxc::SpecificDllLoader &m_support;
+  dxc::DxcDllExtValidationLoader m_extval_support;
   PluginToolsPaths *m_pPluginToolsPaths;
   LPCWSTR m_dumpName = nullptr;
   // keep track of virtual files for duration of this test (for all RUN lines)
@@ -1332,7 +1333,7 @@ class FileRunTestResultImpl : public FileRunTestResult {
   }
 
 public:
-  FileRunTestResultImpl(dxc::DxCompilerDllLoader &support,
+  FileRunTestResultImpl(dxc::SpecificDllLoader &support,
                         PluginToolsPaths *pPluginToolsPaths = nullptr,
                         LPCWSTR dumpName = nullptr)
       : m_support(support), m_pPluginToolsPaths(pPluginToolsPaths),
@@ -1389,7 +1390,7 @@ FileRunTestResult FileRunTestResult::RunFromFileCommands(
 }
 
 FileRunTestResult FileRunTestResult::RunFromFileCommands(
-    LPCWSTR fileName, dxc::DxCompilerDllLoader &dllSupport,
+    LPCWSTR fileName, dxc::SpecificDllLoader &dllSupport,
     PluginToolsPaths *pPluginToolsPaths /*=nullptr*/,
     LPCWSTR dumpName /*=nullptr*/) {
   FileRunTestResultImpl result(dllSupport, pPluginToolsPaths, dumpName);