Skip to content

Commit 5526801

Browse files
AnastaZIukAnastaZIuk
authored
Merge pull request #1029 from Devsh-Graphics-Programming/unroll
Speed up HLSL preprocessing and prepared SPIR-V hot paths
2 parents e11b118 + 9164943 commit 5526801

34 files changed

Lines changed: 1005 additions & 263 deletions

‎.gitmodules‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -72,7 +72,7 @@
7272
[submodule "3rdparty/dxc/dxc"]
7373
path = 3rdparty/dxc/dxc
7474
url = [email protected]:Devsh-Graphics-Programming/DirectXShaderCompiler.git
75-
branch = devshFixes
75+
branch = unroll-devshFixes
7676
[submodule "3rdparty/imgui"]
7777
path = 3rdparty/imgui
7878
url = [email protected]:Devsh-Graphics-Programming/imgui.git

‎3rdparty/dxc/dxc‎

Submodule dxc updated 353 files

‎cmake/common.cmake‎

Lines changed: 21 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1191,6 +1191,7 @@ option(NSC_DEBUG_EDIF_LINE_BIT "Add \"-fspv-debug=line\" to NSC Debug CLI" OFF)
11911191
option(NSC_DEBUG_EDIF_TOOL_BIT "Add \"-fspv-debug=tool\" to NSC Debug CLI" ON)
11921192
option(NSC_DEBUG_EDIF_NON_SEMANTIC_BIT "Add \"-fspv-debug=vulkan-with-source\" to NSC Debug CLI" OFF)
11931193
option(NSC_USE_DEPFILE "Generate depfiles for NSC custom commands" ON)
1194+
option(NBL_DISABLE_EXPERIMENTAL_OPTS "Disable -O1experimental for all NSC compile rules globally" OFF)
11941195

11951196
function(NBL_CREATE_NSC_COMPILE_RULES)
11961197
set(COMMENT "this code has been autogenerated with Nabla CMake NBL_CREATE_HLSL_COMPILE_RULES utility")
@@ -1230,10 +1231,7 @@ struct DeviceConfigCaps
12301231
-fspv-target-env=vulkan1.3
12311232
-Wshadow
12321233
-Wconversion
1233-
-Wno-local-type-template-args
1234-
$<$<CONFIG:Debug>:-O0>
1235-
$<$<CONFIG:Release>:-O3>
1236-
$<$<CONFIG:RelWithDebInfo>:-O3>
1234+
-Wno-local-type-template-args
12371235
)
12381236

12391237
if(NSC_DEBUG_EDIF_FILE_BIT)
@@ -1259,18 +1257,32 @@ struct DeviceConfigCaps
12591257
if(NOT NBL_EMBED_BUILTIN_RESOURCES)
12601258
list(APPEND REQUIRED_OPTIONS
12611259
-no-nbl-builtins
1262-
-I "${NBL_ROOT_PATH}/include"
1263-
-I "${NBL_ROOT_PATH}/3rdparty/dxc/dxc/external/SPIRV-Headers/include"
1264-
-I "${NBL_ROOT_PATH}/3rdparty/boost/superproject/libs/preprocessor/include"
1265-
-I "${NBL_ROOT_PATH_BINARY}/src/nbl/device/include"
1260+
-isystem "${NBL_ROOT_PATH}/include"
1261+
-isystem "${NBL_ROOT_PATH}/3rdparty/dxc/dxc/external/SPIRV-Headers/include"
1262+
-isystem "${NBL_ROOT_PATH}/3rdparty/boost/superproject/libs/preprocessor/include"
1263+
-isystem "${NBL_ROOT_PATH_BINARY}/src/nbl/device/include"
12661264
)
12671265
endif()
12681266

12691267
set(REQUIRED_SINGLE_ARGS TARGET BINARY_DIR OUTPUT_VAR INPUTS INCLUDE NAMESPACE MOUNT_POINT_DEFINE)
12701268
set(OPTIONAL_SINGLE_ARGS GLOB_DIR)
1271-
cmake_parse_arguments(IMPL "DISCARD_DEFAULT_GLOB" "${REQUIRED_SINGLE_ARGS};${OPTIONAL_SINGLE_ARGS};LINK_TO" "COMMON_OPTIONS;DEPENDS" ${ARGV})
1269+
cmake_parse_arguments(IMPL "DISCARD_DEFAULT_GLOB;DISABLE_EXPERIMENTAL_OPTS" "${REQUIRED_SINGLE_ARGS};${OPTIONAL_SINGLE_ARGS};LINK_TO" "COMMON_OPTIONS;DEPENDS" ${ARGV})
12721270
NBL_PARSE_REQUIRED(IMPL ${REQUIRED_SINGLE_ARGS})
12731271

1272+
if(NBL_DISABLE_EXPERIMENTAL_OPTS OR IMPL_DISABLE_EXPERIMENTAL_OPTS)
1273+
list(APPEND REQUIRED_OPTIONS
1274+
$<$<CONFIG:Debug>:-O0>
1275+
$<$<CONFIG:Release>:-O3>
1276+
$<$<CONFIG:RelWithDebInfo>:-O3>
1277+
)
1278+
else()
1279+
list(APPEND REQUIRED_OPTIONS
1280+
$<$<CONFIG:Debug>:-O1experimental>
1281+
$<$<CONFIG:Release>:-O3>
1282+
$<$<CONFIG:RelWithDebInfo>:-O1experimental>
1283+
)
1284+
endif()
1285+
12741286
set(IMPL_HLSL_GLOB "")
12751287
if(NOT IMPL_DISCARD_DEFAULT_GLOB)
12761288
set(GLOB_ROOT "${CMAKE_CURRENT_SOURCE_DIR}")
@@ -1306,12 +1318,6 @@ struct DeviceConfigCaps
13061318
TARGET ${IMPL_TARGET}
13071319
)
13081320

1309-
target_sources(${IMPL_TARGET} PUBLIC ${INCLUDE_FILE})
1310-
set_source_files_properties(${INCLUDE_FILE} PROPERTIES
1311-
HEADER_FILE_ONLY ON
1312-
VS_TOOL_OVERRIDE None
1313-
)
1314-
13151321
target_compile_definitions(${IMPL_TARGET} INTERFACE $<TARGET_PROPERTY:${IMPL_TARGET},NBL_MOUNT_POINT_DEFINES>)
13161322
target_include_directories(${IMPL_TARGET} INTERFACE ${INCLUDE_DIR})
13171323
set_target_properties(${IMPL_TARGET} PROPERTIES NBL_HEADER_GENERATED_RULE ON)

‎examples_tests‎

Submodule examples_tests updated 38 files

‎include/nbl/asset/utils/ISPIRVEntryPointTrimmer.h‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,10 +7,12 @@
77

88
#include "nbl/system/ILogger.h"
99

10+
#include <mutex>
11+
1012
namespace nbl::asset
1113
{
1214

13-
class ISPIRVEntryPointTrimmer final : public core::IReferenceCounted
15+
class NBL_API2 ISPIRVEntryPointTrimmer final : public core::IReferenceCounted
1416
{
1517
public:
1618
ISPIRVEntryPointTrimmer();
@@ -46,6 +48,8 @@ class ISPIRVEntryPointTrimmer final : public core::IReferenceCounted
4648
};
4749

4850
Result trim(const ICPUBuffer* spirvBuffer, const core::set<EntryPoint>& entryPoints, system::logger_opt_ptr logger = nullptr) const;
51+
bool ensureValidated(const ICPUBuffer* spirvBuffer, system::logger_opt_ptr logger = nullptr) const;
52+
void markValidated(const ICPUBuffer* spirvBuffer) const;
4953

5054
inline core::smart_refctd_ptr<const IShader> trim(const IShader* shader, const core::set<EntryPoint>& entryPoints, system::logger_opt_ptr logger = nullptr) const
5155
{
@@ -72,6 +76,8 @@ class ISPIRVEntryPointTrimmer final : public core::IReferenceCounted
7276

7377
private:
7478
core::smart_refctd_ptr<ISPIRVOptimizer> m_optimizer;
79+
mutable std::mutex m_validationCacheMutex;
80+
mutable core::unordered_set<core::blake3_hash_t> m_validatedSpirvHashes;
7581
};
7682

7783
}

‎include/nbl/asset/utils/IShaderCompiler.h‎

Lines changed: 84 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,8 @@
1717
#include "nbl/builtin/hlsl/enums.hlsl"
1818

1919
#include <functional>
20+
#include <mutex>
21+
#include <unordered_set>
2022

2123
namespace nbl::asset
2224
{
@@ -26,6 +28,25 @@ class NBL_API2 IShaderCompiler : public core::IReferenceCounted
2628
public:
2729
IShaderCompiler(core::smart_refctd_ptr<system::ISystem>&& system);
2830

31+
enum class IncludeRootOrigin : uint8_t
32+
{
33+
User,
34+
Builtin,
35+
Generated
36+
};
37+
38+
enum class HeaderClass : uint8_t
39+
{
40+
User,
41+
System
42+
};
43+
44+
struct IncludeClassification
45+
{
46+
IncludeRootOrigin origin = IncludeRootOrigin::User;
47+
HeaderClass headerClass = HeaderClass::User;
48+
};
49+
2950
class NBL_API2 IIncludeLoader : public core::IReferenceCounted
3051
{
3152
public:
@@ -34,12 +55,13 @@ class NBL_API2 IShaderCompiler : public core::IReferenceCounted
3455
system::path absolutePath = {};
3556
std::string contents = {};
3657
core::blake3_hash_t hash = {}; // TODO: we're not yet using IFile::getPrecomputedHash(), so for builtins we can maybe use that in the future
58+
IncludeClassification classification = {};
3759
// Could be used in the future for early rejection of cache hit
3860
//nbl::system::IFileBase::time_point_t lastWriteTime = {};
3961

4062
explicit inline operator bool() const {return !absolutePath.empty();}
4163
};
42-
virtual found_t getInclude(const system::path& searchPath, const std::string& includeName) const = 0;
64+
virtual found_t getInclude(const system::path& searchPath, const std::string& includeName, bool needHash = true) const = 0;
4365
};
4466

4567
class NBL_API2 IIncludeGenerator : public core::IReferenceCounted
@@ -65,7 +87,7 @@ class NBL_API2 IShaderCompiler : public core::IReferenceCounted
6587
public:
6688
CFileSystemIncludeLoader(core::smart_refctd_ptr<system::ISystem>&& system);
6789

68-
IIncludeLoader::found_t getInclude(const system::path& searchPath, const std::string& includeName) const override;
90+
IIncludeLoader::found_t getInclude(const system::path& searchPath, const std::string& includeName, bool needHash = true) const override;
6991

7092
protected:
7193
core::smart_refctd_ptr<system::ISystem> m_system;
@@ -74,37 +96,88 @@ class NBL_API2 IShaderCompiler : public core::IReferenceCounted
7496
class NBL_API2 CIncludeFinder : public core::IReferenceCounted
7597
{
7698
public:
99+
struct SSessionCache
100+
{
101+
struct Stats
102+
{
103+
uint64_t lookupFound = 0ull;
104+
uint64_t lookupMissing = 0ull;
105+
uint64_t lookupMiss = 0ull;
106+
uint64_t storeFound = 0ull;
107+
uint64_t storeMissing = 0ull;
108+
};
109+
110+
enum class LookupResult : uint8_t
111+
{
112+
Miss,
113+
Missing,
114+
Found
115+
};
116+
117+
explicit SSessionCache(const bool threadSafe = false) : threadSafe(threadSafe) {}
118+
119+
void clear();
120+
LookupResult lookup(const std::string& key, IIncludeLoader::found_t& result) const;
121+
void store(const std::string& key, IIncludeLoader::found_t result);
122+
Stats snapshotStats() const;
123+
124+
bool threadSafe = false;
125+
126+
mutable std::mutex mutex;
127+
mutable Stats stats;
128+
core::unordered_map<std::string, IIncludeLoader::found_t> found;
129+
core::unordered_set<std::string> missing;
130+
};
131+
77132
CIncludeFinder(core::smart_refctd_ptr<system::ISystem>&& system);
78133

79134
// ! includes within <>
80135
// @param requestingSourceDir: the directory where the incude was requested
81136
// @param includeName: the string within <> of the include preprocessing directive
82-
IIncludeLoader::found_t getIncludeStandard(const system::path& requestingSourceDir, const std::string& includeName) const;
137+
IIncludeLoader::found_t getIncludeStandard(const system::path& requestingSourceDir, const std::string& includeName, bool needHash = true, SSessionCache* readSessionCache = nullptr, SSessionCache* writeSessionCache = nullptr) const;
83138

84139
// ! includes within ""
85140
// @param requestingSourceDir: the directory where the incude was requested
86141
// @param includeName: the string within "" of the include preprocessing directive
87-
IIncludeLoader::found_t getIncludeRelative(const system::path& requestingSourceDir, const std::string& includeName) const;
142+
IIncludeLoader::found_t getIncludeRelative(const system::path& requestingSourceDir, const std::string& includeName, bool needHash = true, SSessionCache* readSessionCache = nullptr, SSessionCache* writeSessionCache = nullptr) const;
88143

89144
inline core::smart_refctd_ptr<CFileSystemIncludeLoader> getDefaultFileSystemLoader() const { return m_defaultFileSystemLoader; }
90145

91-
void addSearchPath(const std::string& searchPath, const core::smart_refctd_ptr<IIncludeLoader>& loader);
146+
void addSearchPath(const std::string& searchPath, const core::smart_refctd_ptr<IIncludeLoader>& loader, IncludeClassification classification = {});
92147

93-
void addGenerator(const core::smart_refctd_ptr<IIncludeGenerator>& generator);
148+
void addGenerator(const core::smart_refctd_ptr<IIncludeGenerator>& generator, IncludeClassification classification = {IncludeRootOrigin::Generated,HeaderClass::System});
149+
150+
bool isKnownGlobalInclude(std::string_view includeName) const;
151+
IIncludeLoader::found_t classifyFound(IIncludeLoader::found_t found) const;
94152

95153
protected:
96-
IIncludeLoader::found_t trySearchPaths(const std::string& includeName) const;
154+
IIncludeLoader::found_t trySearchPaths(const std::string& includeName, bool needHash) const;
97155

98156
IIncludeLoader::found_t tryIncludeGenerators(const std::string& includeName) const;
157+
void registerHeaderRoot(std::string rootPath, IncludeClassification classification);
99158

100159
struct LoaderSearchPath
101160
{
102161
core::smart_refctd_ptr<IIncludeLoader> loader = nullptr;
103162
std::string searchPath = {};
163+
IncludeClassification classification = {};
164+
};
165+
166+
struct GeneratorEntry
167+
{
168+
core::smart_refctd_ptr<IIncludeGenerator> generator = nullptr;
169+
IncludeClassification classification = {IncludeRootOrigin::Generated,HeaderClass::System};
170+
};
171+
172+
struct HeaderRoot
173+
{
174+
std::string path = {};
175+
IncludeClassification classification = {};
104176
};
105177

106178
std::vector<LoaderSearchPath> m_loaders;
107-
std::vector<core::smart_refctd_ptr<IIncludeGenerator>> m_generators;
179+
std::vector<GeneratorEntry> m_generators;
180+
std::vector<HeaderRoot> m_headerRoots;
108181
core::smart_refctd_ptr<CFileSystemIncludeLoader> m_defaultFileSystemLoader;
109182
};
110183

@@ -134,9 +207,12 @@ class NBL_API2 IShaderCompiler : public core::IReferenceCounted
134207
std::string_view sourceIdentifier = "";
135208
system::logger_opt_ptr logger = nullptr;
136209
const CIncludeFinder* includeFinder = nullptr;
210+
CIncludeFinder::SSessionCache* readIncludeSessionCache = nullptr;
211+
CIncludeFinder::SSessionCache* writeIncludeSessionCache = nullptr;
137212
std::span<const SMacroDefinition> extraDefines = {};
138213
E_SPIRV_VERSION targetSpirvVersion = E_SPIRV_VERSION::ESV_1_6;
139214
bool depfile = false;
215+
bool preserveComments = false;
140216
system::path depfilePath = {};
141217
std::function<void(std::string_view)> onPartialOutputOnFailure = {};
142218
};

‎include/nbl/builtin/hlsl/bxdf/reflection/beckmann.hlsl‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,9 +4,7 @@
44
#ifndef _NBL_BUILTIN_HLSL_BXDF_REFLECTION_BECKMANN_INCLUDED_
55
#define _NBL_BUILTIN_HLSL_BXDF_REFLECTION_BECKMANN_INCLUDED_
66

7-
#include "nbl/builtin/hlsl/bxdf/common.hlsl"
87
#include "nbl/builtin/hlsl/bxdf/bxdf_traits.hlsl"
9-
#include "nbl/builtin/hlsl/sampling/cos_weighted_spheres.hlsl"
108
#include "nbl/builtin/hlsl/bxdf/ndf/beckmann.hlsl"
119
#include "nbl/builtin/hlsl/bxdf/base/cook_torrance_base.hlsl"
1210

‎include/nbl/builtin/hlsl/bxdf/reflection/ggx.hlsl‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,9 +4,7 @@
44
#ifndef _NBL_BUILTIN_HLSL_BXDF_REFLECTION_GGX_INCLUDED_
55
#define _NBL_BUILTIN_HLSL_BXDF_REFLECTION_GGX_INCLUDED_
66

7-
#include "nbl/builtin/hlsl/bxdf/common.hlsl"
87
#include "nbl/builtin/hlsl/bxdf/bxdf_traits.hlsl"
9-
#include "nbl/builtin/hlsl/sampling/cos_weighted_spheres.hlsl"
108
#include "nbl/builtin/hlsl/bxdf/ndf/ggx.hlsl"
119
#include "nbl/builtin/hlsl/bxdf/base/cook_torrance_base.hlsl"
1210

‎include/nbl/builtin/hlsl/bxdf/transmission/beckmann.hlsl‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,10 +4,8 @@
44
#ifndef _NBL_BUILTIN_HLSL_BXDF_TRANSMISSION_BECKMANN_INCLUDED_
55
#define _NBL_BUILTIN_HLSL_BXDF_TRANSMISSION_BECKMANN_INCLUDED_
66

7-
#include "nbl/builtin/hlsl/bxdf/common.hlsl"
87
#include "nbl/builtin/hlsl/bxdf/bxdf_traits.hlsl"
9-
#include "nbl/builtin/hlsl/sampling/cos_weighted_spheres.hlsl"
10-
#include "nbl/builtin/hlsl/bxdf/reflection.hlsl"
8+
#include "nbl/builtin/hlsl/bxdf/ndf/beckmann.hlsl"
119
#include "nbl/builtin/hlsl/bxdf/base/cook_torrance_base.hlsl"
1210

1311
namespace nbl

‎include/nbl/builtin/hlsl/bxdf/transmission/ggx.hlsl‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,10 +4,7 @@
44
#ifndef _NBL_BUILTIN_HLSL_BXDF_TRANSMISSION_GGX_INCLUDED_
55
#define _NBL_BUILTIN_HLSL_BXDF_TRANSMISSION_GGX_INCLUDED_
66

7-
#include "nbl/builtin/hlsl/bxdf/common.hlsl"
87
#include "nbl/builtin/hlsl/bxdf/bxdf_traits.hlsl"
9-
#include "nbl/builtin/hlsl/sampling/cos_weighted_spheres.hlsl"
10-
#include "nbl/builtin/hlsl/bxdf/reflection.hlsl"
118
#include "nbl/builtin/hlsl/bxdf/base/cook_torrance_base.hlsl"
129

1310
namespace nbl

0 commit comments

Comments
 (0)