Skip to content

Commit daf1fe3

Browse files
committed
Add include session cache plumbing
1 parent 40e1e1e commit daf1fe3

5 files changed

Lines changed: 147 additions & 22 deletions

File tree

include/nbl/asset/utils/IShaderCompiler.h

Lines changed: 27 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,8 @@
1717
#include "nbl/builtin/hlsl/enums.hlsl"
1818

1919
#include <functional>
20+
#include <mutex>
21+
#include <unordered_set>
2022

2123
namespace nbl::asset
2224
{
@@ -74,17 +76,39 @@ class NBL_API2 IShaderCompiler : public core::IReferenceCounted
7476
class NBL_API2 CIncludeFinder : public core::IReferenceCounted
7577
{
7678
public:
79+
struct SSessionCache
80+
{
81+
enum class E_LOOKUP_RESULT : uint8_t
82+
{
83+
Miss,
84+
Missing,
85+
Found
86+
};
87+
88+
explicit SSessionCache(const bool threadSafe = false) : threadSafe(threadSafe) {}
89+
90+
void clear();
91+
E_LOOKUP_RESULT lookup(const std::string& key, IIncludeLoader::found_t& result) const;
92+
void store(const std::string& key, IIncludeLoader::found_t result);
93+
94+
bool threadSafe = false;
95+
96+
mutable std::mutex mutex;
97+
core::unordered_map<std::string, IIncludeLoader::found_t> found;
98+
core::unordered_set<std::string> missing;
99+
};
100+
77101
CIncludeFinder(core::smart_refctd_ptr<system::ISystem>&& system);
78102

79103
// ! includes within <>
80104
// @param requestingSourceDir: the directory where the incude was requested
81105
// @param includeName: the string within <> of the include preprocessing directive
82-
IIncludeLoader::found_t getIncludeStandard(const system::path& requestingSourceDir, const std::string& includeName, bool needHash = true) const;
106+
IIncludeLoader::found_t getIncludeStandard(const system::path& requestingSourceDir, const std::string& includeName, bool needHash = true, SSessionCache* sessionCache = nullptr) const;
83107

84108
// ! includes within ""
85109
// @param requestingSourceDir: the directory where the incude was requested
86110
// @param includeName: the string within "" of the include preprocessing directive
87-
IIncludeLoader::found_t getIncludeRelative(const system::path& requestingSourceDir, const std::string& includeName, bool needHash = true) const;
111+
IIncludeLoader::found_t getIncludeRelative(const system::path& requestingSourceDir, const std::string& includeName, bool needHash = true, SSessionCache* sessionCache = nullptr) const;
88112

89113
inline core::smart_refctd_ptr<CFileSystemIncludeLoader> getDefaultFileSystemLoader() const { return m_defaultFileSystemLoader; }
90114

@@ -134,6 +158,7 @@ class NBL_API2 IShaderCompiler : public core::IReferenceCounted
134158
std::string_view sourceIdentifier = "";
135159
system::logger_opt_ptr logger = nullptr;
136160
const CIncludeFinder* includeFinder = nullptr;
161+
mutable CIncludeFinder::SSessionCache* includeSessionCache = nullptr;
137162
std::span<const SMacroDefinition> extraDefines = {};
138163
E_SPIRV_VERSION targetSpirvVersion = E_SPIRV_VERSION::ESV_1_6;
139164
bool depfile = false;

src/nbl/asset/utils/CGLSLCompiler.cpp

Lines changed: 14 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -44,11 +44,12 @@ namespace nbl::asset::impl
4444
class Includer : public shaderc::CompileOptions::IncluderInterface
4545
{
4646
const IShaderCompiler::CIncludeFinder* m_defaultIncludeFinder;
47+
IShaderCompiler::CIncludeFinder::SSessionCache* m_includeSessionCache;
4748
const system::ISystem* m_system;
4849
const uint32_t m_maxInclCnt;
4950

5051
public:
51-
Includer(const IShaderCompiler::CIncludeFinder* _inclFinder, const system::ISystem* _fs, uint32_t _maxInclCnt) : m_defaultIncludeFinder(_inclFinder), m_system(_fs), m_maxInclCnt{ _maxInclCnt } {}
52+
Includer(const IShaderCompiler::CIncludeFinder* _inclFinder, IShaderCompiler::CIncludeFinder::SSessionCache* _includeSessionCache, const system::ISystem* _fs, uint32_t _maxInclCnt) : m_defaultIncludeFinder(_inclFinder), m_includeSessionCache(_includeSessionCache), m_system(_fs), m_maxInclCnt{ _maxInclCnt } {}
5253

5354
//_requesting_source in top level #include's is what shaderc::Compiler's compiling functions get as `input_file_name` parameter
5455
//so in order for properly working relative #include's (""-type) `input_file_name` has to be path to file from which the GLSL source really come from
@@ -81,11 +82,11 @@ namespace nbl::asset::impl
8182
IShaderCompiler::IIncludeLoader::found_t result;
8283
if (_type == shaderc_include_type_relative)
8384
{
84-
result = m_defaultIncludeFinder->getIncludeRelative(relDir, _requested_source);
85+
result = m_defaultIncludeFinder->getIncludeRelative(relDir, _requested_source, true, m_includeSessionCache);
8586
}
8687
else //shaderc_include_type_standard
8788
{
88-
result = m_defaultIncludeFinder->getIncludeStandard(relDir, _requested_source);
89+
result = m_defaultIncludeFinder->getIncludeStandard(relDir, _requested_source, true, m_includeSessionCache);
8990
}
9091

9192
if (!result)
@@ -136,10 +137,15 @@ CGLSLCompiler::CGLSLCompiler(core::smart_refctd_ptr<system::ISystem>&& system)
136137

137138
std::string CGLSLCompiler::preprocessShader(std::string&& code, IShader::E_SHADER_STAGE& stage, const SPreprocessorOptions& preprocessOptions, std::vector<CCache::SEntry::SPreprocessingDependency>* dependencies) const
138139
{
140+
auto effectiveOptions = preprocessOptions;
141+
IShaderCompiler::CIncludeFinder::SSessionCache localIncludeSessionCache;
142+
if (effectiveOptions.includeFinder && !effectiveOptions.includeSessionCache)
143+
effectiveOptions.includeSessionCache = &localIncludeSessionCache;
144+
139145
if (!preprocessOptions.extraDefines.empty())
140146
{
141147
std::ostringstream insertion;
142-
for (const auto& define : preprocessOptions.extraDefines)
148+
for (const auto& define : effectiveOptions.extraDefines)
143149
insertion << "#define " << define.identifier << " " << define.definition << "\n";
144150
insertIntoStart(code,std::move(insertion));
145151
}
@@ -149,15 +155,15 @@ std::string CGLSLCompiler::preprocessShader(std::string&& code, IShader::E_SHADE
149155
shaderc::CompileOptions options;
150156
options.SetTargetSpirv(shaderc_spirv_version_1_6);
151157

152-
if (preprocessOptions.includeFinder != nullptr)
158+
if (effectiveOptions.includeFinder != nullptr)
153159
{
154-
options.SetIncluder(std::make_unique<impl::Includer>(preprocessOptions.includeFinder, m_system.get(), /*maxSelfInclusionCount*/5));//custom #include handler
160+
options.SetIncluder(std::make_unique<impl::Includer>(effectiveOptions.includeFinder, effectiveOptions.includeSessionCache, m_system.get(), /*maxSelfInclusionCount*/5));//custom #include handler
155161
}
156162
const shaderc_shader_kind scstage = stage == IShader::E_SHADER_STAGE::ESS_UNKNOWN ? shaderc_glsl_infer_from_source : ESStoShadercEnum(stage);
157-
auto res = comp.PreprocessGlsl(code, scstage, preprocessOptions.sourceIdentifier.data(), options);
163+
auto res = comp.PreprocessGlsl(code, scstage, effectiveOptions.sourceIdentifier.data(), options);
158164

159165
if (res.GetCompilationStatus() != shaderc_compilation_status_success) {
160-
preprocessOptions.logger.log("%s\n", system::ILogger::ELL_ERROR, res.GetErrorMessage().c_str());
166+
effectiveOptions.logger.log("%s\n", system::ILogger::ELL_ERROR, res.GetErrorMessage().c_str());
161167
return nullptr;
162168
}
163169

src/nbl/asset/utils/CHLSLCompiler.cpp

Lines changed: 12 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -454,12 +454,17 @@ static std::string preprocessShaderImpl(
454454
std::vector<IShaderCompiler::CCache::SEntry::SPreprocessingDependency>* dependencies,
455455
system::ISystem* system)
456456
{
457-
const bool depfileEnabled = preprocessOptions.depfile;
457+
auto effectiveOptions = preprocessOptions;
458+
IShaderCompiler::CIncludeFinder::SSessionCache localIncludeSessionCache;
459+
if (effectiveOptions.includeFinder && !effectiveOptions.includeSessionCache)
460+
effectiveOptions.includeSessionCache = &localIncludeSessionCache;
461+
462+
const bool depfileEnabled = effectiveOptions.depfile;
458463
if (depfileEnabled)
459464
{
460-
if (preprocessOptions.depfilePath.empty())
465+
if (effectiveOptions.depfilePath.empty())
461466
{
462-
preprocessOptions.logger.log("Depfile path is empty.", system::ILogger::ELL_ERROR);
467+
effectiveOptions.logger.log("Depfile path is empty.", system::ILogger::ELL_ERROR);
463468
return {};
464469
}
465470
}
@@ -473,7 +478,7 @@ static std::string preprocessShaderImpl(
473478
ensureTrailingNewline(code);
474479

475480
// preprocess
476-
core::string resolvedString = nbl::wave::preprocess(code, preprocessOptions, bool(dependenciesOut),
481+
core::string resolvedString = nbl::wave::preprocess(code, effectiveOptions, bool(dependenciesOut),
477482
[&dxc_compile_flags_override, &stage, &dependenciesOut](nbl::wave::context& context) -> void
478483
{
479484
if (context.get_hooks().m_dxc_compile_flags_override.size() != 0)
@@ -494,13 +499,13 @@ static std::string preprocessShaderImpl(
494499
if (depfileEnabled)
495500
{
496501
IShaderCompiler::DepfileWriteParams params = {};
497-
const std::string depfilePathString = preprocessOptions.depfilePath.generic_string();
502+
const std::string depfilePathString = effectiveOptions.depfilePath.generic_string();
498503
params.depfilePath = depfilePathString;
499-
params.sourceIdentifier = preprocessOptions.sourceIdentifier;
504+
params.sourceIdentifier = effectiveOptions.sourceIdentifier;
500505
if (!params.sourceIdentifier.empty())
501506
params.workingDirectory = std::filesystem::path(std::string(params.sourceIdentifier)).parent_path();
502507
params.system = system;
503-
if (!IShaderCompiler::writeDepfile(params, *dependenciesOut, preprocessOptions.includeFinder, preprocessOptions.logger))
508+
if (!IShaderCompiler::writeDepfile(params, *dependenciesOut, effectiveOptions.includeFinder, effectiveOptions.logger))
504509
return {};
505510
}
506511

src/nbl/asset/utils/IShaderCompiler.cpp

Lines changed: 90 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -651,6 +651,68 @@ std::string normalizeIncludeLookupName(const std::string& includeName)
651651
return includeName.substr(1ull);
652652
}
653653

654+
template<typename Func>
655+
auto withSessionCacheLock(IShaderCompiler::CIncludeFinder::SSessionCache* cache, Func&& func) -> decltype(func())
656+
{
657+
if (cache && cache->threadSafe)
658+
{
659+
std::lock_guard lock(cache->mutex);
660+
return func();
661+
}
662+
return func();
663+
}
664+
665+
std::string makeIncludeSessionCacheKey(const system::path& requestingSourceDir, std::string_view includeName, const bool needHash, const char mode)
666+
{
667+
const auto requestingDir = requestingSourceDir.generic_string();
668+
std::string key;
669+
key.reserve(requestingDir.size() + includeName.size() + 4ull);
670+
key.push_back(mode);
671+
key.push_back('\n');
672+
key.push_back(needHash ? 'H' : 'N');
673+
key.push_back('\n');
674+
key.append(requestingDir);
675+
key.push_back('\n');
676+
key.append(includeName.data(), includeName.size());
677+
return key;
678+
}
679+
680+
}
681+
682+
void IShaderCompiler::CIncludeFinder::SSessionCache::clear()
683+
{
684+
withSessionCacheLock(this, [&]()
685+
{
686+
found.clear();
687+
missing.clear();
688+
});
689+
}
690+
691+
auto IShaderCompiler::CIncludeFinder::SSessionCache::lookup(const std::string& key, IIncludeLoader::found_t& result) const -> E_LOOKUP_RESULT
692+
{
693+
return withSessionCacheLock(const_cast<SSessionCache*>(this), [&]() -> E_LOOKUP_RESULT
694+
{
695+
if (const auto foundIt = found.find(key); foundIt != found.end())
696+
{
697+
result = foundIt->second;
698+
return E_LOOKUP_RESULT::Found;
699+
}
700+
if (missing.contains(key))
701+
return E_LOOKUP_RESULT::Missing;
702+
return E_LOOKUP_RESULT::Miss;
703+
});
704+
}
705+
706+
void IShaderCompiler::CIncludeFinder::SSessionCache::store(const std::string& key, IIncludeLoader::found_t result)
707+
{
708+
withSessionCacheLock(this, [&]()
709+
{
710+
missing.erase(key);
711+
if (result)
712+
found.insert_or_assign(key, std::move(result));
713+
else
714+
missing.insert(key);
715+
});
654716
}
655717

656718
IShaderCompiler::CIncludeFinder::CIncludeFinder(core::smart_refctd_ptr<system::ISystem>&& system)
@@ -663,10 +725,21 @@ IShaderCompiler::CIncludeFinder::CIncludeFinder(core::smart_refctd_ptr<system::I
663725
// @param requestingSourceDir: the directory where the incude was requested
664726
// @param includeName: the string within <> of the include preprocessing directive
665727
// @param
666-
auto IShaderCompiler::CIncludeFinder::getIncludeStandard(const system::path& requestingSourceDir, const std::string& includeName, bool needHash) const -> IIncludeLoader::found_t
728+
auto IShaderCompiler::CIncludeFinder::getIncludeStandard(const system::path& requestingSourceDir, const std::string& includeName, bool needHash, SSessionCache* sessionCache) const -> IIncludeLoader::found_t
667729
{
668730
const auto lookupName = normalizeIncludeLookupName(includeName);
731+
const auto cacheKey = makeIncludeSessionCacheKey(requestingSourceDir, lookupName, needHash, 'S');
669732
IShaderCompiler::IIncludeLoader::found_t retVal;
733+
switch (sessionCache ? sessionCache->lookup(cacheKey, retVal) : SSessionCache::E_LOOKUP_RESULT::Miss)
734+
{
735+
case SSessionCache::E_LOOKUP_RESULT::Found:
736+
return retVal;
737+
case SSessionCache::E_LOOKUP_RESULT::Missing:
738+
return {};
739+
case SSessionCache::E_LOOKUP_RESULT::Miss:
740+
break;
741+
}
742+
670743
if (auto contents = tryIncludeGenerators(lookupName))
671744
retVal = std::move(contents);
672745
else if (auto contents = trySearchPaths(lookupName, needHash))
@@ -680,16 +753,29 @@ auto IShaderCompiler::CIncludeFinder::getIncludeStandard(const system::path& req
680753
hasher.update(reinterpret_cast<uint8_t*>(retVal.contents.data()), retVal.contents.size() * (sizeof(char) / sizeof(uint8_t)));
681754
retVal.hash = static_cast<core::blake3_hash_t>(hasher);
682755
}
756+
if (sessionCache)
757+
sessionCache->store(cacheKey, retVal);
683758
return retVal;
684759
}
685760

686761
// ! includes within ""
687762
// @param requestingSourceDir: the directory where the incude was requested
688763
// @param includeName: the string within "" of the include preprocessing directive
689-
auto IShaderCompiler::CIncludeFinder::getIncludeRelative(const system::path& requestingSourceDir, const std::string& includeName, bool needHash) const -> IIncludeLoader::found_t
764+
auto IShaderCompiler::CIncludeFinder::getIncludeRelative(const system::path& requestingSourceDir, const std::string& includeName, bool needHash, SSessionCache* sessionCache) const -> IIncludeLoader::found_t
690765
{
691766
const auto lookupName = normalizeIncludeLookupName(includeName);
767+
const auto cacheKey = makeIncludeSessionCacheKey(requestingSourceDir, lookupName, needHash, 'R');
692768
IShaderCompiler::IIncludeLoader::found_t retVal;
769+
switch (sessionCache ? sessionCache->lookup(cacheKey, retVal) : SSessionCache::E_LOOKUP_RESULT::Miss)
770+
{
771+
case SSessionCache::E_LOOKUP_RESULT::Found:
772+
return retVal;
773+
case SSessionCache::E_LOOKUP_RESULT::Missing:
774+
return {};
775+
case SSessionCache::E_LOOKUP_RESULT::Miss:
776+
break;
777+
}
778+
693779
if (auto contents = m_defaultFileSystemLoader->getInclude(requestingSourceDir.string(), lookupName, needHash))
694780
retVal = std::move(contents);
695781
else retVal = std::move(trySearchPaths(lookupName, needHash));
@@ -700,6 +786,8 @@ auto IShaderCompiler::CIncludeFinder::getIncludeRelative(const system::path& req
700786
hasher.update(reinterpret_cast<uint8_t*>(retVal.contents.data()), retVal.contents.size() * (sizeof(char) / sizeof(uint8_t)));
701787
retVal.hash = static_cast<core::blake3_hash_t>(hasher);
702788
}
789+
if (sessionCache)
790+
sessionCache->store(cacheKey, retVal);
703791
return retVal;
704792
}
705793

src/nbl/asset/utils/waveContext.h

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -275,7 +275,7 @@ struct load_to_string final
275275
struct preprocessing_hooks final : public boost::wave::context_policies::default_preprocessing_hooks
276276
{
277277
preprocessing_hooks(const nbl::asset::IShaderCompiler::SPreprocessorOptions& _preprocessOptions)
278-
: m_includeFinder(_preprocessOptions.includeFinder), m_logger(_preprocessOptions.logger), m_preserveComments(_preprocessOptions.preserveComments), m_pragmaStage(nbl::asset::IShader::E_SHADER_STAGE::ESS_UNKNOWN), m_dxc_compile_flags_override()
278+
: m_includeFinder(_preprocessOptions.includeFinder), m_includeSessionCache(_preprocessOptions.includeSessionCache), m_logger(_preprocessOptions.logger), m_preserveComments(_preprocessOptions.preserveComments), m_pragmaStage(nbl::asset::IShader::E_SHADER_STAGE::ESS_UNKNOWN), m_dxc_compile_flags_override()
279279
{
280280
hash_token_occurences = 0;
281281
}
@@ -382,6 +382,7 @@ struct preprocessing_hooks final : public boost::wave::context_policies::default
382382
}
383383

384384
const asset::IShaderCompiler::CIncludeFinder* m_includeFinder;
385+
asset::IShaderCompiler::CIncludeFinder::SSessionCache* m_includeSessionCache;
385386
system::logger_opt_ptr m_logger;
386387
bool m_preserveComments;
387388
asset::IShader::E_SHADER_STAGE m_pragmaStage;
@@ -832,11 +833,11 @@ template<> inline bool boost::wave::impl::pp_iterator_functor<nbl::wave::context
832833
if (perfStats.enabled)
833834
++perfStats.includeLookupCount;
834835
if (is_system) {
835-
result = includeFinder->getIncludeStandard(ctx.get_current_directory(), file_path, needHash);
836+
result = includeFinder->getIncludeStandard(ctx.get_current_directory(), file_path, needHash, ctx.get_hooks().m_includeSessionCache);
836837
standardInclude = true;
837838
}
838839
else {
839-
result = includeFinder->getIncludeRelative(ctx.get_current_directory(), file_path, needHash);
840+
result = includeFinder->getIncludeRelative(ctx.get_current_directory(), file_path, needHash, ctx.get_hooks().m_includeSessionCache);
840841
standardInclude = false;
841842
}
842843
}

0 commit comments

Comments
 (0)