Skip to content

Commit d9d9603

Browse files
keptsecretkeptsecret
committed
merge master, fix conflicts
2 parents 965e9fc + 9cfb3f0 commit d9d9603

8 files changed

Lines changed: 139 additions & 20 deletions

File tree

‎examples_tests‎

‎include/nbl/asset/utils/IShaderCompiler.h‎

Lines changed: 18 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -245,6 +245,9 @@ class NBL_API2 IShaderCompiler : public core::IReferenceCounted
245245
@includeFinder Optional parameter; if not nullptr, it will resolve the includes in the code
246246
@maxSelfInclusionCount used only when includeFinder is not nullptr
247247
@extraDefines adds extra defines to the shader before compilation
248+
@optimizerIsExtraPasses Instead of entirely replacing the default optimization passes, run the provided passes after the default ones
249+
@preprocessedOutputPath If not empty, the preprocessed shader will be saved to this path
250+
@spvOutputPath If not empty, the compiled SPIR-V binary will be saved to this path
248251
*/
249252
struct SCompilerOptions
250253
{
@@ -261,6 +264,9 @@ class NBL_API2 IShaderCompiler : public core::IReferenceCounted
261264
SPreprocessorOptions preprocessorOptions = {};
262265
CCache* readCache = nullptr;
263266
CCache* writeCache = nullptr;
267+
bool optimizerIsExtraPasses = false; // Instead of disabling the default opt passes, run the provided optimization passes at the end
268+
std::string preprocessedOutputPath = "";
269+
std::string spvOutputPath = "";
264270
};
265271

266272
class CCache final : public IReferenceCounted
@@ -269,7 +275,7 @@ class NBL_API2 IShaderCompiler : public core::IReferenceCounted
269275

270276
public:
271277
// Used to check compatibility of Caches before reading
272-
constexpr static inline std::string_view VERSION = "1.1.0";
278+
constexpr static inline std::string_view VERSION = "1.1.1";
273279

274280
static auto const SHADER_BUFFER_SIZE_BYTES = sizeof(uint64_t) / sizeof(uint8_t); // It's obviously 8
275281

@@ -360,16 +366,16 @@ class NBL_API2 IShaderCompiler : public core::IReferenceCounted
360366
{
361367
public:
362368
inline bool operator==(const SCompilerArgs& other) const {
363-
bool retVal = true;
364-
if (stage != other.stage || targetSpirvVersion != other.targetSpirvVersion || debugInfoFlags != other.debugInfoFlags || preprocessorArgs != other.preprocessorArgs) retVal = false;
365-
if (optimizerPasses.size() != other.optimizerPasses.size()) retVal = false;
369+
if (stage != other.stage || targetSpirvVersion != other.targetSpirvVersion || debugInfoFlags != other.debugInfoFlags || preprocessorArgs != other.preprocessorArgs || optimizerIsExtraPasses != other.optimizerIsExtraPasses)
370+
return false;
371+
if (optimizerPasses.size() != other.optimizerPasses.size())
372+
return false;
366373
for (auto passesIt = optimizerPasses.begin(), otherPassesIt = other.optimizerPasses.begin(); passesIt != optimizerPasses.end(); passesIt++, otherPassesIt++) {
367374
if (*passesIt != *otherPassesIt) {
368-
retVal = false;
369-
break;
375+
return false;
370376
}
371377
}
372-
return retVal;
378+
return true;
373379
}
374380

375381
private:
@@ -389,6 +395,7 @@ class NBL_API2 IShaderCompiler : public core::IReferenceCounted
389395
// Only SEntry should instantiate this struct
390396
SCompilerArgs(const SCompilerOptions& options)
391397
: stage(options.stage), targetSpirvVersion(options.preprocessorOptions.targetSpirvVersion), debugInfoFlags(options.debugInfoFlags), preprocessorArgs(options.preprocessorOptions)
398+
, optimizerIsExtraPasses(options.optimizerIsExtraPasses)
392399
{
393400
if (options.spirvOptimizer) {
394401
for (auto pass : options.spirvOptimizer->getPasses())
@@ -399,6 +406,7 @@ class NBL_API2 IShaderCompiler : public core::IReferenceCounted
399406
IShader::E_SHADER_STAGE stage;
400407
E_SPIRV_VERSION targetSpirvVersion;
401408
std::vector<ISPIRVOptimizer::E_OPTIMIZER_PASS> optimizerPasses;
409+
bool optimizerIsExtraPasses;
402410
core::bitflag<E_DEBUG_INFO_FLAGS> debugInfoFlags;
403411
SPreprocessorArgs preprocessorArgs;
404412
};
@@ -458,6 +466,9 @@ class NBL_API2 IShaderCompiler : public core::IReferenceCounted
458466

459467
bool setContent(const asset::ICPUBuffer* uncompressedSpirvBuffer);
460468

469+
IShader::E_SHADER_STAGE getShaderStage() const { return compilerArgs.stage; }
470+
SPreprocessorArgs getPreprocessorArgs() const { return compilerArgs.preprocessorArgs; }
471+
461472
core::smart_refctd_ptr<IShader> decompressShader() const;
462473

463474
// TODO: make some of these private

‎include/nbl/builtin/hlsl/bxdf/config.hlsl‎

Lines changed: 12 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ namespace bxdf
1616
// TODO should just check `is_base_of` (but not possible right now cause of DXC limitations)
1717
namespace config_concepts
1818
{
19+
// TODO: need to require either anisotropic_interaction_type or isotropic_interaction_type
1920
#define NBL_CONCEPT_NAME BasicConfiguration
2021
#define NBL_CONCEPT_TPLT_PRM_KINDS (typename)
2122
#define NBL_CONCEPT_TPLT_PRM_NAMES (T)
@@ -26,10 +27,14 @@ NBL_CONCEPT_END(
2627
((NBL_CONCEPT_REQ_TYPE)(T::anisotropic_interaction_type))
2728
((NBL_CONCEPT_REQ_TYPE)(T::sample_type))
2829
((NBL_CONCEPT_REQ_TYPE)(T::spectral_type))
30+
((NBL_CONCEPT_REQ_TYPE_ALIAS_CONCEPT)(surface_interactions::Isotropic, typename T::anisotropic_interaction_type))
31+
((NBL_CONCEPT_REQ_TYPE_ALIAS_CONCEPT)(LightSample, typename T::sample_type))
32+
((NBL_CONCEPT_REQ_TYPE_ALIAS_CONCEPT)(concepts::FloatingPointLikeVectorial, typename T::spectral_type))
2933
);
3034
#undef conf
3135
#include <nbl/builtin/hlsl/concepts/__end.hlsl>
3236

37+
// TODO: need to require either anisocache_type or isocache_type, but require anisocache_type if anisotropic_interaction_type was provided
3338
#define NBL_CONCEPT_NAME MicrofacetConfiguration
3439
#define NBL_CONCEPT_TPLT_PRM_KINDS (typename)
3540
#define NBL_CONCEPT_TPLT_PRM_NAMES (T)
@@ -39,6 +44,7 @@ NBL_CONCEPT_BEGIN(1)
3944
NBL_CONCEPT_END(
4045
((NBL_CONCEPT_REQ_TYPE_ALIAS_CONCEPT)(BasicConfiguration, T))
4146
((NBL_CONCEPT_REQ_TYPE)(T::anisocache_type))
47+
((NBL_CONCEPT_REQ_TYPE_ALIAS_CONCEPT)(ReadableIsotropicMicrofacetCache, typename T::anisocache_type))
4248
);
4349
#undef conf
4450
#include <nbl/builtin/hlsl/concepts/__end.hlsl>
@@ -49,8 +55,8 @@ struct SConfiguration;
4955

5056
#define CONF_ISO LightSample<LS> && surface_interactions::Isotropic<Interaction> && !surface_interactions::Anisotropic<Interaction> && concepts::FloatingPointLikeVectorial<Spectrum>
5157

52-
template<class LS, class Interaction, class Spectrum>
53-
NBL_PARTIAL_REQ_TOP(CONF_ISO)
58+
// TODO: remove the `IsAnisotropic` and deduce it from whether `anisotropic_interaction_type` was provided
59+
template<class LS, class Interaction, class Spectrum> NBL_PARTIAL_REQ_TOP(CONF_ISO)
5460
struct SConfiguration<LS,Interaction,Spectrum NBL_PARTIAL_REQ_BOT(CONF_ISO) >
5561
#undef CONF_ISO
5662
{
@@ -63,8 +69,7 @@ struct SConfiguration<LS,Interaction,Spectrum NBL_PARTIAL_REQ_BOT(CONF_ISO) >
6369

6470
#define CONF_ANISO LightSample<LS> && surface_interactions::Anisotropic<Interaction> && concepts::FloatingPointLikeVectorial<Spectrum>
6571

66-
template<class LS, class Interaction, class Spectrum>
67-
NBL_PARTIAL_REQ_TOP(CONF_ANISO)
72+
template<class LS, class Interaction, class Spectrum> NBL_PARTIAL_REQ_TOP(CONF_ANISO)
6873
struct SConfiguration<LS,Interaction,Spectrum NBL_PARTIAL_REQ_BOT(CONF_ANISO) >
6974
#undef CONF_ANISO
7075
{
@@ -81,8 +86,7 @@ struct SMicrofacetConfiguration;
8186

8287
#define MICROFACET_CONF_ISO LightSample<LS> && surface_interactions::Isotropic<Interaction> && !surface_interactions::Anisotropic<Interaction> && CreatableIsotropicMicrofacetCache<MicrofacetCache> && !AnisotropicMicrofacetCache<MicrofacetCache> && concepts::FloatingPointLikeVectorial<Spectrum>
8388

84-
template<class LS, class Interaction, class MicrofacetCache, class Spectrum>
85-
NBL_PARTIAL_REQ_TOP(MICROFACET_CONF_ISO)
89+
template<class LS, class Interaction, class MicrofacetCache, class Spectrum> NBL_PARTIAL_REQ_TOP(MICROFACET_CONF_ISO)
8690
struct SMicrofacetConfiguration<LS,Interaction,MicrofacetCache,Spectrum NBL_PARTIAL_REQ_BOT(MICROFACET_CONF_ISO) > : SConfiguration<LS, Interaction, Spectrum>
8791
#undef MICROFACET_CONF_ISO
8892
{
@@ -95,8 +99,7 @@ struct SMicrofacetConfiguration<LS,Interaction,MicrofacetCache,Spectrum NBL_PART
9599

96100
#define MICROFACET_CONF_ANISO LightSample<LS> && surface_interactions::Anisotropic<Interaction> && AnisotropicMicrofacetCache<MicrofacetCache> && concepts::FloatingPointLikeVectorial<Spectrum>
97101

98-
template<class LS, class Interaction, class MicrofacetCache, class Spectrum>
99-
NBL_PARTIAL_REQ_TOP(MICROFACET_CONF_ANISO)
102+
template<class LS, class Interaction, class MicrofacetCache, class Spectrum> NBL_PARTIAL_REQ_TOP(MICROFACET_CONF_ANISO)
100103
struct SMicrofacetConfiguration<LS,Interaction,MicrofacetCache,Spectrum NBL_PARTIAL_REQ_BOT(MICROFACET_CONF_ANISO) > : SConfiguration<LS, Interaction, Spectrum>
101104
#undef MICROFACET_CONF_ANISO
102105
{
@@ -109,7 +112,7 @@ struct SMicrofacetConfiguration<LS,Interaction,MicrofacetCache,Spectrum NBL_PART
109112

110113
#define NBL_BXDF_CONFIG_ALIAS(TYPE,CONFIG) using TYPE = typename CONFIG::TYPE
111114

112-
115+
// TODO: make the `monochrome_type` take `scalar_type` of spectral_type
113116
#define BXDF_CONFIG_TYPE_ALIASES(Config) NBL_BXDF_CONFIG_ALIAS(sample_type, Config);\
114117
using scalar_type = typename sample_type::scalar_type;\
115118
using vector2_type = vector<scalar_type,2>;\

‎include/nbl/video/ILogicalDevice.h‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -833,6 +833,9 @@ class NBL_API2 ILogicalDevice : public core::IReferenceCounted, public IDeviceMe
833833
std::span<const asset::IShaderCompiler::SMacroDefinition> extraDefines = {};
834834
hlsl::ShaderStage stage = hlsl::ShaderStage::ESS_ALL_OR_LIBRARY;
835835
core::bitflag<asset::IShaderCompiler::E_DEBUG_INFO_FLAGS> debugInfoFlags = asset::IShaderCompiler::E_DEBUG_INFO_FLAGS::EDIF_NONE;
836+
bool optimizerIsExtraPasses = false; // Instead of disabling the default opt passes, run the provided optimization passes at the end
837+
std::string preprocessedOutputPath = "";
838+
std::string spvOutputPath = "";
836839
};
837840
core::smart_refctd_ptr<asset::IShader> compileShader(const SShaderCreationParameters& creationParams);
838841

‎src/nbl/asset/utils/CHLSLCompiler.cpp‎

Lines changed: 45 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -546,6 +546,46 @@ core::smart_refctd_ptr<IShader> CHLSLCompiler::compileToSPIRV_impl(const std::st
546546
auto newCode = preprocessShader(std::string(code), stage, hlslOptions.preprocessorOptions, dxc_compile_flags, dependencies);
547547
if (newCode.empty()) return nullptr;
548548

549+
auto saveToFile = [&](const std::string& filePath, const void* buffer, size_t size, const char* loggerName)
550+
{
551+
core::smart_refctd_ptr<system::IFile> saveFile;
552+
553+
system::ISystem::future_t<core::smart_refctd_ptr<system::IFile>> future;
554+
// Ensure it doesn't exist
555+
m_system->deleteFile(filePath + "_temp");
556+
m_system->createFile(future, filePath + "_temp", system::IFile::ECF_WRITE);
557+
if (future.wait())
558+
{
559+
future.acquire().move_into(saveFile);
560+
if (saveFile)
561+
{
562+
system::IFile::success_t succ;
563+
saveFile->write(succ, buffer, 0, size);
564+
if (!succ)
565+
{
566+
logger.log(std::string("Failed Writing To Temp ") + loggerName + " File.", nbl::system::ILogger::ELL_ERROR);
567+
return;
568+
}
569+
saveFile = nullptr; // drop handle first
570+
m_system->deleteFile(filePath); // safe to delete + rename
571+
auto renameResult = m_system->moveFileOrDirectory(filePath + "_temp", filePath);
572+
if (renameResult)
573+
{
574+
logger.log(std::string("Failed Saving ") + loggerName + " File. Check it's not open.", nbl::system::ILogger::ELL_ERROR);
575+
// Clean up
576+
m_system->deleteFile(filePath + "_temp");
577+
}
578+
}
579+
else
580+
logger.log(std::string("Failed Creating ") + loggerName + " File.", nbl::system::ILogger::ELL_ERROR);
581+
}
582+
else
583+
logger.log(std::string("Failed Creating ") + loggerName + " File.", nbl::system::ILogger::ELL_ERROR);
584+
};
585+
586+
if (!options.preprocessedOutputPath.empty())
587+
saveToFile(options.preprocessedOutputPath, newCode.data(), newCode.size(), "Preprocessed Output");
588+
549589
// Suffix is the shader model version
550590
std::wstring targetProfile(SHADER_MODEL_PROFILE);
551591

@@ -584,13 +624,13 @@ core::smart_refctd_ptr<IShader> CHLSLCompiler::compileToSPIRV_impl(const std::st
584624
// TODO: add entry point to `CHLSLCompiler::SOptions` and handle it properly in `dxc_compile_flags.empty()`
585625
arguments.push_back(L"main");
586626
}
587-
// If a custom SPIR-V optimizer is specified, use that instead of DXC's spirv-opt.
627+
// If a custom SPIR-V optimizer is specified and set to replace default optimization passes, use that instead of DXC's spirv-opt.
588628
// This is how we can get more optimizer options.
589629
//
590630
// Optimization is also delegated to SPIRV-Tools. Right now there are no difference between
591631
// optimization levels greater than zero; they will all invoke the same optimization recipe.
592632
// https://github.com/Microsoft/DirectXShaderCompiler/blob/main/docs/SPIR-V.rst#optimization
593-
if (hlslOptions.spirvOptimizer)
633+
if (hlslOptions.spirvOptimizer && !hlslOptions.optimizerIsExtraPasses)
594634
arguments.push_back(L"-O0");
595635
}
596636
//
@@ -651,6 +691,9 @@ core::smart_refctd_ptr<IShader> CHLSLCompiler::compileToSPIRV_impl(const std::st
651691
if (hlslOptions.spirvOptimizer)
652692
outSpirv = hlslOptions.spirvOptimizer->optimize(outSpirv.get(), logger);
653693

694+
if (outSpirv && !options.spvOutputPath.empty())
695+
saveToFile(options.spvOutputPath, outSpirv->getPointer(), outSpirv->getSize(), "SPIR-V Output");
696+
654697
return core::make_smart_refctd_ptr<asset::IShader>(std::move(outSpirv), IShader::E_CONTENT_TYPE::ECT_SPIRV, hlslOptions.preprocessorOptions.sourceIdentifier.data());
655698
}
656699

‎src/nbl/asset/utils/IShaderCompiler.cpp‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -111,6 +111,7 @@ namespace nbl::system::json {
111111
{ "shaderStage", shaderStage },
112112
{ "spirvVersion", spirvVersion },
113113
{ "optimizerPasses", p.optimizerPasses },
114+
{ "optimizerIsExtraPasses", p.optimizerIsExtraPasses },
114115
{ "debugFlags", debugFlags },
115116
{ "preprocessorArgs", p.preprocessorArgs },
116117
};
@@ -122,6 +123,7 @@ namespace nbl::system::json {
122123
j.at("shaderStage").get_to(shaderStage);
123124
j.at("spirvVersion").get_to(spirvVersion);
124125
j.at("optimizerPasses").get_to(p.optimizerPasses);
126+
j.at("optimizerIsExtraPasses").get_to(p.optimizerIsExtraPasses);
125127
j.at("debugFlags").get_to(debugFlags);
126128
j.at("preprocessorArgs").get_to(p.preprocessorArgs);
127129
p.stage = static_cast<IShader::E_SHADER_STAGE>(shaderStage);
@@ -502,6 +504,43 @@ core::smart_refctd_ptr<IShader> nbl::asset::IShaderCompiler::compileToSPIRV(cons
502504
return IShaderCompiler::writeDepfile(params, dependencies, options.preprocessorOptions.includeFinder, options.preprocessorOptions.logger);
503505
};
504506

507+
auto saveToFile = [&](const std::string& filePath, const void* buffer, size_t size, const char* loggerName)
508+
{
509+
core::smart_refctd_ptr<system::IFile> saveFile;
510+
511+
system::ISystem::future_t<core::smart_refctd_ptr<system::IFile>> future;
512+
// Ensure it doesn't exist
513+
m_system->deleteFile(filePath + "_temp");
514+
m_system->createFile(future, filePath + "_temp", system::IFile::ECF_WRITE);
515+
if (future.wait())
516+
{
517+
future.acquire().move_into(saveFile);
518+
if (saveFile)
519+
{
520+
system::IFile::success_t succ;
521+
saveFile->write(succ, buffer, 0, size);
522+
if (!succ)
523+
{
524+
options.preprocessorOptions.logger.log(std::string("Failed Writing To Temp ") + loggerName + " File.", nbl::system::ILogger::ELL_ERROR);
525+
return;
526+
}
527+
saveFile = nullptr; // drop handle first
528+
m_system->deleteFile(filePath);
529+
auto renameResult = m_system->moveFileOrDirectory(filePath + "_temp", filePath);
530+
if (renameResult)
531+
{
532+
options.preprocessorOptions.logger.log(std::string("Failed Saving ") + loggerName + " File. Check it's not open.", nbl::system::ILogger::ELL_ERROR);
533+
// Clean up
534+
m_system->deleteFile(filePath + "_temp");
535+
}
536+
}
537+
else
538+
options.preprocessorOptions.logger.log(std::string("Failed Creating ") + loggerName + " File.", nbl::system::ILogger::ELL_ERROR);
539+
}
540+
else
541+
options.preprocessorOptions.logger.log(std::string("Failed Creating ") + loggerName + " File.", nbl::system::ILogger::ELL_ERROR);
542+
};
543+
505544
CCache::SEntry entry;
506545
if (options.readCache || options.writeCache)
507546
entry = CCache::SEntry(code, options);
@@ -519,6 +558,22 @@ core::smart_refctd_ptr<IShader> nbl::asset::IShaderCompiler::compileToSPIRV(cons
519558
auto shader = found->decompressShader();
520559
if (depfileEnabled && !writeDepfileFromDependencies(found->dependencies))
521560
return nullptr;
561+
if (!options.spvOutputPath.empty())
562+
saveToFile(options.spvOutputPath, shader->getContent()->getPointer(), shader->getContent()->getSize(), "SPIR-V");
563+
// Entry doesn't store preprocessed shader, so preprocess it
564+
if (!options.preprocessedOutputPath.empty())
565+
{
566+
// copy shader contents
567+
std::string shaderContents = found->mainFileContents;
568+
auto stage = options.stage;
569+
// Create a copy, cache hit means dependencies haven't changed but there is no function that takes a const pointer. This is meant for debug anyway.
570+
auto deps = found->dependencies;
571+
auto preprocessedShader = preprocessShader(std::move(shaderContents), stage, options.preprocessorOptions, &deps);
572+
if (preprocessedShader.empty())
573+
options.preprocessorOptions.logger.log("Failed to preprocess shader for output.", nbl::system::ILogger::ELL_ERROR);
574+
else
575+
saveToFile(options.preprocessedOutputPath, preprocessedShader.data(), preprocessedShader.size(), "Preprocessed Shader");
576+
}
522577
return shader;
523578
}
524579
}

‎src/nbl/system/ISystem.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -125,7 +125,7 @@ bool ISystem::deleteDirectory(const system::path& p)
125125
bool nbl::system::ISystem::deleteFile(const system::path& p)
126126
{
127127
if (std::filesystem::exists(p) && !std::filesystem::is_directory(p))
128-
return std::filesystem::remove(p);
128+
return std::filesystem::remove(p);
129129
else
130130
return false;
131131
}

‎src/nbl/video/ILogicalDevice.cpp‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -362,11 +362,15 @@ core::smart_refctd_ptr<asset::IShader> ILogicalDevice::compileShader(const SShad
362362
commonCompileOptions.stage = creationParams.stage;
363363
commonCompileOptions.debugInfoFlags = creationParams.debugInfoFlags;
364364
commonCompileOptions.spirvOptimizer = creationParams.optimizer;
365+
commonCompileOptions.optimizerIsExtraPasses = creationParams.optimizerIsExtraPasses;
365366
commonCompileOptions.preprocessorOptions.targetSpirvVersion = m_physicalDevice->getLimits().spirvVersion;
366367

367368
commonCompileOptions.readCache = creationParams.readCache;
368369
commonCompileOptions.writeCache = creationParams.writeCache;
369370

371+
commonCompileOptions.preprocessedOutputPath = creationParams.preprocessedOutputPath;
372+
commonCompileOptions.spvOutputPath = creationParams.spvOutputPath;
373+
370374
if (sourceContent==asset::IShader::E_CONTENT_TYPE::ECT_HLSL)
371375
{
372376
// TODO: add specific HLSLCompiler::SOption params

0 commit comments

Comments
 (0)