Merge branch 'shader_caching_fixes'

Author: Erfan-Ahmadi

include/nbl/asset/utils/IShaderCompiler.h

@@ -179,10 +179,12 @@ class NBL_API2 IShaderCompiler : public core::IReferenceCounted
 
 		class CCache final : public IReferenceCounted
 		{
+			friend class IShaderCompiler;
+
 			public:
 				// Used to check compatibility of Caches before reading
 				constexpr static inline std::string_view VERSION = "1.0.0";
-        
+
 				using hash_t = std::array<uint64_t,4>;
 				static auto const SHADER_BUFFER_SIZE_BYTES = sizeof(uint64_t) / sizeof(uint8_t); // It's obviously 8
 
@@ -362,7 +364,7 @@ class NBL_API2 IShaderCompiler : public core::IReferenceCounted
 					// Making the copy constructor deep-copy everything but the shader 
 					inline SEntry(const SEntry& other) 
 						: mainFileContents(other.mainFileContents), compilerArgs(other.compilerArgs), hash(other.hash), lookupHash(other.lookupHash), 
-						  dependencies(other.dependencies), value(other.value) {}
+						  dependencies(other.dependencies), cpuShader(other.cpuShader) {}
 
 					inline SEntry& operator=(SEntry& other) = delete;
 					inline SEntry(SEntry&& other) = default;
@@ -375,7 +377,7 @@ class NBL_API2 IShaderCompiler : public core::IReferenceCounted
 					std::array<uint64_t,4> hash;
 					size_t lookupHash;
 					dependency_container_t dependencies;
-					core::smart_refctd_ptr<asset::ICPUShader> value;
+					core::smart_refctd_ptr<asset::ICPUShader> cpuShader;
 				};
 
 				inline void insert(SEntry&& entry)
@@ -426,31 +428,45 @@ class NBL_API2 IShaderCompiler : public core::IReferenceCounted
 					}
 
 				};
-				core::unordered_multiset<SEntry,Hash,KeyEqual> m_container;
+
+				using EntrySet = core::unordered_multiset<SEntry, Hash, KeyEqual>;
+				EntrySet m_container;
+
+				NBL_API2 EntrySet::const_iterator find_impl(const SEntry& mainFile, const CIncludeFinder* finder) const;
 		};
 
 		inline core::smart_refctd_ptr<ICPUShader> compileToSPIRV(const std::string_view code, const SCompilerOptions& options) const
 		{
 			CCache::SEntry entry;
 			std::vector<CCache::SEntry::SPreprocessingDependency> dependencies;
-			if (options.readCache or options.writeCache)
+			if (options.readCache || options.writeCache)
 				entry = std::move(CCache::SEntry(code, options));
+			
 			if (options.readCache)
 			{
-				auto found = options.readCache->find(entry, options.preprocessorOptions.includeFinder);
-				if (found)
-					return found;
+				auto found = options.readCache->find_impl(entry, options.preprocessorOptions.includeFinder);
+				if (found != options.readCache->m_container.end())
+				{
+					if (options.writeCache)
+					{
+						CCache::SEntry writeEntry = *found;
+						options.writeCache->insert(std::move(writeEntry));
+					}
+					return found->cpuShader;
+				}
 			}
+
 			auto retVal = compileToSPIRV_impl(code, options, options.writeCache ? &dependencies : nullptr);
 			// compute the SPIR-V shader content hash
 			{
 				auto backingBuffer = retVal->getContent();
 				const_cast<ICPUBuffer*>(backingBuffer)->setContentHash(backingBuffer->computeContentHash());
 			}
+
 			if (options.writeCache)
 			{
 				entry.dependencies = std::move(dependencies);
-				entry.value = retVal;
+				entry.cpuShader = retVal;
 				options.writeCache->insert(std::move(entry));
 			}
 			return retVal;

include/nbl/system/ISystem.h

@@ -94,6 +94,9 @@ class NBL_API2 ISystem : public core::IReferenceCounted
 
         // can only perform operations on non-virtual filesystem paths
         bool deleteDirectory(const system::path& p);
+        
+        // can only perform operations on non-virtual filesystem paths
+        bool deleteFile(const system::path& p);
 
         // can only perform operations on non-virtual filesystem paths
         std::error_code moveFileOrDirectory(const system::path& oldPath, const system::path& newPath);

src/nbl/asset/utils/IShaderCompiler.cpp

@@ -217,6 +217,11 @@ auto IShaderCompiler::CIncludeFinder::tryIncludeGenerators(const std::string& in
 }
 
 core::smart_refctd_ptr<asset::ICPUShader> IShaderCompiler::CCache::find(const SEntry& mainFile, const IShaderCompiler::CIncludeFinder* finder) const
+{
+    return find_impl(mainFile, finder)->cpuShader;
+}
+
+IShaderCompiler::CCache::EntrySet::const_iterator IShaderCompiler::CCache::find_impl(const SEntry& mainFile, const IShaderCompiler::CIncludeFinder* finder) const
 {
     auto foundRange = m_container.equal_range(mainFile);
     for (auto& found = foundRange.first; found != foundRange.second; found++)
@@ -239,11 +244,10 @@ core::smart_refctd_ptr<asset::ICPUShader> IShaderCompiler::CCache::find(const SE
                 break;
             }
         }
-        if (allDependenciesMatch) {
-            return found->value;
-        }
+        if (allDependenciesMatch)
+            return found;
     }
-    return nullptr;
+    return m_container.end();
 }
 
 core::smart_refctd_ptr<ICPUBuffer> IShaderCompiler::CCache::serialize() const
@@ -263,10 +267,10 @@ core::smart_refctd_ptr<ICPUBuffer> IShaderCompiler::CCache::serialize() const
         // We keep a copy of the offsets and the sizes of each shader. This is so that later on, when we add the shaders to the buffer after json creation
         // (where the params array has been moved) we don't have to read the json to get the offsets again
         offsets[i] = shaderBufferSize;
-        sizes[i] = entry.value->getContent()->getSize();
+        sizes[i] = entry.cpuShader->getContent()->getSize();
 
         // And add the params to the shader creation parameters array
-        shaderCreationParams.emplace_back(entry.value->getStage(), entry.value->getContentType(), entry.value->getFilepathHint(), sizes[i], shaderBufferSize);
+        shaderCreationParams.emplace_back(entry.cpuShader->getStage(), entry.cpuShader->getContentType(), entry.cpuShader->getFilepathHint(), sizes[i], shaderBufferSize);
         // Enlarge the shader buffer by the size of the current shader
         shaderBufferSize += sizes[i];
         i++;
@@ -290,7 +294,7 @@ core::smart_refctd_ptr<ICPUBuffer> IShaderCompiler::CCache::serialize() const
     // Loop over entries again, adding each one's shader to the buffer. 
     i = 0u;
     for (auto& entry : m_container) {
-        memcpy(retVal.data() + SHADER_BUFFER_SIZE_BYTES + offsets[i], entry.value->getContent()->getPointer(), sizes[i]);
+        memcpy(retVal.data() + SHADER_BUFFER_SIZE_BYTES + offsets[i], entry.cpuShader->getContent()->getPointer(), sizes[i]);
         i++;
     }
 
@@ -336,9 +340,7 @@ core::smart_refctd_ptr<IShaderCompiler::CCache> IShaderCompiler::CCache::deseria
         memcpy(code->getPointer(), serializedCache.data() + SHADER_BUFFER_SIZE_BYTES + shaderCreationParams[i].offset, shaderCreationParams[i].codeByteSize);
         code->setContentHash(code->computeContentHash());
         // Create the ICPUShader
-        auto value = core::make_smart_refctd_ptr<ICPUShader>(std::move(code), shaderCreationParams[i].stage, shaderCreationParams[i].contentType, std::move(shaderCreationParams[i].filepathHint));
-
-        entries[i].value = std::move(value);
+        entries[i].cpuShader = core::make_smart_refctd_ptr<ICPUShader>(std::move(code), shaderCreationParams[i].stage, shaderCreationParams[i].contentType, std::move(shaderCreationParams[i].filepathHint));
 
         retVal->insert(std::move(entries[i]));
     }

src/nbl/asset/utils/shaderCompiler_serialization.h

@@ -192,7 +192,7 @@ inline void from_json(const json& j, SEntry& entry)
     j.at("hash").get_to(entry.hash);
     j.at("lookupHash").get_to(entry.lookupHash);
     j.at("dependencies").get_to(entry.dependencies);
-    entry.value = nullptr;
+    entry.cpuShader = nullptr;
 }
 
 }

src/nbl/system/ISystem.cpp

@@ -114,12 +114,20 @@ bool ISystem::createDirectory(const system::path& p)
 
 bool ISystem::deleteDirectory(const system::path& p)
 {
-    if (std::filesystem::exists(p))
+    if (std::filesystem::exists(p) && std::filesystem::is_directory(p))
         return std::filesystem::remove_all(p);
     else
         return false;
 }
 
+bool nbl::system::ISystem::deleteFile(const system::path& p)
+{
+    if (std::filesystem::exists(p) && !std::filesystem::is_directory(p)))
+        return std::filesystem::remove(p);
+    else
+        return false;
+}
+
 std::error_code ISystem::moveFileOrDirectory(const system::path& oldPath, const system::path& newPath)
 {
     std::error_code ec;