Skip to content

Commit fdea055

Browse files
AnastaZIukAnastaZIuk
committed
Fix NSC preprocess output
1 parent ebe6429 commit fdea055

5 files changed

Lines changed: 180 additions & 76 deletions

File tree

‎include/nbl/asset/utils/CHLSLCompiler.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -155,4 +155,4 @@ class NBL_API2 CHLSLCompiler final : public IShaderCompiler
155155

156156
#endif
157157

158-
#endif
158+
#endif

‎src/nbl/asset/utils/CHLSLCompiler.cpp‎

Lines changed: 14 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -363,7 +363,13 @@ namespace nbl::wave
363363
extern nbl::core::string preprocess(std::string& code, const IShaderCompiler::SPreprocessorOptions& preprocessOptions, bool withCaching, std::function<void(nbl::wave::context&)> post);
364364
}
365365

366-
std::string CHLSLCompiler::preprocessShader(std::string&& code, IShader::E_SHADER_STAGE& stage, const SPreprocessorOptions& preprocessOptions, std::vector<std::string>& dxc_compile_flags_override, std::vector<CCache::SEntry::SPreprocessingDependency>* dependencies) const
366+
static std::string preprocessShaderImpl(
367+
std::string&& code,
368+
IShader::E_SHADER_STAGE& stage,
369+
const CHLSLCompiler::SPreprocessorOptions& preprocessOptions,
370+
std::vector<std::string>& dxc_compile_flags_override,
371+
std::vector<IShaderCompiler::CCache::SEntry::SPreprocessingDependency>* dependencies,
372+
system::ISystem* system)
367373
{
368374
const bool depfileEnabled = preprocessOptions.depfile;
369375
if (depfileEnabled)
@@ -375,7 +381,7 @@ std::string CHLSLCompiler::preprocessShader(std::string&& code, IShader::E_SHADE
375381
}
376382
}
377383

378-
std::vector<CCache::SEntry::SPreprocessingDependency> localDependencies;
384+
std::vector<IShaderCompiler::CCache::SEntry::SPreprocessingDependency> localDependencies;
379385
auto* dependenciesOut = dependencies;
380386
if (depfileEnabled && !dependenciesOut)
381387
dependenciesOut = &localDependencies;
@@ -409,19 +415,6 @@ std::string CHLSLCompiler::preprocessShader(std::string&& code, IShader::E_SHADE
409415
}
410416
);
411417

412-
// for debugging cause MSVC doesn't like to show more than 21k LoC in TextVisualizer
413-
if constexpr (false)
414-
{
415-
system::ISystem::future_t<core::smart_refctd_ptr<system::IFile>> future;
416-
m_system->createFile(future,system::path(preprocessOptions.sourceIdentifier).parent_path()/"preprocessed.hlsl",system::IFileBase::ECF_WRITE);
417-
if (auto file=future.acquire(); file&&bool(*file))
418-
{
419-
system::IFile::success_t succ;
420-
(*file)->write(succ,resolvedString.data(),0,resolvedString.size()+1);
421-
succ.getBytesProcessed(true);
422-
}
423-
}
424-
425418
if (resolvedString.empty())
426419
return resolvedString;
427420

@@ -433,14 +426,19 @@ std::string CHLSLCompiler::preprocessShader(std::string&& code, IShader::E_SHADE
433426
params.sourceIdentifier = preprocessOptions.sourceIdentifier;
434427
if (!params.sourceIdentifier.empty())
435428
params.workingDirectory = std::filesystem::path(std::string(params.sourceIdentifier)).parent_path();
436-
params.system = m_system.get();
429+
params.system = system;
437430
if (!IShaderCompiler::writeDepfile(params, *dependenciesOut, preprocessOptions.includeFinder, preprocessOptions.logger))
438431
return {};
439432
}
440433

441434
return resolvedString;
442435
}
443436

437+
std::string CHLSLCompiler::preprocessShader(std::string&& code, IShader::E_SHADER_STAGE& stage, const SPreprocessorOptions& preprocessOptions, std::vector<std::string>& dxc_compile_flags_override, std::vector<CCache::SEntry::SPreprocessingDependency>* dependencies) const
438+
{
439+
return preprocessShaderImpl(std::move(code), stage, preprocessOptions, dxc_compile_flags_override, dependencies, m_system.get());
440+
}
441+
444442
std::string CHLSLCompiler::preprocessShader(std::string&& code, IShader::E_SHADER_STAGE& stage, const SPreprocessorOptions& preprocessOptions, std::vector<CCache::SEntry::SPreprocessingDependency>* dependencies) const
445443
{
446444
std::vector<std::string> extra_dxc_compile_flags = {};

‎src/nbl/asset/utils/CWaveStringResolver.cpp‎

Lines changed: 129 additions & 58 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,8 @@
3939

4040
#include "nabla.h"
4141
#include <boost/exception/diagnostic_information.hpp>
42+
#include <boost/wave/util/insert_whitespace_detection.hpp>
43+
#include <optional>
4244

4345
using namespace nbl;
4446
using namespace nbl::asset;
@@ -115,75 +117,144 @@ std::string makeWaveFailureContext(
115117

116118
return stream.str();
117119
}
120+
121+
template<typename TokenT>
122+
bool isWhitespaceLikeToken(const TokenT& token)
123+
{
124+
using namespace boost::wave;
125+
126+
const auto id = token_id(token);
127+
return id == T_NEWLINE || id == T_GENERATEDNEWLINE || id == T_CONTLINE || IS_CATEGORY(token, WhiteSpaceTokenType);
118128
}
119129

120-
namespace nbl::wave
130+
template<typename TokenT>
131+
std::string tokenValueToString(const TokenT& token)
121132
{
122-
std::string preprocess(std::string& code, const nbl::asset::IShaderCompiler::SPreprocessorOptions& preprocessOptions, bool withCaching, std::function<void(nbl::wave::context&)> post)
133+
const auto& value = token.get_value();
134+
return std::string(value.data(), value.size());
135+
}
136+
137+
core::string renderPreprocessedOutput(nbl::wave::context& context)
138+
{
139+
using namespace boost::wave;
140+
141+
core::string output;
142+
util::insert_whitespace_detection whitespace(true);
143+
std::optional<nbl::wave::context::position_type> previousPosition = std::nullopt;
144+
bool previousWasExplicitWhitespace = false;
145+
146+
for (auto it = context.begin(); it != context.end(); ++it)
123147
{
124-
nbl::wave::context context(code.begin(), code.end(), preprocessOptions.sourceIdentifier.data(), { preprocessOptions });
125-
context.set_caching(withCaching);
126-
context.add_macro_definition("__HLSL_VERSION");
127-
context.add_macro_definition("__SPIRV_MAJOR_VERSION__=" + std::to_string(IShaderCompiler::getSpirvMajor(preprocessOptions.targetSpirvVersion)));
128-
context.add_macro_definition("__SPIRV_MINOR_VERSION__=" + std::to_string(IShaderCompiler::getSpirvMinor(preprocessOptions.targetSpirvVersion)));
129-
130-
// instead of defining extraDefines as "NBL_GLSL_LIMIT_MAX_IMAGE_DIMENSION_1D 32768",
131-
// now define them as "NBL_GLSL_LIMIT_MAX_IMAGE_DIMENSION_1D=32768"
132-
// to match boost wave syntax
133-
// https://www.boost.org/doc/libs/1_82_0/libs/wave/doc/class_reference_context.html#:~:text=Maintain%20defined%20macros-,add_macro_definition,-bool%20add_macro_definition
134-
135-
// preprocess
136-
core::string resolvedString;
137-
const char* phase = "registering built-in macros";
138-
std::string activeMacroDefinition;
139-
try
148+
const auto& token = *it;
149+
const auto id = token_id(token);
150+
if (id == T_EOF || id == T_EOI)
151+
continue;
152+
153+
const auto explicitWhitespace = isWhitespaceLikeToken(token);
154+
const auto& position = token.get_position();
155+
const auto value = tokenValueToString(token);
156+
157+
if (previousPosition.has_value() && !explicitWhitespace)
140158
{
141-
phase = "registering extra macro definitions";
142-
for (const auto& define : preprocessOptions.extraDefines)
159+
const auto movedToNewLogicalLine =
160+
position.get_file() != previousPosition->get_file() ||
161+
position.get_line() > previousPosition->get_line();
162+
163+
if (movedToNewLogicalLine)
143164
{
144-
activeMacroDefinition = define.identifier;
145-
activeMacroDefinition.push_back('=');
146-
activeMacroDefinition.append(define.definition);
147-
context.add_macro_definition(activeMacroDefinition);
165+
if (output.empty() || output.back() != '\n')
166+
{
167+
output.push_back('\n');
168+
whitespace.shift_tokens(T_NEWLINE);
169+
}
170+
}
171+
else if (!previousWasExplicitWhitespace && whitespace.must_insert(id, value))
172+
{
173+
if (output.empty() || (output.back() != ' ' && output.back() != '\n' && output.back() != '\r' && output.back() != '\t'))
174+
{
175+
output.push_back(' ');
176+
whitespace.shift_tokens(T_SPACE);
177+
}
148178
}
149-
activeMacroDefinition.clear();
150-
151-
phase = "expanding translation unit";
152-
auto stream = std::stringstream();
153-
for (auto i = context.begin(); i != context.end(); i++)
154-
stream << i->get_value();
155-
resolvedString = stream.str();
156-
}
157-
catch (boost::wave::preprocess_exception& e)
158-
{
159-
const auto failureContext = makeWaveFailureContext(preprocessOptions, code, phase, activeMacroDefinition, e.file_name(), e.line_no(), e.column_no());
160-
preprocessOptions.logger.log("%s exception caught. %s [%s:%d:%d]\n%s", system::ILogger::ELL_ERROR, e.what(), e.description(), e.file_name(), e.line_no(), e.column_no(), failureContext.c_str());
161-
preprocessOptions.logger.log("Boost diagnostic information:\n%s", system::ILogger::ELL_ERROR, boost::diagnostic_information(e).c_str());
162-
return {};
163-
}
164-
catch (const boost::exception& e)
165-
{
166-
const auto failureContext = makeWaveFailureContext(preprocessOptions, code, phase, activeMacroDefinition, preprocessOptions.sourceIdentifier.data(), 0, 0);
167-
preprocessOptions.logger.log("Boost exception caught during Wave preprocessing.\n%s", system::ILogger::ELL_ERROR, failureContext.c_str());
168-
preprocessOptions.logger.log("Boost diagnostic information:\n%s", system::ILogger::ELL_ERROR, boost::diagnostic_information(e).c_str());
169-
return {};
170-
}
171-
catch (const std::exception& e)
172-
{
173-
const auto failureContext = makeWaveFailureContext(preprocessOptions, code, phase, activeMacroDefinition, preprocessOptions.sourceIdentifier.data(), 0, 0);
174-
preprocessOptions.logger.log("std::exception caught during Wave preprocessing. %s\n%s", system::ILogger::ELL_ERROR, e.what(), failureContext.c_str());
175-
return {};
176179
}
177-
catch (...)
180+
181+
output += value;
182+
whitespace.shift_tokens(id);
183+
previousPosition = position;
184+
previousWasExplicitWhitespace = explicitWhitespace;
185+
}
186+
187+
return output;
188+
}
189+
190+
std::string preprocessImpl(
191+
std::string& code,
192+
const nbl::asset::IShaderCompiler::SPreprocessorOptions& preprocessOptions,
193+
const bool withCaching,
194+
std::function<void(nbl::wave::context&)> post)
195+
{
196+
nbl::wave::context context(code.begin(), code.end(), preprocessOptions.sourceIdentifier.data(), { preprocessOptions });
197+
context.set_caching(withCaching);
198+
context.add_macro_definition("__HLSL_VERSION");
199+
context.add_macro_definition("__SPIRV_MAJOR_VERSION__=" + std::to_string(IShaderCompiler::getSpirvMajor(preprocessOptions.targetSpirvVersion)));
200+
context.add_macro_definition("__SPIRV_MINOR_VERSION__=" + std::to_string(IShaderCompiler::getSpirvMinor(preprocessOptions.targetSpirvVersion)));
201+
202+
core::string resolvedString;
203+
const char* phase = "registering built-in macros";
204+
std::string activeMacroDefinition;
205+
try
206+
{
207+
phase = "registering extra macro definitions";
208+
for (const auto& define : preprocessOptions.extraDefines)
178209
{
179-
const auto failureContext = makeWaveFailureContext(preprocessOptions, code, phase, activeMacroDefinition, preprocessOptions.sourceIdentifier.data(), 0, 0);
180-
preprocessOptions.logger.log("Unknown exception caught during Wave preprocessing.\n%s", system::ILogger::ELL_ERROR, failureContext.c_str());
181-
preprocessOptions.logger.log("Current exception diagnostic information:\n%s", system::ILogger::ELL_ERROR, boost::current_exception_diagnostic_information().c_str());
182-
return {};
210+
activeMacroDefinition = define.identifier;
211+
activeMacroDefinition.push_back('=');
212+
activeMacroDefinition.append(define.definition);
213+
context.add_macro_definition(activeMacroDefinition);
183214
}
215+
activeMacroDefinition.clear();
216+
217+
phase = "expanding translation unit";
218+
resolvedString = renderPreprocessedOutput(context);
219+
}
220+
catch (boost::wave::preprocess_exception& e)
221+
{
222+
const auto failureContext = makeWaveFailureContext(preprocessOptions, code, phase, activeMacroDefinition, e.file_name(), e.line_no(), e.column_no());
223+
preprocessOptions.logger.log("%s exception caught. %s [%s:%d:%d]\n%s", system::ILogger::ELL_ERROR, e.what(), e.description(), e.file_name(), e.line_no(), e.column_no(), failureContext.c_str());
224+
preprocessOptions.logger.log("Boost diagnostic information:\n%s", system::ILogger::ELL_ERROR, boost::diagnostic_information(e).c_str());
225+
return {};
226+
}
227+
catch (const boost::exception& e)
228+
{
229+
const auto failureContext = makeWaveFailureContext(preprocessOptions, code, phase, activeMacroDefinition, preprocessOptions.sourceIdentifier.data(), 0, 0);
230+
preprocessOptions.logger.log("Boost exception caught during Wave preprocessing.\n%s", system::ILogger::ELL_ERROR, failureContext.c_str());
231+
preprocessOptions.logger.log("Boost diagnostic information:\n%s", system::ILogger::ELL_ERROR, boost::diagnostic_information(e).c_str());
232+
return {};
233+
}
234+
catch (const std::exception& e)
235+
{
236+
const auto failureContext = makeWaveFailureContext(preprocessOptions, code, phase, activeMacroDefinition, preprocessOptions.sourceIdentifier.data(), 0, 0);
237+
preprocessOptions.logger.log("std::exception caught during Wave preprocessing. %s\n%s", system::ILogger::ELL_ERROR, e.what(), failureContext.c_str());
238+
return {};
239+
}
240+
catch (...)
241+
{
242+
const auto failureContext = makeWaveFailureContext(preprocessOptions, code, phase, activeMacroDefinition, preprocessOptions.sourceIdentifier.data(), 0, 0);
243+
preprocessOptions.logger.log("Unknown exception caught during Wave preprocessing.\n%s", system::ILogger::ELL_ERROR, failureContext.c_str());
244+
preprocessOptions.logger.log("Current exception diagnostic information:\n%s", system::ILogger::ELL_ERROR, boost::current_exception_diagnostic_information().c_str());
245+
return {};
246+
}
184247

185-
post(context);
248+
post(context);
186249

187-
return resolvedString;
250+
return resolvedString;
251+
}
252+
}
253+
254+
namespace nbl::wave
255+
{
256+
std::string preprocess(std::string& code, const nbl::asset::IShaderCompiler::SPreprocessorOptions& preprocessOptions, bool withCaching, std::function<void(nbl::wave::context&)> post)
257+
{
258+
return preprocessImpl(code, preprocessOptions, withCaching, std::move(post));
188259
}
189260
}

‎src/nbl/asset/utils/waveContext.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -570,4 +570,4 @@ template<> inline bool boost::wave::impl::pp_iterator_functor<nbl::wave::context
570570
return true;
571571
}
572572

573-
#endif
573+
#endif

‎tools/nsc/main.cpp‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -436,6 +436,39 @@ class ShaderCompiler final : public IApplicationFramework
436436
return out;
437437
}
438438

439+
static std::string stripStandalonePragmaOnceLines(std::string_view text)
440+
{
441+
std::string out;
442+
out.reserve(text.size());
443+
444+
size_t lineStart = 0u;
445+
while (lineStart < text.size())
446+
{
447+
const auto lineEnd = text.find('\n', lineStart);
448+
const auto lineSize = lineEnd == std::string_view::npos ? text.size() - lineStart : lineEnd - lineStart;
449+
const auto line = text.substr(lineStart, lineSize);
450+
451+
auto trimmed = line;
452+
while (!trimmed.empty() && (trimmed.front() == ' ' || trimmed.front() == '\t'))
453+
trimmed.remove_prefix(1u);
454+
while (!trimmed.empty() && trimmed.back() == '\r')
455+
trimmed.remove_suffix(1u);
456+
457+
const auto isStandalonePragmaOnce = trimmed == "#pragma once";
458+
if (!isStandalonePragmaOnce)
459+
out.append(line.data(), line.size());
460+
461+
if (lineEnd != std::string_view::npos)
462+
out.push_back('\n');
463+
464+
if (lineEnd == std::string_view::npos)
465+
break;
466+
lineStart = lineEnd + 1u;
467+
}
468+
469+
return out;
470+
}
471+
439472
static void dumpBuildInfo(const argparse::ArgumentParser& program)
440473
{
441474
::json j;
@@ -549,6 +582,8 @@ class ShaderCompiler final : public IApplicationFramework
549582
std::string_view code(codePtr, std::strlen(codePtr));
550583

551584
r.text = hlslcompiler->preprocessShader(std::string(code), shaderStage, opt, nullptr);
585+
if (!r.text.empty())
586+
r.text = stripStandalonePragmaOnceLines(r.text);
552587
r.ok = !r.text.empty();
553588
r.view = r.text;
554589
return r;

0 commit comments

Comments
 (0)