Skip to content

Commit e4df2b2

Browse files
AnastaZIukAnastaZIuk
authored
Merge pull request #1025 from Devsh-Graphics-Programming/preprocessUpdates
Fix remaining preprocess bugs and backport Wave pragma fix
2 parents c6e5d16 + 1f73d6a commit e4df2b2

6 files changed

Lines changed: 376 additions & 92 deletions

File tree

‎3rdparty/boost/superproject‎

‎include/nbl/asset/utils/IShaderCompiler.h‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,8 @@
1616

1717
#include "nbl/builtin/hlsl/enums.hlsl"
1818

19+
#include <functional>
20+
1921
namespace nbl::asset
2022
{
2123

@@ -136,6 +138,7 @@ class NBL_API2 IShaderCompiler : public core::IReferenceCounted
136138
E_SPIRV_VERSION targetSpirvVersion = E_SPIRV_VERSION::ESV_1_6;
137139
bool depfile = false;
138140
system::path depfilePath = {};
141+
std::function<void(std::string_view)> onPartialOutputOnFailure = {};
139142
};
140143

141144
// https://github.com/microsoft/DirectXShaderCompiler/blob/main/docs/SPIR-V.rst#debugging

‎src/nbl/asset/utils/CWaveStringResolver.cpp‎

Lines changed: 170 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@
4040
#include "nabla.h"
4141
#include <boost/exception/diagnostic_information.hpp>
4242
#include <boost/wave/util/insert_whitespace_detection.hpp>
43+
#include <algorithm>
4344
#include <optional>
4445

4546
using namespace nbl;
@@ -49,6 +50,22 @@ using namespace nbl::asset;
4950

5051
namespace
5152
{
53+
constexpr size_t kWaveFailureLogOutputTailMaxChars = 4096ull;
54+
constexpr size_t kWaveFailureLogOutputTailMaxLines = 16ull;
55+
constexpr size_t kWaveFailureLogTokenPreviewMaxChars = 160ull;
56+
57+
struct WaveRenderProgress
58+
{
59+
core::string output;
60+
std::optional<nbl::wave::context::position_type> previousPosition = std::nullopt;
61+
bool previousWasExplicitWhitespace = false;
62+
size_t emittedTokenCount = 0ull;
63+
std::string lastTokenFile;
64+
int lastTokenLine = 0;
65+
int lastTokenColumn = 0;
66+
std::string lastTokenValue;
67+
};
68+
5269
std::string getLineSnippet(std::string_view text, const int lineNo)
5370
{
5471
if (lineNo <= 0)
@@ -85,8 +102,94 @@ std::string makeCaretLine(const int columnNo)
85102
return std::string(static_cast<size_t>(columnNo - 1), ' ') + '^';
86103
}
87104

105+
size_t countLogicalLines(const std::string_view text)
106+
{
107+
if (text.empty())
108+
return 0ull;
109+
110+
size_t lines = static_cast<size_t>(std::count(text.begin(), text.end(), '\n'));
111+
if (text.back() != '\n')
112+
++lines;
113+
return lines;
114+
}
115+
116+
std::string truncateEscapedPreview(std::string value, const size_t maxChars)
117+
{
118+
if (value.size() <= maxChars)
119+
return value;
120+
121+
if (maxChars <= 3ull)
122+
return value.substr(0ull, maxChars);
123+
124+
value.resize(maxChars - 3ull);
125+
value += "...";
126+
return value;
127+
}
128+
129+
std::string indentMultiline(std::string_view text, std::string_view indent)
130+
{
131+
if (text.empty())
132+
return {};
133+
134+
std::string out;
135+
out.reserve(text.size() + indent.size() * 4ull);
136+
137+
size_t lineStart = 0ull;
138+
while (lineStart < text.size())
139+
{
140+
out.append(indent.data(), indent.size());
141+
142+
const auto lineEnd = text.find('\n', lineStart);
143+
if (lineEnd == std::string_view::npos)
144+
{
145+
out.append(text.data() + lineStart, text.size() - lineStart);
146+
break;
147+
}
148+
149+
out.append(text.data() + lineStart, lineEnd - lineStart + 1ull);
150+
lineStart = lineEnd + 1ull;
151+
}
152+
153+
return out;
154+
}
155+
156+
std::string makeFailureLogOutputTail(std::string_view text)
157+
{
158+
if (text.empty())
159+
return {};
160+
161+
size_t start = text.size();
162+
size_t chars = 0ull;
163+
size_t newlines = 0ull;
164+
while (start > 0ull)
165+
{
166+
--start;
167+
++chars;
168+
if (text[start] == '\n')
169+
{
170+
++newlines;
171+
if (newlines > kWaveFailureLogOutputTailMaxLines)
172+
{
173+
++start;
174+
break;
175+
}
176+
}
177+
178+
if (chars >= kWaveFailureLogOutputTailMaxChars)
179+
break;
180+
}
181+
182+
std::string tail;
183+
if (start > 0ull)
184+
tail = "[truncated]\n";
185+
tail.append(text.data() + start, text.size() - start);
186+
return tail;
187+
}
188+
88189
std::string makeWaveFailureContext(
89190
const nbl::asset::IShaderCompiler::SPreprocessorOptions& preprocessOptions,
191+
const nbl::wave::context& context,
192+
const WaveRenderProgress& renderProgress,
90193
const std::string_view code,
91194
const char* const phase,
92195
const std::string_view activeMacroDefinition,
@@ -97,14 +200,28 @@ std::string makeWaveFailureContext(
97200
std::ostringstream stream;
98201
stream << "Wave preprocessing context:";
99202
if (!preprocessOptions.sourceIdentifier.empty())
100-
stream << "\n source: " << preprocessOptions.sourceIdentifier;
203+
stream << "\n source: " << nbl::wave::detail::escape_control_chars(preprocessOptions.sourceIdentifier);
101204
stream << "\n phase: " << phase;
102205
stream << "\n extra_define_count: " << preprocessOptions.extraDefines.size();
206+
stream << "\n source_length_bytes: " << code.size();
103207
stream << "\n source_has_trailing_newline: " << ((!code.empty() && code.back() == '\n') ? "yes" : "no");
208+
stream << "\n include_depth: " << context.get_iteration_depth();
209+
stream << "\n current_include_spelling: " << nbl::wave::detail::escape_control_chars(context.get_current_relative_filename());
210+
stream << "\n current_directory: " << nbl::wave::detail::escape_control_chars(context.get_current_directory().string());
211+
#if BOOST_WAVE_SUPPORT_PRAGMA_ONCE != 0
212+
stream << "\n current_include_absolute_path: " << nbl::wave::detail::escape_control_chars(context.get_current_filename());
213+
#endif
104214
if (!activeMacroDefinition.empty())
105-
stream << "\n active_macro_definition: " << activeMacroDefinition;
215+
stream << "\n active_macro_definition: " << nbl::wave::detail::escape_control_chars(activeMacroDefinition);
106216
if (fileName && fileName[0] != '\0')
107-
stream << "\n location: " << fileName << ':' << lineNo << ':' << columnNo;
217+
stream << "\n location: " << nbl::wave::detail::escape_control_chars(fileName) << ':' << lineNo << ':' << columnNo;
218+
stream << "\n emitted_output_bytes: " << renderProgress.output.size();
219+
stream << "\n emitted_output_lines: " << countLogicalLines(renderProgress.output);
220+
stream << "\n emitted_token_count: " << renderProgress.emittedTokenCount;
221+
if (!renderProgress.lastTokenFile.empty())
222+
stream << "\n last_emitted_token_location: " << nbl::wave::detail::escape_control_chars(renderProgress.lastTokenFile) << ':' << renderProgress.lastTokenLine << ':' << renderProgress.lastTokenColumn;
223+
if (!renderProgress.lastTokenValue.empty())
224+
stream << "\n last_emitted_token_value: " << truncateEscapedPreview(nbl::wave::detail::escape_control_chars(renderProgress.lastTokenValue), kWaveFailureLogTokenPreviewMaxChars);
108225

109226
const auto snippet = getLineSnippet(code, lineNo);
110227
if (!snippet.empty() && fileName && preprocessOptions.sourceIdentifier == fileName)
@@ -115,6 +232,10 @@ std::string makeWaveFailureContext(
115232
stream << "\n " << caret;
116233
}
117234

235+
const auto outputTail = makeFailureLogOutputTail(renderProgress.output);
236+
if (!outputTail.empty())
237+
stream << "\n partial_output_tail:\n" << indentMultiline(outputTail, " ");
238+
118239
return stream.str();
119240
}
120241

@@ -134,14 +255,11 @@ std::string tokenValueToString(const TokenT& token)
134255
return std::string(value.data(), value.size());
135256
}
136257

137-
core::string renderPreprocessedOutput(nbl::wave::context& context)
258+
void renderPreprocessedOutput(nbl::wave::context& context, WaveRenderProgress& renderProgress)
138259
{
139260
using namespace boost::wave;
140261

141-
core::string output;
142262
util::insert_whitespace_detection whitespace(true);
143-
std::optional<nbl::wave::context::position_type> previousPosition = std::nullopt;
144-
bool previousWasExplicitWhitespace = false;
145263

146264
for (auto it = context.begin(); it != context.end(); ++it)
147265
{
@@ -154,37 +272,41 @@ core::string renderPreprocessedOutput(nbl::wave::context& context)
154272
const auto& position = token.get_position();
155273
const auto value = tokenValueToString(token);
156274

157-
if (previousPosition.has_value() && !explicitWhitespace)
275+
if (renderProgress.previousPosition.has_value() && !explicitWhitespace)
158276
{
159277
const auto movedToNewLogicalLine =
160-
position.get_file() != previousPosition->get_file() ||
161-
position.get_line() > previousPosition->get_line();
278+
position.get_file() != renderProgress.previousPosition->get_file() ||
279+
position.get_line() > renderProgress.previousPosition->get_line();
162280

163281
if (movedToNewLogicalLine)
164282
{
165-
if (output.empty() || output.back() != '\n')
283+
if (renderProgress.output.empty() || renderProgress.output.back() != '\n')
166284
{
167-
output.push_back('\n');
285+
renderProgress.output.push_back('\n');
168286
whitespace.shift_tokens(T_NEWLINE);
169287
}
170288
}
171-
else if (!previousWasExplicitWhitespace && whitespace.must_insert(id, value))
289+
else if (!renderProgress.previousWasExplicitWhitespace && whitespace.must_insert(id, value))
172290
{
173-
if (output.empty() || (output.back() != ' ' && output.back() != '\n' && output.back() != '\r' && output.back() != '\t'))
291+
if (renderProgress.output.empty() || (renderProgress.output.back() != ' ' && renderProgress.output.back() != '\n' && renderProgress.output.back() != '\r' && renderProgress.output.back() != '\t'))
174292
{
175-
output.push_back(' ');
293+
renderProgress.output.push_back(' ');
176294
whitespace.shift_tokens(T_SPACE);
177295
}
178296
}
179297
}
180298

181-
output += value;
299+
renderProgress.output += value;
182300
whitespace.shift_tokens(id);
183-
previousPosition = position;
184-
previousWasExplicitWhitespace = explicitWhitespace;
301+
renderProgress.previousPosition = position;
302+
renderProgress.previousWasExplicitWhitespace = explicitWhitespace;
303+
const auto& file = position.get_file();
304+
renderProgress.lastTokenFile.assign(file.c_str(), file.size());
305+
renderProgress.lastTokenLine = position.get_line();
306+
renderProgress.lastTokenColumn = position.get_column();
307+
renderProgress.lastTokenValue = value;
308+
++renderProgress.emittedTokenCount;
185309
}
186-
187-
return output;
188310
}
189311

190312
std::string preprocessImpl(
@@ -194,16 +316,26 @@ std::string preprocessImpl(
194316
std::function<void(nbl::wave::context&)> post)
195317
{
196318
nbl::wave::context context(code.begin(), code.end(), preprocessOptions.sourceIdentifier.data(), { preprocessOptions });
197-
context.set_caching(withCaching);
198-
context.add_macro_definition("__HLSL_VERSION");
199-
context.add_macro_definition("__SPIRV_MAJOR_VERSION__=" + std::to_string(IShaderCompiler::getSpirvMajor(preprocessOptions.targetSpirvVersion)));
200-
context.add_macro_definition("__SPIRV_MINOR_VERSION__=" + std::to_string(IShaderCompiler::getSpirvMinor(preprocessOptions.targetSpirvVersion)));
201319

202-
core::string resolvedString;
320+
WaveRenderProgress renderProgress;
203321
const char* phase = "registering built-in macros";
204322
std::string activeMacroDefinition;
323+
const auto reportPartialOutputOnFailure = [&]()
324+
{
325+
if (preprocessOptions.onPartialOutputOnFailure)
326+
preprocessOptions.onPartialOutputOnFailure(renderProgress.output);
327+
};
328+
const auto makeFailureContext = [&](const char* const fileName, const int lineNo, const int columnNo)
329+
{
330+
return makeWaveFailureContext(preprocessOptions, context, renderProgress, code, phase, activeMacroDefinition, fileName, lineNo, columnNo);
331+
};
205332
try
206333
{
334+
context.set_caching(withCaching);
335+
context.add_macro_definition("__HLSL_VERSION");
336+
context.add_macro_definition("__SPIRV_MAJOR_VERSION__=" + std::to_string(IShaderCompiler::getSpirvMajor(preprocessOptions.targetSpirvVersion)));
337+
context.add_macro_definition("__SPIRV_MINOR_VERSION__=" + std::to_string(IShaderCompiler::getSpirvMinor(preprocessOptions.targetSpirvVersion)));
338+
207339
phase = "registering extra macro definitions";
208340
for (const auto& define : preprocessOptions.extraDefines)
209341
{
@@ -215,39 +347,45 @@ std::string preprocessImpl(
215347
activeMacroDefinition.clear();
216348

217349
phase = "expanding translation unit";
218-
resolvedString = renderPreprocessedOutput(context);
350+
renderPreprocessedOutput(context, renderProgress);
219351
}
220352
catch (boost::wave::preprocess_exception& e)
221353
{
222-
const auto failureContext = makeWaveFailureContext(preprocessOptions, code, phase, activeMacroDefinition, e.file_name(), e.line_no(), e.column_no());
223-
preprocessOptions.logger.log("%s exception caught. %s [%s:%d:%d]\n%s", system::ILogger::ELL_ERROR, e.what(), e.description(), e.file_name(), e.line_no(), e.column_no(), failureContext.c_str());
354+
reportPartialOutputOnFailure();
355+
const auto escapedDescription = nbl::wave::detail::escape_control_chars(e.description());
356+
const auto escapedFileName = nbl::wave::detail::escape_control_chars(e.file_name());
357+
const auto failureContext = makeFailureContext(e.file_name(), e.line_no(), e.column_no());
358+
preprocessOptions.logger.log("%s exception caught. %s [%s:%d:%d]\n%s", system::ILogger::ELL_ERROR, e.what(), escapedDescription.c_str(), escapedFileName.c_str(), e.line_no(), e.column_no(), failureContext.c_str());
224359
preprocessOptions.logger.log("Boost diagnostic information:\n%s", system::ILogger::ELL_ERROR, boost::diagnostic_information(e).c_str());
225360
return {};
226361
}
227362
catch (const boost::exception& e)
228363
{
229-
const auto failureContext = makeWaveFailureContext(preprocessOptions, code, phase, activeMacroDefinition, preprocessOptions.sourceIdentifier.data(), 0, 0);
364+
reportPartialOutputOnFailure();
365+
const auto failureContext = makeFailureContext(preprocessOptions.sourceIdentifier.data(), 0, 0);
230366
preprocessOptions.logger.log("Boost exception caught during Wave preprocessing.\n%s", system::ILogger::ELL_ERROR, failureContext.c_str());
231367
preprocessOptions.logger.log("Boost diagnostic information:\n%s", system::ILogger::ELL_ERROR, boost::diagnostic_information(e).c_str());
232368
return {};
233369
}
234370
catch (const std::exception& e)
235371
{
236-
const auto failureContext = makeWaveFailureContext(preprocessOptions, code, phase, activeMacroDefinition, preprocessOptions.sourceIdentifier.data(), 0, 0);
372+
reportPartialOutputOnFailure();
373+
const auto failureContext = makeFailureContext(preprocessOptions.sourceIdentifier.data(), 0, 0);
237374
preprocessOptions.logger.log("std::exception caught during Wave preprocessing. %s\n%s", system::ILogger::ELL_ERROR, e.what(), failureContext.c_str());
238375
return {};
239376
}
240377
catch (...)
241378
{
242-
const auto failureContext = makeWaveFailureContext(preprocessOptions, code, phase, activeMacroDefinition, preprocessOptions.sourceIdentifier.data(), 0, 0);
379+
reportPartialOutputOnFailure();
380+
const auto failureContext = makeFailureContext(preprocessOptions.sourceIdentifier.data(), 0, 0);
243381
preprocessOptions.logger.log("Unknown exception caught during Wave preprocessing.\n%s", system::ILogger::ELL_ERROR, failureContext.c_str());
244382
preprocessOptions.logger.log("Current exception diagnostic information:\n%s", system::ILogger::ELL_ERROR, boost::current_exception_diagnostic_information().c_str());
245383
return {};
246384
}
247385

248386
post(context);
249387

250-
return resolvedString;
388+
return std::move(renderProgress.output);
251389
}
252390
}
253391

0 commit comments

Comments
 (0)