Clean up Wave preprocessor handling

Author: AnastaZIuk

src/nbl/asset/utils/CHLSLCompiler.cpp

@@ -20,8 +20,6 @@
 #include <combaseapi.h>
 #include <sstream>
 #include <dxc/dxcapi.h>
-#include <boost/algorithm/string/predicate.hpp>
-#include <boost/algorithm/string/trim.hpp>
 
 using namespace nbl;
 using namespace nbl::asset;
@@ -363,6 +361,85 @@ namespace nbl::wave
     extern nbl::core::string preprocess(std::string& code, const IShaderCompiler::SPreprocessorOptions& preprocessOptions, bool withCaching, std::function<void(nbl::wave::context&)> post);
 }
 
+static bool isHorizontalWhitespace(const char c)
+{
+    return c == ' ' || c == '\t' || c == '\r' || c == '\v' || c == '\f';
+}
+
+static bool consumeIdentifier(std::string_view line, size_t& pos, const std::string_view identifier)
+{
+    if (line.substr(pos, identifier.size()) != identifier)
+        return false;
+
+    const size_t end = pos + identifier.size();
+    if (end < line.size())
+    {
+        const char next = line[end];
+        if ((next >= 'a' && next <= 'z') || (next >= 'A' && next <= 'Z') || (next >= '0' && next <= '9') || next == '_')
+            return false;
+    }
+
+    pos = end;
+    return true;
+}
+
+static void normalizeLegacyShaderStagePragmas(std::string& code)
+{
+    std::string normalized;
+    normalized.reserve(code.size());
+
+    size_t lineStart = 0u;
+    while (lineStart < code.size())
+    {
+        const size_t lineEnd = code.find('\n', lineStart);
+        const bool hasNewline = lineEnd != std::string::npos;
+        const size_t lineSize = hasNewline ? (lineEnd - lineStart) : (code.size() - lineStart);
+        const std::string_view line(code.data() + lineStart, lineSize);
+
+        size_t pos = 0u;
+        while (pos < line.size() && isHorizontalWhitespace(line[pos]))
+            ++pos;
+
+        bool normalizedLegacyPragma = false;
+        if (pos < line.size() && line[pos] == '#')
+        {
+            ++pos;
+            while (pos < line.size() && isHorizontalWhitespace(line[pos]))
+                ++pos;
+
+            if (consumeIdentifier(line, pos, "pragma"))
+            {
+                while (pos < line.size() && isHorizontalWhitespace(line[pos]))
+                    ++pos;
+
+                const size_t pragmaArgumentPos = pos;
+                if (consumeIdentifier(line, pos, "shader_stage"))
+                    normalizedLegacyPragma = true;
+                else
+                    pos = pragmaArgumentPos;
+            }
+        }
+
+        if (normalizedLegacyPragma)
+        {
+            normalized.append(line.substr(0u, pos - std::string_view("shader_stage").size()));
+            normalized += "wave ";
+            normalized.append(line.substr(pos - std::string_view("shader_stage").size()));
+        }
+        else
+        {
+            normalized.append(line);
+        }
+
+        if (hasNewline)
+            normalized.push_back('\n');
+
+        lineStart = hasNewline ? (lineEnd + 1u) : code.size();
+    }
+
+    code = std::move(normalized);
+}
+
 static std::string preprocessShaderImpl(
     std::string&& code,
     IShader::E_SHADER_STAGE& stage,
@@ -386,18 +463,7 @@ static std::string preprocessShaderImpl(
     if (depfileEnabled && !dependenciesOut)
         dependenciesOut = &localDependencies;
 
-    // HACK: we do a pre-pre-process here to add \n after every #pragma to neutralize boost::wave's actions
-    // See https://github.com/Devsh-Graphics-Programming/Nabla/issues/746
-    size_t line_index = 0;
-    for (size_t i = 0; i < code.size(); i++) {
-        if (code[i] == '\n') {
-            auto line = code.substr(line_index, i - line_index);
-            boost::trim(line);
-            if (boost::starts_with(line, "#pragma"))
-                code.insert(i++, 1, '\n');
-            line_index = i;
-        }
-    }
+    normalizeLegacyShaderStagePragmas(code);
 
     // preprocess
     core::string resolvedString = nbl::wave::preprocess(code, preprocessOptions, bool(dependenciesOut),

src/nbl/asset/utils/CWaveStringResolver.cpp

@@ -134,48 +134,6 @@ std::string tokenValueToString(const TokenT& token)
     return std::string(value.data(), value.size());
 }
 
-bool isBinaryLiteralTail(const std::string_view value)
-{
-    if (value.size() < 2u)
-        return false;
-
-    if (value.front() != 'b' && value.front() != 'B')
-        return false;
-
-    bool hasBinaryDigit = false;
-    size_t i = 1u;
-    for (; i < value.size(); ++i)
-    {
-        const char c = value[i];
-        if (c == '0' || c == '1')
-        {
-            hasBinaryDigit = true;
-            continue;
-        }
-        if (c == '\'')
-            continue;
-        break;
-    }
-
-    if (!hasBinaryDigit)
-        return false;
-
-    for (; i < value.size(); ++i)
-    {
-        const char c = value[i];
-        const bool isIntegerSuffix = c == 'u' || c == 'U' || c == 'l' || c == 'L' || c == 'z' || c == 'Z';
-        if (!isIntegerSuffix)
-            return false;
-    }
-
-    return true;
-}
-
-bool shouldConcatenateWithoutWhitespace(const std::string_view previousValue, const std::string_view currentValue)
-{
-    return previousValue == "0" && isBinaryLiteralTail(currentValue);
-}
-
 core::string renderPreprocessedOutput(nbl::wave::context& context)
 {
     using namespace boost::wave;
@@ -184,7 +142,6 @@ core::string renderPreprocessedOutput(nbl::wave::context& context)
     util::insert_whitespace_detection whitespace(true);
     std::optional<nbl::wave::context::position_type> previousPosition = std::nullopt;
     bool previousWasExplicitWhitespace = false;
-    std::string previousTokenValue;
 
     for (auto it = context.begin(); it != context.end(); ++it)
     {
@@ -211,7 +168,7 @@ core::string renderPreprocessedOutput(nbl::wave::context& context)
                     whitespace.shift_tokens(T_NEWLINE);
                 }
             }
-            else if (!previousWasExplicitWhitespace && !shouldConcatenateWithoutWhitespace(previousTokenValue, value) && whitespace.must_insert(id, value))
+            else if (!previousWasExplicitWhitespace && whitespace.must_insert(id, value))
             {
                 if (output.empty() || (output.back() != ' ' && output.back() != '\n' && output.back() != '\r' && output.back() != '\t'))
                 {
@@ -225,8 +182,6 @@ core::string renderPreprocessedOutput(nbl::wave::context& context)
         whitespace.shift_tokens(id);
         previousPosition = position;
         previousWasExplicitWhitespace = explicitWhitespace;
-        if (!explicitWhitespace)
-            previousTokenValue = value;
     }
 
     return output;

src/nbl/asset/utils/waveContext.h

@@ -72,7 +72,6 @@ struct preprocessing_hooks final : public boost::wave::context_policies::default
         typename ContextT::token_type const& act_token
     )
     {
-        hash_token_occurences++;
         auto optionStr = option.get_value().c_str();
         if (strcmp(optionStr,"shader_stage")==0) 
         {
@@ -118,12 +117,13 @@ struct preprocessing_hooks final : public boost::wave::context_policies::default
                 std::string compiler_option_s = std::string(valueIter->get_value().c_str());
                 // the compiler_option_s is a token thus can be only part of the actual argument, i.e. "-spirv" will be split into tokens [ "-", "spirv" ]
                 // for dxc_compile_flags just join the strings until it finds a whitespace or end of args
-
-                if (compiler_option_s == " ") 
+                if (IS_CATEGORY(*valueIter, WhiteSpaceTokenType))
                 {
-                    // push argument and reset
-                    m_dxc_compile_flags_override.push_back(arg);
-                    arg.clear();
+                    if (!arg.empty())
+                    {
+                        m_dxc_compile_flags_override.push_back(arg);
+                        arg.clear();
+                    }
                 }
                 else 
                 {
@@ -196,12 +196,13 @@ class context : private boost::noncopyable
             , current_relative_filename(fname)
             , macros(*this_())
             , language(language_support(
-                support_cpp20
-                | support_option_preserve_comments
-                | support_option_emit_line_directives
-                | support_option_emit_pragma_directives
-//                | support_option_emit_contnewlines
-//                | support_option_insert_whitespace
+                support_cpp20 // C++20 lexer mode. https://github.com/Devsh-Graphics-Programming/wave/blob/e02cda69e4d070fd9b16a39282d6b5c717cb3da4/include/boost/wave/language_support.hpp#L56-L59
+                | support_option_prefer_pp_numbers // Prefer pp-number lexing before retokenization. https://github.com/Devsh-Graphics-Programming/wave/blob/e02cda69e4d070fd9b16a39282d6b5c717cb3da4/include/boost/wave/language_support.hpp#L71
+                | support_option_preserve_comments // Keep comments in the token stream. https://github.com/Devsh-Graphics-Programming/wave/blob/e02cda69e4d070fd9b16a39282d6b5c717cb3da4/include/boost/wave/language_support.hpp#L67
+                | support_option_emit_line_directives // Emit #line directives in the output. https://github.com/Devsh-Graphics-Programming/wave/blob/e02cda69e4d070fd9b16a39282d6b5c717cb3da4/include/boost/wave/language_support.hpp#L72
+                | support_option_emit_pragma_directives // Keep pragma directives in the output. https://github.com/Devsh-Graphics-Programming/wave/blob/e02cda69e4d070fd9b16a39282d6b5c717cb3da4/include/boost/wave/language_support.hpp#L74
+//                | support_option_emit_contnewlines // Emit escaped line continuations. https://github.com/Devsh-Graphics-Programming/wave/blob/e02cda69e4d070fd9b16a39282d6b5c717cb3da4/include/boost/wave/language_support.hpp#L65
+//                | support_option_insert_whitespace // Let Wave inject separator whitespace. https://github.com/Devsh-Graphics-Programming/wave/blob/e02cda69e4d070fd9b16a39282d6b5c717cb3da4/include/boost/wave/language_support.hpp#L66
             ))
             , hooks(hooks_)
         {