Merge branch 'Devsh-Graphics-Programming:master' into master

kpentaris · web-flow · commit 2d87fb62b4ca · 2024-01-21T16:38:04.000+02:00
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -352,10 +352,8 @@ add_subdirectory(${THIRD_PARTY_SOURCE_DIR})
 
 add_subdirectory(src/nbl)
 add_subdirectory("${NBL_PYTHON_MODULE_ROOT_PATH}" tests) # Python Framework
-
-if(NBL_BUILD_EXAMPLES)
-	add_subdirectory(examples_tests)
-endif()
+add_subdirectory(examples_tests)
+add_subdirectory(tools)
 
 if(NBL_BUILD_DOCS)
 	add_subdirectory(docs)
diff --git a/include/nbl/asset/utils/CHLSLCompiler.h b/include/nbl/asset/utils/CHLSLCompiler.h
@@ -47,7 +47,8 @@ class NBL_API2 CHLSLCompiler final : public IShaderCompiler
 		//}
 
 		std::string preprocessShader(std::string&& code, IShader::E_SHADER_STAGE& stage, const SPreprocessorOptions& preprocessOptions) const override;
-
+		std::string preprocessShader(std::string&& code, IShader::E_SHADER_STAGE& stage, std::vector<std::string>& dxc_compile_flags_override, const SPreprocessorOptions& preprocessOptions) const;
+							
 		void insertIntoStart(std::string& code, std::ostringstream&& ins) const override;
 	protected:
 
diff --git a/src/nbl/asset/utils/CHLSLCompiler.cpp b/src/nbl/asset/utils/CHLSLCompiler.cpp
@@ -83,7 +83,7 @@ DxcCompilationResult dxcCompile(const CHLSLCompiler* compiler, nbl::asset::impl:
     if ((options.debugInfoFlags.value & sourceEmittingFlags) != CHLSLCompiler::E_DEBUG_INFO_FLAGS::EDIF_NONE)
     {
         std::ostringstream insertion;
-        insertion << "//#pragma compile_flags ";
+        insertion << "#pragma compile_flags ";
 
         std::wstring_convert<std::codecvt_utf8<wchar_t>, wchar_t> conv;
         for (uint32_t arg = 0; arg < argCount; arg ++)
@@ -149,7 +149,8 @@ DxcCompilationResult dxcCompile(const CHLSLCompiler* compiler, nbl::asset::impl:
 
 #include "nbl/asset/utils/waveContext.h"
 
-std::string CHLSLCompiler::preprocessShader(std::string&& code, IShader::E_SHADER_STAGE& stage, const SPreprocessorOptions& preprocessOptions) const
+
+std::string CHLSLCompiler::preprocessShader(std::string&& code, IShader::E_SHADER_STAGE& stage, std::vector<std::string>& dxc_compile_flags_override, const SPreprocessorOptions& preprocessOptions) const
 {
     nbl::wave::context context(code.begin(),code.end(),preprocessOptions.sourceIdentifier.data(),{preprocessOptions});
     context.add_macro_definition("__HLSL_VERSION");
@@ -192,12 +193,22 @@ std::string CHLSLCompiler::preprocessShader(std::string&& code, IShader::E_SHADE
         }
     }
 
+    if (context.get_hooks().m_dxc_compile_flags_override.size() != 0)
+        dxc_compile_flags_override = context.get_hooks().m_dxc_compile_flags_override;
+
     if(context.get_hooks().m_pragmaStage != IShader::ESS_UNKNOWN)
         stage = context.get_hooks().m_pragmaStage;
 
+
+
     return resolvedString;
 }
 
+std::string CHLSLCompiler::preprocessShader(std::string&& code, IShader::E_SHADER_STAGE& stage, const SPreprocessorOptions& preprocessOptions) const
+{
+    std::vector<std::string> extra_dxc_compile_flags = {};
+    return preprocessShader(std::move(code), stage, extra_dxc_compile_flags, preprocessOptions);
+}
 
 core::smart_refctd_ptr<ICPUShader> CHLSLCompiler::compileToSPIRV(const char* code, const IShaderCompiler::SCompilerOptions& options) const
 {
@@ -208,9 +219,9 @@ core::smart_refctd_ptr<ICPUShader> CHLSLCompiler::compileToSPIRV(const char* cod
         hlslOptions.preprocessorOptions.logger.log("code is nullptr", system::ILogger::ELL_ERROR);
         return nullptr;
     }
-
+    std::vector<std::string> dxc_compile_flags = {};
     auto stage = hlslOptions.stage;
-    auto newCode = preprocessShader(code, stage, hlslOptions.preprocessorOptions);
+    auto newCode = preprocessShader(code, stage, dxc_compile_flags, hlslOptions.preprocessorOptions);
 
     // Suffix is the shader model version
     // TODO: Figure out a way to get the shader model version automatically
@@ -256,7 +267,21 @@ core::smart_refctd_ptr<ICPUShader> CHLSLCompiler::compileToSPIRV(const char* cod
             return nullptr;
     };
 
-    std::vector<LPCWSTR> arguments = {
+    std::wstring* arg_storage = NULL;
+    std::vector<LPCWSTR> arguments;
+
+    if (dxc_compile_flags.size()) { // #pragma wave overrides compile flags
+        size_t arg_size = dxc_compile_flags.size();
+        arguments = {};
+        arguments.reserve(arg_size);
+        arg_storage = new std::wstring[arg_size]; // prevent deallocation before shader compilation
+        for (size_t i = 0; i < dxc_compile_flags.size(); i++) {
+            arg_storage[i] = std::wstring(dxc_compile_flags[i].begin(), dxc_compile_flags[i].end());
+            arguments.push_back(arg_storage[i].c_str());
+        }
+    }
+    else {
+        arguments = {
         L"-spirv",
         L"-HV", L"202x",
         L"-T", targetProfile.c_str(),
@@ -267,40 +292,43 @@ core::smart_refctd_ptr<ICPUShader> CHLSLCompiler::compileToSPIRV(const char* cod
         L"-Wno-c++1z-extensions",
         L"-Wno-gnu-static-float-init",
         L"-fspv-target-env=vulkan1.3"
-    };
+        };
+        // If a custom SPIR-V optimizer is specified, use that instead of DXC's spirv-opt.
+        // This is how we can get more optimizer options.
+        // 
+        // Optimization is also delegated to SPIRV-Tools. Right now there are no difference between 
+        // optimization levels greater than zero; they will all invoke the same optimization recipe. 
+        // https://github.com/Microsoft/DirectXShaderCompiler/blob/main/docs/SPIR-V.rst#optimization
+        if (hlslOptions.spirvOptimizer)
+        {
+            arguments.push_back(L"-O0");
+        }
 
-    // If a custom SPIR-V optimizer is specified, use that instead of DXC's spirv-opt.
-    // This is how we can get more optimizer options.
-    // 
-    // Optimization is also delegated to SPIRV-Tools. Right now there are no difference between 
-    // optimization levels greater than zero; they will all invoke the same optimization recipe. 
-    // https://github.com/Microsoft/DirectXShaderCompiler/blob/main/docs/SPIR-V.rst#optimization
-    if (hlslOptions.spirvOptimizer)
-    {
-        arguments.push_back(L"-O0");
+        // Debug only values
+        if (hlslOptions.debugInfoFlags.hasFlags(E_DEBUG_INFO_FLAGS::EDIF_FILE_BIT))
+            arguments.push_back(L"-fspv-debug=file");
+        if (hlslOptions.debugInfoFlags.hasFlags(E_DEBUG_INFO_FLAGS::EDIF_SOURCE_BIT))
+            arguments.push_back(L"-fspv-debug=source");
+        if (hlslOptions.debugInfoFlags.hasFlags(E_DEBUG_INFO_FLAGS::EDIF_LINE_BIT))
+            arguments.push_back(L"-fspv-debug=line");
+        if (hlslOptions.debugInfoFlags.hasFlags(E_DEBUG_INFO_FLAGS::EDIF_TOOL_BIT))
+            arguments.push_back(L"-fspv-debug=tool");
+        if (hlslOptions.debugInfoFlags.hasFlags(E_DEBUG_INFO_FLAGS::EDIF_NON_SEMANTIC_BIT))
+            arguments.push_back(L"-fspv-debug=vulkan-with-source");
     }
 
-    // Debug only values
-    if (hlslOptions.debugInfoFlags.hasFlags(E_DEBUG_INFO_FLAGS::EDIF_FILE_BIT))
-        arguments.push_back(L"-fspv-debug=file");
-    if (hlslOptions.debugInfoFlags.hasFlags(E_DEBUG_INFO_FLAGS::EDIF_SOURCE_BIT))
-        arguments.push_back(L"-fspv-debug=source");
-    if (hlslOptions.debugInfoFlags.hasFlags(E_DEBUG_INFO_FLAGS::EDIF_LINE_BIT))
-        arguments.push_back(L"-fspv-debug=line");
-    if (hlslOptions.debugInfoFlags.hasFlags(E_DEBUG_INFO_FLAGS::EDIF_TOOL_BIT))
-        arguments.push_back(L"-fspv-debug=tool");
-    if (hlslOptions.debugInfoFlags.hasFlags(E_DEBUG_INFO_FLAGS::EDIF_NON_SEMANTIC_BIT))
-        arguments.push_back(L"-fspv-debug=vulkan-with-source");
-
     auto compileResult = dxcCompile(
         this, 
         m_dxcCompilerTypes, 
         newCode,
-        &arguments[0],
+        arguments.data(),
         arguments.size(),
         hlslOptions
     );
 
+    if (arg_storage)
+        delete[] arg_storage;
+
     if (!compileResult.objectBlob)
     {
         return nullptr;
@@ -322,4 +350,6 @@ void CHLSLCompiler::insertIntoStart(std::string& code, std::ostringstream&& ins)
 {
     code.insert(0u, ins.str());
 }
+
+
 #endif
diff --git a/src/nbl/asset/utils/waveContext.h b/src/nbl/asset/utils/waveContext.h
@@ -47,7 +47,10 @@ struct load_to_string final
 struct preprocessing_hooks final : public boost::wave::context_policies::default_preprocessing_hooks
 {
     preprocessing_hooks(const IShaderCompiler::SPreprocessorOptions& _preprocessOptions) 
-        : m_includeFinder(_preprocessOptions.includeFinder), m_logger(_preprocessOptions.logger), m_pragmaStage(IShader::ESS_UNKNOWN) {}
+        : m_includeFinder(_preprocessOptions.includeFinder), m_logger(_preprocessOptions.logger), m_pragmaStage(IShader::ESS_UNKNOWN), m_dxc_compile_flags_override() 
+    {
+        hash_token_occurences = 0;
+    }
 
     template <typename ContextT>
     bool locate_include_file(ContextT& ctx, std::string& file_path, bool is_system, char const* current_name, std::string& dir_path, std::string& native_name)
@@ -56,6 +59,7 @@ struct preprocessing_hooks final : public boost::wave::context_policies::default
         return false;
     }
 
+
     // interpretation of #pragma's of the form 'wave option[(value)]'
     template <typename ContextT, typename ContainerT>
     bool interpret_pragma(
@@ -64,6 +68,7 @@ struct preprocessing_hooks final : public boost::wave::context_policies::default
         typename ContextT::token_type const& act_token
     )
     {
+        hash_token_occurences++;
         auto optionStr = option.get_value().c_str();
         if (strcmp(optionStr,"shader_stage")==0) 
         {
@@ -97,6 +102,23 @@ struct preprocessing_hooks final : public boost::wave::context_policies::default
             m_pragmaStage = found->second;
             return true;
         }
+        
+        if (strcmp(optionStr, "dxc_compile_flags") == 0 && hash_token_occurences == 1) {
+            m_dxc_compile_flags_override.clear();
+            for (auto valueIter = values.begin(); valueIter != values.end(); valueIter++) {
+                std::string compiler_option_s = std::string(valueIter->get_value().c_str());
+                // if this compiler flag is encapsulated in quotation marks, strip them
+                //if (compiler_option_s[0] == '"' && compiler_option_s[compiler_option_s.length() - 1] == '"')
+                //    compiler_option_s = compiler_option_s.substr(1, compiler_option_s.length() - 2);
+
+                // if the compiler flag is a separator that is a comma or whitespace, do not add to list of flag overrides 
+                if (compiler_option_s == "," || compiler_option_s == " ")
+                    continue;
+                m_dxc_compile_flags_override.push_back(compiler_option_s);
+            }
+            return true;
+        }
+
         return false;
     }
 
@@ -114,6 +136,9 @@ struct preprocessing_hooks final : public boost::wave::context_policies::default
     const IShaderCompiler::CIncludeFinder* m_includeFinder;
     system::logger_opt_ptr m_logger;
     IShader::E_SHADER_STAGE m_pragmaStage;
+    int hash_token_occurences;
+    std::vector<std::string> m_dxc_compile_flags_override;
+
 };
 
 class context : private boost::noncopyable
diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt
@@ -0,0 +1 @@
+add_subdirectory(ndt)
diff --git a/tools/ndt/CMakeLists.txt b/tools/ndt/CMakeLists.txt
@@ -0,0 +1,5 @@
+set(NBL_EXTRA_SOURCES
+	# TODO: Cypi list extra sources if you need
+)
+
+nbl_create_executable_project("${NBL_EXTRA_SOURCES}" "" "" "" "${NBL_EXECUTABLE_PROJECT_CREATION_PCH_TARGET}")
diff --git a/tools/ndt/__init__.py b/tools/ndt/__init__.py
diff --git a/tools/ndt/config.json.template b/tools/ndt/config.json.template
@@ -0,0 +1,24 @@
+{
+  "enableParallelBuild": true,
+  "threadsPerBuildProcess" : 2,
+  "isExecuted": true,
+  "scriptPath": "test/test.py",
+  "cmake": {
+    "configurations": [ "Release", "Debug", "RelWithDebInfo" ],
+    "buildModes": [],
+    "requiredOptions": []
+  },
+  "profiles": [
+    {
+      "backend": "vulkan",
+      "platform": "windows",
+      "buildModes": [],
+      "runConfiguration": "Release",
+      "gpuArchitectures": [],
+	    "inputs": [
+	      "test/config/release.json.template"
+	    ]
+    }
+  ],
+  "inputs": []
+}
diff --git a/tools/ndt/main.cpp b/tools/ndt/main.cpp
@@ -0,0 +1,60 @@
+// TODO: Cypi
+
+
+// nsc input/simple_shader.hlsl -T ps_6_0 -E Main -Fo output/shader.ps
+
+#include "nbl/system/IApplicationFramework.h"
+
+#include <iostream>
+#include <cstdlib>
+
+
+
+class ShaderCompiler final : public system::IApplicationFramework
+{
+    using base_t = system::IApplicationFramework;
+
+public:
+    using base_t::base_t;
+
+    bool onAppInitialized(smart_refctd_ptr<ISystem>&& system) override
+    {
+        if (!base_t::onAppInitialized(std::move(system)))
+            return false;
+        
+        auto argc = argv.size();
+
+        core::vector<std::string> arguments(argv + 1, argv + argc);
+
+        for (auto i=0; i<argc; i++)
+        {
+            if (argv[i] == "-no-nbl-builtins")
+            {
+                no_nbl_builtins = true;
+                break;
+            }
+        }
+
+        std::string command = "dxc.exe";
+        for (std::string arg : arguments)
+        {
+            command.append(" ").append(arg);
+        }
+
+        int execute = std::system(command.c_str());
+
+        std::cout << "-no-nbl-builtins - " << no_nbl_builtins;
+
+
+        return true;
+    }
+
+
+    void workLoopBody() override {}
+
+    bool keepRunning() override { return false; }
+
+
+private:
+    bool no_nbl_builtins{false};
+};
diff --git a/tools/ndt/test/config/release.json.template b/tools/ndt/test/config/release.json.template
@@ -0,0 +1,11 @@
+{
+  "dependencies": [],
+  "data": [
+    {
+      "dependencies": [],
+      "command": [
+        "${NBL_EXECUTABLE_GEN_EXP_FILEPATH}"
+      ]
+    }
+  ]
+}
diff --git a/tools/ndt/test/test.py b/tools/ndt/test/test.py
@@ -0,0 +1,11 @@
+import framework as nbl
+
+def main(args = None, config_json_filepaths = None, nabla_dir = None, warnings = None):
+    if config_json_filepaths is None:
+        args, config_json_filepaths, nabla_dir, warnings = nbl.get_args()
+    if nbl.ExpectedFileAsDependencyTest("Nabla DXC Tool", config_json_filepaths, nabla_dir, warnings).run():
+        print("Test finished, passed")
+        exit(0)
+    else:
+        print()
+        exit(1)
