diff --git a/Makefile.libretro b/Makefile.libretro index 13e81443..29175e31 100644 --- a/Makefile.libretro +++ b/Makefile.libretro @@ -331,6 +331,7 @@ LDFLAGS += -ld3d12 SOURCES_CXX += \ $(COMMON_DIR)/d3d12/context.cpp \ $(COMMON_DIR)/d3d12/descriptor_heap_manager.cpp \ + $(COMMON_DIR)/d3d12/embedded_shaders.cpp \ $(COMMON_DIR)/d3d12/shader_cache.cpp \ $(COMMON_DIR)/d3d12/staging_texture.cpp \ $(COMMON_DIR)/d3d12/stream_buffer.cpp \ diff --git a/data/shaders/d3d12/README.md b/data/shaders/d3d12/README.md new file mode 100644 index 00000000..da35ffef --- /dev/null +++ b/data/shaders/d3d12/README.md @@ -0,0 +1,86 @@ +# D3D12 shader sources (pre-baked DXBC) + +This directory holds the HLSL sources for shaders used by the D3D12 backend +that have been moved off the runtime `ShaderGen` path. Each `.hlsl` file here +has a matching pre-compiled DXBC blob checked in to +`src/common/d3d12/embedded_dxbc/`. + +## Why this exists + +Historically the D3D12 backend generated HLSL at runtime via +`src/core/shadergen.cpp` and `src/core/gpu_hw_shadergen.cpp`, then handed the +text to `D3DCompile` (the in-process HLSL -> DXBC compiler from `d3dcompiler_47.dll`) +to produce a DXBC blob for each PSO. On a cold shader cache, walking the +1164-cell batch pipeline matrix in `precompile_sync` mode takes on the order +of one minute, which is the worst entry in the post-dim-cache cost matrix +(see the cost-matrix comments in 10c53b8, 57ac62e, and the patches that +followed). The runtime cache hides this on warm boots, but cold boots and +shader cache invalidations re-pay the full per-cell `D3DCompile` cost every +time the user changes a setting that hits the full-flush branch. + +Pre-baking the DXBC removes: + +* per-launch HLSL compilation stall on shader cache misses, +* a class of "stale cache after `d3dcompiler_47.dll` version drift" failures, +* the implicit dependency on `D3DCompile` being available at runtime for + the cells that have been pre-baked. + +The OpenGL / OpenGL ES / D3D11 / Vulkan backends still use the runtime +`ShaderGen` path; only the D3D12 backend reads from the embedded DXBC. (The +Vulkan equivalent of this infrastructure lives at `data/shaders/vulkan/` / +`src/common/vulkan/embedded_spirv/` / `tools/regen_vulkan_spirv.py`; this +directory is the mirror.) + +## Build-time constraint + +The core build (`Makefile.libretro` and `src/common/CMakeLists.txt`) must +never invoke `fxc.exe` or any other external shader compiler. The `.inc` +files under `src/common/d3d12/embedded_dxbc/` are checked into the +repository and compiled like any other C++ source. The +`tools/regen_d3d12_dxbc.py` helper exists for contributors who edit the +`.hlsl` sources in this directory; it is not part of the build. + +## Regenerating the DXBC blobs + +After editing a `.hlsl` file here, run: + + python3 tools/regen_d3d12_dxbc.py + +This invokes `fxc.exe` (Shader Model 5.0, `/O3`, `/E main`) on each `.hlsl` +file and rewrites the matching `.inc` under `src/common/d3d12/embedded_dxbc/`. +Both the `.hlsl` and the regenerated `.inc` must be committed together so the +two stay in sync. + +`fxc.exe` ships with the Windows 10 SDK and is **not** invoked by the core +build. On Linux, run the script under Wine - it will look for `fxc.exe` in +the standard SDK install path under `$WINEPREFIX/drive_c/Program Files (x86)/Windows Kits/10/bin//x64/fxc.exe`, +or you can pass `--fxc PATH` explicitly. The `--wine PATH` flag lets you +point at a specific wine binary if `wine` isn't on `PATH`. + +## Filename convention + +Stage is encoded in the suffix: + +* `.vs.hlsl` -> vertex shader (compiled as `vs_5_0`) +* `.ps.hlsl` -> pixel shader (compiled as `ps_5_0`) +* `.cs.hlsl` -> compute shader (compiled as `cs_5_0`) +* `.gs.hlsl` -> geometry shader (compiled as `gs_5_0`) + +The `_5_0` shader model targets `D3D_FEATURE_LEVEL_11_0`, matching the +runtime baseline of the D3D12 backend (see the shared compile path in +`src/common/d3d11/shader_compiler.cpp` which both D3D11 and D3D12 used +prior to the pre-bake migration). + +## Template variants + +Shaders that need multiple pre-baked variants (e.g. one per `BatchRenderMode` +or per `GPUTextureFilter`) are listed in the `TEMPLATE_VARIANTS` dict at the +top of `tools/regen_d3d12_dxbc.py`. Each entry maps an `.hlsl` filename to +the list of `(variant_suffix, [/D defines])` tuples that the regen script +should produce. + +If you add a template variant to the dict, add the matching extern +declaration to `src/common/d3d12/embedded_shaders.h` AND the matching slot +lookup to `src/common/d3d12/embedded_shaders.cpp` - the index ordering at +the runtime call site must match the order in the dict so the correct +blob gets bound at PSO-creation time. diff --git a/src/common/CMakeLists.txt b/src/common/CMakeLists.txt index d1812779..1b679843 100644 --- a/src/common/CMakeLists.txt +++ b/src/common/CMakeLists.txt @@ -117,6 +117,8 @@ if(WIN32) d3d12/context.h d3d12/descriptor_heap_manager.cpp d3d12/descriptor_heap_manager.h + d3d12/embedded_shaders.cpp + d3d12/embedded_shaders.h d3d12/shader_cache.cpp d3d12/shader_cache.h d3d12/staging_texture.cpp diff --git a/src/common/d3d12/embedded_shaders.cpp b/src/common/d3d12/embedded_shaders.cpp new file mode 100644 index 00000000..8511627d --- /dev/null +++ b/src/common/d3d12/embedded_shaders.cpp @@ -0,0 +1,25 @@ +// Copyright 2026 LibretroAdmin +// Licensed under GPLv2+ +// Refer to the LICENSE file included. + +#include "embedded_shaders.h" + +// Storage for the embedded DXBC blobs. Each .inc supplies a definition of +// the array and the matching size constant declared in embedded_shaders.h. +// The .inc files MUST be included inside the namespace block below so the +// names resolve to the header's extern declarations. +// +// Mirror of src/common/vulkan/embedded_shaders.cpp. The Vulkan side wraps +// its embedded blobs in a CreateShaderModule helper because vkCreateShaderModule +// is the API hand-off; the D3D12 side has no equivalent API call - the +// caller drops the (data, size) pair straight into a D3D12_SHADER_BYTECODE +// aggregate at PSO-creation time. So this TU stays simple: just the .inc +// includes and the namespace. +namespace D3D12::EmbeddedShaders { + +// (No .inc includes yet - this foundation patch lays down the storage TU. +// Subsequent patches add HLSL sources under data/shaders/d3d12/ and the +// matching generated .inc under src/common/d3d12/embedded_dxbc/, each +// included from inside this namespace block.) + +} // namespace D3D12::EmbeddedShaders diff --git a/src/common/d3d12/embedded_shaders.h b/src/common/d3d12/embedded_shaders.h new file mode 100644 index 00000000..7a4849f6 --- /dev/null +++ b/src/common/d3d12/embedded_shaders.h @@ -0,0 +1,47 @@ +// Copyright 2026 LibretroAdmin +// Licensed under GPLv2+ +// Refer to the LICENSE file included. + +#pragma once + +#include +#include + +// Pre-compiled DXBC blobs for the D3D12 backend. +// +// Each blob is generated offline by tools/regen_d3d12_dxbc.py from the +// matching HLSL source under data/shaders/d3d12/, and checked into +// src/common/d3d12/embedded_dxbc/. Nothing in the build system invokes +// fxc.exe / D3DCompile - the .inc files are consumed as plain C++ arrays. +// See data/shaders/d3d12/README.md for the editing workflow. +// +// This path replaces the runtime HLSL -> DXBC step done in +// D3D11::ShaderCompiler::CompileShader (shared between the D3D11 and D3D12 +// backends) for the shaders that have been pre-baked. Until every shader +// has been pre-baked, the runtime path and D3DCompile still cover the +// remainder. +// +// Mirror of src/common/vulkan/embedded_shaders.h for the Vulkan backend. +// As individual shaders get migrated off the runtime ShaderGen path, this +// header gains a pair of extern declarations per blob: +// +// extern const uint8_t k__ps[]; +// extern const size_t k__ps_size_bytes; +// +// (or _vs / _cs / _gs depending on stage). Blob variants with /D defines +// get a suffix per the TEMPLATE_VARIANTS table in +// tools/regen_d3d12_dxbc.py. +// +// Constructing a D3D12_SHADER_BYTECODE at the PSO creation site is a +// direct aggregate-initialiser - no wrapper helper needed: +// +// D3D12_SHADER_BYTECODE bc = { k__ps, k__ps_size_bytes }; +// +namespace D3D12::EmbeddedShaders { + +// (No extern declarations yet - this foundation patch lays down the +// directory layout, regen script, and namespace scaffolding. Subsequent +// patches add HLSL sources, generated .inc files, and the matching +// extern declarations here.) + +} // namespace D3D12::EmbeddedShaders diff --git a/tools/regen_d3d12_dxbc.py b/tools/regen_d3d12_dxbc.py new file mode 100755 index 00000000..a3b45681 --- /dev/null +++ b/tools/regen_d3d12_dxbc.py @@ -0,0 +1,244 @@ +#!/usr/bin/env python3 +# Regenerate embedded D3D12 DXBC blobs from HLSL sources. +# +# This script is NOT invoked by the core build. It is run by contributors who +# edit shader sources under data/shaders/d3d12/. The build itself only sees +# the resulting .inc files (checked into src/common/d3d12/embedded_dxbc/), so +# the libretro core has zero fxc.exe / D3DCompile dependency at build time. +# +# Mirror of tools/regen_vulkan_spirv.py for the Vulkan backend. The same +# editing workflow applies: edit .hlsl source -> rerun this script -> commit +# both .hlsl and the regenerated .inc together so the two stay in sync. +# +# Usage: +# python3 tools/regen_d3d12_dxbc.py [--fxc PATH] [--wine PATH] +# +# Requires fxc.exe from the Windows 10 SDK. On Linux, requires Wine to run +# fxc.exe under emulation - pass the wine binary via --wine or set up +# `wine fxc.exe` to be invokable through the script's auto-detection. On +# Windows, the script auto-detects fxc.exe in the Windows 10 SDK install +# location (one of the kit / bin / x64 / fxc.exe paths) or via PATH. + +import argparse +import os +import re +import shutil +import struct +import subprocess +import sys +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parent.parent +HLSL_DIR = REPO_ROOT / "data" / "shaders" / "d3d12" +INC_DIR = REPO_ROOT / "src" / "common" / "d3d12" / "embedded_dxbc" + +# Map HLSL filename suffix to (fxc target prefix, identifier suffix). The +# first element is what we pass to fxc /T (e.g. "ps_5_0"); the second is the +# C++ identifier tail (e.g. k__ps). Anything not in this table is +# skipped with a warning. Shader Model 5.0 matches the runtime baseline of +# D3D_FEATURE_LEVEL_11_0 which the D3D12 backend targets via the shared +# D3D11::ShaderCompiler path; see src/common/d3d11/shader_compiler.cpp. +STAGE_FROM_SUFFIX = { + ".vs.hlsl": ("vs_5_0", "vs"), + ".ps.hlsl": ("ps_5_0", "ps"), + ".cs.hlsl": ("cs_5_0", "cs"), + ".gs.hlsl": ("gs_5_0", "gs"), +} + +# Templates: shader sources that produce more than one .inc output, each +# with a different combination of /D preprocessor defines. Index ordering +# here MUST match the helper in src/common/d3d12/embedded_shaders.cpp that +# picks the right blob at runtime - if you add a variant here, add the +# matching entry there too. +# +# For each entry: filename -> [(variant_suffix, [/D defines]), ...] +# +# Empty for now - this dict grows as batch / per-filter / per-pgxp variants +# get pre-baked. Foundation patches lay the infrastructure; subsequent +# patches add HLSL sources and the corresponding variant tables alongside. +TEMPLATE_VARIANTS = {} + + +def find_fxc(explicit_fxc, explicit_wine): + # On Windows, look for fxc.exe in the Windows 10 SDK install location + # first, then PATH. On Linux, require Wine + explicit --fxc pointing at + # the Windows fxc.exe install. + if explicit_fxc: + fxc = Path(explicit_fxc) + if not fxc.exists(): + sys.stderr.write(f"error: --fxc path does not exist: {fxc}\n") + sys.exit(1) + return ([str(fxc)] if sys.platform == "win32" else + [explicit_wine or "wine", str(fxc)]) + + # PATH lookup (Windows: fxc.exe; Linux/macOS: unusual but supported). + for name in ("fxc.exe", "fxc"): + path = shutil.which(name) + if path: + return [path] + + # Try the common Windows 10 SDK install location even when not on + # Windows (in case the user has the SDK mounted under Wine prefix). + sdk_roots = [ + Path("C:/Program Files (x86)/Windows Kits/10/bin"), + Path("C:/Program Files/Windows Kits/10/bin"), + ] + # Add Wine-prefix style path if we have one. + wine_prefix = os.environ.get("WINEPREFIX") + if wine_prefix: + sdk_roots.append(Path(wine_prefix) / "drive_c" / "Program Files (x86)" / "Windows Kits" / "10" / "bin") + sdk_roots.append(Path(wine_prefix) / "drive_c" / "Program Files" / "Windows Kits" / "10" / "bin") + + for root in sdk_roots: + if not root.exists(): + continue + # SDK layout: //x64/fxc.exe - pick the highest version. + versions = sorted([p for p in root.iterdir() if p.is_dir() and p.name[0].isdigit()], + reverse=True) + for v in versions: + candidate = v / "x64" / "fxc.exe" + if candidate.exists(): + return ([str(candidate)] if sys.platform == "win32" else + [explicit_wine or "wine", str(candidate)]) + + sys.stderr.write( + "error: fxc.exe not found. Install the Windows 10 SDK (or a newer\n" + " Windows SDK that still ships fxc.exe), or pass --fxc PATH\n" + " pointing at a working fxc.exe binary. On Linux pass --wine\n" + " PATH if `wine` isn't on PATH.\n") + sys.exit(1) + + +def sanitize_identifier(stem): + # Map "vram_read.ps" -> "vram_read_ps"; only [A-Za-z0-9_] survive. + name = re.sub(r"[^A-Za-z0-9_]", "_", stem) + return name + + +def compile_one(fxc_cmd, hlsl_path, target, defines=None): + # fxc /O3 /T /E main /Fo [/D KEY=VALUE]... + # Output to a temp .dxbc.tmp under INC_DIR so we don't pollute /tmp on + # restricted-filesystem hosts. Matches the regen_vulkan_spirv.py pattern. + dxbc_path = INC_DIR / (hlsl_path.stem + ".dxbc.tmp") + cmd = list(fxc_cmd) + [ + "/T", target, + "/E", "main", + "/O3", + "/nologo", + "/Fo", str(dxbc_path), + str(hlsl_path), + ] + for d in defines or (): + cmd.append(f"/D{d}") + result = subprocess.run(cmd, capture_output=True, text=True) + if result.returncode != 0: + sys.stderr.write(f"fxc failed for {hlsl_path.name}:\n") + sys.stderr.write(result.stdout) + sys.stderr.write(result.stderr) + if dxbc_path.exists(): + dxbc_path.unlink() + sys.exit(1) + data = dxbc_path.read_bytes() + dxbc_path.unlink() + if len(data) < 4: + sys.stderr.write( + f"error: {hlsl_path.name} produced {len(data)} bytes of output " + f"(too short to be DXBC)\n") + sys.exit(1) + # DXBC magic: "DXBC" = 0x43425844 little-endian in the first 4 bytes. + magic = struct.unpack("