From 28091f1c27ed1a01900ecad083613f8e42a392cc Mon Sep 17 00:00:00 2001 From: Einar Forselv Date: Mon, 23 Mar 2020 21:08:19 +0100 Subject: [PATCH] Support #69 --- docs/reference/loaders/separate.rst | 5 - moderngl_window/loaders/program/separate.py | 33 ++++- moderngl_window/loaders/program/single.py | 1 + moderngl_window/opengl/program.py | 132 ++++++++++++++++-- .../resources/programs/include_test.glsl | 24 ++++ .../programs/includes/blend_functions.glsl | 8 ++ .../resources/programs/includes/utils.glsl | 2 + .../resources/programs/includes/utils_1.glsl | 2 + .../resources/programs/includes/utils_2.glsl | 2 + tests/test_loaders_program.py | 4 + tests/test_shader_source.py | 61 ++++++++ 11 files changed, 248 insertions(+), 26 deletions(-) create mode 100644 tests/fixtures/resources/programs/include_test.glsl create mode 100644 tests/fixtures/resources/programs/includes/blend_functions.glsl create mode 100644 tests/fixtures/resources/programs/includes/utils.glsl create mode 100644 tests/fixtures/resources/programs/includes/utils_1.glsl create mode 100644 tests/fixtures/resources/programs/includes/utils_2.glsl diff --git a/docs/reference/loaders/separate.rst b/docs/reference/loaders/separate.rst index 84fe1968..80541816 100644 --- a/docs/reference/loaders/separate.rst +++ b/docs/reference/loaders/separate.rst @@ -17,11 +17,6 @@ Method .. automethod:: Loader.find_texture .. automethod:: Loader.find_scene -Loader Specific Methods ------------------------ - -.. automethod:: Loader.load_shader - Attributes ---------- diff --git a/moderngl_window/loaders/program/separate.py b/moderngl_window/loaders/program/separate.py index c60a37c8..db8b24d7 100644 --- a/moderngl_window/loaders/program/separate.py +++ b/moderngl_window/loaders/program/separate.py @@ -26,12 +26,12 @@ def load(self) -> Union[ """ prog = None - vs_source = self._load_source("vertex", self.meta.vertex_shader) - geo_source = self._load_source("geometry", self.meta.geometry_shader) - fs_source = self._load_source("fragment", self.meta.fragment_shader) - tc_source = self._load_source("tess_control", self.meta.tess_control_shader) - te_source = self._load_source("tess_evaluation", self.meta.tess_evaluation_shader) - cs_source = self._load_source("compute", self.meta.compute_shader) + vs_source = self._load_shader("vertex", self.meta.vertex_shader) + geo_source = self._load_shader("geometry", self.meta.geometry_shader) + fs_source = self._load_shader("fragment", self.meta.fragment_shader) + tc_source = self._load_shader("tess_control", self.meta.tess_control_shader) + te_source = self._load_shader("tess_evaluation", self.meta.tess_evaluation_shader) + cs_source = self._load_shader("compute", self.meta.compute_shader) if vs_source: shaders = program.ProgramShaders.from_separate( @@ -42,6 +42,7 @@ def load(self) -> Union[ tess_control_source=tc_source, tess_evaluation_source=te_source, ) + shaders.handle_includes(self._load_source) prog = shaders.create() # Wrap the program if reloadable is set @@ -52,13 +53,14 @@ def load(self) -> Union[ prog = program.ReloadableProgram(self.meta, prog) elif cs_source: shaders = program.ProgramShaders.compute_shader(self.meta, cs_source) + shaders.handle_includes(self._load_source) prog = shaders.create_compute_shader() else: raise ImproperlyConfigured("Cannot find a shader source to load") return prog - def _load_source(self, shader_type: str, path: str): + def _load_shader(self, shader_type: str, path: str): """Load a single shader source""" if path: resolved_path = self.find_program(path) @@ -69,3 +71,20 @@ def _load_source(self, shader_type: str, path: str): with open(str(resolved_path), 'r') as fd: return fd.read() + + def _load_source(self, path): + """Finds and loads a single source file. + + Args: + path: Path to resource + Returns: + Tuple[resolved_path, source]: The resolved path and the source + """ + resolved_path = self.find_program(path) + if not resolved_path: + raise ImproperlyConfigured("Cannot find program '{}'".format(path)) + + logger.info("Loading: %s", path) + + with open(str(resolved_path), 'r') as fd: + return resolved_path, fd.read() diff --git a/moderngl_window/loaders/program/single.py b/moderngl_window/loaders/program/single.py index f857f70d..28613c75 100644 --- a/moderngl_window/loaders/program/single.py +++ b/moderngl_window/loaders/program/single.py @@ -55,6 +55,7 @@ def load(self) -> moderngl.Program: """ self.meta.resolved_path, source = self._load_source(self.meta.path) shaders = program.ProgramShaders.from_single(self.meta, source) + shaders.handle_includes(self._load_source) prog = shaders.create() # Wrap the program if reloadable is set diff --git a/moderngl_window/opengl/program.py b/moderngl_window/opengl/program.py index 181f4a9b..a2be4f77 100644 --- a/moderngl_window/opengl/program.py +++ b/moderngl_window/opengl/program.py @@ -167,38 +167,142 @@ def create(self): program.extra = {'meta': self.meta} return program + def handle_includes(self, load_source_func): + """Resolves ``#include`` preprocessors + + Args: + load_source_func (func): A function for finding and loading a source + """ + if self.vertex_source: + self.vertex_source.handle_includes(load_source_func) + if self.geometry_source: + self.geometry_source.handle_includes(load_source_func) + if self.fragment_source: + self.fragment_source.handle_includes(load_source_func) + if self.tess_control_source: + self.tess_control_source.handle_includes(load_source_func) + if self.tess_evaluation_source: + self.tess_evaluation_source.handle_includes(load_source_func) + if self.compute_shader_source: + self.compute_shader_source.handle_includes(load_source_func) + class ShaderSource: """ - Helper class representing a single shader type + Helper class representing a single shader type. + + It ensures the source has the right format, injects ``#define`` preprocessors, + resolves ``#include`` preprocessors etc. + + A ``ShaderSource`` can be the base/root shader or a source referenced in an ``#include``. """ - def __init__(self, shader_type: str, name: str, source: str, defines: dict = None): - self.type = shader_type - self.name = name - self.source = source.strip() - self.lines = self.source.split('\n') - - # Make sure version is present - if not self.lines[0].startswith("#version"): + def __init__(self, shader_type: str, name: str, source: str, defines: dict = None, id=0, root=True): + """Create shader source. + + Args: + shader_type (str): A preprocessor name for setting the shader type + name (str): A string (usually the path) so we can give useful error messages to the user + source (str): The raw source for the shader + Keyword Args: + id (int): The source number. Used when shader consists of multiple sources through includes + root (bool): If this shader source is the root shader (Not an include) + """ + self._id = id + self._root = root + self._source_list = [self] # List of sources this shader consists of (original source + includes) + self._type = shader_type + self._name = name + self._defines = defines or {} + if root: + source = source.strip() + self._lines = source.split('\n') + + # Make sure version is present (only if root shader) + if self._root and not self._lines[0].startswith("#version"): self.print() raise ShaderError( - "Missing #version in {}. A version must be defined in the first line".format(self.name), + "Missing #version in {}. A version must be defined in the first line".format(self._name), ) self.apply_defines(defines) # Inject source with shade type - self.lines.insert(1, "#define {} 1".format(self.type)) - self.lines.insert(2, "#line 2") + if self._root: + self._lines.insert(1, "#define {} 1".format(self._type)) + self._lines.insert(2, "#line 2") + + @property + def id(self) -> int: + """int: The shader number/id""" + return self._id - self.source = '\n'.join(self.lines) + @property + def source(self) -> str: + """str: The source lines as a string""" + return '\n'.join(self._lines) + + @property + def source_list(self) -> List['ShaderSource']: + """List[ShaderSource]: List of all shader sources""" + return self._source_list + + @property + def name(self) -> str: + """str: a path or name for this shader""" + return self._name + + @property + def lines(self) -> List[str]: + """List[str]: The lines in this shader""" + return self._lines + + @property + def line_count(self) -> int: + """int: Number of lines in this source (stripped)""" + return len(self._lines) + + @property + def defines(self) -> dict: + """dict: Defines configured for this shader""" + return self._defines + + def handle_includes(self, load_source_func, depth=0): + """Inject includes into the shader source. + This happens recursively up to a max level in case the users has circular includes. + We also build up a list of all the included sources in the root shader. + + Args: + load_source_func (func): A function for finding and loading a source + depth (int): The current include depth (incease by 1 for every call) + """ + if depth > 100: + raise ShaderError("Reaching an include depth of 100. You probably have circular includes") + + current_id = 0 + while True: + for nr, line in enumerate(self._lines): + line = line.strip() + if line.startswith('#include'): + path = line[9:] + current_id += 1 + source = ShaderSource( + None, path, load_source_func(path)[1], + defines=self._defines, id=current_id, root=False, + ) + source.handle_includes(load_source_func, depth=depth + 1) + self._lines = self.lines[:nr] + source.lines + self.lines[nr + 1:] + self._source_list += source.source_list + current_id = self._source_list[-1].id + break + else: + break def apply_defines(self, defines: dict): """Apply the configured define values""" if not defines: return - for nr, line in enumerate(self.lines): + for nr, line in enumerate(self._lines): line = line.strip() if line.startswith('#define'): try: diff --git a/tests/fixtures/resources/programs/include_test.glsl b/tests/fixtures/resources/programs/include_test.glsl new file mode 100644 index 00000000..b58580b2 --- /dev/null +++ b/tests/fixtures/resources/programs/include_test.glsl @@ -0,0 +1,24 @@ +#version 330 +#include programs/includes/blend_functions.glsl +#include programs/includes/utils.glsl + +#define TEST 1 + +#if defined VERTEX_SHADER + +in vec3 in_position; + +void main() { + gl_Position = vec4(in_position, 1.0); +} + + +#elif defined FRAGMENT_SHADER + +out vec4 fragColor; + +void main() { + fragColor = vec4(1.0); +} + +#endif diff --git a/tests/fixtures/resources/programs/includes/blend_functions.glsl b/tests/fixtures/resources/programs/includes/blend_functions.glsl new file mode 100644 index 00000000..45142585 --- /dev/null +++ b/tests/fixtures/resources/programs/includes/blend_functions.glsl @@ -0,0 +1,8 @@ + +vec3 blendMultiply(vec3 base, vec3 blend) { + return base * blend; +} + +vec3 blendMultiply(vec3 base, vec3 blend, float opacity) { + return (blendMultiply(base, blend) * opacity + base * (1.0 - opacity)); +} diff --git a/tests/fixtures/resources/programs/includes/utils.glsl b/tests/fixtures/resources/programs/includes/utils.glsl new file mode 100644 index 00000000..1a1e8287 --- /dev/null +++ b/tests/fixtures/resources/programs/includes/utils.glsl @@ -0,0 +1,2 @@ +#include programs/includes/utils_1.glsl +#include programs/includes/utils_2.glsl diff --git a/tests/fixtures/resources/programs/includes/utils_1.glsl b/tests/fixtures/resources/programs/includes/utils_1.glsl new file mode 100644 index 00000000..6cc40e84 --- /dev/null +++ b/tests/fixtures/resources/programs/includes/utils_1.glsl @@ -0,0 +1,2 @@ +// Utils 1 +#define TEST 1 diff --git a/tests/fixtures/resources/programs/includes/utils_2.glsl b/tests/fixtures/resources/programs/includes/utils_2.glsl new file mode 100644 index 00000000..968b2c39 --- /dev/null +++ b/tests/fixtures/resources/programs/includes/utils_2.glsl @@ -0,0 +1,2 @@ +// Utils 2 +#define TEST 1 diff --git a/tests/test_loaders_program.py b/tests/test_loaders_program.py index 7f3bcecf..b16672d6 100644 --- a/tests/test_loaders_program.py +++ b/tests/test_loaders_program.py @@ -111,3 +111,7 @@ def test_compute_shader(self): self.assertEqual(descr.compute_shader, path) program = resources.programs.load(descr) self.assertIsInstance(program, moderngl.ComputeShader) + + def test_include(self): + program = resources.programs.load(ProgramDescription(path='programs/include_test.glsl')) + self.assertIsInstance(program, moderngl.Program) diff --git a/tests/test_shader_source.py b/tests/test_shader_source.py index 0c4d441e..29ac3e34 100644 --- a/tests/test_shader_source.py +++ b/tests/test_shader_source.py @@ -1,8 +1,14 @@ +from pathlib import Path from unittest import TestCase from moderngl_window.opengl import program +from moderngl_window import resources +from moderngl_window.meta import DataDescription + +resources.register_dir((Path(__file__).parent / 'fixtures/resources').resolve()) class ShaderSourceTestCase(TestCase): + maxDiff = None def test_no_version(self): """Missing #version statement should raise a shader error""" @@ -70,3 +76,58 @@ def test_define(self): ) self.assertTrue("#define NUM_THINGS 100" in shader.source) self.assertTrue("#define SCALE 2.0" in shader.source) + + def test_include(self): + """Test include""" + def load_source(path): + return path, resources.data.load(DataDescription(path, kind='text')) + + path = 'programs/include_test.glsl' + source = load_source(path)[1] + source_vs = program.ShaderSource(program.VERTEX_SHADER, path, source, defines={'TEST': '100'}) + source_vs.handle_includes(load_source) + + # print(source_vs.source) + self.assertEqual(source_vs.source, INCLUDE_RESULT) + + +INCLUDE_RESULT = """#version 330 +#define VERTEX_SHADER 1 +#line 2 + +vec3 blendMultiply(vec3 base, vec3 blend) { + return base * blend; +} + +vec3 blendMultiply(vec3 base, vec3 blend, float opacity) { + return (blendMultiply(base, blend) * opacity + base * (1.0 - opacity)); +} + +// Utils 1 +#define TEST 100 + +// Utils 2 +#define TEST 100 + + + +#define TEST 100 + +#if defined VERTEX_SHADER + +in vec3 in_position; + +void main() { + gl_Position = vec4(in_position, 1.0); +} + + +#elif defined FRAGMENT_SHADER + +out vec4 fragColor; + +void main() { + fragColor = vec4(1.0); +} + +#endif"""