From 2ca886f64f56f47fffc9b4125458b728352c9e7e Mon Sep 17 00:00:00 2001
From: offtkp <parisoplop@gmail.com>
Date: Wed, 17 Jul 2024 22:08:48 +0300
Subject: [PATCH] Move normal calculation to the fragment shader

---
 src/host_shaders/opengl_fragment_shader.frag | 30 +++++++++++++++-----
 src/host_shaders/opengl_vertex_shader.vert   | 16 ++---------
 2 files changed, 26 insertions(+), 20 deletions(-)

diff --git a/src/host_shaders/opengl_fragment_shader.frag b/src/host_shaders/opengl_fragment_shader.frag
index ae43d9937..582d6eefe 100644
--- a/src/host_shaders/opengl_fragment_shader.frag
+++ b/src/host_shaders/opengl_fragment_shader.frag
@@ -1,8 +1,6 @@
 #version 410 core
 
-in vec3 v_tangent;
-in vec3 v_normal;
-in vec3 v_bitangent;
+in vec4 v_quaternion;
 in vec4 v_colour;
 in vec3 v_texcoord0;
 in vec2 v_texcoord1;
@@ -37,6 +35,7 @@ uint readPicaReg(uint reg_addr) { return u_picaRegs[reg_addr - 0x48u]; }
 vec4 tevSources[16];
 vec4 tevNextPreviousBuffer;
 bool tevUnimplementedSourceFlag = false;
+vec3 normal;
 
 // Holds the enabled state of the lighting samples for various PICA configurations
 // As explained in https://www.3dbrew.org/wiki/GPU/Internal_Registers#GPUREG_LIGHTING_CONFIG0
@@ -255,7 +254,7 @@ float lightLutLookup(uint environment_id, uint lut_id, uint light_id, vec3 light
 	uint input_id = bitfieldExtract(GPUREG_LIGHTING_LUTINPUT_SELECT, int(lut_id) << 2, 3);
 	switch (input_id) {
 		case 0u: {
-			delta = dot(v_normal, normalize(half_vector));
+			delta = dot(normal, normalize(half_vector));
 			break;
 		}
 		case 1u: {
@@ -263,11 +262,11 @@ float lightLutLookup(uint environment_id, uint lut_id, uint light_id, vec3 light
 			break;
 		}
 		case 2u: {
-			delta = dot(v_normal, normalize(v_view));
+			delta = dot(normal, normalize(v_view));
 			break;
 		}
 		case 3u: {
-			delta = dot(light_vector, v_normal);
+			delta = dot(light_vector, normal);
 			break;
 		}
 		case 4u: {
@@ -313,6 +312,12 @@ float lightLutLookup(uint environment_id, uint lut_id, uint light_id, vec3 light
 	}
 }
 
+vec3 rotateVec3ByQuaternion(vec3 v, vec4 q) {
+	vec3 u = q.xyz;
+	float s = q.w;
+	return 2.0 * dot(u, v) * u + (s * s - dot(u, u)) * v + 2.0 * s * cross(u, v);
+}
+
 // Implements the following algorthm: https://mathb.in/26766
 void calcLighting(out vec4 primary_color, out vec4 secondary_color) {
 	error_unimpl = false;
@@ -336,6 +341,17 @@ void calcLighting(out vec4 primary_color, out vec4 secondary_color) {
 	GPUREG_LIGHTING_LUTINPUT_ABS = readPicaReg(0x01D0u);
 	GPUREG_LIGHTING_LUTINPUT_SELECT = readPicaReg(0x01D1u);
 
+	uint bump_mode = bitfieldExtract(GPUREG_LIGHTING_CONFIG0, 28, 2);
+
+	// Bump mode is ignored for now because it breaks some games ie. Toad Treasure Tracker
+	// Could be because the texture is not sampled correctly, may need the clamp/border color configurations
+	switch (bump_mode) {
+		default: {
+			normal = rotateVec3ByQuaternion(vec3(0.0, 0.0, 1.0), v_quaternion);
+			break;
+		}
+	}
+
 	vec4 diffuse_sum = vec4(0.0, 0.0, 0.0, 1.0);
 	vec4 specular_sum = vec4(0.0, 0.0, 0.0, 1.0);
 
@@ -377,7 +393,7 @@ void calcLighting(out vec4 primary_color, out vec4 secondary_color) {
 		light_vector = normalize(light_vector);
 		half_vector = light_vector + normalize(v_view);
 
-		float NdotL = dot(v_normal, light_vector);  // N dot Li
+		float NdotL = dot(normal, light_vector);  // N dot Li
 
 		// Two sided diffuse
 		if (bitfieldExtract(GPUREG_LIGHTi_CONFIG, 1, 1) == 0u)
diff --git a/src/host_shaders/opengl_vertex_shader.vert b/src/host_shaders/opengl_vertex_shader.vert
index a25d7a6d7..057f9a88b 100644
--- a/src/host_shaders/opengl_vertex_shader.vert
+++ b/src/host_shaders/opengl_vertex_shader.vert
@@ -9,9 +9,7 @@ layout(location = 5) in float a_texcoord0_w;
 layout(location = 6) in vec3 a_view;
 layout(location = 7) in vec2 a_texcoord2;
 
-out vec3 v_normal;
-out vec3 v_tangent;
-out vec3 v_bitangent;
+out vec4 v_quaternion;
 out vec4 v_colour;
 out vec3 v_texcoord0;
 out vec2 v_texcoord1;
@@ -35,12 +33,6 @@ vec4 abgr8888ToVec4(uint abgr) {
 	return scale * vec4(float(abgr & 0xffu), float((abgr >> 8) & 0xffu), float((abgr >> 16) & 0xffu), float(abgr >> 24));
 }
 
-vec3 rotateVec3ByQuaternion(vec3 v, vec4 q) {
-	vec3 u = q.xyz;
-	float s = q.w;
-	return 2.0 * dot(u, v) * u + (s * s - dot(u, u)) * v + 2.0 * s * cross(u, v);
-}
-
 // Convert an arbitrary-width floating point literal to an f32
 float decodeFP(uint hex, uint E, uint M) {
 	uint width = M + E + 1u;
@@ -73,10 +65,6 @@ void main() {
 	v_texcoord2 = vec2(a_texcoord2.x, 1.0 - a_texcoord2.y);
 	v_view = a_view;
 
-	v_normal = normalize(rotateVec3ByQuaternion(vec3(0.0, 0.0, 1.0), a_quaternion));
-	v_tangent = normalize(rotateVec3ByQuaternion(vec3(1.0, 0.0, 0.0), a_quaternion));
-	v_bitangent = normalize(rotateVec3ByQuaternion(vec3(0.0, 1.0, 0.0), a_quaternion));
-
 	for (int i = 0; i < 6; i++) {
 		v_textureEnvColor[i] = abgr8888ToVec4(u_textureEnvColor[i]);
 	}
@@ -95,4 +83,6 @@ void main() {
 	// There's also another, always-on clipping plane based on vertex z
 	gl_ClipDistance[0] = -a_coords.z;
 	gl_ClipDistance[1] = dot(clipData, a_coords);
+
+	v_quaternion = a_quaternion;
 }