Fix normals under nonuniform scale

include/MeshInstanceRenderer.hpp

@@ -25,7 +25,7 @@ class MeshInstanceRenderer {
 private:
     struct InstanceData {
         glm::mat4 world;
-        glm::mat4 rotation;
+        glm::mat4 normal;
     };
     GLuint buffer_{};
     std::vector<const MeshRenderCommand*> sorted_;

include/Transform.hpp

@@ -1,29 +1,44 @@
 #ifndef TRANSFORM_HPP_INCLUDED
 #define TRANSFORM_HPP_INCLUDED
 
+#include <cmath>
 #include <glm/ext/matrix_float4x4.hpp>
 #include <glm/ext/matrix_transform.hpp>
 #include <glm/ext/vector_float3.hpp>
 #include <glm/trigonometric.hpp>
 
 // Position uses world units; Euler angles use degrees in Z * Y * X order.
 struct Transform {
-  glm::vec3 position{0.0F};
-  glm::vec3 rotationDegrees{0.0F};
-  glm::vec3 scale{1.0F};
+    glm::vec3 position{0.0F};
+    glm::vec3 rotationDegrees{0.0F};
+    glm::vec3 scale{1.0F};
 
-  [[nodiscard]] glm::mat4 RotationMatrix() const {
-    return glm::rotate(glm::mat4(1.0F), glm::radians(rotationDegrees.z),
-                       glm::vec3(0.0F, 0.0F, 1.0F)) *
-           glm::rotate(glm::mat4(1.0F), glm::radians(rotationDegrees.y),
-                       glm::vec3(0.0F, 1.0F, 0.0F)) *
-           glm::rotate(glm::mat4(1.0F), glm::radians(rotationDegrees.x),
-                       glm::vec3(1.0F, 0.0F, 0.0F));
-  }
+    [[nodiscard]] glm::mat4 RotationMatrix() const {
+        return glm::rotate(glm::mat4(1.0F), glm::radians(rotationDegrees.z),
+                           glm::vec3(0.0F, 0.0F, 1.0F)) *
+               glm::rotate(glm::mat4(1.0F), glm::radians(rotationDegrees.y),
+                           glm::vec3(0.0F, 1.0F, 0.0F)) *
+               glm::rotate(glm::mat4(1.0F), glm::radians(rotationDegrees.x),
+                           glm::vec3(1.0F, 0.0F, 0.0F));
+    }
 
     [[nodiscard]] glm::mat4 ModelMatrix() const {
-      return glm::translate(glm::mat4(1.0F), position) * RotationMatrix() *
-             glm::scale(glm::mat4(1.0F), scale);
+        return glm::translate(glm::mat4(1.0F), position) * RotationMatrix() *
+               glm::scale(glm::mat4(1.0F), scale);
+    }
+
+    [[nodiscard]] glm::mat4 NormalMatrix() const {
+        constexpr float kMinimumScale = 1e-6F;
+        if (std::abs(scale.x) <= kMinimumScale ||
+            std::abs(scale.y) <= kMinimumScale ||
+            std::abs(scale.z) <= kMinimumScale) {
+            // A collapsed surface has no unique normal. Keep the orientation
+            // finite and stable instead of producing infinities or NaNs.
+            return RotationMatrix();
+        }
+        const glm::vec3 inverseScale(1.0F / scale.x, 1.0F / scale.y,
+                                     1.0F / scale.z);
+        return RotationMatrix() * glm::scale(glm::mat4(1.0F), inverseScale);
     }
 };
 

shaders/DSGeometryPass.vsh

@@ -21,10 +21,10 @@ in vec3 s_vTangent;
 void main()                                                                   
 {                                                                             
 	mat4 world = instanced ? instanceWorld : gWorld;
-	mat4 rotation = instanced ? instanceRotation : mRotate;
+	mat4 normalMatrix = instanced ? instanceRotation : mRotate;
 	gl_Position = gVC*world *vec4(s_vPosition, 1.0);
 	//gl_Position = gWorld *vec4(s_vPosition, 1.0);	
-	fN=(rotation*s_vNormal).xyz;
+	fN = (normalMatrix * vec4(s_vNormal.xyz, 0.0)).xyz;
 	//fL=-dLightDir;
 	//fE=(gVC*gWorld*vec4(s_vCamPos,1)).xyz;
 	UV=s_vUV;

shaders/vertexShader.vsh

@@ -39,10 +39,10 @@ uniform float pLightInt;
 void main()                                                                   
 {                                                                             
 	mat4 world = instanced ? instanceWorld : gWorld;
-	mat4 rotation = instanced ? instanceRotation : mRotate;
+	mat4 normalMatrix = instanced ? instanceRotation : mRotate;
 	vec3 worldPosition = (world * vec4(s_vPosition, 1.0)).xyz;
 	gl_Position = gVC * vec4(worldPosition, 1.0);
-	fN=(rotation*s_vNormal).xyz;
+	fN = (normalMatrix * vec4(s_vNormal.xyz, 0.0)).xyz;
 	fL=-dLightDir;
 	fE=worldPosition-s_vCamPos;
 	UV=s_vUV;

src/Mesh.cpp

@@ -175,7 +175,7 @@ void Mesh::Draw(const Transform& transform, const Material& material,
                 const MeshShaderBindings& bindings, float opacity,
                 GLuint instanceBuffer, GLsizei instanceCount) const {
     const glm::mat4 modelMatrix = transform.ModelMatrix();
-    const glm::mat4 rotationMatrix = transform.RotationMatrix();
+    const glm::mat4 normalMatrix = transform.NormalMatrix();
 
     material.Use(opacity);
     glUniform1i(bindings.instanced, instanceBuffer != 0);
@@ -196,7 +196,7 @@ void Mesh::Draw(const Transform& transform, const Material& material,
     glUniformMatrix4fv(bindings.world, 1, GL_FALSE,
                        glm::value_ptr(modelMatrix));
     glUniformMatrix4fv(bindings.rotation, 1, GL_FALSE,
-                       glm::value_ptr(rotationMatrix));  // rotation of model
+                       glm::value_ptr(normalMatrix));
 
     glEnableVertexAttribArray(bindings.position);
     glEnableVertexAttribArray(bindings.normal);

src/MeshInstanceRenderer.cpp

@@ -44,7 +44,7 @@ void MeshInstanceRenderer::Render(std::span<const MeshRenderCommand> commands,
             for (std::size_t index = begin; index < end; ++index) {
                 const auto& transform = sorted_[index]->transform;
                 instances_.push_back(
-                    {transform.ModelMatrix(), transform.RotationMatrix()});
+                    {transform.ModelMatrix(), transform.NormalMatrix()});
             }
             if (instances_.size() > static_cast<std::size_t>(
                                         std::numeric_limits<GLsizei>::max()) ||

tests/MathMigrationTests.cpp

@@ -1,15 +1,16 @@
 #include <GLFW/glfw3.h>
 #include <gtest/gtest.h>
 
+#include <cmath>
 #include <glm/ext/matrix_float4x4.hpp>
 #include <glm/ext/vector_float3.hpp>
 #include <glm/geometric.hpp>
 #include <glm/gtc/type_ptr.hpp>
 
 #include "Camera.hpp"
 #include "FontMetrics.hpp"
-#include "RenderQueue.hpp"
 #include "RenderMode.hpp"
+#include "RenderQueue.hpp"
 #include "Text2D.hpp"
 #include "TextLayout.hpp"
 #include "Transform.hpp"
@@ -79,6 +80,36 @@ TEST(ObjectTransform, RotationMatrixOmitsTranslationAndScale) {
     EXPECT_NEAR(rotated.w, 1.0F, kTolerance);
 }
 
+TEST(ObjectTransform,
+     NormalMatrixPreservesPerpendicularityAfterNonuniformScale) {
+    const Transform transform{.rotationDegrees = glm::vec3(20, 35, 10),
+                              .scale = glm::vec3(2.0F, 1.0F, 0.5F)};
+    const glm::vec3 localNormal = glm::normalize(glm::vec3(1, 1, 0));
+    const glm::vec3 localTangent = glm::normalize(glm::vec3(1, -1, 0));
+
+    const glm::vec3 worldNormal = glm::normalize(
+        glm::vec3(transform.NormalMatrix() * glm::vec4(localNormal, 0.0F)));
+    const glm::vec3 worldTangent = glm::normalize(
+        glm::vec3(transform.ModelMatrix() * glm::vec4(localTangent, 0.0F)));
+
+    EXPECT_NEAR(glm::dot(worldNormal, worldTangent), 0.0F, kTolerance);
+}
+
+TEST(ObjectTransform, NormalMatrixUsesFiniteRotationFallbackForCollapsedScale) {
+    const Transform transform{.rotationDegrees = glm::vec3(20, 35, 10),
+                              .scale = glm::vec3(0.0F, 2.0F, 3.0F)};
+    const glm::mat4 normalMatrix = transform.NormalMatrix();
+    const glm::mat4 rotationMatrix = transform.RotationMatrix();
+
+    for (glm::length_t column = 0; column < 4; ++column) {
+        for (glm::length_t row = 0; row < 4; ++row) {
+            EXPECT_TRUE(std::isfinite(normalMatrix[column][row]));
+            EXPECT_NEAR(normalMatrix[column][row], rotationMatrix[column][row],
+                        kTolerance);
+        }
+    }
+}
+
 TEST(ObjectTransform, TransformableKeepsSpatialStateTogether) {
     Transformable object;
     object.SetPosition(1.0F, 2.0F, 3.0F);