Geometry Shader

[SpiderPig]

Just got a geometry shader working to only calculate normals for flat shading.  The only part that I have ever been and continue to be stumped at is the tangent and bitangent.  Am I on the right track using the edges of the triangle or is it more complex than that?

#version 450
#extension GL_GOOGLE_include_directive : enable
#extension GL_ARB_separate_shader_objects : enable

layout (triangles) in;
layout (triangle_strip, max_vertices = 3) out;

layout(location = 0) in vec4 color[];
layout(location = 1) in vec3 normal[];
layout(location = 2) in vec4 texcoords[];
layout(location = 3) in vec3 tangent[];
layout(location = 4) in vec3 bitangent[];
layout(location = 5) flat in uint materialID[];
layout(location = 6) in vec4 vertexCameraPosition[];
layout(location = 7) in vec4 vertexWorldPosition[];
layout(location = 9) in flat uint entityflags[];

layout(location = 0) out vec4 _color;
layout(location = 1) out vec3 _normal;
layout(location = 2) out vec4 _texcoords;
layout(location = 3) out vec3 _tangent;
layout(location = 4) out vec3 _bitangent;
layout(location = 5) flat out uint _materialID;
layout(location = 6) out vec4 _vertexCameraPosition;
layout(location = 7) out vec4 _vertexWorldPosition;
layout(location = 9) out flat uint _entityflags;
  
void main() {
	vec3 e1 = normalize(gl_in[1].gl_Position.xyz - gl_in[0].gl_Position.xyz);
	vec3 e2 = normalize(gl_in[2].gl_Position.xyz - gl_in[0].gl_Position.xyz);
	vec3 norm = normalize(cross(e1, e2));

	for(int v = 0; v < 3; v++){
		gl_Position = gl_in[v].gl_Position; 
		_color = color[v];
		_normal = norm;
		_tangent = e1;
		_bitangent = e2;
		_texcoords = texcoords[v];
		_materialID = materialID[v];
		_vertexWorldPosition = vertexWorldPosition[v];
		_vertexCameraPosition = vertexCameraPosition[v];
		_entityflags = entityflags[v];
		EmitVertex();
	}
    EndPrimitive();
} 

[StOneDOes]

Tangents run perpendicular to the normal in the direction of the texture U coordinate, and the bitangent runs perpendicular to the normal and tangent in the direction of texture V coordinate. You need to calculate the TU vector and TV vector, and you can then use these vectors and your cross vectors to derive the tangent and bitangent. I couldn't remember the exact formula, but I think this thread should provide the answer: https://stackoverflow.com/questions/5255806/how-to-calculate-tangent-and-binormal

I hope this helps.

 

[SpiderPig]

That should get me on the right track, thanks for that.

[Josh]

This is how the engine does it, but if you are procedurally generating the texture coordinates, it's a lot easier to reuse those vectors for the binormal/tangent, instead of reconstructing them from the texcoords:

bool Mesh::UpdateTangents()
{
	Vec3 s, t, result;
	float sgna, sgnb, sgnc, m, r;
	float v1x,v1y,v1z,v1u,v1v;
	float v2x,v2y,v2z,v2u,v2v;
	float v3x,v3y,v3z,v3u,v3v;
	float x1,x2,y1,y2,z1,z2,s1,s2,t1,t2;
	int a, b, c, i, d;
	Vec3 position;
	Vec2 texcoords;
	std::vector<Vec3> normals(vertices.size());
	for (int v = 0; v < vertices.size(); ++v)
	{
		normals[v] = vertices[v].normal;
	}
	std::vector<bool> vertexupdated(vertices.size());

	if (polygonpoints < 3) return false;
	auto polys = CountPrimitives();
	for (i = 0; i < polys; i++)
	{
		a = GetPrimitiveVertex(i,0);
		b = GetPrimitiveVertex(i,1);
		c = GetPrimitiveVertex(i,2);
		d = 0;
		if (polygonpoints == 4)
		{
			d = GetPrimitiveVertex(i, 3);
		}

		if ((vertexupdated[a]==false) or (vertexupdated[b]==false) or (vertexupdated[c]==false) or (polygonpoints == 4 and vertexupdated[d] == false))
		{
			//This should never happen, but skip if it does
			if (a == b or b == c or c == a) continue;

			v1x = vertices[a].position.x; v1y = vertices[a].position.y; v1z = vertices[a].position.z; v1u = m_vertices[a].texcoords[0]; v1v = m_vertices[a].texcoords[1];
			v2x = vertices[b].position.x; v2y = vertices[b].position.y; v2z = vertices[b].position.z; v2u = m_vertices[b].texcoords[0]; v2v = m_vertices[b].texcoords[1];
			v3x = vertices[c].position.x; v3y = vertices[c].position.y; v3z = vertices[c].position.z; v3u = m_vertices[c].texcoords[0]; v3v = m_vertices[c].texcoords[1];

			x1 = v2x - v1x; x2 = v3x - v1x;
			y1 = v2y - v1y; y2 = v3y - v1y;
			z1 = v2z - v1z; z2 = v3z - v1z;
			s1 = v2u - v1u; s2 = v3u - v1u;
			t1 = v2v - v1v; t2 = v3v - v1v;
			
			m = (s1 * t2 - s2 * t1);
			if (m == 0.0f) continue;
			r = 1.0f / m;
			s.x = (t2*x1-t1*x2) * Sign(r); s.y = (t2*y1-t1*y2)*Sign(r); s.z = (t2*z1-t1*z2)*Sign(r);
			t.x = (s1*x2-s2*x1) * Sign(r); t.y = (s1*y2-s2*y1)*Sign(r); t.z = (s1*z2-s2*z1)*Sign(r);
			m = s.Length();
			if (m == 0.0f) continue;
			s /= m;
			m = t.Length();
			if (m == 0.0f) continue;
			t /= m;
			s.Cross(normals[a], result);
			sgna = Sign(t.Dot(result));
			s.Cross(normals[b], result);
			sgnb = Sign(t.Dot(result));
			s.Cross(normals[c], result);
			sgnc = Sign(t.Dot(result));
			if (vertexupdated[a]==false)
			{
				vertexupdated[a]=true;
				m_vertices[a].tangent = s;
			}
			if (vertexupdated[b]==false)
			{
				vertexupdated[b]=true;
				m_vertices[b].tangent = s;
			}
			if (vertexupdated[c]==false)
			{
				vertexupdated[c]=true;
				m_vertices[c].tangent = s;
			}
			if (polygonpoints == 4)
			{
				if (vertexupdated[d] == false)
				{
					vertexupdated[d] = true;
					m_vertices[d].tangent = s;
				}
			}
		}
	}
	Finalize();
	return true;
}

 

[SpiderPig]

1 hour ago, Josh said:

This is how the engine does it, but if you are procedurally generating the texture coordinates, it's a lot easier to reuse those vectors for the binormal/tangent, instead of reconstructing them from the texcoords:

Thanks that'll help a lot.  I'm using the xyz position as the texcoords and will setup triplaner texturing soon.  Is that what you mean by procedually generated texcoords?  I need to learn more about what the tangent and bitangent actually represent.  Slowly getting a grasp on it I think.

[Josh]

Yes, if you're doing tri-planar mapping, then you know the vectors you are using to generate the texture coordinates, so you can use them for tangent/bitangent . These just mean "the horizontal and vertical directions the texture is oriented with in 3D space relative to the mesh".

If you are just using the X and Z positions for texcoords, for example, then your tangent and bitangent would just be (1,0,0) and (0,0,1).

"Binormal" is sometimes used instead of "bitangent", and accidentally say it sometimes, but it's not the correct term actually.

[SpiderPig]

Had to use the vertex world position for normal calculations instead of gl_Position.  This enables flat shading on shared vertices.

#version 450
#extension GL_GOOGLE_include_directive : enable
#extension GL_ARB_separate_shader_objects : enable

#include "../../Shaders/Base/EntityInfo.glsl"

layout (triangles) in;
layout (triangle_strip, max_vertices = 3) out;

layout(location = 0) in vec4 color[];
layout(location = 1) in vec3 normal[];
layout(location = 2) in vec4 texcoords[];
layout(location = 3) in vec3 tangent[];
layout(location = 4) in vec3 bitangent[];
layout(location = 5) flat in uint materialID[];
layout(location = 6) in vec4 vertexCameraPosition[];
layout(location = 7) in vec4 vertexWorldPosition[];
layout(location = 9) in flat uint entityflags[];
layout(location = 25) in flat uint entityID[];
layout(location = 23) in vec3 screenvelocity[];

layout(location = 0) out vec4 _color;
layout(location = 1) out vec3 _normal;
layout(location = 2) out vec4 _texcoords;
layout(location = 3) out vec3 _tangent;
layout(location = 4) out vec3 _bitangent;
layout(location = 5) flat out uint _materialID;
layout(location = 6) out vec4 _vertexCameraPosition;
layout(location = 7) out vec4 _vertexWorldPosition;
layout(location = 9) out flat uint _entityflags;
layout(location = 25) out flat uint _entityID;
layout(location = 23) out vec3 _screenvelocity;


void main() {
	vec3 e1 = normalize(vertexWorldPosition[1].xyz - vertexWorldPosition[0].xyz);
	vec3 e2 = normalize(vertexWorldPosition[2].xyz - vertexWorldPosition[0].xyz);
	vec3 flat_norm = normalize(cross(e1, e2));

	for(int v = 0; v < 3; v++){
	
		gl_Position = gl_in[v].gl_Position; 
		_color = color[v];
		_normal = flat_norm;
		_tangent = tangent[v];
		_bitangent = bitangent[v];
		_texcoords = texcoords[v];
		_materialID = materialID[v];
		_vertexWorldPosition = vertexWorldPosition[v];
		_vertexCameraPosition = vertexCameraPosition[v];
		_entityflags = entityflags[v];
		_entityID = entityID[v];
		_screenvelocity = screenvelocity[v];
		EmitVertex();
	}
    EndPrimitive();
}