rsx: Improve vertex textures support

- Adds proper support for vertex textures, including dimensions other than 2D textures
- Minor analyser fixup, removes spurious 'analyser failed' errors
- Minor optimizations for program state tracking

rpcs3/Emu/RSX/Common/GLSLCommon.h

@@ -604,8 +604,13 @@ namespace glsl
 			return "dFdx($0)";
 		case FUNCTION::FUNCTION_DFDY:
 			return "dFdy($0)";
+		case FUNCTION::FUNCTION_VERTEX_TEXTURE_FETCH1D:
+			return "textureLod($t, $0.x, 0)";
 		case FUNCTION::FUNCTION_VERTEX_TEXTURE_FETCH2D:
 			return "textureLod($t, $0.xy, 0)";
+		case FUNCTION::FUNCTION_VERTEX_TEXTURE_FETCH3D:
+		case FUNCTION::FUNCTION_VERTEX_TEXTURE_FETCHCUBE:
+			return "textureLod($t, $0.xyz, 0)";
 		case FUNCTION::FUNCTION_TEXTURE_SAMPLE2D_DEPTH_RGBA:
 			return "TEX2D_DEPTH_RGBA8($_i, $t, $0.xy)";
 		}

rpcs3/Emu/RSX/Common/ProgramStateCache.cpp

@@ -31,7 +31,7 @@ size_t vertex_program_utils::get_vertex_program_ucode_hash(const RSXVertexProgra
 
 vertex_program_utils::vertex_program_metadata vertex_program_utils::analyse_vertex_program(const u32* data, u32 entry, RSXVertexProgram& dst_prog)
 {
-	vertex_program_utils::vertex_program_metadata result;
+	vertex_program_utils::vertex_program_metadata result{};
 	u32 last_instruction_address = 0;
 	u32 first_instruction_address = entry;
 
@@ -79,6 +79,17 @@ vertex_program_utils::vertex_program_metadata vertex_program_utils::analyse_vert
 			instruction_range.first = std::min(current_instrution, instruction_range.first);
 			instruction_range.second = std::max(current_instrution, instruction_range.second);
 
+			// Basic vec op analysis, must be done before flow analysis
+			switch (d1.vec_opcode)
+			{
+			case RSX_VEC_OPCODE_TXL:
+			{
+				d2.HEX = instruction->word[2];
+				result.referenced_textures_mask |= (1 << d2.tex_num);
+				break;
+			}
+			}
+
 			bool static_jump = false;
 			bool function_call = true;
 
@@ -223,7 +234,7 @@ vertex_program_utils::vertex_program_metadata vertex_program_utils::analyse_vert
 		// Verification
 		for (const u32 target : dst_prog.jump_table)
 		{
-			if (!result.instruction_mask[target])
+			if (!dst_prog.instruction_mask[target])
 			{
 				LOG_ERROR(RSX, "vp_analyser: Failed, branch target 0x%x was not resolved", target);
 			}
@@ -237,13 +248,16 @@ size_t vertex_program_storage_hash::operator()(const RSXVertexProgram &program)
 {
 	size_t hash = vertex_program_utils::get_vertex_program_ucode_hash(program);
 	hash ^= program.output_mask;
+	hash ^= program.texture_dimensions;
 	return hash;
 }
 
 bool vertex_program_compare::operator()(const RSXVertexProgram &binary1, const RSXVertexProgram &binary2) const
 {
 	if (binary1.output_mask != binary2.output_mask)
 		return false;
+	if (binary1.texture_dimensions != binary2.texture_dimensions)
+		return false;
 	if (binary1.data.size() != binary2.data.size())
 		return false;
 	if (binary1.jump_table != binary2.jump_table)

rpcs3/Emu/RSX/Common/ProgramStateCache.h

@@ -31,6 +31,7 @@ namespace program_hash_util
 		{
 			std::bitset<512> instruction_mask;
 			u32 ucode_length;
+			u32 referenced_textures_mask;
 		};
 
 		static size_t get_vertex_program_ucode_hash(const RSXVertexProgram &program);

rpcs3/Emu/RSX/Common/VertexProgramDecompiler.cpp

@@ -176,7 +176,24 @@ void VertexProgramDecompiler::SetDST(bool is_sca, std::string value)
 
 std::string VertexProgramDecompiler::GetTex()
 {
-	return m_parr.AddParam(PF_PARAM_UNIFORM, "sampler2D", std::string("vtex") + std::to_string(d2.tex_num));
+	std::string sampler;
+	switch (m_prog.get_texture_dimension(d2.tex_num))
+	{
+	case rsx::texture_dimension_extended::texture_dimension_1d:
+		sampler = "sampler1D";
+		break;
+	case rsx::texture_dimension_extended::texture_dimension_2d:
+		sampler = "sampler2D";
+		break;
+	case rsx::texture_dimension_extended::texture_dimension_3d:
+		sampler = "sampler3D";
+		break;
+	case rsx::texture_dimension_extended::texture_dimension_cubemap:
+		sampler = "samplerCube";
+		break;
+	}
+
+	return m_parr.AddParam(PF_PARAM_UNIFORM, sampler, std::string("vtex") + std::to_string(d2.tex_num));
 }
 
 std::string VertexProgramDecompiler::Format(const std::string& code)
@@ -612,8 +629,26 @@ std::string VertexProgramDecompiler::Decompile()
 		case RSX_VEC_OPCODE_SNE: SetDSTVec(getFloatTypeName(4) + "(" + compareFunction(COMPARE::FUNCTION_SNE, "$0", "$1") + ")"); break;
 		case RSX_VEC_OPCODE_STR: SetDSTVec(getFunction(FUNCTION::FUNCTION_STR)); break;
 		case RSX_VEC_OPCODE_SSG: SetDSTVec("sign($0)"); break;
-		case RSX_VEC_OPCODE_TXL: SetDSTVec(getFunction(FUNCTION::FUNCTION_VERTEX_TEXTURE_FETCH2D)); break;
+		case RSX_VEC_OPCODE_TXL:
+		{
+			switch (m_prog.get_texture_dimension(d2.tex_num))
+			{
+			case rsx::texture_dimension_extended::texture_dimension_1d:
+				SetDSTVec(getFunction(FUNCTION::FUNCTION_VERTEX_TEXTURE_FETCH1D));
+				break;
+			case rsx::texture_dimension_extended::texture_dimension_2d:
+				SetDSTVec(getFunction(FUNCTION::FUNCTION_VERTEX_TEXTURE_FETCH2D));
+				break;
+			case rsx::texture_dimension_extended::texture_dimension_3d:
+				SetDSTVec(getFunction(FUNCTION::FUNCTION_VERTEX_TEXTURE_FETCH3D));
+				break;
+			case rsx::texture_dimension_extended::texture_dimension_cubemap:
+				SetDSTVec(getFunction(FUNCTION::FUNCTION_VERTEX_TEXTURE_FETCHCUBE));
+				break;
+			}
 
+			break;
+		}
 		default:
 			AddCode(fmt::format("//Unknown vp opcode 0x%x", u32{ d1.vec_opcode }));
 			LOG_ERROR(RSX, "Unknown vp opcode 0x%x", u32{ d1.vec_opcode });

rpcs3/Emu/RSX/D3D12/D3D12CommonDecompiler.cpp

@@ -25,7 +25,7 @@ std::string getFunctionImp(FUNCTION f)
 	switch (f)
 	{
 	default:
-		abort();
+		fmt::throw_exception("Unsupported program function %d", (u32)f);
 	case FUNCTION::FUNCTION_DP2:
 		return "dot($0.xy, $1.xy).xxxx";
 	case FUNCTION::FUNCTION_DP2A:

rpcs3/Emu/RSX/D3D12/D3D12PipelineState.cpp

@@ -6,6 +6,7 @@
 #include "D3D12Formats.h"
 #include "../rsx_methods.h"
 #include "../rsx_utils.h"
+#include "../Common/TextureUtils.h"
 
 #define TO_STRING(x) #x
 
@@ -53,7 +54,7 @@ void D3D12GSRender::load_program()
 		return std::make_tuple(true, native_pitch);
 	};
 
-	get_current_vertex_program(false);
+	get_current_vertex_program({}, true);
 	get_current_fragment_program_legacy(rtt_lookup_func);
 
 	if (!current_fragment_program.valid)

rpcs3/Emu/RSX/GL/GLGSRender.cpp

@@ -24,7 +24,7 @@ namespace
 
 GLGSRender::GLGSRender() : GSRender()
 {
-	m_shaders_cache.reset(new gl::shader_cache(m_prog_buffer, "opengl", "v1.5"));
+	m_shaders_cache.reset(new gl::shader_cache(m_prog_buffer, "opengl", "v1.6"));
 
 	if (g_cfg.video.disable_vertex_cache)
 		m_vertex_cache.reset(new gl::null_vertex_cache());
@@ -330,7 +330,7 @@ void GLGSRender::end()
 					*sampler_state = m_gl_texture_cache.upload_texture(unused, rsx::method_registers.fragment_textures[i], m_rtts);
 
 					if (m_textures_dirty[i])
-						m_gl_sampler_states[i].apply(rsx::method_registers.fragment_textures[i], fs_sampler_state[i].get());
+						m_fs_sampler_states[i].apply(rsx::method_registers.fragment_textures[i], fs_sampler_state[i].get());
 				}
 				else
 				{
@@ -354,6 +354,9 @@ void GLGSRender::end()
 				if (rsx::method_registers.vertex_textures[i].enabled())
 				{
 					*sampler_state = m_gl_texture_cache.upload_texture(unused, rsx::method_registers.vertex_textures[i], m_rtts);
+
+					if (m_vertex_textures_dirty[i])
+						m_vs_sampler_states[i].apply(rsx::method_registers.vertex_textures[i], vs_sampler_state[i].get());
 				}
 				else
 					*sampler_state = {};
@@ -783,8 +786,14 @@ void GLGSRender::on_init_thread()
 
 	for (int i = 0; i < rsx::limits::fragment_textures_count; ++i)
 	{
-		m_gl_sampler_states[i].create();
-		m_gl_sampler_states[i].bind(i);
+		m_fs_sampler_states[i].create();
+		m_fs_sampler_states[i].bind(i);
+	}
+
+	for (int i = 0; i < rsx::limits::vertex_textures_count; ++i)
+	{
+		m_vs_sampler_states[i].create();
+		m_vs_sampler_states[i].bind(rsx::limits::fragment_textures_count + i);
 	}
 
 	//Occlusion query
@@ -917,7 +926,12 @@ void GLGSRender::on_exit()
 	m_gl_persistent_stream_buffer.reset();
 	m_gl_volatile_stream_buffer.reset();
 
-	for (auto &sampler : m_gl_sampler_states)
+	for (auto &sampler : m_fs_sampler_states)
+	{
+		sampler.remove();
+	}
+
+	for (auto &sampler : m_vs_sampler_states)
 	{
 		sampler.remove();
 	}
@@ -1101,7 +1115,7 @@ void GLGSRender::load_program(const gl::vertex_upload_info& upload_info)
 		get_current_fragment_program(fs_sampler_state);
 		verify(HERE), current_fragment_program.valid;
 
-		get_current_vertex_program();
+		get_current_vertex_program(vs_sampler_state);
 
 		current_vertex_program.skip_vertex_input_check = true;	//not needed for us since decoding is done server side
 		current_fragment_program.unnormalized_coords = 0; //unused

rpcs3/Emu/RSX/GL/GLGSRender.h

@@ -279,7 +279,8 @@ class GLGSRender : public GSRender, public ::rsx::reports::ZCULL_control
 	GLFragmentProgram m_fragment_prog;
 	GLVertexProgram m_vertex_prog;
 
-	gl::sampler_state m_gl_sampler_states[rsx::limits::fragment_textures_count];
+	gl::sampler_state m_fs_sampler_states[rsx::limits::fragment_textures_count];
+	gl::sampler_state m_vs_sampler_states[rsx::limits::vertex_textures_count];
 
 	gl::glsl::program *m_program;
 

rpcs3/Emu/RSX/GL/GLProgramBuffer.h

@@ -48,14 +48,14 @@ struct GLTraits
 		for (int i = 0; i < rsx::limits::fragment_textures_count; ++i)
 		{
 			int location;
-			if (result.uniforms.has_location("tex" + std::to_string(i), &location))
+			if (result.uniforms.has_location(rsx::constants::fragment_texture_names[i], &location))
 				result.uniforms[location] = i;
 		}
 
 		for (int i = 0; i < rsx::limits::vertex_textures_count; ++i)
 		{
 			int location;
-			if (result.uniforms.has_location("vtex" + std::to_string(i), &location))
+			if (result.uniforms.has_location(rsx::constants::vertex_texture_names[i], &location))
 				result.uniforms[location] = (i + rsx::limits::fragment_textures_count);
 		}
 

rpcs3/Emu/RSX/GL/GLTexture.cpp

@@ -204,7 +204,7 @@ namespace gl
 	}
 
 	//Apply sampler state settings
-	void sampler_state::apply(rsx::fragment_texture& tex, const rsx::sampled_image_descriptor_base* sampled_image)
+	void sampler_state::apply(const rsx::fragment_texture& tex, const rsx::sampled_image_descriptor_base* sampled_image)
 	{
 		const color4f border_color = rsx::decode_border_color(tex.border_color());
 
@@ -274,6 +274,22 @@ namespace gl
 			glSamplerParameteri(samplerHandle, GL_TEXTURE_COMPARE_MODE, GL_NONE);
 	}
 
+	void sampler_state::apply(const rsx::vertex_texture& tex, const rsx::sampled_image_descriptor_base* /*sampled_image*/)
+	{
+		const color4f border_color = rsx::decode_border_color(tex.border_color());
+		glSamplerParameterfv(samplerHandle, GL_TEXTURE_BORDER_COLOR, border_color.rgba);
+
+		glSamplerParameteri(samplerHandle, GL_TEXTURE_WRAP_S, wrap_mode(tex.wrap_s()));
+		glSamplerParameteri(samplerHandle, GL_TEXTURE_WRAP_T, wrap_mode(tex.wrap_t()));
+		glSamplerParameteri(samplerHandle, GL_TEXTURE_WRAP_R, wrap_mode(tex.wrap_r()));
+		glSamplerParameteri(samplerHandle, GL_TEXTURE_MIN_FILTER, GL_NEAREST);
+		glSamplerParameteri(samplerHandle, GL_TEXTURE_MAG_FILTER, GL_NEAREST);
+		glSamplerParameterf(samplerHandle, GL_TEXTURE_LOD_BIAS, tex.bias());
+		glSamplerParameteri(samplerHandle, GL_TEXTURE_MIN_LOD, (tex.min_lod() >> 8));
+		glSamplerParameteri(samplerHandle, GL_TEXTURE_MAX_LOD, (tex.max_lod() >> 8));
+		glSamplerParameteri(samplerHandle, GL_TEXTURE_COMPARE_MODE, GL_NONE);
+	}
+
 	bool is_compressed_format(u32 texture_format)
 	{
 		switch (texture_format)

rpcs3/Emu/RSX/GL/GLTexture.h

@@ -57,6 +57,7 @@ namespace gl
 			glBindSampler(index, samplerHandle);
 		}
 
-		void apply(rsx::fragment_texture& tex, const rsx::sampled_image_descriptor_base* sampled_image);
+		void apply(const rsx::fragment_texture& tex, const rsx::sampled_image_descriptor_base* sampled_image);
+		void apply(const rsx::vertex_texture& tex, const rsx::sampled_image_descriptor_base* sampled_image);
 	};
 }

rpcs3/Emu/RSX/GL/GLVertexProgram.cpp

@@ -191,17 +191,6 @@ void GLVertexDecompilerThread::insertMainStart(std::stringstream & OS)
 			OS << "	vec4 " << PI.name << "= read_location(" << std::to_string(PI.location) << ");\n";
 		}
 	}
-
-	for (const ParamType &PT : m_parr.params[PF_PARAM_UNIFORM])
-	{
-		if (PT.type == "sampler2D")
-		{
-			for (const ParamItem &PI : PT.items)
-			{
-				OS << "	vec2 " << PI.name << "_coord_scale = vec2(1.);\n";
-			}
-		}
-	}
 }
 
 void GLVertexDecompilerThread::insertMainEnd(std::stringstream & OS)